How to run deep networks in browser

Original author	Dmitry Kurtaev
Compatibility	OpenCV >= 3.3.1

Introduction

This tutorial will show us how to run deep learning models using OpenCV.js right in a browser. Tutorial refers a sample of face detection and face recognition models pipeline.

Face detection

Face detection network gets BGR image as input and produces set of bounding boxes that might contain faces. All that we need is just select the boxes with a strong confidence.

Face recognition

Network is called OpenFace (project cmusatyalab/openface). Face recognition model receives RGB face image of size 96x96. Then it returns 128-dimensional unit vector that represents input face as a point on the unit multidimensional sphere. So difference between two faces is an angle between two output vectors.

Sample

All the sample is an HTML page that has JavaScript code to use OpenCV.js functionality. You may see an insertion of this page below. Press Start button to begin a demo. Press Add a person to name a person that is recognized as an unknown one. Next we’ll discuss main parts of the code.

Start

Add a person

1. Run face detection network to detect faces on input image.

function detectFaces(img) {
  netDet.setInputSize(new cv.Size(img.cols, img.rows));
  var out = new cv.Mat();
  netDet.detect(img, out);
  var faces = [];
  for (var i = 0, n = out.data32F.length; i < n; i += 15) {
    var left = out.data32F[i];
    var top = out.data32F[i + 1];
    var right = (out.data32F[i] + out.data32F[i + 2]);
    var bottom = (out.data32F[i + 1] + out.data32F[i + 3]);
    left = Math.min(Math.max(0, left), img.cols - 1);
    top = Math.min(Math.max(0, top), img.rows - 1);
    right = Math.min(Math.max(0, right), img.cols - 1);
    bottom = Math.min(Math.max(0, bottom), img.rows - 1);

    if (left < right && top < bottom) {
      faces.push({
        x: left,
        y: top,
        width: right - left,
        height: bottom - top,
        x1: out.data32F[i + 4] < 0 || out.data32F[i + 4] > img.cols - 1 ? -1 : out.data32F[i + 4],
        y1: out.data32F[i + 5] < 0 || out.data32F[i + 5] > img.rows - 1 ? -1 : out.data32F[i + 5],
        x2: out.data32F[i + 6] < 0 || out.data32F[i + 6] > img.cols - 1 ? -1 : out.data32F[i + 6],
        y2: out.data32F[i + 7] < 0 || out.data32F[i + 7] > img.rows - 1 ? -1 : out.data32F[i + 7],
        x3: out.data32F[i + 8] < 0 || out.data32F[i + 8] > img.cols - 1 ? -1 : out.data32F[i + 8],
        y3: out.data32F[i + 9] < 0 || out.data32F[i + 9] > img.rows - 1 ? -1 : out.data32F[i + 9],
        x4: out.data32F[i + 10] < 0 || out.data32F[i + 10] > img.cols - 1 ? -1 : out.data32F[i + 10],
        y4: out.data32F[i + 11] < 0 || out.data32F[i + 11] > img.rows - 1 ? -1 : out.data32F[i + 11],
        x5: out.data32F[i + 12] < 0 || out.data32F[i + 12] > img.cols - 1 ? -1 : out.data32F[i + 12],
        y5: out.data32F[i + 13] < 0 || out.data32F[i + 13] > img.rows - 1 ? -1 : out.data32F[i + 13],
        confidence: out.data32F[i + 14]
      })
    }
  }
  out.delete();
  return faces;
};

You may play with input blob sizes to balance detection quality and efficiency. The bigger input blob the smaller faces may be detected.

1. Run face recognition network to receive 128-dimensional unit feature vector by input face image.

function face2vec(face) {
  var blob = cv.blobFromImage(face, 1.0, {width: 112, height: 112}, [0, 0, 0, 0], true, false)
  netRecogn.setInput(blob);
  var vec = netRecogn.forward();
  blob.delete();
  return vec;
};

1. Perform a recognition.

function recognize(face) {
  var vec = face2vec(face);

  var bestMatchName = 'unknown';
  var bestMatchScore = 30;  // Threshold for face recognition.
  for (name in persons) {
    var personVec = persons[name];
    var score = vec.dot(personVec);
    if (score > bestMatchScore) {
      bestMatchScore = score;
      bestMatchName = name;
    }
  }
  vec.delete();
  return bestMatchName;
};

Match a new feature vector with registered ones. Return a name of the best matched person.

1. The main loop.

var isRunning = false;
const FPS = 30;  // Target number of frames processed per second.
function captureFrame() {
  var begin = Date.now();
  cap.read(frame);  // Read a frame from camera
  cv.cvtColor(frame, frameBGR, cv.COLOR_RGBA2BGR);

  var faces = detectFaces(frameBGR);
  faces.forEach(function(rect) {
    cv.rectangle(frame, {x: rect.x, y: rect.y}, {x: rect.x + rect.width, y: rect.y + rect.height}, [0, 255, 0, 255]);
    if(rect.x1>0 && rect.y1>0)
      cv.circle(frame, {x: rect.x1, y: rect.y1}, 2, [255, 0, 0, 255], 2)
    if(rect.x2>0 && rect.y2>0)
      cv.circle(frame, {x: rect.x2, y: rect.y2}, 2, [0, 0, 255, 255], 2)
    if(rect.x3>0 && rect.y3>0)
      cv.circle(frame, {x: rect.x3, y: rect.y3}, 2, [0, 255, 0, 255], 2)
    if(rect.x4>0 && rect.y4>0)
      cv.circle(frame, {x: rect.x4, y: rect.y4}, 2, [255, 0, 255, 255], 2)
    if(rect.x5>0 && rect.y5>0)
      cv.circle(frame, {x: rect.x5, y: rect.y5}, 2, [0, 255, 255, 255], 2)

    var face = frameBGR.roi(rect);
    var name = recognize(face);
    cv.putText(frame, name, {x: rect.x, y: rect.y}, cv.FONT_HERSHEY_SIMPLEX, 1.0, [0, 255, 0, 255]);
  });

  cv.imshow(output, frame);

  // Loop this function.
  if (isRunning) {
    var delay = 1000 / FPS - (Date.now() - begin);
    setTimeout(captureFrame, delay);
  }
};

A main loop of our application receives a frames from a camera and makes a recognition of an every detected face on the frame. We start this function ones when OpenCV.js was initialized and deep learning models were downloaded.