Capture Gray code pattern tutorial

Goal

In this tutorial you will learn how to use the GrayCodePattern class to:

• Generate a Gray code pattern.
• Project the Gray code pattern.
• Capture the projected Gray code pattern.

It is important to underline that GrayCodePattern class actually implements the 3DUNDERWORLD algorithm described in [101] , which is based on a stereo approach: we need to capture the projected pattern at the same time from two different views if we want to reconstruct the 3D model of the scanned object. Thus, an acquisition set consists of the images captured by each camera for each image in the pattern sequence.

Code

/*M///////////////////////////////////////////////////////////////////////////////////////
 //
 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
 //
 //  By downloading, copying, installing or using the software you agree to this license.
 //  If you do not agree to this license, do not download, install,
 //  copy or use the software.
 //
 //
 //                           License Agreement
 //                For Open Source Computer Vision Library
 //
 // Copyright (C) 2015, OpenCV Foundation, all rights reserved.
 // Third party copyrights are property of their respective owners.
 //
 // Redistribution and use in source and binary forms, with or without modification,
 // are permitted provided that the following conditions are met:
 //
 //   * Redistribution's of source code must retain the above copyright notice,
 //     this list of conditions and the following disclaimer.
 //
 //   * Redistribution's in binary form must reproduce the above copyright notice,
 //     this list of conditions and the following disclaimer in the documentation
 //     and/or other materials provided with the distribution.
 //
 //   * The name of the copyright holders may not be used to endorse or promote products
 //     derived from this software without specific prior written permission.
 //
 // This software is provided by the copyright holders and contributors "as is" and
 // any express or implied warranties, including, but not limited to, the implied
 // warranties of merchantability and fitness for a particular purpose are disclaimed.
 // In no event shall the Intel Corporation or contributors be liable for any direct,
 // indirect, incidental, special, exemplary, or consequential damages
 // (including, but not limited to, procurement of substitute goods or services;
 // loss of use, data, or profits; or business interruption) however caused
 // and on any theory of liability, whether in contract, strict liability,
 // or tort (including negligence or otherwise) arising in any way out of
 // the use of this software, even if advised of the possibility of such damage.
 //
 //M*/

#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/structured_light.hpp>
#include <iostream>
#include <stdio.h>

using namespace cv;
using namespace std;

static const char* keys =
{ "{@path | | Path of the folder where the captured pattern images will be save }"
     "{@proj_width      | | Projector width            }"
     "{@proj_height     | | Projector height           }" };

static void help()
{
    cout << "\nThis example shows how to use the \"Structured Light module\" to acquire a graycode pattern"
         "\nCall (with the two cams connected):\n"
         "./example_structured_light_cap_pattern <path> <proj_width> <proj_height> \n"
         << endl;
}

int main( int argc, char** argv )
{

  structured_light::GrayCodePattern::Params params;

  CommandLineParser parser( argc, argv, keys );
  String path = parser.get<String>( 0 );
  params.width = parser.get<int>( 1 );
  params.height = parser.get<int>( 2 );

  if( path.empty() || params.width < 1 || params.height < 1 )
  {
    help();
    return -1;
  }

  // Set up GraycodePattern with params
  Ptr<structured_light::GrayCodePattern> graycode = structured_light::GrayCodePattern::create( params );

  // Storage for pattern
  vector<Mat> pattern;
  graycode->generate( pattern );

  cout << pattern.size() << " pattern images + 2 images for shadows mask computation to acquire with both cameras"
         << endl;

  // Generate the all-white and all-black images needed for shadows mask computation
  Mat white;
  Mat black;
  graycode->getImagesForShadowMasks( black, white );

  pattern.push_back( white );
  pattern.push_back( black );

  // Setting pattern window on second monitor (the projector's one)
  namedWindow( "Pattern Window", WINDOW_NORMAL );
  resizeWindow( "Pattern Window", params.width, params.height );
  moveWindow( "Pattern Window", params.width + 316, -20 );
  setWindowProperty( "Pattern Window", WND_PROP_FULLSCREEN, WINDOW_FULLSCREEN );

  // Open camera number 1, using libgphoto2
  VideoCapture cap1( CAP_GPHOTO2 );

  if( !cap1.isOpened() )
  {
    // check if cam1 opened
    cout << "cam1 not opened!" << endl;
    help();
    return -1;
  }

  // Open camera number 2
  VideoCapture cap2( 1 );

  if( !cap2.isOpened() )
  {
     // check if cam2 opened
     cout << "cam2 not opened!" << endl;
     help();
     return -1;
  }

  // Turning off autofocus
  cap1.set( CAP_PROP_SETTINGS, 1 );
  cap2.set( CAP_PROP_SETTINGS, 1 );

  int i = 0;
  while( i < (int) pattern.size() )
  {
    cout << "Waiting to save image number " << i + 1 << endl << "Press any key to acquire the photo" << endl;
    imshow( "Pattern Window", pattern[i] );

    Mat frame1;
    Mat frame2;

    cap1 >> frame1;  // get a new frame from camera 1
    cap2 >> frame2;  // get a new frame from camera 2

    if( ( frame1.data ) && ( frame2.data ) )
    {

      Mat tmp;
      cout << "cam 1 size: " << Size( ( int ) cap1.get( CAP_PROP_FRAME_WIDTH ), ( int ) cap1.get( CAP_PROP_FRAME_HEIGHT ) )
           << endl;

      cout << "cam 2 size: " << Size( ( int ) cap2.get( CAP_PROP_FRAME_WIDTH ), ( int ) cap2.get( CAP_PROP_FRAME_HEIGHT ) )
           << endl;

      cout << "zoom cam 1: " << cap1.get( CAP_PROP_ZOOM ) << endl << "zoom cam 2: " << cap2.get( CAP_PROP_ZOOM )
           << endl;

      cout << "focus cam 1: " << cap1.get( CAP_PROP_FOCUS ) << endl << "focus cam 2: " << cap2.get( CAP_PROP_FOCUS )
           << endl;

      cout << "Press enter to save the photo or an other key to re-acquire the photo" << endl;

      namedWindow( "cam1", WINDOW_NORMAL );
      resizeWindow( "cam1", 640, 480 );

      namedWindow( "cam2", WINDOW_NORMAL );
      resizeWindow( "cam2", 640, 480 );

      // Moving window of cam2 to see the image at the same time with cam1
      moveWindow( "cam2", 640 + 75, 0 );

      // Resizing images to avoid issues for high resolution images, visualizing them as grayscale
      resize( frame1, tmp, Size( 640, 480 ), 0, 0, INTER_LINEAR_EXACT);
      cvtColor( tmp, tmp, COLOR_RGB2GRAY );
      imshow( "cam1", tmp );
      resize( frame2, tmp, Size( 640, 480 ), 0, 0, INTER_LINEAR_EXACT);
      cvtColor( tmp, tmp, COLOR_RGB2GRAY );
      imshow( "cam2", tmp );

      bool save1 = false;
      bool save2 = false;

      int key = waitKey( 0 );

      // Pressing enter, it saves the output
      if( key == 13 )
      {
        ostringstream name;
        name << i + 1;

        save1 = imwrite( path + "pattern_cam1_im" + name.str() + ".png", frame1 );
        save2 = imwrite( path + "pattern_cam2_im" + name.str() + ".png", frame2 );

        if( ( save1 ) && ( save2 ) )
        {
          cout << "pattern cam1 and cam2 images number " << i + 1 << " saved" << endl << endl;
          i++;
        }
        else
        {
          cout << "pattern cam1 and cam2 images number " << i + 1 << " NOT saved" << endl << endl << "Retry, check the path"<< endl << endl;
        }
      }
      // Pressing escape, the program closes
      if( key == 27 )
      {
        cout << "Closing program" << endl;
      }
    }
    else
    {
      cout << "No frame data, waiting for new frame" << endl;
    }
  }

  // the camera will be deinitialized automatically in VideoCapture destructor
  return 0;
}

Explanation

First of all the pattern images to project must be generated. Since the number of images is a function of the projector's resolution, GrayCodePattern class parameters must be set with our projector's width and height. In this way the generate method can be called: it fills a vector of Mat with the computed pattern images:

structured_light::GrayCodePattern::Params params;
  ....
params.width = parser.get<int>( 1 );
params.height = parser.get<int>( 2 );
  ....
// Set up GraycodePattern with params
Ptr<structured_light::GrayCodePattern> graycode = structured_light::GrayCodePattern::create( params );
// Storage for pattern
vector<Mat> pattern;
graycode->generate( pattern );

For example, using the default projector resolution (1024 x 768), 40 images have to be projected: 20 for regular color pattern (10 images for the columns sequence and 10 for the rows one) and 20 for the color-inverted pattern, where the inverted pattern images are images with the same structure as the original but with inverted colors. This provides an effective method for easily determining the intensity value of each pixel when it is lit (highest value) and when it is not lit (lowest value) during the decoding step.

Subsequently, to identify shadow regions, the regions of two images where the pixels are not lit by projector's light and thus where there is not code information, the 3DUNDERWORLD algorithm computes a shadow mask for the two cameras views, starting from a white and a black images captured by each camera. So two additional images need to be projected and captured with both cameras:

// Generate the all-white and all-black images needed for shadows mask computation
Mat white;
Mat black;
graycode->getImagesForShadowMasks( black, white );
pattern.push_back( white );
pattern.push_back( black );

Thus, the final projection sequence is projected as follows: first the column and its inverted sequence, then the row and its inverted sequence and finally the white and black images.

Once the pattern images have been generated, they must be projected using the full screen option: the images must fill all the projection area, otherwise the projector full resolution is not exploited, a condition on which is based 3DUNDERWORLD implementation.

// Setting pattern window on second monitor (the projector's one)
namedWindow( "Pattern Window", WINDOW_NORMAL );
resizeWindow( "Pattern Window", params.width, params.height );
moveWindow( "Pattern Window", params.width + 316, -20 );
setWindowProperty( "Pattern Window", WND_PROP_FULLSCREEN, WINDOW_FULLSCREEN );

At this point the images can be captured with our digital cameras, using libgphoto2 library, recently included in OpenCV: remember to turn on gPhoto2 option in Cmake.list when building OpenCV.

// Open camera number 1, using libgphoto2
VideoCapture cap1( CAP_GPHOTO2 );
if( !cap1.isOpened() )
{
  // check if cam1 opened
  cout << "cam1 not opened!" << endl;
  help();
  return -1;
}
// Open camera number 2
VideoCapture cap2( 1 );
if( !cap2.isOpened() )
{
   // check if cam2 opened
   cout << "cam2 not opened!" << endl;
   help();
   return -1;
}

The two cameras must work at the same resolution and must have autofocus option disabled, maintaining the same focus during all acquisition. The projector can be positioned in the middle of the cameras.

However, before to proceed with pattern acquisition, the cameras must be calibrated. Once the calibration is performed, there should be no movement of the cameras, otherwise a new calibration will be needed.

After having connected the cameras and the projector to the computer, cap_pattern demo can be launched giving as parameters the path where to save the images, and the projector's width and height, taking care to use the same focus and cameras settings of calibration.

At this point, to acquire the images with both cameras, the user can press any key.

// Turning off autofocus
cap1.set( CAP_PROP_SETTINGS, 1 );
cap2.set( CAP_PROP_SETTINGS, 1 );
int i = 0;
while( i < (int) pattern.size() )
{
  cout << "Waiting to save image number " << i + 1 << endl << "Press any key to acquire the photo" << endl;
  imshow( "Pattern Window", pattern[i] );
  Mat frame1;
  Mat frame2;
  cap1 >> frame1;  // get a new frame from camera 1
  cap2 >> frame2;  // get a new frame from camera 2
   ...
}

If the captured images are good (the user must take care that the projected pattern is viewed from the two cameras), the user can save them pressing the enter key, otherwise pressing any other key he can take another shot.

// Pressing enter, it saves the output
if( key == 13 )
{
  ostringstream name;
  name << i + 1;
  save1 = imwrite( path + "pattern_cam1_im" + name.str() + ".png", frame1 );
  save2 = imwrite( path + "pattern_cam2_im" + name.str() + ".png", frame2 );
  if( ( save1 ) && ( save2 ) )
  {
    cout << "pattern cam1 and cam2 images number " << i + 1 << " saved" << endl << endl;
    i++;
  }
  else
  {
    cout << "pattern cam1 and cam2 images number " << i + 1 << " NOT saved" << endl << endl << "Retry, check the path"<< endl << endl;
  }
}

The acquistion ends when all the pattern images have saved for both cameras. Then the user can reconstruct the 3D model of the captured scene using the decode method of GrayCodePattern class (see next tutorial).