Line Features Tutorial

In this tutorial it will be shown how to:

• Use the BinaryDescriptor interface to extract the lines and store them in KeyLine objects
• Use the same interface to compute descriptors for every extracted line
• Use the BynaryDescriptorMatcher to determine matches among descriptors obtained from different images

Lines extraction and descriptors computation

In the following snippet of code, it is shown how to detect lines from an image. The LSD extractor is initialized with LSD_REFINE_ADV option; remaining parameters are left to their default values. A mask of ones is used in order to accept all extracted lines, which, at the end, are displayed using random colors for octave 0.

/*M///////////////////////////////////////////////////////////////////////////////////////
 //
 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
 //
 //  By downloading, copying, installing or using the software you agree to this license.
 //  If you do not agree to this license, do not download, install,
 //  copy or use the software.
 //
 //
 //                           License Agreement
 //                For Open Source Computer Vision Library
 //
 // Copyright (C) 2014, Biagio Montesano, all rights reserved.
 // Third party copyrights are property of their respective owners.
 //
 // Redistribution and use in source and binary forms, with or without modification,
 // are permitted provided that the following conditions are met:
 //
 //   * Redistribution's of source code must retain the above copyright notice,
 //     this list of conditions and the following disclaimer.
 //
 //   * Redistribution's in binary form must reproduce the above copyright notice,
 //     this list of conditions and the following disclaimer in the documentation
 //     and/or other materials provided with the distribution.
 //
 //   * The name of the copyright holders may not be used to endorse or promote products
 //     derived from this software without specific prior written permission.
 //
 // This software is provided by the copyright holders and contributors "as is" and
 // any express or implied warranties, including, but not limited to, the implied
 // warranties of merchantability and fitness for a particular purpose are disclaimed.
 // In no event shall the Intel Corporation or contributors be liable for any direct,
 // indirect, incidental, special, exemplary, or consequential damages
 // (including, but not limited to, procurement of substitute goods or services;
 // loss of use, data, or profits; or business interruption) however caused
 // and on any theory of liability, whether in contract, strict liability,
 // or tort (including negligence or otherwise) arising in any way out of
 // the use of this software, even if advised of the possibility of such damage.
 //
 //M*/

#include <iostream>
#include <opencv2/opencv_modules.hpp>

#ifdef HAVE_OPENCV_FEATURES2D

#include <opencv2/line_descriptor.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/highgui.hpp>

using namespace cv;
using namespace cv::line_descriptor;
using namespace std;

static const char* keys =
{ "{@image_path | | Image path }" };

static void help()
{
  cout << "\nThis example shows the functionalities of lines extraction " << "furnished by BinaryDescriptor class\n"
       << "Please, run this sample using a command in the form\n" << "./example_line_descriptor_lines_extraction <path_to_input_image>" << endl;
}

int main( int argc, char** argv )
{
  /* get parameters from comand line */
  CommandLineParser parser( argc, argv, keys );
  String image_path = parser.get<String>( 0 );

  if( image_path.empty() )
  {
    help();
    return -1;
  }

  /* load image */
  cv::Mat imageMat = imread( image_path, 1 );
  if( imageMat.data == NULL )
  {
    std::cout << "Error, image could not be loaded. Please, check its path" << std::endl;
    return -1;
  }

  /* create a random binary mask */
  cv::Mat mask = Mat::ones( imageMat.size(), CV_8UC1 );

  /* create a pointer to a BinaryDescriptor object with deafult parameters */
  Ptr<LSDDetector> bd = LSDDetector::createLSDDetector();

  /* create a structure to store extracted lines */
  vector<KeyLine> lines;

  /* extract lines */
  cv::Mat output = imageMat.clone();
  bd->detect( imageMat, lines, 2, 1, mask );

  /* draw lines extracted from octave 0 */
  if( output.channels() == 1 )
    cvtColor( output, output, COLOR_GRAY2BGR );
  for ( size_t i = 0; i < lines.size(); i++ )
  {
    KeyLine kl = lines[i];
    if( kl.octave == 0)
    {
      /* get a random color */
      int R = ( rand() % (int) ( 255 + 1 ) );
      int G = ( rand() % (int) ( 255 + 1 ) );
      int B = ( rand() % (int) ( 255 + 1 ) );

      /* get extremes of line */
      Point pt1 = Point2f( kl.startPointX, kl.startPointY );
      Point pt2 = Point2f( kl.endPointX, kl.endPointY );

      /* draw line */
      line( output, pt1, pt2, Scalar( B, G, R ), 3 );
    }

  }

  /* show lines on image */
  imshow( "LSD lines", output );
  waitKey();
}

#else

int main()
{
    std::cerr << "OpenCV was built without features2d module" << std::endl;
    return 0;
}

#endif // HAVE_OPENCV_FEATURES2D

This is the result obtained from the famous cameraman image:

alternate text

Another way to extract lines is using LSDDetector class; such class uses the LSD extractor to compute lines. To obtain this result, it is sufficient to use the snippet code seen above, just modifying it by the rows

// create a pointer to an LSDDetector object
Ptr<LSDDetector> lsd = LSDDetector::createLSDDetector();

// compute lines
std::vector<KeyLine> keylines;
lsd->detect( imageMat, keylines, mask );

Here's the result returned by LSD detector again on cameraman picture:

alternate text

Once keylines have been detected, it is possible to compute their descriptors as shown in the following:

/*M///////////////////////////////////////////////////////////////////////////////////////
 //
 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
 //
 //  By downloading, copying, installing or using the software you agree to this license.
 //  If you do not agree to this license, do not download, install,
 //  copy or use the software.
 //
 //
 //                           License Agreement
 //                For Open Source Computer Vision Library
 //
 // Copyright (C) 2014, Biagio Montesano, all rights reserved.
 // Third party copyrights are property of their respective owners.
 //
 // Redistribution and use in source and binary forms, with or without modification,
 // are permitted provided that the following conditions are met:
 //
 //   * Redistribution's of source code must retain the above copyright notice,
 //     this list of conditions and the following disclaimer.
 //
 //   * Redistribution's in binary form must reproduce the above copyright notice,
 //     this list of conditions and the following disclaimer in the documentation
 //     and/or other materials provided with the distribution.
 //
 //   * The name of the copyright holders may not be used to endorse or promote products
 //     derived from this software without specific prior written permission.
 //
 // This software is provided by the copyright holders and contributors "as is" and
 // any express or implied warranties, including, but not limited to, the implied
 // warranties of merchantability and fitness for a particular purpose are disclaimed.
 // In no event shall the Intel Corporation or contributors be liable for any direct,
 // indirect, incidental, special, exemplary, or consequential damages
 // (including, but not limited to, procurement of substitute goods or services;
 // loss of use, data, or profits; or business interruption) however caused
 // and on any theory of liability, whether in contract, strict liability,
 // or tort (including negligence or otherwise) arising in any way out of
 // the use of this software, even if advised of the possibility of such damage.
 //
 //M*/

#include <iostream>
#include <opencv2/opencv_modules.hpp>

#ifdef HAVE_OPENCV_FEATURES2D

#include <opencv2/line_descriptor.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/highgui.hpp>

using namespace cv;
using namespace cv::line_descriptor;


static const char* keys =
{ "{@image_path | | Image path }" };

static void help()
{
  std::cout << "\nThis example shows the functionalities of lines extraction " << "and descriptors computation furnished by BinaryDescriptor class\n"
            << "Please, run this sample using a command in the form\n" << "./example_line_descriptor_compute_descriptors <path_to_input_image>"
            << std::endl;
}

int main( int argc, char** argv )
{
  /* get parameters from command line */
  CommandLineParser parser( argc, argv, keys );
  String image_path = parser.get<String>( 0 );

  if( image_path.empty() )
  {
    help();
    return -1;
  }

  /* load image */
  cv::Mat imageMat = imread( image_path, 1 );
  if( imageMat.data == NULL )
  {
    std::cout << "Error, image could not be loaded. Please, check its path" << std::endl;
  }

  /* create a binary mask */
  cv::Mat mask = Mat::ones( imageMat.size(), CV_8UC1 );

  /* create a pointer to a BinaryDescriptor object with default parameters */
  Ptr<BinaryDescriptor> bd = BinaryDescriptor::createBinaryDescriptor();

  /* compute lines */
  std::vector<KeyLine> keylines;
  bd->detect( imageMat, keylines, mask );

  /* compute descriptors */
  cv::Mat descriptors;

  bd->compute( imageMat, keylines, descriptors);

}

#else

int main()
{
    std::cerr << "OpenCV was built without features2d module" << std::endl;
    return 0;
}

#endif // HAVE_OPENCV_FEATURES2D

Matching among descriptors

If we have extracted descriptors from two different images, it is possible to search for matches among them. One way of doing it is matching exactly a descriptor to each input query descriptor, choosing the one at closest distance:

/*M///////////////////////////////////////////////////////////////////////////////////////
 //
 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
 //
 //  By downloading, copying, installing or using the software you agree to this license.
 //  If you do not agree to this license, do not download, install,
 //  copy or use the software.
 //
 //
 //                           License Agreement
 //                For Open Source Computer Vision Library
 //
 // Copyright (C) 2014, Biagio Montesano, all rights reserved.
 // Third party copyrights are property of their respective owners.
 //
 // Redistribution and use in source and binary forms, with or without modification,
 // are permitted provided that the following conditions are met:
 //
 //   * Redistribution's of source code must retain the above copyright notice,
 //     this list of conditions and the following disclaimer.
 //
 //   * Redistribution's in binary form must reproduce the above copyright notice,
 //     this list of conditions and the following disclaimer in the documentation
 //     and/or other materials provided with the distribution.
 //
 //   * The name of the copyright holders may not be used to endorse or promote products
 //     derived from this software without specific prior written permission.
 //
 // This software is provided by the copyright holders and contributors "as is" and
 // any express or implied warranties, including, but not limited to, the implied
 // warranties of merchantability and fitness for a particular purpose are disclaimed.
 // In no event shall the Intel Corporation or contributors be liable for any direct,
 // indirect, incidental, special, exemplary, or consequential damages
 // (including, but not limited to, procurement of substitute goods or services;
 // loss of use, data, or profits; or business interruption) however caused
 // and on any theory of liability, whether in contract, strict liability,
 // or tort (including negligence or otherwise) arising in any way out of
 // the use of this software, even if advised of the possibility of such damage.
 //
 //M*/

#include <iostream>
#include <opencv2/opencv_modules.hpp>

#ifdef HAVE_OPENCV_FEATURES2D

#include <opencv2/line_descriptor.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/highgui.hpp>

#define MATCHES_DIST_THRESHOLD 25

using namespace cv;
using namespace cv::line_descriptor;

static const char* keys =
{ "{@image_path1 | | Image path 1 }"
    "{@image_path2 | | Image path 2 }" };

static void help()
{
  std::cout << "\nThis example shows the functionalities of lines extraction " << "and descriptors computation furnished by BinaryDescriptor class\n"
            << "Please, run this sample using a command in the form\n" << "./example_line_descriptor_compute_descriptors <path_to_input_image 1>"
            << "<path_to_input_image 2>" << std::endl;

}

int main( int argc, char** argv )
{
  /* get parameters from command line */
  CommandLineParser parser( argc, argv, keys );
  String image_path1 = parser.get<String>( 0 );
  String image_path2 = parser.get<String>( 1 );

  if( image_path1.empty() || image_path2.empty() )
  {
    help();
    return -1;
  }

  /* load image */
  cv::Mat imageMat1 = imread( image_path1, 1 );
  cv::Mat imageMat2 = imread( image_path2, 1 );

  if( imageMat1.data == NULL || imageMat2.data == NULL )
  {
    std::cout << "Error, images could not be loaded. Please, check their path" << std::endl;
  }

  /* create binary masks */
  cv::Mat mask1 = Mat::ones( imageMat1.size(), CV_8UC1 );
  cv::Mat mask2 = Mat::ones( imageMat2.size(), CV_8UC1 );

  /* create a pointer to a BinaryDescriptor object with default parameters */
  Ptr<BinaryDescriptor> bd = BinaryDescriptor::createBinaryDescriptor(  );

  /* compute lines and descriptors */
  std::vector<KeyLine> keylines1, keylines2;
  cv::Mat descr1, descr2;

  ( *bd )( imageMat1, mask1, keylines1, descr1, false, false );
  ( *bd )( imageMat2, mask2, keylines2, descr2, false, false );

  /* select keylines from first octave and their descriptors */
  std::vector<KeyLine> lbd_octave1, lbd_octave2;
  Mat left_lbd, right_lbd;
  for ( int i = 0; i < (int) keylines1.size(); i++ )
  {
    if( keylines1[i].octave == 0 )
    {
      lbd_octave1.push_back( keylines1[i] );
      left_lbd.push_back( descr1.row( i ) );
    }
  }

  for ( int j = 0; j < (int) keylines2.size(); j++ )
  {
    if( keylines2[j].octave == 0 )
    {
      lbd_octave2.push_back( keylines2[j] );
      right_lbd.push_back( descr2.row( j ) );
    }
  }

  /* create a BinaryDescriptorMatcher object */
  Ptr<BinaryDescriptorMatcher> bdm = BinaryDescriptorMatcher::createBinaryDescriptorMatcher();

  /* require match */
  std::vector<DMatch> matches;
  bdm->match( left_lbd, right_lbd, matches );

  /* select best matches */
  std::vector<DMatch> good_matches;
  for ( int i = 0; i < (int) matches.size(); i++ )
  {
    if( matches[i].distance < MATCHES_DIST_THRESHOLD )
      good_matches.push_back( matches[i] );
  }

  /* plot matches */
  cv::Mat outImg;
  cv::Mat scaled1, scaled2;
  std::vector<char> mask( matches.size(), 1 );
  drawLineMatches( imageMat1, lbd_octave1, imageMat2, lbd_octave2, good_matches, outImg, Scalar::all( -1 ), Scalar::all( -1 ), mask,
                     DrawLinesMatchesFlags::DEFAULT );

  imshow( "Matches", outImg );
  waitKey();
  imwrite("/home/ubisum/Desktop/images/env_match/matches.jpg", outImg);
  /* create an LSD detector */
  Ptr<LSDDetector> lsd = LSDDetector::createLSDDetector();

  /* detect lines */
  std::vector<KeyLine> klsd1, klsd2;
  Mat lsd_descr1, lsd_descr2;
  lsd->detect( imageMat1, klsd1, 2, 2, mask1 );
  lsd->detect( imageMat2, klsd2, 2, 2, mask2 );

  /* compute descriptors for lines from first octave */
  bd->compute( imageMat1, klsd1, lsd_descr1 );
  bd->compute( imageMat2, klsd2, lsd_descr2 );

  /* select lines and descriptors from first octave */
  std::vector<KeyLine> octave0_1, octave0_2;
  Mat leftDEscr, rightDescr;
  for ( int i = 0; i < (int) klsd1.size(); i++ )
  {
    if( klsd1[i].octave == 1 )
    {
      octave0_1.push_back( klsd1[i] );
      leftDEscr.push_back( lsd_descr1.row( i ) );
    }
  }

  for ( int j = 0; j < (int) klsd2.size(); j++ )
  {
    if( klsd2[j].octave == 1 )
    {
      octave0_2.push_back( klsd2[j] );
      rightDescr.push_back( lsd_descr2.row( j ) );
    }
  }

  /* compute matches */
  std::vector<DMatch> lsd_matches;
  bdm->match( leftDEscr, rightDescr, lsd_matches );

  /* select best matches */
  good_matches.clear();
  for ( int i = 0; i < (int) lsd_matches.size(); i++ )
  {
    if( lsd_matches[i].distance < MATCHES_DIST_THRESHOLD )
      good_matches.push_back( lsd_matches[i] );
  }

  /* plot matches */
  cv::Mat lsd_outImg;
  resize( imageMat1, imageMat1, Size( imageMat1.cols / 2, imageMat1.rows / 2 ), 0, 0, INTER_LINEAR_EXACT );
  resize( imageMat2, imageMat2, Size( imageMat2.cols / 2, imageMat2.rows / 2 ), 0, 0, INTER_LINEAR_EXACT );
  std::vector<char> lsd_mask( matches.size(), 1 );
  drawLineMatches( imageMat1, octave0_1, imageMat2, octave0_2, good_matches, lsd_outImg, Scalar::all( -1 ), Scalar::all( -1 ), lsd_mask,
                   DrawLinesMatchesFlags::DEFAULT );

  imshow( "LSD matches", lsd_outImg );
  waitKey();


}

#else

int main()
{
    std::cerr << "OpenCV was built without features2d module" << std::endl;
    return 0;
}

#endif // HAVE_OPENCV_FEATURES2D

Sometimes, we could be interested in searching for the closest k descriptors, given an input one. This requires modifying previous code slightly:

// prepare a structure to host matches
std::vector<std::vector<DMatch> > matches;

// require knn match
bdm->knnMatch( descr1, descr2, matches, 6 );

In the above example, the closest 6 descriptors are returned for every query. In some cases, we could have a search radius and look for all descriptors distant at the most r from input query. Previous code must be modified like:

// prepare a structure to host matches
std::vector<std::vector<DMatch> > matches;

// compute matches
bdm->radiusMatch( queries, matches, 30 );

Here's an example of matching among descriptors extracted from original cameraman image and its downsampled (and blurred) version:

alternate text

Querying internal database

The BynaryDescriptorMatcher class owns an internal database that can be populated with descriptors extracted from different images and queried using one of the modalities described in the previous section. Population of internal dataset can be done using the add function; such function doesn't directly add new data to the database, but it just stores it them locally. The real update happens when the function train is invoked or when any querying function is executed, since each of them invokes train before querying. When queried, internal database not only returns required descriptors, but for every returned match, it is able to tell which image matched descriptor was extracted from. An example of internal dataset usage is described in the following code; after adding locally new descriptors, a radius search is invoked. This provokes local data to be transferred to dataset which in turn, is then queried.

/*M///////////////////////////////////////////////////////////////////////////////////////
 //
 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
 //
 //  By downloading, copying, installing or using the software you agree to this license.
 //  If you do not agree to this license, do not download, install,
 //  copy or use the software.
 //
 //
 //                           License Agreement
 //                For Open Source Computer Vision Library
 //
 // Copyright (C) 2014, Biagio Montesano, all rights reserved.
 // Third party copyrights are property of their respective owners.
 //
 // Redistribution and use in source and binary forms, with or without modification,
 // are permitted provided that the following conditions are met:
 //
 //   * Redistribution's of source code must retain the above copyright notice,
 //     this list of conditions and the following disclaimer.
 //
 //   * Redistribution's in binary form must reproduce the above copyright notice,
 //     this list of conditions and the following disclaimer in the documentation
 //     and/or other materials provided with the distribution.
 //
 //   * The name of the copyright holders may not be used to endorse or promote products
 //     derived from this software without specific prior written permission.
 //
 // This software is provided by the copyright holders and contributors "as is" and
 // any express or implied warranties, including, but not limited to, the implied
 // warranties of merchantability and fitness for a particular purpose are disclaimed.
 // In no event shall the Intel Corporation or contributors be liable for any direct,
 // indirect, incidental, special, exemplary, or consequential damages
 // (including, but not limited to, procurement of substitute goods or services;
 // loss of use, data, or profits; or business interruption) however caused
 // and on any theory of liability, whether in contract, strict liability,
 // or tort (including negligence or otherwise) arising in any way out of
 // the use of this software, even if advised of the possibility of such damage.
 //
 //M*/

#include <iostream>
#include <opencv2/opencv_modules.hpp>

#ifdef HAVE_OPENCV_FEATURES2D

#include <opencv2/line_descriptor.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/highgui.hpp>

#include <vector>

using namespace cv;
using namespace cv::line_descriptor;

static const std::string images[] =
{ "cameraman.jpg", "church.jpg", "church2.png", "einstein.jpg", "stuff.jpg" };

static const char* keys =
{ "{@image_path | | Image path }" };

static void help()
{
  std::cout << "\nThis example shows the functionalities of radius matching " << "Please, run this sample using a command in the form\n"
      << "./example_line_descriptor_radius_matching <path_to_input_images>/" << std::endl;
}

int main( int argc, char** argv )
{
  /* get parameters from comand line */
  CommandLineParser parser( argc, argv, keys );
  String pathToImages = parser.get < String > ( 0 );

  /* create structures for hosting KeyLines and descriptors */
  int num_elements = sizeof ( images ) / sizeof ( images[0] );
  std::vector < Mat > descriptorsMat;
  std::vector < std::vector<KeyLine> > linesMat;

  /*create a pointer to a BinaryDescriptor object */
  Ptr < BinaryDescriptor > bd = BinaryDescriptor::createBinaryDescriptor();

  /* compute lines and descriptors */
  for ( int i = 0; i < num_elements; i++ )
  {
    /* get path to image */
    std::stringstream image_path;
    image_path << pathToImages << images[i];
    std::cout << image_path.str().c_str() << std::endl;

    /* load image */
    Mat loadedImage = imread( image_path.str().c_str(), 1 );
    if( loadedImage.data == NULL )
    {
      std::cout << "Could not load images." << std::endl;
      help();
      exit( -1 );
    }

    /* compute lines and descriptors */
    std::vector < KeyLine > lines;
    Mat computedDescr;
    bd->detect( loadedImage, lines );
    bd->compute( loadedImage, lines, computedDescr );

    descriptorsMat.push_back( computedDescr );
    linesMat.push_back( lines );

  }

  /* compose a queries matrix */
  Mat queries;
  for ( size_t j = 0; j < descriptorsMat.size(); j++ )
  {
    if( descriptorsMat[j].rows >= 5 )
      queries.push_back( descriptorsMat[j].rowRange( 0, 5 ) );

    else if( descriptorsMat[j].rows > 0 && descriptorsMat[j].rows < 5 )
      queries.push_back( descriptorsMat[j] );
  }

  std::cout << "It has been generated a matrix of " << queries.rows << " descriptors" << std::endl;

  /* create a BinaryDescriptorMatcher object */
  Ptr < BinaryDescriptorMatcher > bdm = BinaryDescriptorMatcher::createBinaryDescriptorMatcher();

  /* populate matcher */
  bdm->add( descriptorsMat );

  /* compute matches */
  std::vector < std::vector<DMatch> > matches;
  bdm->radiusMatch( queries, matches, 30 );
  std::cout << "size matches sample " << matches.size() << std::endl;

  for ( int i = 0; i < (int) matches.size(); i++ )
  {
    for ( int j = 0; j < (int) matches[i].size(); j++ )
    {
      std::cout << "match: " << matches[i][j].queryIdx << " " << matches[i][j].trainIdx << " " << matches[i][j].distance << std::endl;
    }

  }

}

#else

int main()
{
    std::cerr << "OpenCV was built without features2d module" << std::endl;
    return 0;
}

#endif // HAVE_OPENCV_FEATURES2D