OpenCV  4.3.0-pre
Open Source Computer Vision
Detection of ChArUco Corners

ArUco markers and boards are very useful due to their fast detection and their versatility. However, one of the problems of ArUco markers is that the accuracy of their corner positions is not too high, even after applying subpixel refinement.

On the contrary, the corners of chessboard patterns can be refined more accurately since each corner is surrounded by two black squares. However, finding a chessboard pattern is not as versatile as finding an ArUco board: it has to be completely visible and occlusions are not permitted.

A ChArUco board tries to combine the benefits of these two approaches:

charucodefinition.png
Charuco definition

The ArUco part is used to interpolate the position of the chessboard corners, so that it has the versatility of marker boards, since it allows occlusions or partial views. Moreover, since the interpolated corners belong to a chessboard, they are very accurate in terms of subpixel accuracy.

When high precision is necessary, such as in camera calibration, Charuco boards are a better option than standard Aruco boards.

Goal

In this tutorial you will learn:

Source code

You can find this code in opencv_contrib/modules/aruco/samples/tutorial_charuco_create_detect.cpp

Here's a sample code of how to achieve all the stuff enumerated at the goal list.

#include <iostream>
#include <string>
namespace {
const char* about = "A tutorial code on charuco board creation and detection of charuco board with and without camera caliberation";
const char* keys = "{c | | Put value of c=1 to create charuco board;\nc=2 to detect charuco board without camera calibration;\nc=3 to detect charuco board with camera calibration and Pose Estimation}";
}
void createBoard();
void detectCharucoBoardWithCalibrationPose();
void detectCharucoBoardWithoutCalibration();
static bool readCameraParameters(std::string filename, cv::Mat& camMatrix, cv::Mat& distCoeffs)
{
if (!fs.isOpened())
return false;
fs["camera_matrix"] >> camMatrix;
fs["distortion_coefficients"] >> distCoeffs;
return true;
}
void createBoard()
{
cv::Ptr<cv::aruco::CharucoBoard> board = cv::aruco::CharucoBoard::create(5, 7, 0.04f, 0.02f, dictionary);
cv::Mat boardImage;
board->draw(cv::Size(600, 500), boardImage, 10, 1);
cv::imwrite("BoardImage.jpg", boardImage);
}
void detectCharucoBoardWithCalibrationPose()
{
cv::VideoCapture inputVideo;
inputVideo.open(0);
cv::Mat cameraMatrix, distCoeffs;
std::string filename = "calib.txt";
bool readOk = readCameraParameters(filename, cameraMatrix, distCoeffs);
if (!readOk) {
std::cerr << "Invalid camera file" << std::endl;
} else {
cv::Ptr<cv::aruco::CharucoBoard> board = cv::aruco::CharucoBoard::create(5, 7, 0.04f, 0.02f, dictionary);
while (inputVideo.grab()) {
cv::Mat image;
cv::Mat imageCopy;
inputVideo.retrieve(image);
image.copyTo(imageCopy);
std::vector<int> markerIds;
std::vector<std::vector<cv::Point2f> > markerCorners;
cv::aruco::detectMarkers(image, board->dictionary, markerCorners, markerIds, params);
// if at least one marker detected
if (markerIds.size() > 0) {
cv::aruco::drawDetectedMarkers(imageCopy, markerCorners, markerIds);
std::vector<cv::Point2f> charucoCorners;
std::vector<int> charucoIds;
cv::aruco::interpolateCornersCharuco(markerCorners, markerIds, image, board, charucoCorners, charucoIds, cameraMatrix, distCoeffs);
// if at least one charuco corner detected
if (charucoIds.size() > 0) {
cv::Scalar color = cv::Scalar(255, 0, 0);
cv::aruco::drawDetectedCornersCharuco(imageCopy, charucoCorners, charucoIds, color);
cv::Vec3d rvec, tvec;
// cv::aruco::estimatePoseCharucoBoard(charucoCorners, charucoIds, board, cameraMatrix, distCoeffs, rvec, tvec);
bool valid = cv::aruco::estimatePoseCharucoBoard(charucoCorners, charucoIds, board, cameraMatrix, distCoeffs, rvec, tvec);
// if charuco pose is valid
if (valid)
cv::aruco::drawAxis(imageCopy, cameraMatrix, distCoeffs, rvec, tvec, 0.1f);
}
}
cv::imshow("out", imageCopy);
char key = (char)cv::waitKey(30);
if (key == 27)
break;
}
}
}
void detectCharucoBoardWithoutCalibration()
{
cv::VideoCapture inputVideo;
inputVideo.open(0);
cv::Ptr<cv::aruco::CharucoBoard> board = cv::aruco::CharucoBoard::create(5, 7, 0.04f, 0.02f, dictionary);
params->cornerRefinementMethod = cv::aruco::CORNER_REFINE_NONE;
while (inputVideo.grab()) {
cv::Mat image, imageCopy;
inputVideo.retrieve(image);
image.copyTo(imageCopy);
std::vector<int> markerIds;
std::vector<std::vector<cv::Point2f> > markerCorners;
cv::aruco::detectMarkers(image, board->dictionary, markerCorners, markerIds, params);
//or
//cv::aruco::detectMarkers(image, dictionary, markerCorners, markerIds, params);
// if at least one marker detected
if (markerIds.size() > 0) {
cv::aruco::drawDetectedMarkers(imageCopy, markerCorners, markerIds);
std::vector<cv::Point2f> charucoCorners;
std::vector<int> charucoIds;
cv::aruco::interpolateCornersCharuco(markerCorners, markerIds, image, board, charucoCorners, charucoIds);
// if at least one charuco corner detected
if (charucoIds.size() > 0)
cv::aruco::drawDetectedCornersCharuco(imageCopy, charucoCorners, charucoIds, cv::Scalar(255, 0, 0));
}
cv::imshow("out", imageCopy);
char key = (char)cv::waitKey(30);
if (key == 27)
break;
}
}
int main(int argc, char* argv[])
{
cv::CommandLineParser parser(argc, argv, keys);
parser.about(about);
if (argc < 2) {
parser.printMessage();
return 0;
}
int choose = parser.get<int>("c");
switch (choose) {
case 1:
createBoard();
std::cout << "An image named BoardImg.jpg is generated in folder containing this file" << std::endl;
break;
case 2:
detectCharucoBoardWithoutCalibration();
break;
case 3:
detectCharucoBoardWithCalibrationPose();
break;
default:
break;
}
return 0;
}

ChArUco Board Creation

The aruco module provides the cv::aruco::CharucoBoard class that represents a Charuco Board and which inherits from the Board class.

This class, as the rest of ChArUco functionalities, are defined in:

To define a CharucoBoard, it is necessary:

As for the GridBoard objects, the aruco module provides a function to create CharucoBoards easily. This function is the static function cv::aruco::CharucoBoard::create() :

cv::aruco::CharucoBoard board = cv::aruco::CharucoBoard::create(5, 7, 0.04, 0.02, dictionary);

The ids of each of the markers are assigned by default in ascending order and starting on 0, like in GridBoard::create(). This can be easily customized by accessing to the ids vector through board.ids, like in the Board parent class.

Once we have our CharucoBoard object, we can create an image to print it. This can be done with the CharucoBoard::draw() method:

cv::Ptr<cv::aruco::CharucoBoard> board = cv::aruco::CharucoBoard::create(5, 7, 0.04f, 0.02f, dictionary);
cv::Mat boardImage;
board->draw(cv::Size(600, 500), boardImage, 10, 1);

The output image will be something like this:

charucoboard.jpg

A full working example is included in the create_board_charuco.cpp inside the modules/aruco/samples/create_board_charuco.cpp.

Note: The create_board_charuco.cpp now take input via commandline via the OpenCV Commandline Parser. For this file the example parameters will look like

"_ output path_/chboard.png" -w=5 -h=7 -sl=200 -ml=120 -d=10

ChArUco Board Detection

When you detect a ChArUco board, what you are actually detecting is each of the chessboard corners of the board.

Each corner on a ChArUco board has a unique identifier (id) assigned. These ids go from 0 to the total number of corners in the board. The steps of charuco board detection can be broken down to the following steps:

cv::Mat image;

The original image where the markers are to be detected. The image is necessary to perform subpixel refinement in the ChArUco corners.

cv::Mat cameraMatrix, distCoeffs;
std::string filename = "calib.txt";
bool readOk = readCameraParameters(filename, cameraMatrix, distCoeffs);

The parameters of readCameraParameters are:

This function takes these parameters as input and returns a boolean value of whether the camera calibration parameters are valid or not. For detection of corners without calibration, this step is not required.

std::vector<int> markerIds;
std::vector<std::vector<cv::Point2f> > markerCorners;
cv::aruco::detectMarkers(image, board->dictionary, markerCorners, markerIds, params);

The parameters of detectMarkers are:

For detection with calibration

std::vector<cv::Point2f> charucoCorners;
std::vector<int> charucoIds;
cv::aruco::interpolateCornersCharuco(markerCorners, markerIds, image, board, charucoCorners, charucoIds, cameraMatrix, distCoeffs);

For detection without calibration

std::vector<cv::Point2f> charucoCorners;
std::vector<int> charucoIds;
cv::aruco::interpolateCornersCharuco(markerCorners, markerIds, image, board, charucoCorners, charucoIds);

The function that detect the ChArUco corners is cv::aruco::interpolateCornersCharuco(). This function returns the number of Charuco corners interpolated.

If calibration parameters are provided, the ChArUco corners are interpolated by, first, estimating a rough pose from the ArUco markers and, then, reprojecting the ChArUco corners back to the image.

On the other hand, if calibration parameters are not provided, the ChArUco corners are interpolated by calculating the corresponding homography between the ChArUco plane and the ChArUco image projection.

The main problem of using homography is that the interpolation is more sensible to image distortion. Actually, the homography is only performed using the closest markers of each ChArUco corner to reduce the effect of distortion.

When detecting markers for ChArUco boards, and specially when using homography, it is recommended to disable the corner refinement of markers. The reason of this is that, due to the proximity of the chessboard squares, the subpixel process can produce important deviations in the corner positions and these deviations are propagated to the ChArUco corner interpolation, producing poor results.

Furthermore, only those corners whose two surrounding markers have be found are returned. If any of the two surrounding markers has not been detected, this usually means that there is some occlusion or the image quality is not good in that zone. In any case, it is preferable not to consider that corner, since what we want is to be sure that the interpolated ChArUco corners are very accurate.

After the ChArUco corners have been interpolated, a subpixel refinement is performed.

Once we have interpolated the ChArUco corners, we would probably want to draw them to see if their detections are correct. This can be easily done using the drawDetectedCornersCharuco() function:

cv::aruco::drawDetectedCornersCharuco(imageCopy, charucoCorners, charucoIds, color);

For this image:

choriginal.png
Image with Charuco board

The result will be:

chcorners.png
Charuco board detected

In the presence of occlusion. like in the following image, although some corners are clearly visible, not all their surrounding markers have been detected due occlusion and, thus, they are not interpolated:

chocclusion.png
Charuco detection with occlusion

Finally, this is a full example of ChArUco detection (without using calibration parameters):

void detectCharucoBoardWithoutCalibration()
{
cv::VideoCapture inputVideo;
inputVideo.open(0);
cv::Ptr<cv::aruco::CharucoBoard> board = cv::aruco::CharucoBoard::create(5, 7, 0.04f, 0.02f, dictionary);
params->cornerRefinementMethod = cv::aruco::CORNER_REFINE_NONE;
while (inputVideo.grab()) {
cv::Mat image, imageCopy;
inputVideo.retrieve(image);
image.copyTo(imageCopy);
std::vector<int> markerIds;
std::vector<std::vector<cv::Point2f> > markerCorners;
cv::aruco::detectMarkers(image, board->dictionary, markerCorners, markerIds, params);
//or
//cv::aruco::detectMarkers(image, dictionary, markerCorners, markerIds, params);
// if at least one marker detected
if (markerIds.size() > 0) {
cv::aruco::drawDetectedMarkers(imageCopy, markerCorners, markerIds);
std::vector<cv::Point2f> charucoCorners;
std::vector<int> charucoIds;
cv::aruco::interpolateCornersCharuco(markerCorners, markerIds, image, board, charucoCorners, charucoIds);
// if at least one charuco corner detected
if (charucoIds.size() > 0)
cv::aruco::drawDetectedCornersCharuco(imageCopy, charucoCorners, charucoIds, cv::Scalar(255, 0, 0));
}
cv::imshow("out", imageCopy);
char key = (char)cv::waitKey(30);
if (key == 27)
break;
}
}

Sample video:

A full working example is included in the detect_board_charuco.cpp inside the modules/aruco/samples/detect_board_charuco.cpp.

Note: The samples now take input via commandline via the OpenCV Commandline Parser. For this file the example parameters will look like

-c="_path_/calib.txt" -dp="_path_/detector_params.yml" -w=5 -h=7 -sl=0.04 -ml=0.02 -d=10

Here the calib.txt is the output file generated by the calibrate_camera_charuco.cpp.

ChArUco Pose Estimation

The final goal of the ChArUco boards is finding corners very accurately for a high precision calibration or pose estimation.

The aruco module provides a function to perform ChArUco pose estimation easily. As in the GridBoard, the coordinate system of the CharucoBoard is placed in the board plane with the Z axis pointing out, and centered in the bottom left corner of the board.

The function for pose estimation is estimatePoseCharucoBoard():

// cv::aruco::estimatePoseCharucoBoard(charucoCorners, charucoIds, board, cameraMatrix, distCoeffs, rvec, tvec);

The axis can be drawn using drawAxis() to check the pose is correctly estimated. The result would be: (X:red, Y:green, Z:blue)

chaxis.png
Charuco Board Axis

A full example of ChArUco detection with pose estimation:

void detectCharucoBoardWithCalibrationPose()
{
cv::VideoCapture inputVideo;
inputVideo.open(0);
cv::Mat cameraMatrix, distCoeffs;
std::string filename = "calib.txt";
bool readOk = readCameraParameters(filename, cameraMatrix, distCoeffs);
if (!readOk) {
std::cerr << "Invalid camera file" << std::endl;
} else {
cv::Ptr<cv::aruco::CharucoBoard> board = cv::aruco::CharucoBoard::create(5, 7, 0.04f, 0.02f, dictionary);
while (inputVideo.grab()) {
cv::Mat image;
cv::Mat imageCopy;
inputVideo.retrieve(image);
image.copyTo(imageCopy);
std::vector<int> markerIds;
std::vector<std::vector<cv::Point2f> > markerCorners;
cv::aruco::detectMarkers(image, board->dictionary, markerCorners, markerIds, params);
// if at least one marker detected
if (markerIds.size() > 0) {
cv::aruco::drawDetectedMarkers(imageCopy, markerCorners, markerIds);
std::vector<cv::Point2f> charucoCorners;
std::vector<int> charucoIds;
cv::aruco::interpolateCornersCharuco(markerCorners, markerIds, image, board, charucoCorners, charucoIds, cameraMatrix, distCoeffs);
// if at least one charuco corner detected
if (charucoIds.size() > 0) {
cv::Scalar color = cv::Scalar(255, 0, 0);
cv::aruco::drawDetectedCornersCharuco(imageCopy, charucoCorners, charucoIds, color);
cv::Vec3d rvec, tvec;
// cv::aruco::estimatePoseCharucoBoard(charucoCorners, charucoIds, board, cameraMatrix, distCoeffs, rvec, tvec);
bool valid = cv::aruco::estimatePoseCharucoBoard(charucoCorners, charucoIds, board, cameraMatrix, distCoeffs, rvec, tvec);
// if charuco pose is valid
if (valid)
cv::aruco::drawAxis(imageCopy, cameraMatrix, distCoeffs, rvec, tvec, 0.1f);
}
}
cv::imshow("out", imageCopy);
char key = (char)cv::waitKey(30);
if (key == 27)
break;
}
}
}

A full working example is included in the detect_board_charuco.cpp inside the modules/aruco/samples/detect_board_charuco.cpp.

Note: The samples now take input via commandline via the OpenCV Commandline Parser. For this file the example parameters will look like

"_path_/calib.txt" -dp="_path_/detector_params.yml" -w=5 -h=7 -sl=0.04 -ml=0.02 -d=10