OpenCV  4.5.0
Open Source Computer Vision
Super-resolution benchmarking

Benchmarking

The super-resolution module contains sample codes for benchmarking, in order to compare different models and algorithms. Here is presented a sample code for performing benchmarking, and then a few benchmarking results are collected. It was performed on an Intel i7-9700K CPU on an Ubuntu 18.04.02 OS.

Source Code of the sample

1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
4 
5 #include <iostream>
6 #include <opencv2/opencv_modules.hpp>
7 
8 #ifdef HAVE_OPENCV_QUALITY
9 #include <opencv2/dnn_superres.hpp>
10 #include <opencv2/quality.hpp>
11 #include <opencv2/imgproc.hpp>
12 #include <opencv2/highgui.hpp>
13 
14 using namespace std;
15 using namespace cv;
16 using namespace dnn_superres;
17 
18 static void showBenchmark(vector<Mat> images, string title, Size imageSize,
19  const vector<String> imageTitles,
20  const vector<double> psnrValues,
21  const vector<double> ssimValues)
22 {
23  int fontFace = FONT_HERSHEY_COMPLEX_SMALL;
24  int fontScale = 1;
25  Scalar fontColor = Scalar(255, 255, 255);
26 
27  int len = static_cast<int>(images.size());
28 
29  int cols = 2, rows = 2;
30 
31  Mat fullImage = Mat::zeros(Size((cols * 10) + imageSize.width * cols, (rows * 10) + imageSize.height * rows),
32  images[0].type());
33 
34  stringstream ss;
35  int h_ = -1;
36  for (int i = 0; i < len; i++) {
37 
38  int fontStart = 15;
39  int w_ = i % cols;
40  if (i % cols == 0)
41  h_++;
42 
43  Rect ROI((w_ * (10 + imageSize.width)), (h_ * (10 + imageSize.height)), imageSize.width, imageSize.height);
44  Mat tmp;
45  resize(images[i], tmp, Size(ROI.width, ROI.height));
46 
47  ss << imageTitles[i];
48  putText(tmp,
49  ss.str(),
50  Point(5, fontStart),
51  fontFace,
52  fontScale,
53  fontColor,
54  1,
55  16);
56 
57  ss.str("");
58  fontStart += 20;
59 
60  ss << "PSNR: " << psnrValues[i];
61  putText(tmp,
62  ss.str(),
63  Point(5, fontStart),
64  fontFace,
65  fontScale,
66  fontColor,
67  1,
68  16);
69 
70  ss.str("");
71  fontStart += 20;
72 
73  ss << "SSIM: " << ssimValues[i];
74  putText(tmp,
75  ss.str(),
76  Point(5, fontStart),
77  fontFace,
78  fontScale,
79  fontColor,
80  1,
81  16);
82 
83  ss.str("");
84  fontStart += 20;
85 
86  tmp.copyTo(fullImage(ROI));
87  }
88 
89  namedWindow(title, 1);
90  imshow(title, fullImage);
91  waitKey();
92 }
93 
94 static Vec2d getQualityValues(Mat orig, Mat upsampled)
95 {
96  double psnr = PSNR(upsampled, orig);
97  Scalar q = quality::QualitySSIM::compute(upsampled, orig, noArray());
98  double ssim = mean(Vec3d((q[0]), q[1], q[2]))[0];
99  return Vec2d(psnr, ssim);
100 }
101 
102 int main(int argc, char *argv[])
103 {
104  // Check for valid command line arguments, print usage
105  // if insufficient arguments were given.
106  if (argc < 4) {
107  cout << "usage: Arg 1: image path | Path to image" << endl;
108  cout << "\t Arg 2: algorithm | edsr, espcn, fsrcnn or lapsrn" << endl;
109  cout << "\t Arg 3: path to model file 2 \n";
110  cout << "\t Arg 4: scale | 2, 3, 4 or 8 \n";
111  return -1;
112  }
113 
114  string path = string(argv[1]);
115  string algorithm = string(argv[2]);
116  string model = string(argv[3]);
117  int scale = atoi(argv[4]);
118 
119  Mat img = imread(path);
120  if (img.empty()) {
121  cerr << "Couldn't load image: " << img << "\n";
122  return -2;
123  }
124 
125  //Crop the image so the images will be aligned
126  int width = img.cols - (img.cols % scale);
127  int height = img.rows - (img.rows % scale);
128  Mat cropped = img(Rect(0, 0, width, height));
129 
130  //Downscale the image for benchmarking
131  Mat img_downscaled;
132  resize(cropped, img_downscaled, Size(), 1.0 / scale, 1.0 / scale);
133 
134  //Make dnn super resolution instance
135  DnnSuperResImpl sr;
136 
137  vector <Mat> allImages;
138  Mat img_new;
139 
140  //Read and set the dnn model
141  sr.readModel(model);
142  sr.setModel(algorithm, scale);
143  sr.upsample(img_downscaled, img_new);
144 
145  vector<double> psnrValues = vector<double>();
146  vector<double> ssimValues = vector<double>();
147 
148  //DL MODEL
149  Vec2f quality = getQualityValues(cropped, img_new);
150 
151  psnrValues.push_back(quality[0]);
152  ssimValues.push_back(quality[1]);
153 
154  cout << sr.getAlgorithm() << ":" << endl;
155  cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
156  cout << "----------------------" << endl;
157 
158  //BICUBIC
159  Mat bicubic;
160  resize(img_downscaled, bicubic, Size(), scale, scale, INTER_CUBIC);
161  quality = getQualityValues(cropped, bicubic);
162 
163  psnrValues.push_back(quality[0]);
164  ssimValues.push_back(quality[1]);
165 
166  cout << "Bicubic " << endl;
167  cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
168  cout << "----------------------" << endl;
169 
170  //NEAREST NEIGHBOR
171  Mat nearest;
172  resize(img_downscaled, nearest, Size(), scale, scale, INTER_NEAREST);
173  quality = getQualityValues(cropped, nearest);
174 
175  psnrValues.push_back(quality[0]);
176  ssimValues.push_back(quality[1]);
177 
178  cout << "Nearest neighbor" << endl;
179  cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
180  cout << "----------------------" << endl;
181 
182  //LANCZOS
183  Mat lanczos;
184  resize(img_downscaled, lanczos, Size(), scale, scale, INTER_LANCZOS4);
185  quality = getQualityValues(cropped, lanczos);
186 
187  psnrValues.push_back(quality[0]);
188  ssimValues.push_back(quality[1]);
189 
190  cout << "Lanczos" << endl;
191  cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
192  cout << "-----------------------------------------------" << endl;
193 
194  vector <Mat> imgs{img_new, bicubic, nearest, lanczos};
195  vector <String> titles{sr.getAlgorithm(), "Bicubic", "Nearest neighbor", "Lanczos"};
196  showBenchmark(imgs, "Quality benchmark", Size(bicubic.cols, bicubic.rows), titles, psnrValues, ssimValues);
197 
198  waitKey(0);
199 
200  return 0;
201 }
202 #else
203 int main()
204 {
205  std::cout << "This sample requires the OpenCV Quality module." << std::endl;
206  return 0;
207 }
208 #endif
cv::Scalar
Scalar_< double > Scalar
Definition: types.hpp:669
cv::mean
Scalar mean(InputArray src, InputArray mask=noArray())
Calculates an average (mean) of array elements.
cv::INTER_CUBIC
Definition: imgproc.hpp:250
cv::Mat::rows
int rows
the number of rows and columns or (-1, -1) when the matrix has more than 2 dimensions ...
Definition: mat.hpp:2087
cv::Mat::copyTo
void copyTo(OutputArray m) const
Copies the matrix to another one.
cv::imread
Mat imread(const String &filename, int flags=IMREAD_COLOR)
Loads an image from a file.
cv::Size_::width
_Tp width
the width
Definition: types.hpp:339
std
STL namespace.
cv::imshow
void imshow(const String &winname, InputArray mat)
Displays an image in the specified window.
cv
"black box" representation of the file storage associated with a file on disk.
Definition: affine.hpp:51
cv::FONT_HERSHEY_COMPLEX_SMALL
smaller version of FONT_HERSHEY_COMPLEX
Definition: imgproc.hpp:823
cv::PSNR
double PSNR(InputArray src1, InputArray src2, double R=255.)
Computes the Peak Signal-to-Noise Ratio (PSNR) image quality metric.
cv::Size_
Template class for specifying the size of an image or rectangle.
Definition: types.hpp:315
cv::quality::quality_utils::scale
void scale(cv::Mat &mat, const cv::Mat &range, const T min, const T max)
Definition: quality_utils.hpp:90
cv::noArray
InputOutputArray noArray()
cv::Mat::cols
int cols
Definition: mat.hpp:2087
cv::Rect
Rect2i Rect
Definition: types.hpp:462
cv::Rect_
Template class for 2D rectangles.
Definition: types.hpp:420
cv::INTER_LANCZOS4
Definition: imgproc.hpp:256
cv::namedWindow
void namedWindow(const String &winname, int flags=WINDOW_AUTOSIZE)
Creates a window.
cv::Size_::height
_Tp height
the height
Definition: types.hpp:340
cv::Size
Size2i Size
Definition: types.hpp:347
dnn_superres.hpp
cv::Vec2d
Vec< double, 2 > Vec2d
Definition: matx.hpp:432
cv::hal::resize
void resize(int src_type, const uchar *src_data, size_t src_step, int src_width, int src_height, uchar *dst_data, size_t dst_step, int dst_width, int dst_height, double inv_scale_x, double inv_scale_y, int interpolation)
cv::putText
void putText(InputOutputArray img, const String &text, Point org, int fontFace, double fontScale, Scalar color, int thickness=1, int lineType=LINE_8, bool bottomLeftOrigin=false)
Draws a text string.
cv::INTER_NEAREST
Definition: imgproc.hpp:246
cv::Vec3d
Vec< double, 3 > Vec3d
Definition: matx.hpp:433
cv::Scalar_< double >
cv::Mat
n-dimensional dense array class
Definition: mat.hpp:792
cv::Mat::empty
bool empty() const
Returns true if the array has no elements.
cv::Point
Point2i Point
Definition: types.hpp:194
cv::waitKey
int waitKey(int delay=0)
Waits for a pressed key.

Explanation

  1. Read and downscale the image

    int width = img.cols - (img.cols % scale);
    int height = img.rows - (img.rows % scale);
    Mat cropped = img(Rect(0, 0, width, height));
    Mat img_downscaled;
    cv::resize(cropped, img_downscaled, cv::Size(), 1.0 / scale, 1.0 / scale);

    Resize the image by the scaling factor. Before that a cropping is necessary, so the images will align.

  2. Set the model

    DnnSuperResImpl sr;
    sr.readModel(path);
    sr.setModel(algorithm, scale);
    sr.upsample(img_downscaled, img_new);

    Instantiate a dnn super-resolution object. Read and set the algorithm and scaling factor.

  3. Perform benchmarking

    double psnr = PSNR(img_new, cropped);
    Scalar q = cv::quality::QualitySSIM::compute(img_new, cropped, cv::noArray());
    double ssim = mean(cv::Vec3f(q[0], q[1], q[2]))[0];

    Calculate PSNR and SSIM. Use OpenCVs PSNR (core opencv) and SSIM (contrib) functions to compare the images. Repeat it with other upscaling algorithms, such as other DL models or interpolation methods (eg. bicubic, nearest neighbor).

Benchmarking results

Dataset benchmarking

General100 dataset

2x scaling factor
Avg inference time in sec (CPU)Avg PSNR Avg SSIM
ESPCN 0.008795 32.7059 0.9276
EDSR 5.923450 34.1300 0.9447
FSRCNN 0.021741 32.8886 0.9301
LapSRN 0.114812 32.2681 0.9248
Bicubic 0.000208 32.1638 0.9305
Nearest neighbor 0.000114 29.1665 0.9049
Lanczos 0.001094 32.4687 0.9327
3x scaling factor
Avg inference time in sec (CPU)Avg PSNR Avg SSIM
ESPCN 0.005495 28.4229 0.8474
EDSR 2.455510 29.9828 0.8801
FSRCNN 0.008807 28.3068 0.8429
LapSRN 0.282575 26.7330 0.8862
Bicubic 0.000311 26.0635 0.8754
Nearest neighbor 0.000148 23.5628 0.8174
Lanczos 0.001012 25.9115 0.8706
4x scaling factor
Avg inference time in sec (CPU)Avg PSNR Avg SSIM
ESPCN 0.004311 26.6870 0.7891
EDSR 1.607570 28.1552 0.8317
FSRCNN 0.005302 26.6088 0.7863
LapSRN 0.121229 26.7383 0.7896
Bicubic 0.000311 26.0635 0.8754
Nearest neighbor 0.000148 23.5628 0.8174
Lanczos 0.001012 25.9115 0.8706

Images

2x scaling factor

Set5: butterfly.png size: 256x256
orig_butterfly.jpg
Original
bicubic_butterfly.jpg
Bicubic interpolation
nearest_butterfly.jpg
Nearest neighbor interpolation
lanczos_butterfly.jpg
Lanczos interpolation
PSRN / SSIM / Speed (CPU)26.6645 / 0.9048 / 0.000201 23.6854 / 0.8698 / 0.000075 26.9476 / 0.9075 / 0.001039
espcn_butterfly.jpg
ESPCN
fsrcnn_butterfly.jpg
FSRCNN
lapsrn_butterfly.jpg
LapSRN
edsr_butterfly.jpg
EDSR
29.0341 / 0.9354 / 0.00415729.0077 / 0.9345 / 0.006325 27.8212 / 0.9230 / 0.037937 30.0347 / 0.9453 / 2.077280

3x scaling factor

Urban100: img_001.png size: 1024x644
orig_urban.jpg
Original
bicubic_urban.jpg
Bicubic interpolation
nearest_urban.jpg
Nearest neighbor interpolation
lanczos_urban.jpg
Lanczos interpolation
PSRN / SSIM / Speed (CPU)27.0474 / 0.8484 / 0.000391 26.0842 / 0.8353 / 0.000236 27.0704 / 0.8483 / 0.002234
espcn_urban.jpg
ESPCN
fsrcnn_urban.jpg
FSRCNN
LapSRN is not trained for 3x
because of its architecture
edsr_urban.jpg
EDSR
28.0118 / 0.8588 / 0.03074828.0184 / 0.8597 / 0.094173 30.5671 / 0.9019 / 9.517580

4x scaling factor

Set14: comic.png size: 250x361
orig_comic.jpg
Original
bicubic_comic.jpg
Bicubic interpolation
nearest_comic.jpg
Nearest neighbor interpolation
lanczos_comic.jpg
Lanczos interpolation
PSRN / SSIM / Speed (CPU)19.6766 / 0.6413 / 0.000262 18.5106 / 0.5879 / 0.000085 19.4948 / 0.6317 / 0.001098
espcn_comic.jpg
ESPCN
fsrcnn_comic.jpg
FSRCNN
lapsrn_comic.jpg
LapSRN
edsr_comic.jpg
EDSR
20.0417 / 0.6302 / 0.00189420.0885 / 0.6384 / 0.002103 20.0676 / 0.6339 / 0.061640 20.5233 / 0.6901 / 0.665876

8x scaling factor

Div2K: 0006.png size: 1356x2040
orig_div2k.jpg
Original
bicubic_div2k.jpg
Bicubic interpolation
nearest_div2k.jpg
Nearest neighbor interpolation
PSRN / SSIM / Speed (CPU)26.3139 / 0.8033 / 0.00110723.8291 / 0.7340 / 0.000611
lanczos_div2k.jpg
Lanczos interpolation
lapsrn_div2k.jpg
LapSRN
26.1565 / 0.7962 / 0.00478226.7046 / 0.7987 / 2.274290
Generated on Sun Oct 11 2020 21:45:02 for OpenCV by   doxygen 1.8.13