OpenCV  4.1.2
Open Source Computer Vision
Super-resolution benchmarking

Benchmarking

The super-resolution module contains sample codes for benchmarking, in order to compare different models and algorithms. Here is presented a sample code for performing benchmarking, and then a few benchmarking results are collected. It was performed on an Intel i7-9700K CPU on an Ubuntu 18.04.02 OS.

Source Code of the sample

1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
4 
5 #include <iostream>
6 
7 #include <opencv2/dnn_superres.hpp>
8 #include <opencv2/quality.hpp>
9 #include <opencv2/imgproc.hpp>
10 #include <opencv2/highgui.hpp>
11 
12 using namespace std;
13 using namespace cv;
14 using namespace dnn_superres;
15 
16 static void showBenchmark(vector<Mat> images, string title, Size imageSize,
17  const vector<String> imageTitles,
18  const vector<double> psnrValues,
19  const vector<double> ssimValues)
20 {
21  int fontFace = FONT_HERSHEY_COMPLEX_SMALL;
22  int fontScale = 1;
23  Scalar fontColor = Scalar(255, 255, 255);
24 
25  int len = static_cast<int>(images.size());
26 
27  int cols = 2, rows = 2;
28 
29  Mat fullImage = Mat::zeros(Size((cols * 10) + imageSize.width * cols, (rows * 10) + imageSize.height * rows),
30  images[0].type());
31 
32  stringstream ss;
33  int h_ = -1;
34  for (int i = 0; i < len; i++) {
35 
36  int fontStart = 15;
37  int w_ = i % cols;
38  if (i % cols == 0)
39  h_++;
40 
41  Rect ROI((w_ * (10 + imageSize.width)), (h_ * (10 + imageSize.height)), imageSize.width, imageSize.height);
42  Mat tmp;
43  resize(images[i], tmp, Size(ROI.width, ROI.height));
44 
45  ss << imageTitles[i];
46  putText(tmp,
47  ss.str(),
48  Point(5, fontStart),
49  fontFace,
50  fontScale,
51  fontColor,
52  1,
53  16);
54 
55  ss.str("");
56  fontStart += 20;
57 
58  ss << "PSNR: " << psnrValues[i];
59  putText(tmp,
60  ss.str(),
61  Point(5, fontStart),
62  fontFace,
63  fontScale,
64  fontColor,
65  1,
66  16);
67 
68  ss.str("");
69  fontStart += 20;
70 
71  ss << "SSIM: " << ssimValues[i];
72  putText(tmp,
73  ss.str(),
74  Point(5, fontStart),
75  fontFace,
76  fontScale,
77  fontColor,
78  1,
79  16);
80 
81  ss.str("");
82  fontStart += 20;
83 
84  tmp.copyTo(fullImage(ROI));
85  }
86 
87  namedWindow(title, 1);
88  imshow(title, fullImage);
89  waitKey();
90 }
91 
92 static Vec2d getQualityValues(Mat orig, Mat upsampled)
93 {
94  double psnr = PSNR(upsampled, orig);
95  Scalar q = quality::QualitySSIM::compute(upsampled, orig, noArray());
96  double ssim = mean(Vec3d((q[0]), q[1], q[2]))[0];
97  return Vec2d(psnr, ssim);
98 }
99 
100 int main(int argc, char *argv[])
101 {
102  // Check for valid command line arguments, print usage
103  // if insufficient arguments were given.
104  if (argc < 4) {
105  cout << "usage: Arg 1: image path | Path to image" << endl;
106  cout << "\t Arg 2: algorithm | edsr, espcn, fsrcnn or lapsrn" << endl;
107  cout << "\t Arg 3: path to model file 2 \n";
108  cout << "\t Arg 4: scale | 2, 3, 4 or 8 \n";
109  return -1;
110  }
111 
112  string path = string(argv[1]);
113  string algorithm = string(argv[2]);
114  string model = string(argv[3]);
115  int scale = atoi(argv[4]);
116 
117  Mat img = imread(path);
118  if (img.empty()) {
119  cerr << "Couldn't load image: " << img << "\n";
120  return -2;
121  }
122 
123  //Crop the image so the images will be aligned
124  int width = img.cols - (img.cols % scale);
125  int height = img.rows - (img.rows % scale);
126  Mat cropped = img(Rect(0, 0, width, height));
127 
128  //Downscale the image for benchmarking
129  Mat img_downscaled;
130  resize(cropped, img_downscaled, Size(), 1.0 / scale, 1.0 / scale);
131 
132  //Make dnn super resolution instance
133  DnnSuperResImpl sr;
134 
135  vector <Mat> allImages;
136  Mat img_new;
137 
138  //Read and set the dnn model
139  sr.readModel(model);
140  sr.setModel(algorithm, scale);
141  sr.upsample(img_downscaled, img_new);
142 
143  vector<double> psnrValues = vector<double>();
144  vector<double> ssimValues = vector<double>();
145 
146  //DL MODEL
147  Vec2f quality = getQualityValues(cropped, img_new);
148 
149  psnrValues.push_back(quality[0]);
150  ssimValues.push_back(quality[1]);
151 
152  cout << sr.getAlgorithm() << ":" << endl;
153  cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
154  cout << "----------------------" << endl;
155 
156  //BICUBIC
157  Mat bicubic;
158  resize(img_downscaled, bicubic, Size(), scale, scale, INTER_CUBIC);
159  quality = getQualityValues(cropped, bicubic);
160 
161  psnrValues.push_back(quality[0]);
162  ssimValues.push_back(quality[1]);
163 
164  cout << "Bicubic " << endl;
165  cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
166  cout << "----------------------" << endl;
167 
168  //NEAREST NEIGHBOR
169  Mat nearest;
170  resize(img_downscaled, nearest, Size(), scale, scale, INTER_NEAREST);
171  quality = getQualityValues(cropped, nearest);
172 
173  psnrValues.push_back(quality[0]);
174  ssimValues.push_back(quality[1]);
175 
176  cout << "Nearest neighbor" << endl;
177  cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
178  cout << "----------------------" << endl;
179 
180  //LANCZOS
181  Mat lanczos;
182  resize(img_downscaled, lanczos, Size(), scale, scale, INTER_LANCZOS4);
183  quality = getQualityValues(cropped, lanczos);
184 
185  psnrValues.push_back(quality[0]);
186  ssimValues.push_back(quality[1]);
187 
188  cout << "Lanczos" << endl;
189  cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
190  cout << "-----------------------------------------------" << endl;
191 
192  vector <Mat> imgs{img_new, bicubic, nearest, lanczos};
193  vector <String> titles{sr.getAlgorithm(), "Bicubic", "Nearest neighbor", "Lanczos"};
194  showBenchmark(imgs, "Quality benchmark", Size(bicubic.cols, bicubic.rows), titles, psnrValues, ssimValues);
195 
196  waitKey(0);
197 
198  return 0;
199 }
cv::Scalar
Scalar_< double > Scalar
Definition: types.hpp:669
cv::mean
Scalar mean(InputArray src, InputArray mask=noArray())
Calculates an average (mean) of array elements.
cv::INTER_CUBIC
Definition: imgproc.hpp:250
cv::Mat::rows
int rows
the number of rows and columns or (-1, -1) when the matrix has more than 2 dimensions ...
Definition: mat.hpp:2086
cv::Mat::copyTo
void copyTo(OutputArray m) const
Copies the matrix to another one.
cv::imread
Mat imread(const String &filename, int flags=IMREAD_COLOR)
Loads an image from a file.
cv::Size_::width
_Tp width
the width
Definition: types.hpp:339
std
STL namespace.
cv::imshow
void imshow(const String &winname, InputArray mat)
Displays an image in the specified window.
cv
"black box" representation of the file storage associated with a file on disk.
Definition: affine.hpp:51
cv::FONT_HERSHEY_COMPLEX_SMALL
smaller version of FONT_HERSHEY_COMPLEX
Definition: imgproc.hpp:817
cv::PSNR
double PSNR(InputArray src1, InputArray src2, double R=255.)
Computes the Peak Signal-to-Noise Ratio (PSNR) image quality metric.
cv::Size_
Template class for specifying the size of an image or rectangle.
Definition: types.hpp:315
cv::quality::quality_utils::scale
void scale(cv::Mat &mat, const cv::Mat &range, const T min, const T max)
Definition: quality_utils.hpp:90
cv::noArray
InputOutputArray noArray()
cv::Mat::cols
int cols
Definition: mat.hpp:2086
cv::Rect
Rect2i Rect
Definition: types.hpp:462
cv::Rect_
Template class for 2D rectangles.
Definition: types.hpp:420
cv::INTER_LANCZOS4
Definition: imgproc.hpp:256
cv::namedWindow
void namedWindow(const String &winname, int flags=WINDOW_AUTOSIZE)
Creates a window.
cv::Size_::height
_Tp height
the height
Definition: types.hpp:340
cv::Size
Size2i Size
Definition: types.hpp:347
dnn_superres.hpp
cv::Vec2d
Vec< double, 2 > Vec2d
Definition: matx.hpp:423
cv::hal::resize
void resize(int src_type, const uchar *src_data, size_t src_step, int src_width, int src_height, uchar *dst_data, size_t dst_step, int dst_width, int dst_height, double inv_scale_x, double inv_scale_y, int interpolation)
cv::putText
void putText(InputOutputArray img, const String &text, Point org, int fontFace, double fontScale, Scalar color, int thickness=1, int lineType=LINE_8, bool bottomLeftOrigin=false)
Draws a text string.
cv::INTER_NEAREST
Definition: imgproc.hpp:246
cv::Vec3d
Vec< double, 3 > Vec3d
Definition: matx.hpp:424
cv::Scalar_< double >
cv::Mat
n-dimensional dense array class
Definition: mat.hpp:791
cv::Mat::empty
bool empty() const
Returns true if the array has no elements.
cv::Point
Point2i Point
Definition: types.hpp:194
cv::waitKey
int waitKey(int delay=0)
Waits for a pressed key.

Explanation

  1. Read and downscale the image

    int width = img.cols - (img.cols % scale);
    int height = img.rows - (img.rows % scale);
    Mat cropped = img(Rect(0, 0, width, height));
    Mat img_downscaled;
    cv::resize(cropped, img_downscaled, cv::Size(), 1.0 / scale, 1.0 / scale);

    Resize the image by the scaling factor. Before that a cropping is necessary, so the images will align.

  2. Set the model

    DnnSuperResImpl sr;
    sr.readModel(path);
    sr.setModel(algorithm, scale);
    sr.upsample(img_downscaled, img_new);

    Instantiate a dnn super-resolution object. Read and set the algorithm and scaling factor.

  3. Perform benchmarking

    double psnr = PSNR(img_new, cropped);
    Scalar q = cv::quality::QualitySSIM::compute(img_new, cropped, cv::noArray());
    double ssim = mean(cv::Vec3f(q[0], q[1], q[2]))[0];

    Calculate PSNR and SSIM. Use OpenCVs PSNR (core opencv) and SSIM (contrib) functions to compare the images. Repeat it with other upscaling algorithms, such as other DL models or interpolation methods (eg. bicubic, nearest neighbor).

Benchmarking results

Dataset benchmarking

General100 dataset

2x scaling factor
Avg inference time in sec (CPU)Avg PSNR Avg SSIM
ESPCN 0.008795 32.7059 0.9276
EDSR 5.923450 34.1300 0.9447
FSRCNN 0.021741 32.8886 0.9301
LapSRN 0.114812 32.2681 0.9248
Bicubic 0.000208 32.1638 0.9305
Nearest neighbor 0.000114 29.1665 0.9049
Lanczos 0.001094 32.4687 0.9327
3x scaling factor
Avg inference time in sec (CPU)Avg PSNR Avg SSIM
ESPCN 0.005495 28.4229 0.8474
EDSR 2.455510 29.9828 0.8801
FSRCNN 0.008807 28.3068 0.8429
LapSRN 0.282575 26.7330 0.8862
Bicubic 0.000311 26.0635 0.8754
Nearest neighbor 0.000148 23.5628 0.8174
Lanczos 0.001012 25.9115 0.8706
4x scaling factor
Avg inference time in sec (CPU)Avg PSNR Avg SSIM
ESPCN 0.004311 26.6870 0.7891
EDSR 1.607570 28.1552 0.8317
FSRCNN 0.005302 26.6088 0.7863
LapSRN 0.121229 26.7383 0.7896
Bicubic 0.000311 26.0635 0.8754
Nearest neighbor 0.000148 23.5628 0.8174
Lanczos 0.001012 25.9115 0.8706

Images

2x scaling factor

Set5: butterfly.png size: 256x256
orig_butterfly.jpg
Original
bicubic_butterfly.jpg
Bicubic interpolation
nearest_butterfly.jpg
Nearest neighbor interpolation
lanczos_butterfly.jpg
Lanczos interpolation
PSRN / SSIM / Speed (CPU)26.6645 / 0.9048 / 0.000201 23.6854 / 0.8698 / 0.000075 26.9476 / 0.9075 / 0.001039
espcn_butterfly.jpg
ESPCN
fsrcnn_butterfly.jpg
FSRCNN
lapsrn_butterfly.jpg
LapSRN
edsr_butterfly.jpg
EDSR
29.0341 / 0.9354 / 0.00415729.0077 / 0.9345 / 0.006325 27.8212 / 0.9230 / 0.037937 30.0347 / 0.9453 / 2.077280

3x scaling factor

Urban100: img_001.png size: 1024x644
orig_urban.jpg
Original
bicubic_urban.jpg
Bicubic interpolation
nearest_urban.jpg
Nearest neighbor interpolation
lanczos_urban.jpg
Lanczos interpolation
PSRN / SSIM / Speed (CPU)27.0474 / 0.8484 / 0.000391 26.0842 / 0.8353 / 0.000236 27.0704 / 0.8483 / 0.002234
espcn_urban.jpg
ESPCN
fsrcnn_urban.jpg
FSRCNN
LapSRN is not trained for 3x
because of its architecture
edsr_urban.jpg
EDSR
28.0118 / 0.8588 / 0.03074828.0184 / 0.8597 / 0.094173 30.5671 / 0.9019 / 9.517580

4x scaling factor

Set14: comic.png size: 250x361
orig_comic.jpg
Original
bicubic_comic.jpg
Bicubic interpolation
nearest_comic.jpg
Nearest neighbor interpolation
lanczos_comic.jpg
Lanczos interpolation
PSRN / SSIM / Speed (CPU)19.6766 / 0.6413 / 0.000262 18.5106 / 0.5879 / 0.000085 19.4948 / 0.6317 / 0.001098
espcn_comic.jpg
ESPCN
fsrcnn_comic.jpg
FSRCNN
lapsrn_comic.jpg
LapSRN
edsr_comic.jpg
EDSR
20.0417 / 0.6302 / 0.00189420.0885 / 0.6384 / 0.002103 20.0676 / 0.6339 / 0.061640 20.5233 / 0.6901 / 0.665876

8x scaling factor

Div2K: 0006.png size: 1356x2040
orig_div2k.jpg
Original
bicubic_div2k.jpg
Bicubic interpolation
nearest_div2k.jpg
Nearest neighbor interpolation
PSRN / SSIM / Speed (CPU)26.3139 / 0.8033 / 0.00110723.8291 / 0.7340 / 0.000611
lanczos_div2k.jpg
Lanczos interpolation
lapsrn_div2k.jpg
LapSRN
26.1565 / 0.7962 / 0.00478226.7046 / 0.7987 / 2.274290
Generated on Wed Oct 9 2019 23:24:57 for OpenCV by   doxygen 1.8.12