Super-resolution benchmarking#

Benchmarking#

The super-resolution module contains sample codes for benchmarking, in order to compare different models and algorithms. Here is presented a sample code for performing benchmarking, and then a few benchmarking results are collected. It was performed on an Intel i7-9700K CPU on an Ubuntu 18.04.02 OS.

Source Code of the sample#

// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.

#include <iostream>
#include <opencv2/opencv_modules.hpp>

#ifdef HAVE_OPENCV_QUALITY
#include <opencv2/dnn_superres.hpp>
#include <opencv2/quality.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>

using namespace std;
using namespace cv;
using namespace dnn_superres;

static void showBenchmark(vector<Mat> images, string title, Size imageSize,
                          const vector<String> imageTitles,
                          const vector<double> psnrValues,
                          const vector<double> ssimValues)
{
    int fontFace = FONT_HERSHEY_COMPLEX_SMALL;
    int fontScale = 1;
    Scalar fontColor = Scalar(255, 255, 255);

    int len = static_cast<int>(images.size());

    int cols = 2, rows = 2;

    Mat fullImage = Mat::zeros(Size((cols * 10) + imageSize.width * cols, (rows * 10) + imageSize.height * rows),
                               images[0].type());

    stringstream ss;
    int h_ = -1;
    for (int i = 0; i < len; i++) {

        int fontStart = 15;
        int w_ = i % cols;
        if (i % cols == 0)
            h_++;

        Rect ROI((w_ * (10 + imageSize.width)), (h_ * (10 + imageSize.height)), imageSize.width, imageSize.height);
        Mat tmp;
        resize(images[i], tmp, Size(ROI.width, ROI.height));

        ss << imageTitles[i];
        putText(tmp,
                ss.str(),
                Point(5, fontStart),
                fontFace,
                fontScale,
                fontColor,
                1,
                16);

        ss.str("");
        fontStart += 20;

        ss << "PSNR: " << psnrValues[i];
        putText(tmp,
                ss.str(),
                Point(5, fontStart),
                fontFace,
                fontScale,
                fontColor,
                1,
                16);

        ss.str("");
        fontStart += 20;

        ss << "SSIM: " << ssimValues[i];
        putText(tmp,
                ss.str(),
                Point(5, fontStart),
                fontFace,
                fontScale,
                fontColor,
                1,
                16);

        ss.str("");
        fontStart += 20;

        tmp.copyTo(fullImage(ROI));
    }

    namedWindow(title, 1);
    imshow(title, fullImage);
    waitKey();
}

static Vec2d getQualityValues(Mat orig, Mat upsampled)
{
    double psnr = PSNR(upsampled, orig);
    Scalar q = quality::QualitySSIM::compute(upsampled, orig, noArray());
    double ssim = mean(Vec3d((q[0]), q[1], q[2]))[0];
    return Vec2d(psnr, ssim);
}

int main(int argc, char *argv[])
{
    // Check for valid command line arguments, print usage
    // if insufficient arguments were given.
    if (argc < 4) {
        cout << "usage:   Arg 1: image path  | Path to image" << endl;
        cout << "\t Arg 2: algorithm | edsr, espcn, fsrcnn or lapsrn" << endl;
        cout << "\t Arg 3: path to model file 2 \n";
        cout << "\t Arg 4: scale  | 2, 3, 4 or 8 \n";
        return -1;
    }

    string path = string(argv[1]);
    string algorithm = string(argv[2]);
    string model = string(argv[3]);
    int scale = atoi(argv[4]);

    Mat img = imread(path);
    if (img.empty()) {
        cerr << "Couldn't load image: " << img << "\n";
        return -2;
    }

    //Crop the image so the images will be aligned
    int width = img.cols - (img.cols % scale);
    int height = img.rows - (img.rows % scale);
    Mat cropped = img(Rect(0, 0, width, height));

    //Downscale the image for benchmarking
    Mat img_downscaled;
    resize(cropped, img_downscaled, Size(), 1.0 / scale, 1.0 / scale);

    //Make dnn super resolution instance
    DnnSuperResImpl sr;

    vector <Mat> allImages;
    Mat img_new;

    //Read and set the dnn model
    sr.readModel(model);
    sr.setModel(algorithm, scale);
    sr.upsample(img_downscaled, img_new);

    vector<double> psnrValues = vector<double>();
    vector<double> ssimValues = vector<double>();

    //DL MODEL
    Vec2f quality = getQualityValues(cropped, img_new);

    psnrValues.push_back(quality[0]);
    ssimValues.push_back(quality[1]);

    cout << sr.getAlgorithm() << ":" << endl;
    cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
    cout << "----------------------" << endl;

    //BICUBIC
    Mat bicubic;
    resize(img_downscaled, bicubic, Size(), scale, scale, INTER_CUBIC);
    quality = getQualityValues(cropped, bicubic);

    psnrValues.push_back(quality[0]);
    ssimValues.push_back(quality[1]);

    cout << "Bicubic " << endl;
    cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
    cout << "----------------------" << endl;

    //NEAREST NEIGHBOR
    Mat nearest;
    resize(img_downscaled, nearest, Size(), scale, scale, INTER_NEAREST);
    quality = getQualityValues(cropped, nearest);

    psnrValues.push_back(quality[0]);
    ssimValues.push_back(quality[1]);

    cout << "Nearest neighbor" << endl;
    cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
    cout << "----------------------" << endl;

    //LANCZOS
    Mat lanczos;
    resize(img_downscaled, lanczos, Size(), scale, scale, INTER_LANCZOS4);
    quality = getQualityValues(cropped, lanczos);

    psnrValues.push_back(quality[0]);
    ssimValues.push_back(quality[1]);

    cout << "Lanczos" << endl;
    cout << "PSNR: " << quality[0] << " SSIM: " << quality[1] << endl;
    cout << "-----------------------------------------------" << endl;

    vector <Mat> imgs{img_new, bicubic, nearest, lanczos};
    vector <String> titles{sr.getAlgorithm(), "Bicubic", "Nearest neighbor", "Lanczos"};
    showBenchmark(imgs, "Quality benchmark", Size(bicubic.cols, bicubic.rows), titles, psnrValues, ssimValues);

    waitKey(0);

    return 0;
}
#else
int main()
{
    std::cout << "This sample requires the OpenCV Quality module." << std::endl;
    return 0;
}
#endif

Explanation#

  1. Read and downscale the image

    int width = img.cols - (img.cols % scale);
int height = img.rows - (img.rows % scale);
Mat cropped = img(Rect(0, 0, width, height));
Mat img_downscaled;
cv::resize(cropped, img_downscaled, cv::Size(), 1.0 / scale, 1.0 / scale);

    Resize the image by the scaling factor. Before that a cropping is necessary, so the images will align.

  2. Set the model

    DnnSuperResImpl sr;
sr.readModel(path);
sr.setModel(algorithm, scale);
sr.upsample(img_downscaled, img_new);

    Instantiate a dnn super-resolution object. Read and set the algorithm and scaling factor.

  3. Perform benchmarking

    double psnr = PSNR(img_new, cropped);
Scalar q = cv::quality::QualitySSIM::compute(img_new, cropped, cv::noArray());
double ssim = mean(cv::Vec3f(q[0], q[1], q[2]))[0];

    Calculate PSNR and SSIM. Use OpenCVs PSNR (core opencv) and SSIM (contrib) functions to compare the images. Repeat it with other upscaling algorithms, such as other DL models or interpolation methods (eg. bicubic, nearest neighbor).

Benchmarking results#

General100 dataset#

2x scaling factor#

Avg inference time in sec (CPU)

Avg PSNR

Avg SSIM

ESPCN

0.008795

32.7059

0.9276

EDSR

5.923450

34.1300

0.9447

FSRCNN

0.021741

32.8886

0.9301

LapSRN

0.114812

32.2681

0.9248

Bicubic

0.000208

32.1638

0.9305

Nearest neighbor

0.000114

29.1665

0.9049

Lanczos

0.001094

32.4687

0.9327

3x scaling factor#

Avg inference time in sec (CPU)

Avg PSNR

Avg SSIM

ESPCN

0.005495

28.4229

0.8474

EDSR

2.455510

29.9828

0.8801

FSRCNN

0.008807

28.3068

0.8429

LapSRN

0.282575

26.7330

0.8862

Bicubic

0.000311

26.0635

0.8754

Nearest neighbor

0.000148

23.5628

0.8174

Lanczos

0.001012

25.9115

0.8706

4x scaling factor#

Avg inference time in sec (CPU)

Avg PSNR

Avg SSIM

ESPCN

0.004311

26.6870

0.7891

EDSR

1.607570

28.1552

0.8317

FSRCNN

0.005302

26.6088

0.7863

LapSRN

0.121229

26.7383

0.7896

Bicubic

0.000311

26.0635

0.8754

Nearest neighbor

0.000148

23.5628

0.8174

Lanczos

0.001012

25.9115

0.8706

Images#

2x scaling factor#

Set5: butterfly.png

size: 256x256

Original

Bicubic interpolation

Nearest neighbor interpolation

Lanczos interpolation

PSRN / SSIM / Speed (CPU)

26.6645 / 0.9048 / 0.000201

23.6854 / 0.8698 / 0.000075

26.9476 / 0.9075 / 0.001039

ESPCN

FSRCNN

LapSRN

EDSR

29.0341 / 0.9354 / 0.004157

29.0077 / 0.9345 / 0.006325

27.8212 / 0.9230 / 0.037937

30.0347 / 0.9453 / 2.077280

3x scaling factor#

Urban100: img_001.png

size: 1024x644

Original

Bicubic interpolation

Nearest neighbor interpolation

Lanczos interpolation

PSRN / SSIM / Speed (CPU)

27.0474 / 0.8484 / 0.000391

26.0842 / 0.8353 / 0.000236

27.0704 / 0.8483 / 0.002234

ESPCN

FSRCNN

LapSRN is not trained for 3x
because of its architecture

EDSR

28.0118 / 0.8588 / 0.030748

28.0184 / 0.8597 / 0.094173

30.5671 / 0.9019 / 9.517580

4x scaling factor#

Set14: comic.png

size: 250x361

Original

Bicubic interpolation

Nearest neighbor interpolation

Lanczos interpolation

PSRN / SSIM / Speed (CPU)

19.6766 / 0.6413 / 0.000262

18.5106 / 0.5879 / 0.000085

19.4948 / 0.6317 / 0.001098

ESPCN

FSRCNN

LapSRN

EDSR

20.0417 / 0.6302 / 0.001894

20.0885 / 0.6384 / 0.002103

20.0676 / 0.6339 / 0.061640

20.5233 / 0.6901 / 0.665876

8x scaling factor#

Div2K: 0006.png

size: 1356x2040

Original

Bicubic interpolation

Nearest neighbor interpolation

PSRN / SSIM / Speed (CPU)

26.3139 / 0.8033 / 0.001107

23.8291 / 0.7340 / 0.000611

Lanczos interpolation

LapSRN

26.1565 / 0.7962 / 0.004782

26.7046 / 0.7987 / 2.274290