File Input and Output using XML and YAML files

Prev Tutorial: Discrete Fourier Transform

Next Tutorial: Interoperability with OpenCV 1

Goal

You'll find answers for the following questions:

• How to print and read text entries to a file and OpenCV using YAML or XML files?
• How to do the same for OpenCV data structures?
• How to do this for your data structures?
• Usage of OpenCV data structures such as cv::FileStorage , cv::FileNode or cv::FileNodeIterator .

Source code

C++

You can download this from here or find it in the samples/cpp/tutorial_code/core/file_input_output/file_input_output.cpp of the OpenCV source code library.

Here's a sample code of how to achieve all the stuff enumerated at the goal list.

#include <opencv2/core.hpp>
#include <iostream>
#include <string>

using namespace cv;
using namespace std;

static void help(char** av)
{
    cout << endl
        << av[0] << " shows the usage of the OpenCV serialization functionality."         << endl
        << "usage: "                                                                      << endl
        <<  av[0] << " outputfile.yml.gz"                                                 << endl
        << "The output file may be either XML (xml) or YAML (yml/yaml). You can even compress it by "
        << "specifying this in its extension like xml.gz yaml.gz etc... "                  << endl
        << "With FileStorage you can serialize objects in OpenCV by using the << and >> operators" << endl
        << "For example: - create a class and have it serialized"                         << endl
        << "             - use it to read and write matrices."                            << endl;
}

class MyData
{
public:
    MyData() : A(0), X(0), id()
    {}
    explicit MyData(int) : A(97), X(CV_PI), id("mydata1234") // explicit to avoid implicit conversion
    {}
    void write(FileStorage& fs) const                        //Write serialization for this class
    {
        fs << "{" << "A" << A << "X" << X << "id" << id << "}";
    }
    void read(const FileNode& node)                          //Read serialization for this class
    {
        A = (int)node["A"];
        X = (double)node["X"];
        id = (string)node["id"];
    }
public:   // Data Members
    int A;
    double X;
    string id;
};

//These write and read functions must be defined for the serialization in FileStorage to work
static void write(FileStorage& fs, const std::string&, const MyData& x)
{
    x.write(fs);
}
static void read(const FileNode& node, MyData& x, const MyData& default_value = MyData()){
    if(node.empty())
        x = default_value;
    else
        x.read(node);
}

// This function will print our custom class to the console
static ostream& operator<<(ostream& out, const MyData& m)
{
    out << "{ id = " << m.id << ", ";
    out << "X = " << m.X << ", ";
    out << "A = " << m.A << "}";
    return out;
}

int main(int ac, char** av)
{
    if (ac != 2)
    {
        help(av);
        return 1;
    }

    string filename = av[1];
    { //write
        Mat R = Mat_<uchar>::eye(3, 3),
            T = Mat_<double>::zeros(3, 1);
        MyData m(1);

        FileStorage fs(filename, FileStorage::WRITE);
        // or:
        // FileStorage fs;
        // fs.open(filename, FileStorage::WRITE);

        fs << "iterationNr" << 100;
        fs << "strings" << "[";                              // text - string sequence
        fs << "image1.jpg" << "Awesomeness" << "../data/baboon.jpg";
        fs << "]";                                           // close sequence

        fs << "Mapping";                              // text - mapping
        fs << "{" << "One" << 1;
        fs <<        "Two" << 2 << "}";

        fs << "R" << R;                                      // cv::Mat
        fs << "T" << T;

        fs << "MyData" << m;                                // your own data structures

        fs.release();                                       // explicit close
        cout << "Write Done." << endl;
    }

    {//read
        cout << endl << "Reading: " << endl;
        FileStorage fs;
        fs.open(filename, FileStorage::READ);

        int itNr;
        //fs["iterationNr"] >> itNr;
        itNr = (int) fs["iterationNr"];
        cout << itNr;
        if (!fs.isOpened())
        {
            cerr << "Failed to open " << filename << endl;
            help(av);
            return 1;
        }

        FileNode n = fs["strings"];                         // Read string sequence - Get node
        if (n.type() != FileNode::SEQ)
        {
            cerr << "strings is not a sequence! FAIL" << endl;
            return 1;
        }

        FileNodeIterator it = n.begin(), it_end = n.end(); // Go through the node
        for (; it != it_end; ++it)
            cout << (string)*it << endl;


        n = fs["Mapping"];                                // Read mappings from a sequence
        cout << "Two  " << (int)(n["Two"]) << "; ";
        cout << "One  " << (int)(n["One"]) << endl << endl;


        MyData m;
        Mat R, T;

        fs["R"] >> R;                                      // Read cv::Mat
        fs["T"] >> T;
        fs["MyData"] >> m;                                 // Read your own structure_

        cout << endl
            << "R = " << R << endl;
        cout << "T = " << T << endl << endl;
        cout << "MyData = " << endl << m << endl << endl;

        //Show default behavior for non existing nodes
        cout << "Attempt to read NonExisting (should initialize the data structure with its default).";
        fs["NonExisting"] >> m;
        cout << endl << "NonExisting = " << endl << m << endl;
    }

    cout << endl
        << "Tip: Open up " << filename << " with a text editor to see the serialized data." << endl;

    return 0;
}

Python

You can download this from here or find it in the samples/python/tutorial_code/core/file_input_output/file_input_output.py of the OpenCV source code library.

Here's a sample code of how to achieve all the stuff enumerated at the goal list.

from __future__ import print_function

import numpy as np
import cv2 as cv
import sys

def help(filename):
    print (
        '''
        {0} shows the usage of the OpenCV serialization functionality. \n\n
        usage:\n
            python3 {0} outputfile.yml.gz\n\n
        The output file may be either in XML, YAML or JSON. You can even compress it\n
        by specifying this in its extension like xml.gz yaml.gz etc... With\n
        FileStorage you can serialize objects in OpenCV.\n\n
        For example: - create a class and have it serialized\n
                     - use it to read and write matrices.\n
        '''.format(filename)
    )

class MyData:
    A = 97
    X = np.pi
    name = 'mydata1234'

    def __repr__(self):
        s = '{ name = ' + self.name + ', X = ' + str(self.X)
        s = s + ', A = ' +  str(self.A) + '}'
        return s

    
    def write(self, fs, name):
        fs.startWriteStruct(name, cv.FileNode_MAP|cv.FileNode_FLOW)
        fs.write('A', self.A)
        fs.write('X', self.X)
        fs.write('name', self.name)
        fs.endWriteStruct()

    def read(self, node):
        if (not node.empty()):
            self.A = int(node.getNode('A').real())
            self.X = node.getNode('X').real()
            self.name = node.getNode('name').string()
        else:
            self.A = self.X = 0
            self.name = ''
    

def main(argv):
    if len(argv) != 2:
        help(argv[0])
        exit(1)

    # write
    
    R = np.eye(3,3)
    T = np.zeros((3,1))
    
    m = MyData()
    

    filename = argv[1]

    
    s = cv.FileStorage(filename, cv.FileStorage_WRITE)
    # or:
    # s = cv.FileStorage()
    # s.open(filename, cv.FileStorage_WRITE)
    

    
    s.write('iterationNr', 100)
    

    
    s.startWriteStruct('strings', cv.FileNode_SEQ)
    for elem in ['image1.jpg', 'Awesomeness', '../data/baboon.jpg']:
        s.write('', elem)
    s.endWriteStruct()
    

    
    s.startWriteStruct('Mapping', cv.FileNode_MAP)
    s.write('One', 1)
    s.write('Two', 2)
    s.endWriteStruct()
    

    
    s.write('R_MAT', R)
    s.write('T_MAT', T)
    

    
    m.write(s, 'MyData')
    
    s.release()
    
    print ('Write Done.')

    # read
    print ('\nReading: ')
    s = cv.FileStorage()
    s.open(filename, cv.FileStorage_READ)

    
    n = s.getNode('iterationNr')
    itNr = int(n.real())
    
    print (itNr)

    if (not s.isOpened()):
        print ('Failed to open ', filename, file=sys.stderr)
        help(argv[0])
        exit(1)

    
    n = s.getNode('strings')
    if (not n.isSeq()):
        print ('strings is not a sequence! FAIL', file=sys.stderr)
        exit(1)

    for i in range(n.size()):
        print (n.at(i).string())
    

    
    n = s.getNode('Mapping')
    print ('Two',int(n.getNode('Two').real()),'; ')
    print ('One',int(n.getNode('One').real()),'\n')
    

    
    R = s.getNode('R_MAT').mat()
    T = s.getNode('T_MAT').mat()
    
    m.read(s.getNode('MyData'))
    

    print ('\nR =',R)
    print ('T =',T,'\n')
    print ('MyData =','\n',m,'\n')

    
    print ('Attempt to read NonExisting (should initialize the data structure',
            'with its default).')
    m.read(s.getNode('NonExisting'))
    print ('\nNonExisting =','\n',m)
    

    print ('\nTip: Open up',filename,'with a text editor to see the serialized data.')

if __name__ == '__main__':
    main(sys.argv)

Explanation

Here we talk only about XML and YAML file inputs. Your output (and its respective input) file may have only one of these extensions and the structure coming from this. They are two kinds of data structures you may serialize: mappings (like the STL map and the Python dictionary) and element sequence (like the STL vector). The difference between these is that in a map every element has a unique name through what you may access it. For sequences you need to go through them to query a specific item.

1. XML/YAML File Open and Close. Before you write any content to such file you need to open it and at the end to close it. The XML/YAML data structure in OpenCV is cv::FileStorage . To specify that this structure to which file binds on your hard drive you can use either its constructor or the open() function of this:

   C++

           FileStorage fs(filename, FileStorage::WRITE);
           // or:
           // FileStorage fs;
           // fs.open(filename, FileStorage::WRITE);

   Python

       s = cv.FileStorage(filename, cv.FileStorage_WRITE)
       # or:
       # s = cv.FileStorage()
       # s.open(filename, cv.FileStorage_WRITE)

   Either one of this you use the second argument is a constant specifying the type of operations you'll be able to on them: WRITE, READ or APPEND. The extension specified in the file name also determinates the output format that will be used. The output may be even compressed if you specify an extension such as *.xml.gz*.

   The file automatically closes when the cv::FileStorage objects is destroyed. However, you may explicitly call for this by using the release function:

   C++

           fs.release();                                       // explicit close

   Python

       s.release()

2. Input and Output of text and numbers. In C++, the data structure uses the << output operator in the STL library. In Python, cv::FileStorage::write() is used instead. For outputting any type of data structure we need first to specify its name. We do this by just simply pushing the name of this to the stream in C++. In Python, the first parameter for the write function is the name. For basic types you may follow this with the print of the value :
   C++

           fs << "iterationNr" << 100;

   Python

       s.write('iterationNr', 100)

   Reading in is a simple addressing (via the [] operator) and casting operation or a read via the >> operator. In Python, we address with getNode() and use real() :
   C++

           int itNr;
           //fs["iterationNr"] >> itNr;
           itNr = (int) fs["iterationNr"];

   Python

           int itNr;
           //fs["iterationNr"] >> itNr;
           itNr = (int) fs["iterationNr"];

3. Input/Output of OpenCV Data structures. Well these behave exactly just as the basic C++ and Python types:
   C++

           Mat R = Mat_<uchar>::eye(3, 3),
               T = Mat_<double>::zeros(3, 1);

           fs << "R" << R;                                      // cv::Mat
           fs << "T" << T;

           fs["R"] >> R;                                      // Read cv::Mat
           fs["T"] >> T;

   Python

       R = np.eye(3,3)
       T = np.zeros((3,1))

       s.write('R_MAT', R)
       s.write('T_MAT', T)

       R = s.getNode('R_MAT').mat()
       T = s.getNode('T_MAT').mat()

4. Input/Output of vectors (arrays) and associative maps. As I mentioned beforehand, we can output maps and sequences (array, vector) too. Again we first print the name of the variable and then we have to specify if our output is either a sequence or map.

   For sequence before the first element print the "[" character and after the last one the "]" character. With Python, call FileStorage.startWriteStruct(structure_name, struct_type), where struct_type is cv2.FileNode_MAP or cv2.FileNode_SEQ to start writing the structure. Call FileStorage.endWriteStruct() to finish the structure:

   C++

           fs << "strings" << "[";                              // text - string sequence
           fs << "image1.jpg" << "Awesomeness" << "../data/baboon.jpg";
           fs << "]";                                           // close sequence

   Python

       s.startWriteStruct('strings', cv.FileNode_SEQ)
       for elem in ['image1.jpg', 'Awesomeness', '../data/baboon.jpg']:
           s.write('', elem)
       s.endWriteStruct()

   For maps the drill is the same however now we use the "{" and "}" delimiter characters:

   C++

           fs << "Mapping";                              // text - mapping
           fs << "{" << "One" << 1;
           fs <<        "Two" << 2 << "}";

   Python

       s.startWriteStruct('Mapping', cv.FileNode_MAP)
       s.write('One', 1)
       s.write('Two', 2)
       s.endWriteStruct()

   To read from these we use the cv::FileNode and the cv::FileNodeIterator data structures. The [] operator of the cv::FileStorage class (or the getNode() function in Python) returns a cv::FileNode data type. If the node is sequential we can use the cv::FileNodeIterator to iterate through the items. In Python, the at() function can be used to address elements of the sequence and the size() function returns the length of the sequence:

   C++

           FileNode n = fs["strings"];                         // Read string sequence - Get node
           if (n.type() != FileNode::SEQ)
           {
               cerr << "strings is not a sequence! FAIL" << endl;
               return 1;
           }
   
           FileNodeIterator it = n.begin(), it_end = n.end(); // Go through the node
           for (; it != it_end; ++it)
               cout << (string)*it << endl;

   Python

       n = s.getNode('strings')
       if (not n.isSeq()):
           print ('strings is not a sequence! FAIL', file=sys.stderr)
           exit(1)
   
       for i in range(n.size()):
           print (n.at(i).string())

   For maps you can use the [] operator (at() function in Python) again to access the given item (or the >> operator too):

   C++

           n = fs["Mapping"];                                // Read mappings from a sequence
           cout << "Two  " << (int)(n["Two"]) << "; ";
           cout << "One  " << (int)(n["One"]) << endl << endl;

   Python

       n = s.getNode('Mapping')
       print ('Two',int(n.getNode('Two').real()),'; ')
       print ('One',int(n.getNode('One').real()),'\n')

5. Read and write your own data structures. Suppose you have a data structure such as:

   C++

   class MyData
   {
   public:
         MyData() : A(0), X(0), id() {}
   public:   // Data Members
      int A;
      double X;
      string id;
   };

   Python

   class MyData:
       def __init__(self):
           self.A = self.X = 0
           self.name = ''

   In C++, it's possible to serialize this through the OpenCV I/O XML/YAML interface (just as in case of the OpenCV data structures) by adding a read and a write function inside and outside of your class. In Python, you can get close to this by implementing a read and write function inside the class. For the inside part:

   C++

       void write(FileStorage& fs) const                        //Write serialization for this class
       {
           fs << "{" << "A" << A << "X" << X << "id" << id << "}";
       }
       void read(const FileNode& node)                          //Read serialization for this class
       {
           A = (int)node["A"];
           X = (double)node["X"];
           id = (string)node["id"];
       }

   Python

       def write(self, fs, name):
           fs.startWriteStruct(name, cv.FileNode_MAP|cv.FileNode_FLOW)
           fs.write('A', self.A)
           fs.write('X', self.X)
           fs.write('name', self.name)
           fs.endWriteStruct()
   
       def read(self, node):
           if (not node.empty()):
               self.A = int(node.getNode('A').real())
               self.X = node.getNode('X').real()
               self.name = node.getNode('name').string()
           else:
               self.A = self.X = 0
               self.name = ''

   C++

   In C++, you need to add the following functions definitions outside the class:

   static void write(FileStorage& fs, const std::string&, const MyData& x)
   {
       x.write(fs);
   }
   static void read(const FileNode& node, MyData& x, const MyData& default_value = MyData()){
       if(node.empty())
           x = default_value;
       else
           x.read(node);
   }

   Here you can observe that in the read section we defined what happens if the user tries to read a non-existing node. In this case we just return the default initialization value, however a more verbose solution would be to return for instance a minus one value for an object ID.

   Once you added these four functions use the >> operator for write and the << operator for read (or the defined input/output functions for Python):

   C++

           MyData m(1);

           fs << "MyData" << m;                                // your own data structures

           fs["MyData"] >> m;                                 // Read your own structure_

   Python

       m = MyData()

       m.write(s, 'MyData')

       m.read(s.getNode('MyData'))

   Or to try out reading a non-existing read:

   C++

           cout << "Attempt to read NonExisting (should initialize the data structure with its default).";
           fs["NonExisting"] >> m;
           cout << endl << "NonExisting = " << endl << m << endl;

   Python

       print ('Attempt to read NonExisting (should initialize the data structure',
               'with its default).')
       m.read(s.getNode('NonExisting'))
       print ('\nNonExisting =','\n',m)

Result

Well mostly we just print out the defined numbers. On the screen of your console you could see:

Write Done.

Reading:
100image1.jpg
Awesomeness
baboon.jpg
Two  2; One  1


R = [1, 0, 0;
  0, 1, 0;
  0, 0, 1]
T = [0; 0; 0]

MyData =
{ id = mydata1234, X = 3.14159, A = 97}

Attempt to read NonExisting (should initialize the data structure with its default).
NonExisting =
{ id = , X = 0, A = 0}

Tip: Open up output.xml with a text editor to see the serialized data.

Nevertheless, it's much more interesting what you may see in the output xml file:

<?xml version="1.0"?>
<opencv_storage>
<iterationNr>100</iterationNr>
<strings>
  image1.jpg Awesomeness baboon.jpg</strings>
<Mapping>
  <One>1</One>
  <Two>2</Two></Mapping>
<R type_id="opencv-matrix">
  <rows>3</rows>
  <cols>3</cols>
  <dt>u</dt>
  <data>
    1 0 0 0 1 0 0 0 1</data></R>
<T type_id="opencv-matrix">
  <rows>3</rows>
  <cols>1</cols>
  <dt>d</dt>
  <data>
    0. 0. 0.</data></T>
<MyData>
  <A>97</A>
  <X>3.1415926535897931e+000</X>
  <id>mydata1234</id></MyData>
</opencv_storage>

Or the YAML file:

%YAML:1.0
iterationNr: 100
strings:
   - "image1.jpg"
   - Awesomeness
   - "baboon.jpg"
Mapping:
   One: 1
   Two: 2
R: !!opencv-matrix
   rows: 3
   cols: 3
   dt: u
   data: [ 1, 0, 0, 0, 1, 0, 0, 0, 1 ]
T: !!opencv-matrix
   rows: 3
   cols: 1
   dt: d
   data: [ 0., 0., 0. ]
MyData:
   A: 97
   X: 3.1415926535897931e+000
   id: mydata1234

You may observe a runtime instance of this on the YouTube here .