1 /*
2  * Software License Agreement (BSD License)
3  *
4  *  Copyright (c) 2009, Willow Garage, Inc.
5  *  All rights reserved.
6  *
7  *  Redistribution and use in source and binary forms, with or without
8  *  modification, are permitted provided that the following conditions
9  *  are met:
10  *
11  *   * Redistributions of source code must retain the above copyright
12  *     notice, this list of conditions and the following disclaimer.
13  *   * Redistributions in binary form must reproduce the above
14  *     copyright notice, this list of conditions and the following
15  *     disclaimer in the documentation and/or other materials provided
16  *     with the distribution.
17  *   * Neither the name of Willow Garage, Inc. nor the names of its
18  *     contributors may be used to endorse or promote products derived
19  *     from this software without specific prior written permission.
20  *
21  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  *  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25  *  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  *  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  *  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29  *  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30  *  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31  *  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32  *  POSSIBILITY OF SUCH DAMAGE.
33  *
34  */
35 /**
36  * @file demo_classify.cpp
37  * @brief Feature extraction and classification.
38  * @author Yida Wang
39  */
40 #include <opencv2/cnn_3dobj.hpp>
41 #include <opencv2/features2d.hpp>
42 #include <iomanip>
43 using namespace cv;
44 using namespace std;
45 using namespace cv::cnn_3dobj;
46 
47 /**
48  * @function listDir
49  * @brief Making all files names under a directory into a list
50  */
listDir(const char * path,std::vector<String> & files,bool r)51 static void listDir(const char *path, std::vector<String>& files, bool r)
52 {
53     DIR *pDir;
54     struct dirent *ent;
55     char childpath[512];
56     pDir = opendir(path);
57     memset(childpath, 0, sizeof(childpath));
58     while ((ent = readdir(pDir)) != NULL)
59     {
60         if (ent->d_type & DT_DIR)
61         {
62             if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0 || strcmp(ent->d_name, ".DS_Store") == 0)
63             {
64                 continue;
65             }
66             if (r)
67             {
68                 sprintf(childpath, "%s/%s", path, ent->d_name);
69                 listDir(childpath,files,false);
70             }
71         }
72         else
73         {
74             if (strcmp(ent->d_name, ".DS_Store") != 0)
75                 files.push_back(ent->d_name);
76         }
77     }
78     sort(files.begin(),files.end());
79 };
80 
81 /**
82  * @function featureWrite
83  * @brief Writing features of gallery images into binary files
84  */
featureWrite(const Mat & features,const String & fname)85 static int featureWrite(const Mat &features, const String &fname)
86 {
87     ofstream ouF;
88     ouF.open(fname.c_str(), std::ofstream::binary);
89     if (!ouF)
90     {
91         cerr << "failed to open the file : " << fname << endl;
92         return 0;
93     }
94     for (int r = 0; r < features.rows; r++)
95     {
96         ouF.write(reinterpret_cast<const char*>(features.ptr(r)), features.cols*features.elemSize());
97     }
98     ouF.close();
99     return 1;
100 }
101 
102 /**
103  * @function main
104  */
main(int argc,char ** argv)105 int main(int argc, char** argv)
106 {
107     const String keys = "{help | | This sample will extract features from reference images and target image for classification. You can add a mean_file if there little variance in data such as human faces, otherwise it is not so useful}"
108     "{src_dir | ../data/images_all/ | Source direction of the images ready for being used for extract feature as gallery.}"
109     "{caffemodel | ../../testdata/cv/3d_triplet_iter_30000.caffemodel | caffe model for feature exrtaction.}"
110     "{network_forIMG | ../../testdata/cv/3d_triplet_testIMG.prototxt | Network definition file used for extracting feature from a single image and making a classification}"
111     "{mean_file | no | The mean file generated by Caffe from all gallery images, this could be used for mean value substraction from all images. If you want to use the mean file, you can set this as ../data/images_mean/triplet_mean.binaryproto.}"
112     "{target_img | ../data/images_all/4_78.png | Path of image waiting to be classified.}"
113     "{feature_blob | feat | Name of layer which will represent as the feature, in this network, ip1 or feat is well.}"
114     "{num_candidate | 15 | Number of candidates in gallery as the prediction result.}"
115     "{device | CPU | Device type: CPU or GPU}"
116     "{dev_id | 0 | Device id}"
117     "{gallery_out | 0 | Option on output binary features on gallery images}";
118     /* get parameters from comand line */
119     cv::CommandLineParser parser(argc, argv, keys);
120     parser.about("Feature extraction and classification");
121     if (parser.has("help"))
122     {
123         parser.printMessage();
124         return 0;
125     }
126     String src_dir = parser.get<String>("src_dir");
127     String caffemodel = parser.get<String>("caffemodel");
128     String network_forIMG   = parser.get<String>("network_forIMG");
129     String mean_file    = parser.get<String>("mean_file");
130     String target_img   = parser.get<String>("target_img");
131     String feature_blob = parser.get<String>("feature_blob");
132     int num_candidate = parser.get<int>("num_candidate");
133     String device = parser.get<String>("device");
134     int gallery_out = parser.get<int>("gallery_out");
135     /* Initialize a net work with Device */
136     cv::cnn_3dobj::descriptorExtractor descriptor(device);
137     std::cout << "Using" << descriptor.getDeviceType() << std::endl;
138     /* Load net with the caffe trained net work parameter and structure */
139     if (strcmp(mean_file.c_str(), "no") == 0)
140         descriptor.loadNet(network_forIMG, caffemodel);
141     else
142         descriptor.loadNet(network_forIMG, caffemodel, mean_file);
143     std::vector<String> name_gallery;
144     /* List the file names under a given path */
145     listDir(src_dir.c_str(), name_gallery, false);
146     if (gallery_out)
147     {
148         ofstream namelist_out("gallelist.txt");
149         /* Writing name of the reference images. */
150         for (unsigned int i = 0; i < name_gallery.size(); i++)
151             namelist_out << name_gallery.at(i) << endl;
152     }
153     for (unsigned int i = 0; i < name_gallery.size(); i++)
154     {
155         name_gallery[i] = src_dir + name_gallery[i];
156     }
157     std::vector<cv::Mat> img_gallery;
158     cv::Mat feature_reference;
159     for (unsigned int i = 0; i < name_gallery.size(); i++)
160     {
161         img_gallery.push_back(cv::imread(name_gallery[i]));
162     }
163     /* Extract feature from a set of images */
164     descriptor.extract(img_gallery, feature_reference, feature_blob);
165     if (gallery_out)
166     {
167         std::cout << std::endl << "---------- Features of gallery images ----------" << std::endl;
168         /* Print features of the reference images. */
169         for (int i = 0; i < feature_reference.rows; i++)
170             std::cout << feature_reference.row(i) << endl;
171         std::cout << std::endl << "---------- Saving features of gallery images into feature.bin ----------" << std::endl;
172         featureWrite(feature_reference, "feature.bin");
173     }
174     else
175     {
176         std::cout << std::endl << "---------- Prediction for " << target_img << " ----------" << std::endl;
177         cv::Mat img = cv::imread(target_img);
178         std::cout << std::endl << "---------- Features of gallery images ----------" << std::endl;
179         std::vector<std::pair<String, float> > prediction;
180         /* Print features of the reference images. */
181         for (int i = 0; i < feature_reference.rows; i++)
182             std::cout << feature_reference.row(i) << endl;
183         cv::Mat feature_test;
184         descriptor.extract(img, feature_test, feature_blob);
185         /* Initialize a matcher which using L2 distance. */
186         cv::BFMatcher matcher(NORM_L2);
187         std::vector<std::vector<cv::DMatch> > matches;
188         /* Have a KNN match on the target and reference images. */
189         matcher.knnMatch(feature_test, feature_reference, matches, num_candidate);
190         /* Print feature of the target image waiting to be classified. */
191         std::cout << std::endl << "---------- Features of target image: " << target_img << "----------" << endl << feature_test << std::endl;
192         /* Print the top N prediction. */
193         std::cout << std::endl << "---------- Prediction result(Distance - File Name in Gallery) ----------" << std::endl;
194         for (size_t i = 0; i < matches[0].size(); ++i)
195         {
196             std::cout << i << " - " << std::fixed << std::setprecision(2) << name_gallery[matches[0][i].trainIdx] << " - \""  << matches[0][i].distance << "\"" << std::endl;
197         }
198     }
199     return 0;
200 }
201