1 /**
2  * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "Loader.h"
18 
19 #include "glow/Base/Image.h"
20 #include "glow/Converter/TypeAToTypeBFunctionConverter.h"
21 #include "glow/Graph/Nodes.h"
22 #include "glow/Importer/Caffe2ModelLoader.h"
23 #include "glow/Importer/ONNXModelLoader.h"
24 
25 #include "llvm/ADT/StringSwitch.h"
26 #include "llvm/Support/CommandLine.h"
27 #include "llvm/Support/Format.h"
28 #include "llvm/Support/Timer.h"
29 #include "llvm/Support/raw_ostream.h"
30 
31 #include <atomic>
32 #include <cfloat>
33 #include <fstream>
34 #include <future>
35 #include <iostream>
36 #include <memory>
37 #include <mutex>
38 #include <queue>
39 #include <sstream>
40 #include <thread>
41 
42 #include "ExecutorCore.h"
43 #include "ExecutorCoreHelperFunctions.h"
44 
45 using namespace glow;
46 
47 namespace {
48 
49 /// Image loader options.
50 llvm::cl::OptionCategory imageClassifierCat("Image Loader Options");
51 
52 llvm::cl::opt<unsigned> labelOffset(
53     "label-offset",
54     llvm::cl::desc("Label offset for TF ONNX models with 1001 classes"),
55     llvm::cl::Optional, llvm::cl::init(0), llvm::cl::cat(imageClassifierCat));
56 
57 llvm::cl::opt<bool>
58     computeSoftmax("compute-softmax",
59                    llvm::cl::desc("Compute softmax of the network output"),
60                    llvm::cl::Optional, llvm::cl::init(false),
61                    llvm::cl::cat(imageClassifierCat));
62 
63 llvm::cl::opt<unsigned>
64     topKCount("topk",
65               llvm::cl::desc("Number of highest likelihood labels to print and "
66                              "match the correspondent expected-lables"),
67               llvm::cl::Optional, llvm::cl::init(1),
68               llvm::cl::cat(imageClassifierCat));
69 
70 llvm::cl::list<unsigned> expectedMatchingLabels(
71     "expected-labels",
72     llvm::cl::desc("The comma delimited list of the matching lables"),
73     llvm::cl::value_desc("int"), llvm::cl::ZeroOrMore, llvm::cl::CommaSeparated,
74     llvm::cl::cat(imageClassifierCat));
75 } // unnamed namespace
76 
77 /// A pair representing a float and the index where the float was found.
78 using FloatIndexPair = std::pair<float, size_t>;
79 
80 /// Given a Handle \p H of a 1D tensor with float elements, \returns the top K
81 /// (topKCount) [float, index] pairs, i.e. the pairs with the highest floats.
82 template <typename ElemTy>
getTopKPairs(Handle<ElemTy> H)83 static std::vector<FloatIndexPair> getTopKPairs(Handle<ElemTy> H) {
84   DCHECK_LE(topKCount, H.size()) << "Function requires k < number of labels.";
85   DCHECK_EQ(H.dims().size(), 1) << "H must be a Handle of a 1d Tensor.";
86 
87   // Use a priority queue of pairs of floats (probabilities) to size_t (indices)
88   // to determine the top K pairs, and then return the indices from it.
89   std::priority_queue<FloatIndexPair, std::vector<FloatIndexPair>,
90                       std::greater<FloatIndexPair>>
91       topKQueue;
92 
93   // Loop over all the probabilites, finding the highest k probability pairs.
94   for (dim_t i = 0, e = H.size(); i < e; i++) {
95     float currProbability = H.at({i});
96     if (topKQueue.size() < topKCount) {
97       // Always push the first k elements.
98       topKQueue.push(std::make_pair(currProbability, i));
99     } else if (topKQueue.top().first < currProbability) {
100       // If the lowest element has lower probability than the current, then pop
101       // the lowest and insert the current pair.
102       topKQueue.pop();
103       topKQueue.push(std::make_pair(currProbability, i));
104     }
105   }
106 
107   // We now have the top K pairs in reverse order.
108   std::vector<FloatIndexPair> res(topKCount);
109   for (size_t i = 0; i < topKCount; i++) {
110     res[topKCount - i - 1] = topKQueue.top();
111     topKQueue.pop();
112   }
113 
114   return res;
115 }
116 
117 /// Print out the top K pairs to stdout, which were passed in via \p topKPairs.
printTopKPairs(const std::vector<FloatIndexPair> & topKPairs)118 static void printTopKPairs(const std::vector<FloatIndexPair> &topKPairs) {
119   for (size_t i = 0; i < topKPairs.size(); i++) {
120     // Some models are trained with more classes. E.g. Some imagenet models
121     // exported from TensorFlow have 1 extra "neutral" class.
122     const size_t label = topKPairs[i].second - labelOffset;
123     // Tab out the label so it aligns nicely with Label-K1.
124     if (i != 0) {
125       llvm::outs() << "\t\t\t\t\t";
126     }
127     llvm::outs() << "\tLabel-K" << i + 1 << ": " << label << " (probability: "
128                  << llvm::format("%0.4f", topKPairs[i].first) << ")\n";
129   }
130 }
131 
132 /// Checks if \p topKPairs have the index that matches the provided index,
133 /// \returns 0 on success and 1 if mismatches found.
checkExpectedLabel(llvm::ArrayRef<FloatIndexPair> topKPairs,llvm::StringRef fileName,unsigned expectedCategoryIndex)134 static int checkExpectedLabel(llvm::ArrayRef<FloatIndexPair> topKPairs,
135                               llvm::StringRef fileName,
136                               unsigned expectedCategoryIndex) {
137   // Loop through pairs and try to find a matching label.
138   for (const auto &p : topKPairs) {
139     if (p.second - labelOffset == expectedCategoryIndex) {
140       return 0;
141     }
142   }
143 
144   llvm::outs() << " File: " << fileName
145                << " doesn't match index: " << expectedCategoryIndex
146                << " in the top " << topKPairs.size() << " pairs\n";
147 
148   return 1;
149 }
150 
151 /// Apply the softmax function to the given handle.
applySoftmax(Handle<ElemTy> H)152 template <typename ElemTy> static void applySoftmax(Handle<ElemTy> H) {
153   DCHECK_EQ(H.dims().size(), 1) << "H must be a Handle of a 1d Tensor.";
154   float denominator = 0.0f;
155 
156   for (auto elem : H) {
157     denominator += std::exp(static_cast<float>(elem));
158   }
159 
160   for (auto &elem : H) {
161     elem = std::exp(static_cast<float>(elem)) / denominator;
162   }
163 }
164 
165 /// Given the output Softmax Tensor \p SMT and \p imageList, prints the
166 /// results of inference and returns number of incorrect predictions,
167 /// \returns the number of found mismatches.
168 template <typename ElemTy>
processAndPrintResultsImpl(Tensor * SMT,llvm::ArrayRef<std::string> imageList)169 static int processAndPrintResultsImpl(Tensor *SMT,
170                                       llvm::ArrayRef<std::string> imageList) {
171   // Softmax should have at least two dimensions: batchSize (first dimension),
172   // numLabels (any other dimension), and optionally - 1 in all other
173   // dimensions. The value of numLabels should be greater than 1.
174   DCHECK_GE(SMT->dims().size(), 2) << "Softmax should have at least 2 dims.";
175   const dim_t batchSize = SMT->dims()[0];
176   DCHECK_EQ(batchSize, imageList.size())
177       << "Softmax batch size must equal the input number of images.";
178   size_t labelsDim = 0;
179   for (size_t i = 1; i < SMT->dims().size(); i++) {
180     if (SMT->dims()[i] > 1) {
181       DCHECK_EQ(labelsDim, 0) << "More than one dimension of size > 1?";
182       labelsDim = i;
183     }
184   }
185   DCHECK_NE(labelsDim, 0) << "Labels dimension not found!";
186   const dim_t numLabels = SMT->dims()[labelsDim];
187   // Get a view with canonical layout {batches, labels}.
188   Tensor canonical = SMT->getUnowned({batchSize, numLabels});
189   SMT = &canonical;
190 
191   std::vector<dim_t> sliceOffset(SMT->dims().size(), 0);
192 
193   int retVal = 0;
194   for (unsigned i = 0; i < imageList.size(); i++) {
195     const auto &fileName = imageList[i];
196     if (topKCount) {
197       llvm::outs() << " File: " << fileName;
198     }
199 
200     // batchSize is the first dimension, so update it to get the next slice.
201     sliceOffset[0] = i;
202     Tensor slice = SMT->getUnowned({numLabels}, sliceOffset);
203     auto SH = slice.getHandle<ElemTy>();
204 
205     if (computeSoftmax) {
206       applySoftmax(SH);
207     }
208 
209     if (topKCount) {
210       auto topKPairs = getTopKPairs(SH);
211       printTopKPairs(topKPairs);
212       if (!expectedMatchingLabels.empty()) {
213         retVal +=
214             checkExpectedLabel(topKPairs, fileName, expectedMatchingLabels[i]);
215       }
216     }
217   }
218 
219   return retVal;
220 }
221 
222 class ImageClassifierProcessResult : public PostProcessOutputDataExtension {
223 public:
224   int processOutputs(
225       const llvm::StringMap<Placeholder *> &PHM, PlaceholderBindings &bindings,
226       llvm::ArrayRef<std::string> inputImageBatchFilenames) override;
227 };
228 
229 /// Given the output PlaceHolder StringMap \p PHM, of size 1, from SoftMax and
230 /// \p functionName, switch between the correct element type to print the
231 /// results of inference as contained in \p PHM, \returns the number of found
232 /// mismatches.
processOutputs(const llvm::StringMap<Placeholder * > & PHM,PlaceholderBindings & bindings,llvm::ArrayRef<std::string> imageList)233 int ImageClassifierProcessResult::processOutputs(
234     const llvm::StringMap<Placeholder *> &PHM, PlaceholderBindings &bindings,
235     llvm::ArrayRef<std::string> imageList) {
236 
237   if (profilingGraph()) {
238     LOG(INFO) << "Graph profiling is ON. Processing of output is disabled.";
239     return 0;
240   }
241 
242   if (PHM.size() > 1) {
243     LOG(FATAL) << "Network has more then one output. Process results not "
244                   "run for ImageClassifier.";
245   }
246 
247   auto *SMT = bindings.get(PHM.begin()->second);
248   switch (SMT->getElementType()) {
249   case ElemKind::FloatTy:
250     return processAndPrintResultsImpl<float>(SMT, imageList);
251   case ElemKind::Float16Ty:
252     return processAndPrintResultsImpl<float16_t>(SMT, imageList);
253   case ElemKind::BFloat16Ty:
254     return processAndPrintResultsImpl<bfloat16_t>(SMT, imageList);
255   default:
256     llvm_unreachable("Type not supported");
257   }
258   return 0;
259 }
260 
main(int argc,char ** argv)261 int main(int argc, char **argv) {
262 
263   if (!expectedMatchingLabels.empty()) {
264     // The number of category indices must match the number of files.
265     if (expectedMatchingLabels.size() != inputImageFilenames.size()) {
266       llvm::errs() << "Number of matching indices: "
267                    << expectedMatchingLabels.size()
268                    << " doesn't match the number of files: "
269                    << inputImageFilenames.size() << "\n";
270       return 1;
271     }
272   }
273 
274   glow::Executor core("ImageClassifier", argc, argv);
275   auto printResultCreator =
276       []() -> std::unique_ptr<PostProcessOutputDataExtension> {
277     return std::make_unique<ImageClassifierProcessResult>();
278   };
279   core.registerPostProcessOutputExtension(printResultCreator);
280   int numErrors = core.executeNetwork();
281   return numErrors;
282 }
283