1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
4 //
5 // Copyright (C) 2018-2019, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
7 #include "test_precomp.hpp"
8 
9 #ifdef HAVE_INF_ENGINE
10 #include <opencv2/core/utils/filesystem.hpp>
11 
12 
13 //
14 // Synchronize headers include statements with src/op_inf_engine.hpp
15 //
16 //#define INFERENCE_ENGINE_DEPRECATED  // turn off deprecation warnings from IE
17 //there is no way to suppress warnings from IE only at this moment, so we are forced to suppress warnings globally
18 #if defined(__GNUC__)
19 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
20 #endif
21 #ifdef _MSC_VER
22 #pragma warning(disable: 4996)  // was declared deprecated
23 #endif
24 
25 #if defined(__GNUC__)
26 #pragma GCC visibility push(default)
27 #endif
28 
29 #include <inference_engine.hpp>
30 #include <ie_icnn_network.hpp>
31 #include <ie_extension.h>
32 
33 #if defined(__GNUC__)
34 #pragma GCC visibility pop
35 #endif
36 
37 
38 namespace opencv_test { namespace {
39 
initDLDTDataPath()40 static void initDLDTDataPath()
41 {
42 #ifndef WINRT
43     static bool initialized = false;
44     if (!initialized)
45     {
46 #if INF_ENGINE_RELEASE <= 2018050000
47         const char* dldtTestDataPath = getenv("INTEL_CVSDK_DIR");
48         if (dldtTestDataPath)
49             cvtest::addDataSearchPath(dldtTestDataPath);
50 #else
51         const char* omzDataPath = getenv("OPENCV_OPEN_MODEL_ZOO_DATA_PATH");
52         if (omzDataPath)
53             cvtest::addDataSearchPath(omzDataPath);
54         const char* dnnDataPath = getenv("OPENCV_DNN_TEST_DATA_PATH");
55         if (dnnDataPath)
56             cvtest::addDataSearchPath(std::string(dnnDataPath) + "/omz_intel_models");
57 #endif
58         initialized = true;
59     }
60 #endif
61 }
62 
63 using namespace cv;
64 using namespace cv::dnn;
65 using namespace InferenceEngine;
66 
67 struct OpenVINOModelTestCaseInfo
68 {
69     const char* modelPathFP32;
70     const char* modelPathFP16;
71 };
72 
getOpenVINOTestModels()73 static const std::map<std::string, OpenVINOModelTestCaseInfo>& getOpenVINOTestModels()
74 {
75     static std::map<std::string, OpenVINOModelTestCaseInfo> g_models {
76 #if INF_ENGINE_RELEASE >= 2018050000 && \
77     INF_ENGINE_RELEASE <= 2020999999  // don't use IRv5 models with 2020.1+
78         // layout is defined by open_model_zoo/model_downloader
79         // Downloaded using these parameters for Open Model Zoo downloader (2019R1):
80         // ./downloader.py -o ${OPENCV_DNN_TEST_DATA_PATH}/omz_intel_models --cache_dir ${OPENCV_DNN_TEST_DATA_PATH}/.omz_cache/ \
81         //     --name face-person-detection-retail-0002,face-person-detection-retail-0002-fp16,age-gender-recognition-retail-0013,age-gender-recognition-retail-0013-fp16,head-pose-estimation-adas-0001,head-pose-estimation-adas-0001-fp16,person-detection-retail-0002,person-detection-retail-0002-fp16,vehicle-detection-adas-0002,vehicle-detection-adas-0002-fp16
82         { "age-gender-recognition-retail-0013", {
83             "Retail/object_attributes/age_gender/dldt/age-gender-recognition-retail-0013",
84             "Retail/object_attributes/age_gender/dldt/age-gender-recognition-retail-0013-fp16"
85         }},
86         { "face-person-detection-retail-0002", {
87             "Retail/object_detection/face_pedestrian/rmnet-ssssd-2heads/0002/dldt/face-person-detection-retail-0002",
88             "Retail/object_detection/face_pedestrian/rmnet-ssssd-2heads/0002/dldt/face-person-detection-retail-0002-fp16"
89         }},
90         { "head-pose-estimation-adas-0001", {
91             "Transportation/object_attributes/headpose/vanilla_cnn/dldt/head-pose-estimation-adas-0001",
92             "Transportation/object_attributes/headpose/vanilla_cnn/dldt/head-pose-estimation-adas-0001-fp16"
93         }},
94         { "person-detection-retail-0002", {
95             "Retail/object_detection/pedestrian/hypernet-rfcn/0026/dldt/person-detection-retail-0002",
96             "Retail/object_detection/pedestrian/hypernet-rfcn/0026/dldt/person-detection-retail-0002-fp16"
97         }},
98         { "vehicle-detection-adas-0002", {
99             "Transportation/object_detection/vehicle/mobilenet-reduced-ssd/dldt/vehicle-detection-adas-0002",
100             "Transportation/object_detection/vehicle/mobilenet-reduced-ssd/dldt/vehicle-detection-adas-0002-fp16"
101         }},
102 #endif
103 #if INF_ENGINE_RELEASE >= 2020010000
104         // Downloaded using these parameters for Open Model Zoo downloader (2020.1):
105         // ./downloader.py -o ${OPENCV_DNN_TEST_DATA_PATH}/omz_intel_models --cache_dir ${OPENCV_DNN_TEST_DATA_PATH}/.omz_cache/ \
106         //     --name person-detection-retail-0013,age-gender-recognition-retail-0013
107         { "person-detection-retail-0013", {  // IRv10
108             "intel/person-detection-retail-0013/FP32/person-detection-retail-0013",
109             "intel/person-detection-retail-0013/FP16/person-detection-retail-0013"
110         }},
111         { "age-gender-recognition-retail-0013", {
112             "intel/age-gender-recognition-retail-0013/FP16/age-gender-recognition-retail-0013",
113             "intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013"
114         }},
115 #endif
116     };
117 
118     return g_models;
119 }
120 
getOpenVINOTestModelsList()121 static const std::vector<std::string> getOpenVINOTestModelsList()
122 {
123     std::vector<std::string> result;
124     const std::map<std::string, OpenVINOModelTestCaseInfo>& models = getOpenVINOTestModels();
125     for (const auto& it : models)
126         result.push_back(it.first);
127     return result;
128 }
129 
getOpenVINOModel(const std::string & modelName,bool isFP16)130 inline static std::string getOpenVINOModel(const std::string &modelName, bool isFP16)
131 {
132     const std::map<std::string, OpenVINOModelTestCaseInfo>& models = getOpenVINOTestModels();
133     const auto it = models.find(modelName);
134     if (it != models.end())
135     {
136         OpenVINOModelTestCaseInfo modelInfo = it->second;
137         if (isFP16 && modelInfo.modelPathFP16)
138             return std::string(modelInfo.modelPathFP16);
139         else if (!isFP16 && modelInfo.modelPathFP32)
140             return std::string(modelInfo.modelPathFP32);
141     }
142     return std::string();
143 }
144 
genData(const InferenceEngine::TensorDesc & desc,Mat & m,Blob::Ptr & dataPtr)145 static inline void genData(const InferenceEngine::TensorDesc& desc, Mat& m, Blob::Ptr& dataPtr)
146 {
147     const std::vector<size_t>& dims = desc.getDims();
148     m.create(std::vector<int>(dims.begin(), dims.end()), CV_32F);
149     randu(m, -1, 1);
150 
151     dataPtr = make_shared_blob<float>(desc, (float*)m.data);
152 }
153 
runIE(Target target,const std::string & xmlPath,const std::string & binPath,std::map<std::string,cv::Mat> & inputsMap,std::map<std::string,cv::Mat> & outputsMap)154 void runIE(Target target, const std::string& xmlPath, const std::string& binPath,
155            std::map<std::string, cv::Mat>& inputsMap, std::map<std::string, cv::Mat>& outputsMap)
156 {
157     SCOPED_TRACE("runIE");
158 
159     std::string device_name;
160 
161 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GT(2019010000)
162     Core ie;
163 #else
164     InferenceEnginePluginPtr enginePtr;
165     InferencePlugin plugin;
166 #endif
167 
168 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GT(2019030000)
169     CNNNetwork net = ie.ReadNetwork(xmlPath, binPath);
170 #else
171     CNNNetReader reader;
172     reader.ReadNetwork(xmlPath);
173     reader.ReadWeights(binPath);
174 
175     CNNNetwork net = reader.getNetwork();
176 #endif
177 
178     ExecutableNetwork netExec;
179     InferRequest infRequest;
180 
181     try
182     {
183         switch (target)
184         {
185             case DNN_TARGET_CPU:
186                 device_name = "CPU";
187                 break;
188             case DNN_TARGET_OPENCL:
189             case DNN_TARGET_OPENCL_FP16:
190                 device_name = "GPU";
191                 break;
192             case DNN_TARGET_MYRIAD:
193                 device_name = "MYRIAD";
194                 break;
195             case DNN_TARGET_FPGA:
196                 device_name = "FPGA";
197                 break;
198             default:
199                 CV_Error(Error::StsNotImplemented, "Unknown target");
200         };
201 
202 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000)
203         auto dispatcher = InferenceEngine::PluginDispatcher({""});
204         enginePtr = dispatcher.getPluginByDevice(device_name);
205 #endif
206         if (target == DNN_TARGET_CPU || target == DNN_TARGET_FPGA)
207         {
208             std::string suffixes[] = {"_avx2", "_sse4", ""};
209             bool haveFeature[] = {
210                 checkHardwareSupport(CPU_AVX2),
211                 checkHardwareSupport(CPU_SSE4_2),
212                 true
213             };
214             for (int i = 0; i < 3; ++i)
215             {
216                 if (!haveFeature[i])
217                     continue;
218 #ifdef _WIN32
219                 std::string libName = "cpu_extension" + suffixes[i] + ".dll";
220 #elif defined(__APPLE__)
221                 std::string libName = "libcpu_extension" + suffixes[i] + ".dylib";
222 #else
223                 std::string libName = "libcpu_extension" + suffixes[i] + ".so";
224 #endif  // _WIN32
225                 try
226                 {
227                     IExtensionPtr extension = make_so_pointer<IExtension>(libName);
228 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GT(2019010000)
229                     ie.AddExtension(extension, device_name);
230 #else
231                     enginePtr->AddExtension(extension, 0);
232 #endif
233                     break;
234                 }
235                 catch(...) {}
236             }
237             // Some of networks can work without a library of extra layers.
238         }
239 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GT(2019010000)
240         netExec = ie.LoadNetwork(net, device_name);
241 #else
242         plugin = InferencePlugin(enginePtr);
243         netExec = plugin.LoadNetwork(net, {});
244 #endif
245         infRequest = netExec.CreateInferRequest();
246     }
247     catch (const std::exception& ex)
248     {
249         CV_Error(Error::StsAssert, format("Failed to initialize Inference Engine backend: %s", ex.what()));
250     }
251 
252     // Fill input blobs.
253     inputsMap.clear();
254     BlobMap inputBlobs;
255     for (auto& it : net.getInputsInfo())
256     {
257         genData(it.second->getTensorDesc(), inputsMap[it.first], inputBlobs[it.first]);
258     }
259     infRequest.SetInput(inputBlobs);
260 
261     // Fill output blobs.
262     outputsMap.clear();
263     BlobMap outputBlobs;
264     for (auto& it : net.getOutputsInfo())
265     {
266         genData(it.second->getTensorDesc(), outputsMap[it.first], outputBlobs[it.first]);
267     }
268     infRequest.SetOutput(outputBlobs);
269 
270     infRequest.Infer();
271 }
272 
runCV(Backend backendId,Target targetId,const std::string & xmlPath,const std::string & binPath,const std::map<std::string,cv::Mat> & inputsMap,std::map<std::string,cv::Mat> & outputsMap)273 void runCV(Backend backendId, Target targetId, const std::string& xmlPath, const std::string& binPath,
274            const std::map<std::string, cv::Mat>& inputsMap,
275            std::map<std::string, cv::Mat>& outputsMap)
276 {
277     SCOPED_TRACE("runOCV");
278 
279     Net net = readNet(xmlPath, binPath);
280     for (auto& it : inputsMap)
281         net.setInput(it.second, it.first);
282 
283     net.setPreferableBackend(backendId);
284     net.setPreferableTarget(targetId);
285 
286     std::vector<String> outNames = net.getUnconnectedOutLayersNames();
287     std::vector<Mat> outs;
288     net.forward(outs, outNames);
289 
290     outputsMap.clear();
291     EXPECT_EQ(outs.size(), outNames.size());
292     for (int i = 0; i < outs.size(); ++i)
293     {
294         EXPECT_TRUE(outputsMap.insert({outNames[i], outs[i]}).second);
295     }
296 }
297 
298 typedef TestWithParam<tuple< tuple<Backend, Target>, std::string> > DNNTestOpenVINO;
TEST_P(DNNTestOpenVINO,models)299 TEST_P(DNNTestOpenVINO, models)
300 {
301     initDLDTDataPath();
302 
303     const Backend backendId = get<0>(get<0>(GetParam()));
304     const Target targetId = get<1>(get<0>(GetParam()));
305     std::string modelName = get<1>(GetParam());
306 
307     ASSERT_FALSE(backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) <<
308         "Inference Engine backend is required";
309 
310 #if INF_ENGINE_VER_MAJOR_EQ(2021040000)
311     if (targetId == DNN_TARGET_MYRIAD && (
312             modelName == "person-detection-retail-0013" ||  // ncDeviceOpen:1013 Failed to find booted device after boot
313             modelName == "age-gender-recognition-retail-0013"  // ncDeviceOpen:1013 Failed to find booted device after boot
314         )
315     )
316         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
317 #endif
318 
319 #if INF_ENGINE_VER_MAJOR_GE(2020020000)
320     if (targetId == DNN_TARGET_MYRIAD && backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
321     {
322         if (modelName == "person-detection-retail-0013")  // IRv10
323             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
324     }
325 #endif
326 
327 #if INF_ENGINE_VER_MAJOR_EQ(2020040000)
328     if (targetId == DNN_TARGET_MYRIAD && modelName == "person-detection-retail-0002")  // IRv5, OpenVINO 2020.4 regression
329         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
330 #endif
331 
332     if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
333         setInferenceEngineBackendType(CV_DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_API);
334     else if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
335         setInferenceEngineBackendType(CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH);
336     else
337         FAIL() << "Unknown backendId";
338 
339     bool isFP16 = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD);
340 
341     const std::string modelPath = getOpenVINOModel(modelName, isFP16);
342     ASSERT_FALSE(modelPath.empty()) << modelName;
343 
344     std::string xmlPath = findDataFile(modelPath + ".xml", false);
345     std::string binPath = findDataFile(modelPath + ".bin", false);
346 
347     std::map<std::string, cv::Mat> inputsMap;
348     std::map<std::string, cv::Mat> ieOutputsMap, cvOutputsMap;
349     // Single Myriad device cannot be shared across multiple processes.
350     if (targetId == DNN_TARGET_MYRIAD)
351         resetMyriadDevice();
352     if (targetId == DNN_TARGET_HDDL)
353         releaseHDDLPlugin();
354     EXPECT_NO_THROW(runIE(targetId, xmlPath, binPath, inputsMap, ieOutputsMap)) << "runIE";
355     EXPECT_NO_THROW(runCV(backendId, targetId, xmlPath, binPath, inputsMap, cvOutputsMap)) << "runCV";
356 
357     double eps = 0;
358 #if INF_ENGINE_VER_MAJOR_GE(2020010000)
359     if (targetId == DNN_TARGET_CPU && checkHardwareSupport(CV_CPU_AVX_512F))
360         eps = 1e-5;
361 #endif
362 
363     EXPECT_EQ(ieOutputsMap.size(), cvOutputsMap.size());
364     for (auto& srcIt : ieOutputsMap)
365     {
366         auto dstIt = cvOutputsMap.find(srcIt.first);
367         CV_Assert(dstIt != cvOutputsMap.end());
368         double normInf = cvtest::norm(srcIt.second, dstIt->second, cv::NORM_INF);
369         EXPECT_LE(normInf, eps) << "output=" << srcIt.first;
370     }
371 }
372 
373 
374 INSTANTIATE_TEST_CASE_P(/**/,
375     DNNTestOpenVINO,
376     Combine(dnnBackendsAndTargetsIE(),
377             testing::ValuesIn(getOpenVINOTestModelsList())
378     )
379 );
380 
381 typedef TestWithParam<Target> DNNTestHighLevelAPI;
TEST_P(DNNTestHighLevelAPI,predict)382 TEST_P(DNNTestHighLevelAPI, predict)
383 {
384     initDLDTDataPath();
385 
386     Target target = (dnn::Target)(int)GetParam();
387     bool isFP16 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD);
388     const std::string modelName = "age-gender-recognition-retail-0013";
389     const std::string modelPath = getOpenVINOModel(modelName, isFP16);
390     ASSERT_FALSE(modelPath.empty()) << modelName;
391 
392     std::string xmlPath = findDataFile(modelPath + ".xml");
393     std::string binPath = findDataFile(modelPath + ".bin");
394 
395     Model model(xmlPath, binPath);
396     Mat frame = imread(findDataFile("dnn/googlenet_1.png"));
397     std::vector<Mat> outs;
398     model.setPreferableBackend(DNN_BACKEND_INFERENCE_ENGINE);
399     model.setPreferableTarget(target);
400     model.predict(frame, outs);
401 
402     Net net = readNet(xmlPath, binPath);
403     Mat input = blobFromImage(frame, 1.0, Size(62, 62));
404     net.setInput(input);
405     net.setPreferableBackend(DNN_BACKEND_INFERENCE_ENGINE);
406     net.setPreferableTarget(target);
407 
408     std::vector<String> outNames = net.getUnconnectedOutLayersNames();
409     std::vector<Mat> refs;
410     net.forward(refs, outNames);
411 
412     CV_Assert(refs.size() == outs.size());
413     for (int i = 0; i < refs.size(); ++i)
414         normAssert(outs[i], refs[i]);
415 }
416 
417 INSTANTIATE_TEST_CASE_P(/**/,
418     DNNTestHighLevelAPI, testing::ValuesIn(getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE))
419 );
420 
421 }}
422 #endif  // HAVE_INF_ENGINE
423