1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
4 //
5 // Copyright (C) 2018-2019, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
7 #include "test_precomp.hpp"
8
9 #ifdef HAVE_INF_ENGINE
10 #include <opencv2/core/utils/filesystem.hpp>
11
12
13 //
14 // Synchronize headers include statements with src/op_inf_engine.hpp
15 //
16 //#define INFERENCE_ENGINE_DEPRECATED // turn off deprecation warnings from IE
17 //there is no way to suppress warnings from IE only at this moment, so we are forced to suppress warnings globally
18 #if defined(__GNUC__)
19 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
20 #endif
21 #ifdef _MSC_VER
22 #pragma warning(disable: 4996) // was declared deprecated
23 #endif
24
25 #if defined(__GNUC__)
26 #pragma GCC visibility push(default)
27 #endif
28
29 #include <inference_engine.hpp>
30 #include <ie_icnn_network.hpp>
31 #include <ie_extension.h>
32
33 #if defined(__GNUC__)
34 #pragma GCC visibility pop
35 #endif
36
37
38 namespace opencv_test { namespace {
39
initDLDTDataPath()40 static void initDLDTDataPath()
41 {
42 #ifndef WINRT
43 static bool initialized = false;
44 if (!initialized)
45 {
46 #if INF_ENGINE_RELEASE <= 2018050000
47 const char* dldtTestDataPath = getenv("INTEL_CVSDK_DIR");
48 if (dldtTestDataPath)
49 cvtest::addDataSearchPath(dldtTestDataPath);
50 #else
51 const char* omzDataPath = getenv("OPENCV_OPEN_MODEL_ZOO_DATA_PATH");
52 if (omzDataPath)
53 cvtest::addDataSearchPath(omzDataPath);
54 const char* dnnDataPath = getenv("OPENCV_DNN_TEST_DATA_PATH");
55 if (dnnDataPath)
56 cvtest::addDataSearchPath(std::string(dnnDataPath) + "/omz_intel_models");
57 #endif
58 initialized = true;
59 }
60 #endif
61 }
62
63 using namespace cv;
64 using namespace cv::dnn;
65 using namespace InferenceEngine;
66
67 struct OpenVINOModelTestCaseInfo
68 {
69 const char* modelPathFP32;
70 const char* modelPathFP16;
71 };
72
getOpenVINOTestModels()73 static const std::map<std::string, OpenVINOModelTestCaseInfo>& getOpenVINOTestModels()
74 {
75 static std::map<std::string, OpenVINOModelTestCaseInfo> g_models {
76 #if INF_ENGINE_RELEASE >= 2018050000 && \
77 INF_ENGINE_RELEASE <= 2020999999 // don't use IRv5 models with 2020.1+
78 // layout is defined by open_model_zoo/model_downloader
79 // Downloaded using these parameters for Open Model Zoo downloader (2019R1):
80 // ./downloader.py -o ${OPENCV_DNN_TEST_DATA_PATH}/omz_intel_models --cache_dir ${OPENCV_DNN_TEST_DATA_PATH}/.omz_cache/ \
81 // --name face-person-detection-retail-0002,face-person-detection-retail-0002-fp16,age-gender-recognition-retail-0013,age-gender-recognition-retail-0013-fp16,head-pose-estimation-adas-0001,head-pose-estimation-adas-0001-fp16,person-detection-retail-0002,person-detection-retail-0002-fp16,vehicle-detection-adas-0002,vehicle-detection-adas-0002-fp16
82 { "age-gender-recognition-retail-0013", {
83 "Retail/object_attributes/age_gender/dldt/age-gender-recognition-retail-0013",
84 "Retail/object_attributes/age_gender/dldt/age-gender-recognition-retail-0013-fp16"
85 }},
86 { "face-person-detection-retail-0002", {
87 "Retail/object_detection/face_pedestrian/rmnet-ssssd-2heads/0002/dldt/face-person-detection-retail-0002",
88 "Retail/object_detection/face_pedestrian/rmnet-ssssd-2heads/0002/dldt/face-person-detection-retail-0002-fp16"
89 }},
90 { "head-pose-estimation-adas-0001", {
91 "Transportation/object_attributes/headpose/vanilla_cnn/dldt/head-pose-estimation-adas-0001",
92 "Transportation/object_attributes/headpose/vanilla_cnn/dldt/head-pose-estimation-adas-0001-fp16"
93 }},
94 { "person-detection-retail-0002", {
95 "Retail/object_detection/pedestrian/hypernet-rfcn/0026/dldt/person-detection-retail-0002",
96 "Retail/object_detection/pedestrian/hypernet-rfcn/0026/dldt/person-detection-retail-0002-fp16"
97 }},
98 { "vehicle-detection-adas-0002", {
99 "Transportation/object_detection/vehicle/mobilenet-reduced-ssd/dldt/vehicle-detection-adas-0002",
100 "Transportation/object_detection/vehicle/mobilenet-reduced-ssd/dldt/vehicle-detection-adas-0002-fp16"
101 }},
102 #endif
103 #if INF_ENGINE_RELEASE >= 2020010000
104 // Downloaded using these parameters for Open Model Zoo downloader (2020.1):
105 // ./downloader.py -o ${OPENCV_DNN_TEST_DATA_PATH}/omz_intel_models --cache_dir ${OPENCV_DNN_TEST_DATA_PATH}/.omz_cache/ \
106 // --name person-detection-retail-0013,age-gender-recognition-retail-0013
107 { "person-detection-retail-0013", { // IRv10
108 "intel/person-detection-retail-0013/FP32/person-detection-retail-0013",
109 "intel/person-detection-retail-0013/FP16/person-detection-retail-0013"
110 }},
111 { "age-gender-recognition-retail-0013", {
112 "intel/age-gender-recognition-retail-0013/FP16/age-gender-recognition-retail-0013",
113 "intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013"
114 }},
115 #endif
116 };
117
118 return g_models;
119 }
120
getOpenVINOTestModelsList()121 static const std::vector<std::string> getOpenVINOTestModelsList()
122 {
123 std::vector<std::string> result;
124 const std::map<std::string, OpenVINOModelTestCaseInfo>& models = getOpenVINOTestModels();
125 for (const auto& it : models)
126 result.push_back(it.first);
127 return result;
128 }
129
getOpenVINOModel(const std::string & modelName,bool isFP16)130 inline static std::string getOpenVINOModel(const std::string &modelName, bool isFP16)
131 {
132 const std::map<std::string, OpenVINOModelTestCaseInfo>& models = getOpenVINOTestModels();
133 const auto it = models.find(modelName);
134 if (it != models.end())
135 {
136 OpenVINOModelTestCaseInfo modelInfo = it->second;
137 if (isFP16 && modelInfo.modelPathFP16)
138 return std::string(modelInfo.modelPathFP16);
139 else if (!isFP16 && modelInfo.modelPathFP32)
140 return std::string(modelInfo.modelPathFP32);
141 }
142 return std::string();
143 }
144
genData(const InferenceEngine::TensorDesc & desc,Mat & m,Blob::Ptr & dataPtr)145 static inline void genData(const InferenceEngine::TensorDesc& desc, Mat& m, Blob::Ptr& dataPtr)
146 {
147 const std::vector<size_t>& dims = desc.getDims();
148 m.create(std::vector<int>(dims.begin(), dims.end()), CV_32F);
149 randu(m, -1, 1);
150
151 dataPtr = make_shared_blob<float>(desc, (float*)m.data);
152 }
153
runIE(Target target,const std::string & xmlPath,const std::string & binPath,std::map<std::string,cv::Mat> & inputsMap,std::map<std::string,cv::Mat> & outputsMap)154 void runIE(Target target, const std::string& xmlPath, const std::string& binPath,
155 std::map<std::string, cv::Mat>& inputsMap, std::map<std::string, cv::Mat>& outputsMap)
156 {
157 SCOPED_TRACE("runIE");
158
159 std::string device_name;
160
161 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GT(2019010000)
162 Core ie;
163 #else
164 InferenceEnginePluginPtr enginePtr;
165 InferencePlugin plugin;
166 #endif
167
168 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GT(2019030000)
169 CNNNetwork net = ie.ReadNetwork(xmlPath, binPath);
170 #else
171 CNNNetReader reader;
172 reader.ReadNetwork(xmlPath);
173 reader.ReadWeights(binPath);
174
175 CNNNetwork net = reader.getNetwork();
176 #endif
177
178 ExecutableNetwork netExec;
179 InferRequest infRequest;
180
181 try
182 {
183 switch (target)
184 {
185 case DNN_TARGET_CPU:
186 device_name = "CPU";
187 break;
188 case DNN_TARGET_OPENCL:
189 case DNN_TARGET_OPENCL_FP16:
190 device_name = "GPU";
191 break;
192 case DNN_TARGET_MYRIAD:
193 device_name = "MYRIAD";
194 break;
195 case DNN_TARGET_FPGA:
196 device_name = "FPGA";
197 break;
198 default:
199 CV_Error(Error::StsNotImplemented, "Unknown target");
200 };
201
202 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000)
203 auto dispatcher = InferenceEngine::PluginDispatcher({""});
204 enginePtr = dispatcher.getPluginByDevice(device_name);
205 #endif
206 if (target == DNN_TARGET_CPU || target == DNN_TARGET_FPGA)
207 {
208 std::string suffixes[] = {"_avx2", "_sse4", ""};
209 bool haveFeature[] = {
210 checkHardwareSupport(CPU_AVX2),
211 checkHardwareSupport(CPU_SSE4_2),
212 true
213 };
214 for (int i = 0; i < 3; ++i)
215 {
216 if (!haveFeature[i])
217 continue;
218 #ifdef _WIN32
219 std::string libName = "cpu_extension" + suffixes[i] + ".dll";
220 #elif defined(__APPLE__)
221 std::string libName = "libcpu_extension" + suffixes[i] + ".dylib";
222 #else
223 std::string libName = "libcpu_extension" + suffixes[i] + ".so";
224 #endif // _WIN32
225 try
226 {
227 IExtensionPtr extension = make_so_pointer<IExtension>(libName);
228 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GT(2019010000)
229 ie.AddExtension(extension, device_name);
230 #else
231 enginePtr->AddExtension(extension, 0);
232 #endif
233 break;
234 }
235 catch(...) {}
236 }
237 // Some of networks can work without a library of extra layers.
238 }
239 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GT(2019010000)
240 netExec = ie.LoadNetwork(net, device_name);
241 #else
242 plugin = InferencePlugin(enginePtr);
243 netExec = plugin.LoadNetwork(net, {});
244 #endif
245 infRequest = netExec.CreateInferRequest();
246 }
247 catch (const std::exception& ex)
248 {
249 CV_Error(Error::StsAssert, format("Failed to initialize Inference Engine backend: %s", ex.what()));
250 }
251
252 // Fill input blobs.
253 inputsMap.clear();
254 BlobMap inputBlobs;
255 for (auto& it : net.getInputsInfo())
256 {
257 genData(it.second->getTensorDesc(), inputsMap[it.first], inputBlobs[it.first]);
258 }
259 infRequest.SetInput(inputBlobs);
260
261 // Fill output blobs.
262 outputsMap.clear();
263 BlobMap outputBlobs;
264 for (auto& it : net.getOutputsInfo())
265 {
266 genData(it.second->getTensorDesc(), outputsMap[it.first], outputBlobs[it.first]);
267 }
268 infRequest.SetOutput(outputBlobs);
269
270 infRequest.Infer();
271 }
272
runCV(Backend backendId,Target targetId,const std::string & xmlPath,const std::string & binPath,const std::map<std::string,cv::Mat> & inputsMap,std::map<std::string,cv::Mat> & outputsMap)273 void runCV(Backend backendId, Target targetId, const std::string& xmlPath, const std::string& binPath,
274 const std::map<std::string, cv::Mat>& inputsMap,
275 std::map<std::string, cv::Mat>& outputsMap)
276 {
277 SCOPED_TRACE("runOCV");
278
279 Net net = readNet(xmlPath, binPath);
280 for (auto& it : inputsMap)
281 net.setInput(it.second, it.first);
282
283 net.setPreferableBackend(backendId);
284 net.setPreferableTarget(targetId);
285
286 std::vector<String> outNames = net.getUnconnectedOutLayersNames();
287 std::vector<Mat> outs;
288 net.forward(outs, outNames);
289
290 outputsMap.clear();
291 EXPECT_EQ(outs.size(), outNames.size());
292 for (int i = 0; i < outs.size(); ++i)
293 {
294 EXPECT_TRUE(outputsMap.insert({outNames[i], outs[i]}).second);
295 }
296 }
297
298 typedef TestWithParam<tuple< tuple<Backend, Target>, std::string> > DNNTestOpenVINO;
TEST_P(DNNTestOpenVINO,models)299 TEST_P(DNNTestOpenVINO, models)
300 {
301 initDLDTDataPath();
302
303 const Backend backendId = get<0>(get<0>(GetParam()));
304 const Target targetId = get<1>(get<0>(GetParam()));
305 std::string modelName = get<1>(GetParam());
306
307 ASSERT_FALSE(backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) <<
308 "Inference Engine backend is required";
309
310 #if INF_ENGINE_VER_MAJOR_EQ(2021040000)
311 if (targetId == DNN_TARGET_MYRIAD && (
312 modelName == "person-detection-retail-0013" || // ncDeviceOpen:1013 Failed to find booted device after boot
313 modelName == "age-gender-recognition-retail-0013" // ncDeviceOpen:1013 Failed to find booted device after boot
314 )
315 )
316 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
317 #endif
318
319 #if INF_ENGINE_VER_MAJOR_GE(2020020000)
320 if (targetId == DNN_TARGET_MYRIAD && backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
321 {
322 if (modelName == "person-detection-retail-0013") // IRv10
323 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
324 }
325 #endif
326
327 #if INF_ENGINE_VER_MAJOR_EQ(2020040000)
328 if (targetId == DNN_TARGET_MYRIAD && modelName == "person-detection-retail-0002") // IRv5, OpenVINO 2020.4 regression
329 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
330 #endif
331
332 if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
333 setInferenceEngineBackendType(CV_DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_API);
334 else if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
335 setInferenceEngineBackendType(CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH);
336 else
337 FAIL() << "Unknown backendId";
338
339 bool isFP16 = (targetId == DNN_TARGET_OPENCL_FP16 || targetId == DNN_TARGET_MYRIAD);
340
341 const std::string modelPath = getOpenVINOModel(modelName, isFP16);
342 ASSERT_FALSE(modelPath.empty()) << modelName;
343
344 std::string xmlPath = findDataFile(modelPath + ".xml", false);
345 std::string binPath = findDataFile(modelPath + ".bin", false);
346
347 std::map<std::string, cv::Mat> inputsMap;
348 std::map<std::string, cv::Mat> ieOutputsMap, cvOutputsMap;
349 // Single Myriad device cannot be shared across multiple processes.
350 if (targetId == DNN_TARGET_MYRIAD)
351 resetMyriadDevice();
352 if (targetId == DNN_TARGET_HDDL)
353 releaseHDDLPlugin();
354 EXPECT_NO_THROW(runIE(targetId, xmlPath, binPath, inputsMap, ieOutputsMap)) << "runIE";
355 EXPECT_NO_THROW(runCV(backendId, targetId, xmlPath, binPath, inputsMap, cvOutputsMap)) << "runCV";
356
357 double eps = 0;
358 #if INF_ENGINE_VER_MAJOR_GE(2020010000)
359 if (targetId == DNN_TARGET_CPU && checkHardwareSupport(CV_CPU_AVX_512F))
360 eps = 1e-5;
361 #endif
362
363 EXPECT_EQ(ieOutputsMap.size(), cvOutputsMap.size());
364 for (auto& srcIt : ieOutputsMap)
365 {
366 auto dstIt = cvOutputsMap.find(srcIt.first);
367 CV_Assert(dstIt != cvOutputsMap.end());
368 double normInf = cvtest::norm(srcIt.second, dstIt->second, cv::NORM_INF);
369 EXPECT_LE(normInf, eps) << "output=" << srcIt.first;
370 }
371 }
372
373
374 INSTANTIATE_TEST_CASE_P(/**/,
375 DNNTestOpenVINO,
376 Combine(dnnBackendsAndTargetsIE(),
377 testing::ValuesIn(getOpenVINOTestModelsList())
378 )
379 );
380
381 typedef TestWithParam<Target> DNNTestHighLevelAPI;
TEST_P(DNNTestHighLevelAPI,predict)382 TEST_P(DNNTestHighLevelAPI, predict)
383 {
384 initDLDTDataPath();
385
386 Target target = (dnn::Target)(int)GetParam();
387 bool isFP16 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD);
388 const std::string modelName = "age-gender-recognition-retail-0013";
389 const std::string modelPath = getOpenVINOModel(modelName, isFP16);
390 ASSERT_FALSE(modelPath.empty()) << modelName;
391
392 std::string xmlPath = findDataFile(modelPath + ".xml");
393 std::string binPath = findDataFile(modelPath + ".bin");
394
395 Model model(xmlPath, binPath);
396 Mat frame = imread(findDataFile("dnn/googlenet_1.png"));
397 std::vector<Mat> outs;
398 model.setPreferableBackend(DNN_BACKEND_INFERENCE_ENGINE);
399 model.setPreferableTarget(target);
400 model.predict(frame, outs);
401
402 Net net = readNet(xmlPath, binPath);
403 Mat input = blobFromImage(frame, 1.0, Size(62, 62));
404 net.setInput(input);
405 net.setPreferableBackend(DNN_BACKEND_INFERENCE_ENGINE);
406 net.setPreferableTarget(target);
407
408 std::vector<String> outNames = net.getUnconnectedOutLayersNames();
409 std::vector<Mat> refs;
410 net.forward(refs, outNames);
411
412 CV_Assert(refs.size() == outs.size());
413 for (int i = 0; i < refs.size(); ++i)
414 normAssert(outs[i], refs[i]);
415 }
416
417 INSTANTIATE_TEST_CASE_P(/**/,
418 DNNTestHighLevelAPI, testing::ValuesIn(getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE))
419 );
420
421 }}
422 #endif // HAVE_INF_ENGINE
423