1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
4 //
5 // Copyright (C) 2018-2019, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
7
8 #include "precomp.hpp"
9 #include "op_inf_engine.hpp"
10 #include <opencv2/dnn/shape_utils.hpp>
11
12 #ifdef HAVE_INF_ENGINE
13 #include <ie_extension.h>
14 #endif // HAVE_INF_ENGINE
15
16 #include <opencv2/core/utils/configuration.private.hpp>
17 #include <opencv2/core/utils/logger.hpp>
18
19 namespace cv { namespace dnn {
20
21 #ifdef HAVE_INF_ENGINE
22
parseInferenceEngineBackendType(const cv::String & backend)23 static Backend parseInferenceEngineBackendType(const cv::String& backend)
24 {
25 CV_Assert(!backend.empty());
26 if (backend == CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
27 return DNN_BACKEND_INFERENCE_ENGINE_NGRAPH;
28 if (backend == CV_DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_API)
29 return DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019;
30 CV_Error(Error::StsBadArg, cv::format("Unknown IE backend: %s", backend.c_str()));
31 }
dumpInferenceEngineBackendType(Backend backend)32 static const char* dumpInferenceEngineBackendType(Backend backend)
33 {
34 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
35 return CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH;
36 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
37 return CV_DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_API;
38 CV_Error(Error::StsBadArg, cv::format("Invalid backend ID for IE: %d", backend));
39 }
getInferenceEngineBackendTypeParam()40 Backend& getInferenceEngineBackendTypeParam()
41 {
42 static Backend param = parseInferenceEngineBackendType(
43 utils::getConfigurationParameterString("OPENCV_DNN_BACKEND_INFERENCE_ENGINE_TYPE",
44 #ifdef HAVE_DNN_NGRAPH
45 CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH
46 #elif defined(HAVE_DNN_IE_NN_BUILDER_2019)
47 CV_DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_API
48 #else
49 #error "Build configuration error: nGraph or NN Builder API backend should be enabled"
50 #endif
51 )
52 );
53 return param;
54 }
55
56 CV__DNN_INLINE_NS_BEGIN
57
getInferenceEngineBackendType()58 cv::String getInferenceEngineBackendType()
59 {
60 return dumpInferenceEngineBackendType(getInferenceEngineBackendTypeParam());
61 }
setInferenceEngineBackendType(const cv::String & newBackendType)62 cv::String setInferenceEngineBackendType(const cv::String& newBackendType)
63 {
64 Backend newBackend = parseInferenceEngineBackendType(newBackendType);
65 Backend& param = getInferenceEngineBackendTypeParam();
66 Backend old = param;
67 param = newBackend;
68 return dumpInferenceEngineBackendType(old);
69 }
70
71 CV__DNN_INLINE_NS_END
72
73
infEngineBlobToMat(const InferenceEngine::Blob::Ptr & blob)74 Mat infEngineBlobToMat(const InferenceEngine::Blob::Ptr& blob)
75 {
76 // NOTE: Inference Engine sizes are reversed.
77 std::vector<size_t> dims = blob->getTensorDesc().getDims();
78 std::vector<int> size(dims.begin(), dims.end());
79 auto precision = blob->getTensorDesc().getPrecision();
80
81 int type = -1;
82 switch (precision)
83 {
84 case InferenceEngine::Precision::FP32: type = CV_32F; break;
85 case InferenceEngine::Precision::U8: type = CV_8U; break;
86 default:
87 CV_Error(Error::StsNotImplemented, "Unsupported blob precision");
88 }
89 return Mat(size, type, (void*)blob->buffer());
90 }
91
infEngineBlobsToMats(const std::vector<InferenceEngine::Blob::Ptr> & blobs,std::vector<Mat> & mats)92 void infEngineBlobsToMats(const std::vector<InferenceEngine::Blob::Ptr>& blobs,
93 std::vector<Mat>& mats)
94 {
95 mats.resize(blobs.size());
96 for (int i = 0; i < blobs.size(); ++i)
97 mats[i] = infEngineBlobToMat(blobs[i]);
98 }
99
100
101 #ifdef HAVE_DNN_IE_NN_BUILDER_2019
102
103 // For networks with input layer which has an empty name, IE generates a name id[some_number].
104 // OpenCV lets users use an empty input name and to prevent unexpected naming,
105 // we can use some predefined name.
106 static std::string kDefaultInpLayerName = "empty_inp_layer_name";
107 static std::string kOpenCVLayersType = "OpenCVLayer";
108
shapesToStr(const std::vector<Mat> & mats)109 static std::string shapesToStr(const std::vector<Mat>& mats)
110 {
111 std::ostringstream shapes;
112 shapes << mats.size() << " ";
113 for (const Mat& m : mats)
114 {
115 shapes << m.dims << " ";
116 for (int i = 0; i < m.dims; ++i)
117 shapes << m.size[i] << " ";
118 }
119 return shapes.str();
120 }
121
strToShapes(const std::string & str,std::vector<std::vector<size_t>> & shapes)122 static void strToShapes(const std::string& str, std::vector<std::vector<size_t> >& shapes)
123 {
124 std::istringstream ss(str);
125 int num, dims;
126 ss >> num;
127 shapes.resize(num);
128 for (int i = 0; i < num; ++i)
129 {
130 ss >> dims;
131 shapes[i].resize(dims);
132 for (int j = 0; j < dims; ++j)
133 ss >> shapes[i][j];
134 }
135 }
136
137 class InfEngineCustomLayer : public InferenceEngine::ILayerExecImpl
138 {
139 public:
InfEngineCustomLayer(const InferenceEngine::CNNLayer & layer)140 explicit InfEngineCustomLayer(const InferenceEngine::CNNLayer& layer) : cnnLayer(layer)
141 {
142 std::istringstream iss(layer.GetParamAsString("impl"));
143 size_t ptr;
144 iss >> ptr;
145 cvLayer = (Layer*)ptr;
146
147 std::vector<std::vector<size_t> > shapes;
148 strToShapes(layer.GetParamAsString("internals"), shapes);
149 internals.resize(shapes.size());
150 for (int i = 0; i < shapes.size(); ++i)
151 internals[i].create(std::vector<int>(shapes[i].begin(), shapes[i].end()), CV_32F);
152 }
153
execute(std::vector<InferenceEngine::Blob::Ptr> & inputs,std::vector<InferenceEngine::Blob::Ptr> & outputs,InferenceEngine::ResponseDesc * resp)154 virtual InferenceEngine::StatusCode execute(std::vector<InferenceEngine::Blob::Ptr>& inputs,
155 std::vector<InferenceEngine::Blob::Ptr>& outputs,
156 InferenceEngine::ResponseDesc *resp) noexcept
157 {
158 std::vector<Mat> inpMats, outMats;
159 infEngineBlobsToMats(inputs, inpMats);
160 infEngineBlobsToMats(outputs, outMats);
161
162 try
163 {
164 cvLayer->forward(inpMats, outMats, internals);
165 return InferenceEngine::StatusCode::OK;
166 }
167 catch (...)
168 {
169 return InferenceEngine::StatusCode::GENERAL_ERROR;
170 }
171 }
172
173 virtual InferenceEngine::StatusCode
getSupportedConfigurations(std::vector<InferenceEngine::LayerConfig> & conf,InferenceEngine::ResponseDesc * resp)174 getSupportedConfigurations(std::vector<InferenceEngine::LayerConfig>& conf,
175 InferenceEngine::ResponseDesc* resp) noexcept
176 {
177 std::vector<InferenceEngine::DataConfig> inDataConfig;
178 std::vector<InferenceEngine::DataConfig> outDataConfig;
179 for (auto& it : cnnLayer.insData)
180 {
181 InferenceEngine::DataConfig conf;
182 conf.desc = it.lock()->getTensorDesc();
183 inDataConfig.push_back(conf);
184 }
185
186 for (auto& it : cnnLayer.outData)
187 {
188 InferenceEngine::DataConfig conf;
189 conf.desc = it->getTensorDesc();
190 outDataConfig.push_back(conf);
191 }
192
193 InferenceEngine::LayerConfig layerConfig;
194 layerConfig.inConfs = inDataConfig;
195 layerConfig.outConfs = outDataConfig;
196
197 conf.push_back(layerConfig);
198 return InferenceEngine::StatusCode::OK;
199 }
200
init(InferenceEngine::LayerConfig & config,InferenceEngine::ResponseDesc * resp)201 InferenceEngine::StatusCode init(InferenceEngine::LayerConfig& config,
202 InferenceEngine::ResponseDesc *resp) noexcept
203 {
204 return InferenceEngine::StatusCode::OK;
205 }
206
207 private:
208 InferenceEngine::CNNLayer cnnLayer;
209 dnn::Layer* cvLayer;
210 std::vector<Mat> internals;
211 };
212
213 class InfEngineCustomLayerShapeInfer : public InferenceEngine::IShapeInferImpl
214 {
215 public:
216 InferenceEngine::StatusCode
inferShapes(const std::vector<InferenceEngine::Blob::CPtr> & inBlobs,const std::map<std::string,std::string> & params,const std::map<std::string,InferenceEngine::Blob::Ptr> & blobs,std::vector<InferenceEngine::SizeVector> & outShapes,InferenceEngine::ResponseDesc * desc)217 inferShapes(const std::vector<InferenceEngine::Blob::CPtr>& inBlobs,
218 const std::map<std::string, std::string>& params,
219 const std::map<std::string, InferenceEngine::Blob::Ptr>& blobs,
220 std::vector<InferenceEngine::SizeVector>& outShapes,
221 InferenceEngine::ResponseDesc* desc) noexcept override
222 {
223 strToShapes(params.at("outputs"), outShapes);
224 return InferenceEngine::StatusCode::OK;
225 }
226 };
227
228 class InfEngineCustomLayerFactory : public InferenceEngine::ILayerImplFactory {
229 public:
InfEngineCustomLayerFactory(const InferenceEngine::CNNLayer * layer)230 explicit InfEngineCustomLayerFactory(const InferenceEngine::CNNLayer* layer) : cnnLayer(*layer) {}
231
232 InferenceEngine::StatusCode
getImplementations(std::vector<InferenceEngine::ILayerImpl::Ptr> & impls,InferenceEngine::ResponseDesc * resp)233 getImplementations(std::vector<InferenceEngine::ILayerImpl::Ptr>& impls,
234 InferenceEngine::ResponseDesc* resp) noexcept override {
235 impls.push_back(std::make_shared<InfEngineCustomLayer>(cnnLayer));
236 return InferenceEngine::StatusCode::OK;
237 }
238
239 private:
240 InferenceEngine::CNNLayer cnnLayer;
241 };
242
getFactoryFor(InferenceEngine::ILayerImplFactory * & factory,const InferenceEngine::CNNLayer * cnnLayer,InferenceEngine::ResponseDesc * resp)243 InferenceEngine::StatusCode InfEngineExtension::getFactoryFor(
244 InferenceEngine::ILayerImplFactory*& factory,
245 const InferenceEngine::CNNLayer* cnnLayer,
246 InferenceEngine::ResponseDesc* resp
247 ) noexcept
248 {
249 if (cnnLayer->type != kOpenCVLayersType)
250 return InferenceEngine::StatusCode::NOT_IMPLEMENTED;
251 factory = new InfEngineCustomLayerFactory(cnnLayer);
252 return InferenceEngine::StatusCode::OK;
253 }
254
InfEngineBackendNode(const InferenceEngine::Builder::Layer & _layer)255 InfEngineBackendNode::InfEngineBackendNode(const InferenceEngine::Builder::Layer& _layer)
256 : BackendNode(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019), layer(_layer) {}
257
InfEngineBackendNode(Ptr<Layer> & cvLayer_,std::vector<Mat * > & inputs,std::vector<Mat> & outputs,std::vector<Mat> & internals)258 InfEngineBackendNode::InfEngineBackendNode(Ptr<Layer>& cvLayer_, std::vector<Mat*>& inputs,
259 std::vector<Mat>& outputs,
260 std::vector<Mat>& internals)
261 : BackendNode(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019), layer(cvLayer_->name),
262 cvLayer(cvLayer_)
263 {
264 CV_Assert(!cvLayer->name.empty());
265 layer.setName(cvLayer->name);
266 layer.setType(kOpenCVLayersType);
267 layer.getParameters()["impl"] = (size_t)cvLayer.get();
268 layer.getParameters()["outputs"] = shapesToStr(outputs);
269 layer.getParameters()["internals"] = shapesToStr(internals);
270 layer.setInputPorts(std::vector<InferenceEngine::Port>(inputs.size()));
271 layer.setOutputPorts(std::vector<InferenceEngine::Port>(outputs.size()));
272 }
273
274 static std::vector<Ptr<InfEngineBackendWrapper> >
infEngineWrappers(const std::vector<Ptr<BackendWrapper>> & ptrs)275 infEngineWrappers(const std::vector<Ptr<BackendWrapper> >& ptrs)
276 {
277 std::vector<Ptr<InfEngineBackendWrapper> > wrappers(ptrs.size());
278 for (int i = 0; i < ptrs.size(); ++i)
279 {
280 CV_Assert(!ptrs[i].empty());
281 wrappers[i] = ptrs[i].dynamicCast<InfEngineBackendWrapper>();
282 CV_Assert(!wrappers[i].empty());
283 }
284 return wrappers;
285 }
286
InfEngineBackendNet()287 InfEngineBackendNet::InfEngineBackendNet() : netBuilder("")
288 {
289 hasNetOwner = false;
290 device_name = "CPU";
291 }
292
InfEngineBackendNet(InferenceEngine::CNNNetwork & net)293 InfEngineBackendNet::InfEngineBackendNet(InferenceEngine::CNNNetwork& net) : netBuilder(""), cnn(net)
294 {
295 hasNetOwner = true;
296 device_name = "CPU";
297 }
298
connect(const std::vector<Ptr<BackendWrapper>> & inputs,const std::vector<Ptr<BackendWrapper>> & outputs,const std::string & layerName)299 void InfEngineBackendNet::connect(const std::vector<Ptr<BackendWrapper> >& inputs,
300 const std::vector<Ptr<BackendWrapper> >& outputs,
301 const std::string& layerName)
302 {
303 std::vector<Ptr<InfEngineBackendWrapper> > inpWrappers = infEngineWrappers(inputs);
304 std::map<std::string, int>::iterator it = layers.find(layerName);
305 CV_Assert(it != layers.end());
306
307 const int layerId = it->second;
308 for (size_t i = 0; i < inpWrappers.size(); ++i)
309 {
310 const auto& inp = inpWrappers[i];
311 const std::string& inpName = inp->dataPtr->getName();
312
313 std::string inpLayerName = inpName;
314 size_t inpPortId = inpName.rfind('.');
315 if (inpPortId != std::string::npos)
316 {
317 std::string portIdStr = inpName.substr(inpPortId + 1);
318 if (std::all_of(portIdStr.begin(), portIdStr.end(), ::isdigit))
319 {
320 inpLayerName = inpName.substr(0, inpPortId);
321 inpPortId = atoi(portIdStr.c_str());
322 }
323 else
324 inpPortId = 0;
325 }
326 else
327 inpPortId = 0;
328
329 int inpId;
330 it = layers.find(inpLayerName);
331 if (it == layers.end())
332 {
333 InferenceEngine::Builder::InputLayer inpLayer(!inpLayerName.empty() ? inpLayerName : kDefaultInpLayerName);
334 std::vector<size_t> shape(inp->blob->getTensorDesc().getDims());
335 inpLayer.setPort(InferenceEngine::Port(shape));
336 inpId = netBuilder.addLayer(inpLayer);
337
338 layers.insert({inpName, inpId});
339 }
340 else
341 inpId = it->second;
342
343 netBuilder.connect({(size_t)inpId, inpPortId}, {(size_t)layerId, i});
344 unconnectedPorts.erase({inpId, inpPortId});
345 }
346 CV_Assert(!outputs.empty());
347 for (int i = 0; i < outputs.size(); ++i)
348 {
349 InferenceEngine::DataPtr dataPtr = infEngineDataNode(outputs[i]);
350 std::string outputName = outputs.size() > 1 ? (layerName + "." + std::to_string(i)) : layerName;
351 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1)
352 dataPtr->name = outputName;
353 #else
354 dataPtr->setName(outputName);
355 #endif
356 }
357 }
358
init(Target targetId)359 void InfEngineBackendNet::init(Target targetId)
360 {
361 if (!hasNetOwner)
362 {
363 CV_Assert(!unconnectedPorts.empty());
364 for (const auto& port : unconnectedPorts)
365 {
366 InferenceEngine::Builder::OutputLayer outLayer("myconv1");
367 #if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2019R1)
368 // Inference Engine determines network precision by ports.
369 InferenceEngine::Precision p = (targetId == DNN_TARGET_MYRIAD ||
370 targetId == DNN_TARGET_HDDL ||
371 targetId == DNN_TARGET_OPENCL_FP16) ?
372 InferenceEngine::Precision::FP16 :
373 InferenceEngine::Precision::FP32;
374 outLayer.setPort(InferenceEngine::Port({}, p));
375 #endif
376 netBuilder.addLayer({InferenceEngine::PortInfo(port.first, port.second)}, outLayer);
377 }
378 netBuilder.getContext().addShapeInferImpl(kOpenCVLayersType,
379 std::make_shared<InfEngineCustomLayerShapeInfer>());
380 cnn = InferenceEngine::CNNNetwork(InferenceEngine::Builder::convertToICNNNetwork(netBuilder.build()));
381 }
382
383 switch (targetId)
384 {
385 case DNN_TARGET_CPU:
386 device_name = "CPU";
387 break;
388 case DNN_TARGET_OPENCL:
389 case DNN_TARGET_OPENCL_FP16:
390 device_name = "GPU";
391 break;
392 case DNN_TARGET_MYRIAD:
393 device_name = "MYRIAD";
394 break;
395 case DNN_TARGET_HDDL:
396 device_name = "HDDL";
397 break;
398 case DNN_TARGET_FPGA:
399 device_name = "FPGA";
400 break;
401 default:
402 CV_Error(Error::StsNotImplemented, "Unknown target");
403 };
404
405 for (const auto& name : requestedOutputs)
406 {
407 cnn.addOutput(name);
408 }
409
410 for (const auto& it : cnn.getInputsInfo())
411 {
412 const std::string& name = it.first;
413 auto blobIt = allBlobs.find(name);
414 CV_Assert(blobIt != allBlobs.end());
415 it.second->setPrecision(blobIt->second->getTensorDesc().getPrecision());
416 }
417 for (const auto& it : cnn.getOutputsInfo())
418 {
419 const std::string& name = it.first;
420 auto blobIt = allBlobs.find(name);
421 CV_Assert(blobIt != allBlobs.end());
422 it.second->setPrecision(blobIt->second->getTensorDesc().getPrecision()); // Should be always FP32
423 }
424
425 initPlugin(cnn);
426 }
427
addLayer(InferenceEngine::Builder::Layer & layer)428 void InfEngineBackendNet::addLayer(InferenceEngine::Builder::Layer& layer)
429 {
430 #if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2019R1)
431 // Add weights to network and connect them after input blobs.
432 std::map<std::string, InferenceEngine::Parameter>& params = layer.getParameters();
433 std::vector<int> blobsIds;
434 std::vector<int> portIds;
435 for (const std::string& name : {"weights", "biases"})
436 {
437 bool asInput = false;
438 int portId = 0;
439 for (int i = 0; i < layer.getInputPorts().size(); ++i)
440 {
441 const auto& port = layer.getInputPorts()[i];
442 auto it = port.getParameters().find("type");
443 if (it != port.getParameters().end() && it->second == name)
444 {
445 portId = i;
446 asInput = true;
447 break;
448 }
449 }
450
451 if (!asInput)
452 continue;
453
454 auto it = params.find(name);
455 if (it != params.end())
456 {
457 InferenceEngine::Blob::Ptr blob = it->second.as<InferenceEngine::Blob::Ptr>();
458 params.erase(it);
459 int blobId = netBuilder.addLayer(InferenceEngine::Builder::ConstLayer(name).setData(blob));
460 blobsIds.push_back(blobId);
461 portIds.push_back(portId);
462 }
463 }
464 #endif
465
466 int id = netBuilder.addLayer(layer);
467 const std::string& layerName = layer.getName();
468
469 CV_Assert(layers.insert({layerName, id}).second);
470 for (int i = 0; i < layer.getOutputPorts().size(); ++i)
471 unconnectedPorts.insert({id, i});
472
473 #if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2019R1)
474 // By default, all the weights are connected to last ports ids.
475 for (int i = 0; i < blobsIds.size(); ++i)
476 {
477 netBuilder.connect((size_t)blobsIds[i], {(size_t)id, (size_t)portIds[i]});
478 }
479 #endif
480 }
481
addOutput(const std::string & name)482 void InfEngineBackendNet::addOutput(const std::string& name)
483 {
484 requestedOutputs.push_back(name);
485 }
486
estimateLayout(const Mat & m)487 static InferenceEngine::Layout estimateLayout(const Mat& m)
488 {
489 if (m.dims == 4)
490 return InferenceEngine::Layout::NCHW;
491 else if (m.dims == 2)
492 return InferenceEngine::Layout::NC;
493 else
494 return InferenceEngine::Layout::ANY;
495 }
496
wrapToInfEngineDataNode(const Mat & m,const std::string & name="")497 static InferenceEngine::DataPtr wrapToInfEngineDataNode(const Mat& m, const std::string& name = "")
498 {
499 std::vector<size_t> shape = getShape<size_t>(m);
500 if (m.type() == CV_32F)
501 return InferenceEngine::DataPtr(new InferenceEngine::Data(name,
502 {InferenceEngine::Precision::FP32, shape, estimateLayout(m)}));
503 else if (m.type() == CV_8U)
504 return InferenceEngine::DataPtr(new InferenceEngine::Data(name,
505 {InferenceEngine::Precision::U8, shape, estimateLayout(m)}));
506 else
507 CV_Error(Error::StsNotImplemented, format("Unsupported data type %d", m.type()));
508 }
509
wrapToInfEngineBlob(const Mat & m,const std::vector<size_t> & shape,InferenceEngine::Layout layout)510 InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector<size_t>& shape,
511 InferenceEngine::Layout layout)
512 {
513 if (m.type() == CV_32F)
514 return InferenceEngine::make_shared_blob<float>(
515 {InferenceEngine::Precision::FP32, shape, layout}, (float*)m.data);
516 else if (m.type() == CV_8U)
517 return InferenceEngine::make_shared_blob<uint8_t>(
518 {InferenceEngine::Precision::U8, shape, layout}, (uint8_t*)m.data);
519 else
520 CV_Error(Error::StsNotImplemented, format("Unsupported data type %d", m.type()));
521 }
522
wrapToInfEngineBlob(const Mat & m,InferenceEngine::Layout layout)523 InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, InferenceEngine::Layout layout)
524 {
525 std::vector<size_t> shape = getShape<size_t>(m);
526 return wrapToInfEngineBlob(m, shape, layout);
527 }
528
cloneBlob(const InferenceEngine::Blob::Ptr & blob)529 InferenceEngine::Blob::Ptr cloneBlob(const InferenceEngine::Blob::Ptr& blob)
530 {
531 InferenceEngine::Blob::Ptr copy;
532 auto description = blob->getTensorDesc();
533 InferenceEngine::Precision precision = description.getPrecision();
534 if (precision == InferenceEngine::Precision::FP32)
535 {
536 copy = InferenceEngine::make_shared_blob<float>(description);
537 }
538 else if (precision == InferenceEngine::Precision::U8)
539 {
540 copy = InferenceEngine::make_shared_blob<uint8_t>(description);
541 }
542 else
543 CV_Error(Error::StsNotImplemented, "Unsupported blob precision");
544 copy->allocate();
545 return copy;
546 }
547
infEngineDataNode(const Ptr<BackendWrapper> & ptr)548 InferenceEngine::DataPtr infEngineDataNode(const Ptr<BackendWrapper>& ptr)
549 {
550 CV_Assert(!ptr.empty());
551 Ptr<InfEngineBackendWrapper> p = ptr.dynamicCast<InfEngineBackendWrapper>();
552 CV_Assert(!p.empty());
553 return p->dataPtr;
554 }
555
InfEngineBackendWrapper(int targetId,const cv::Mat & m)556 InfEngineBackendWrapper::InfEngineBackendWrapper(int targetId, const cv::Mat& m)
557 : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, targetId)
558 {
559 dataPtr = wrapToInfEngineDataNode(m);
560 blob = wrapToInfEngineBlob(m, estimateLayout(m));
561 }
562
InfEngineBackendWrapper(Ptr<BackendWrapper> wrapper)563 InfEngineBackendWrapper::InfEngineBackendWrapper(Ptr<BackendWrapper> wrapper)
564 : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, wrapper->targetId)
565 {
566 Ptr<InfEngineBackendWrapper> ieWrapper = wrapper.dynamicCast<InfEngineBackendWrapper>();
567 CV_Assert(!ieWrapper.empty());
568 InferenceEngine::DataPtr srcData = ieWrapper->dataPtr;
569
570 dataPtr = InferenceEngine::DataPtr(new InferenceEngine::Data(srcData->getName(), srcData->getTensorDesc()));
571 blob = ieWrapper->blob;
572 }
573
create(Ptr<BackendWrapper> wrapper)574 Ptr<BackendWrapper> InfEngineBackendWrapper::create(Ptr<BackendWrapper> wrapper)
575 {
576 return Ptr<BackendWrapper>(new InfEngineBackendWrapper(wrapper));
577 }
578
~InfEngineBackendWrapper()579 InfEngineBackendWrapper::~InfEngineBackendWrapper()
580 {
581
582 }
583
copyToHost()584 void InfEngineBackendWrapper::copyToHost()
585 {
586
587 }
588
setHostDirty()589 void InfEngineBackendWrapper::setHostDirty()
590 {
591
592 }
593
594 #endif // HAVE_DNN_IE_NN_BUILDER_2019
595
596 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1)
getSharedPlugins()597 static std::map<std::string, InferenceEngine::InferenceEnginePluginPtr>& getSharedPlugins()
598 {
599 static std::map<std::string, InferenceEngine::InferenceEnginePluginPtr> sharedPlugins;
600 return sharedPlugins;
601 }
602 #else
init_IE_plugins()603 static bool init_IE_plugins()
604 {
605 // load and hold IE plugins
606 static InferenceEngine::Core* init_core = new InferenceEngine::Core(); // 'delete' is never called
607 (void)init_core->GetAvailableDevices();
608 return true;
609 }
retrieveIECore(const std::string & id,std::map<std::string,std::shared_ptr<InferenceEngine::Core>> & cores)610 static InferenceEngine::Core& retrieveIECore(const std::string& id, std::map<std::string, std::shared_ptr<InferenceEngine::Core> >& cores)
611 {
612 AutoLock lock(getInitializationMutex());
613 std::map<std::string, std::shared_ptr<InferenceEngine::Core> >::iterator i = cores.find(id);
614 if (i == cores.end())
615 {
616 std::shared_ptr<InferenceEngine::Core> core = std::make_shared<InferenceEngine::Core>();
617 cores[id] = core;
618 return *core.get();
619 }
620 return *(i->second).get();
621 }
create_IE_Core_instance(const std::string & id)622 static InferenceEngine::Core& create_IE_Core_instance(const std::string& id)
623 {
624 static std::map<std::string, std::shared_ptr<InferenceEngine::Core> > cores;
625 return retrieveIECore(id, cores);
626 }
create_IE_Core_pointer(const std::string & id)627 static InferenceEngine::Core& create_IE_Core_pointer(const std::string& id)
628 {
629 // load and hold IE plugins
630 static std::map<std::string, std::shared_ptr<InferenceEngine::Core> >* cores =
631 new std::map<std::string, std::shared_ptr<InferenceEngine::Core> >();
632 return retrieveIECore(id, *cores);
633 }
getCore(const std::string & id)634 InferenceEngine::Core& getCore(const std::string& id)
635 {
636 // to make happy memory leak tools use:
637 // - OPENCV_DNN_INFERENCE_ENGINE_HOLD_PLUGINS=0
638 // - OPENCV_DNN_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND=0
639 static bool param_DNN_INFERENCE_ENGINE_HOLD_PLUGINS = utils::getConfigurationParameterBool("OPENCV_DNN_INFERENCE_ENGINE_HOLD_PLUGINS", true);
640 static bool init_IE_plugins_ = param_DNN_INFERENCE_ENGINE_HOLD_PLUGINS && init_IE_plugins(); CV_UNUSED(init_IE_plugins_);
641
642 static bool param_DNN_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND =
643 utils::getConfigurationParameterBool("OPENCV_DNN_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND",
644 #ifdef _WIN32
645 true
646 #else
647 false
648 #endif
649 );
650
651 InferenceEngine::Core& core = param_DNN_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND
652 ? create_IE_Core_pointer(id)
653 : create_IE_Core_instance(id);
654 return core;
655 }
656 #endif
657
detectArmPlugin_()658 static bool detectArmPlugin_()
659 {
660 InferenceEngine::Core& ie = getCore("CPU");
661 const std::vector<std::string> devices = ie.GetAvailableDevices();
662 for (std::vector<std::string>::const_iterator i = devices.begin(); i != devices.end(); ++i)
663 {
664 if (i->find("CPU") != std::string::npos)
665 {
666 const std::string name = ie.GetMetric(*i, METRIC_KEY(FULL_DEVICE_NAME)).as<std::string>();
667 CV_LOG_INFO(NULL, "CPU plugin: " << name);
668 return name.find("arm_compute::NEON") != std::string::npos;
669 }
670 }
671 return false;
672 }
673
674 #if !defined(OPENCV_DNN_IE_VPU_TYPE_DEFAULT)
detectMyriadX_(std::string device)675 static bool detectMyriadX_(std::string device)
676 {
677 AutoLock lock(getInitializationMutex());
678 #if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2019R3)
679 // Lightweight detection
680 InferenceEngine::Core& ie = getCore(device);
681 const std::vector<std::string> devices = ie.GetAvailableDevices();
682 for (std::vector<std::string>::const_iterator i = devices.begin(); i != devices.end(); ++i)
683 {
684 if (i->find(device) != std::string::npos)
685 {
686 const std::string name = ie.GetMetric(*i, METRIC_KEY(FULL_DEVICE_NAME)).as<std::string>();
687 CV_LOG_INFO(NULL, "Myriad device: " << name);
688 return name.find("MyriadX") != std::string::npos || name.find("Myriad X") != std::string::npos || name.find("HDDL") != std::string::npos;
689 }
690 }
691 return false;
692 #else
693 InferenceEngine::Builder::Network builder("");
694 InferenceEngine::idx_t inpId = builder.addLayer(
695 InferenceEngine::Builder::InputLayer().setPort(InferenceEngine::Port({1})));
696
697 #if INF_ENGINE_RELEASE <= 2018050000
698 InferenceEngine::idx_t clampId;
699 {
700 InferenceEngine::Builder::Layer l = InferenceEngine::Builder::ClampLayer();
701 auto& blobs = l.getConstantData();
702 auto blob = InferenceEngine::make_shared_blob<int16_t>(
703 InferenceEngine::Precision::FP16,
704 InferenceEngine::Layout::C, {1});
705 blob->allocate();
706 blobs[""] = blob;
707 clampId = builder.addLayer({inpId}, l);
708 }
709 builder.addLayer({InferenceEngine::PortInfo(clampId)}, InferenceEngine::Builder::OutputLayer());
710 #else
711
712 InferenceEngine::idx_t clampId = builder.addLayer({inpId}, InferenceEngine::Builder::ClampLayer());
713 builder.addLayer({InferenceEngine::PortInfo(clampId)},
714 InferenceEngine::Builder::OutputLayer().setPort(InferenceEngine::Port({},
715 InferenceEngine::Precision::FP16)));
716 #endif
717
718 InferenceEngine::CNNNetwork cnn = InferenceEngine::CNNNetwork(
719 InferenceEngine::Builder::convertToICNNNetwork(builder.build()));
720
721 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1)
722 InferenceEngine::InferenceEnginePluginPtr enginePtr;
723 {
724 auto& sharedPlugins = getSharedPlugins();
725 auto pluginIt = sharedPlugins.find(device);
726 if (pluginIt != sharedPlugins.end()) {
727 enginePtr = pluginIt->second;
728 } else {
729 auto dispatcher = InferenceEngine::PluginDispatcher({""});
730 enginePtr = dispatcher.getPluginByDevice(device);
731 sharedPlugins[device] = enginePtr;
732 }
733 }
734 auto plugin = InferenceEngine::InferencePlugin(enginePtr);
735 try
736 {
737 auto netExec = plugin.LoadNetwork(cnn, {{"VPU_PLATFORM", "VPU_2480"}});
738 #else
739 try
740 {
741 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R3)
742 auto netExec = getCore(device).LoadNetwork(cnn, device, {{"VPU_PLATFORM", "VPU_2480"}});
743 #else
744 auto netExec = getCore(device).LoadNetwork(cnn, device, {{"VPU_MYRIAD_PLATFORM", "VPU_MYRIAD_2480"}});
745 #endif
746 #endif
747 auto infRequest = netExec.CreateInferRequest();
748 } catch(...) {
749 return false;
750 }
751 return true;
752 #endif
753 }
754 #endif // !defined(OPENCV_DNN_IE_VPU_TYPE_DEFAULT)
755
756
757 #ifdef HAVE_DNN_IE_NN_BUILDER_2019
758
759 void InfEngineBackendNet::initPlugin(InferenceEngine::CNNNetwork& net)
760 {
761 CV_Assert(!isInitialized());
762
763 try
764 {
765 AutoLock lock(getInitializationMutex());
766 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1)
767 auto& sharedPlugins = getSharedPlugins();
768 auto pluginIt = sharedPlugins.find(device_name);
769 if (pluginIt != sharedPlugins.end())
770 {
771 enginePtr = pluginIt->second;
772 }
773 else
774 #else
775 InferenceEngine::Core& ie = getCore(device_name);
776 #endif
777 {
778 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1)
779 auto dispatcher = InferenceEngine::PluginDispatcher({""});
780 if (device_name == "FPGA")
781 enginePtr = dispatcher.getPluginByDevice("HETERO:FPGA,CPU");
782 else
783 enginePtr = dispatcher.getPluginByDevice(device_name);
784 sharedPlugins[device_name] = enginePtr;
785 #else
786 isInit = true;
787 #endif
788 std::vector<std::string> candidates;
789 std::string param_pluginPath = utils::getConfigurationParameterString("OPENCV_DNN_IE_EXTRA_PLUGIN_PATH", "");
790 if (!param_pluginPath.empty())
791 {
792 candidates.push_back(param_pluginPath);
793 }
794 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R3)
795 if (device_name == "CPU" || device_name == "FPGA")
796 {
797 std::string suffixes[] = {"_avx2", "_sse4", ""};
798 bool haveFeature[] = {
799 checkHardwareSupport(CPU_AVX2),
800 checkHardwareSupport(CPU_SSE4_2),
801 true
802 };
803 for (int i = 0; i < 3; ++i)
804 {
805 if (!haveFeature[i])
806 continue;
807 #ifdef _WIN32
808 candidates.push_back("cpu_extension" + suffixes[i] + ".dll");
809 #elif defined(__APPLE__)
810 candidates.push_back("libcpu_extension" + suffixes[i] + ".so"); // built as loadable module
811 candidates.push_back("libcpu_extension" + suffixes[i] + ".dylib"); // built as shared library
812 #else
813 candidates.push_back("libcpu_extension" + suffixes[i] + ".so");
814 #endif // _WIN32
815 }
816 }
817 #endif
818 bool found = false;
819 for (size_t i = 0; i != candidates.size(); ++i)
820 {
821 const std::string& libName = candidates[i];
822 try
823 {
824 InferenceEngine::IExtensionPtr extension =
825 InferenceEngine::make_so_pointer<InferenceEngine::IExtension>(libName);
826
827 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1)
828 enginePtr->AddExtension(extension, 0);
829 #else
830 ie.AddExtension(extension, "CPU");
831 #endif
832 CV_LOG_INFO(NULL, "DNN-IE: Loaded extension plugin: " << libName);
833 found = true;
834 break;
835 }
836 catch(...) {}
837 }
838 if (!found && !candidates.empty())
839 {
840 CV_LOG_WARNING(NULL, "DNN-IE: Can't load extension plugin (extra layers for some networks). Specify path via OPENCV_DNN_IE_EXTRA_PLUGIN_PATH parameter");
841 }
842 // Some of networks can work without a library of extra layers.
843 #if INF_ENGINE_VER_MAJOR_GT(INF_ENGINE_RELEASE_2019R1)
844 // OpenCV fallbacks as extensions.
845 try
846 {
847 ie.AddExtension(std::make_shared<InfEngineExtension>(), "CPU");
848 }
849 catch(const std::exception& e)
850 {
851 CV_LOG_INFO(NULL, "DNN-IE: Can't register OpenCV custom layers extension: " << e.what());
852 }
853 #endif
854 // Limit the number of CPU threads.
855 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1)
856 #ifndef _WIN32
857 enginePtr->SetConfig({{
858 InferenceEngine::PluginConfigParams::KEY_CPU_THREADS_NUM, format("%d", getNumThreads()),
859 }}, 0);
860 #endif // _WIN32
861 #else
862 if (device_name == "CPU")
863 ie.SetConfig({{
864 InferenceEngine::PluginConfigParams::KEY_CPU_THREADS_NUM, format("%d", getNumThreads()),
865 }}, device_name);
866 #endif
867 }
868 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1)
869 plugin = InferenceEngine::InferencePlugin(enginePtr);
870 netExec = plugin.LoadNetwork(net, {});
871 #else
872 bool isHetero = false;
873 if (device_name != "CPU")
874 {
875 isHetero = device_name == "FPGA";
876 for (auto& layer : net)
877 {
878 if (layer->type == kOpenCVLayersType)
879 {
880 isHetero = true;
881 #if INF_ENGINE_VER_MAJOR_LT(INF_ENGINE_RELEASE_2019R3)
882 // Not sure about lower versions but in 2019R3 we do not need this
883 layer->affinity = "CPU";
884 }
885 else
886 {
887 layer->affinity = device_name;
888 #endif
889 }
890 }
891 }
892 if (isHetero)
893 netExec = ie.LoadNetwork(net, "HETERO:" + device_name + ",CPU");
894 else
895 netExec = ie.LoadNetwork(net, device_name);
896 #endif
897 }
898 catch (const std::exception& ex)
899 {
900 CV_Error(Error::StsError, format("Failed to initialize Inference Engine backend (device = %s): %s", device_name.c_str(), ex.what()));
901 }
902 }
903
904 bool InfEngineBackendNet::isInitialized()
905 {
906 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1)
907 return (bool)enginePtr;
908 #else
909 return isInit;
910 #endif
911 }
912
913 void InfEngineBackendNet::reset()
914 {
915 allBlobs.clear();
916 infRequests.clear();
917 isInit = false;
918 }
919
920 void InfEngineBackendNet::addBlobs(const std::vector<cv::Ptr<BackendWrapper> >& ptrs)
921 {
922 auto wrappers = infEngineWrappers(ptrs);
923 for (const auto& wrapper : wrappers)
924 {
925 std::string name = wrapper->dataPtr->getName();
926 name = name.empty() ? kDefaultInpLayerName : name;
927 allBlobs.insert({name, wrapper->blob});
928 }
929 }
930
931 void InfEngineBackendNet::InfEngineReqWrapper::makePromises(const std::vector<Ptr<BackendWrapper> >& outsWrappers)
932 {
933 auto outs = infEngineWrappers(outsWrappers);
934 outProms.clear();
935 outProms.resize(outs.size());
936 outsNames.resize(outs.size());
937 for (int i = 0; i < outs.size(); ++i)
938 {
939 outs[i]->futureMat = outProms[i].getArrayResult();
940 outsNames[i] = outs[i]->dataPtr->getName();
941 }
942 }
943
944 void InfEngineBackendNet::forward(const std::vector<Ptr<BackendWrapper> >& outBlobsWrappers,
945 bool isAsync)
946 {
947 CV_LOG_DEBUG(NULL, "InfEngineBackendNet::forward(" << (isAsync ? "async" : "sync") << ")");
948 // Look for finished requests.
949 Ptr<InfEngineReqWrapper> reqWrapper;
950 for (auto& wrapper : infRequests)
951 {
952 if (wrapper->isReady)
953 {
954 reqWrapper = wrapper;
955 break;
956 }
957 }
958 if (reqWrapper.empty())
959 {
960 reqWrapper = Ptr<InfEngineReqWrapper>(new InfEngineReqWrapper());
961 try
962 {
963 reqWrapper->req = netExec.CreateInferRequest();
964 }
965 catch (const std::exception& ex)
966 {
967 CV_Error(Error::StsAssert, format("Failed to initialize Inference Engine backend: %s", ex.what()));
968 }
969 infRequests.push_back(reqWrapper);
970
971 InferenceEngine::BlobMap inpBlobs, outBlobs;
972 for (const auto& it : cnn.getInputsInfo())
973 {
974 const std::string& name = it.first;
975 auto blobIt = allBlobs.find(name);
976 CV_Assert(blobIt != allBlobs.end());
977 inpBlobs[name] = isAsync ? cloneBlob(blobIt->second) : blobIt->second;
978 }
979 for (const auto& it : cnn.getOutputsInfo())
980 {
981 const std::string& name = it.first;
982 auto blobIt = allBlobs.find(name);
983 CV_Assert(blobIt != allBlobs.end());
984 outBlobs[name] = isAsync ? cloneBlob(blobIt->second) : blobIt->second;
985 }
986 reqWrapper->req.SetInput(inpBlobs);
987 reqWrapper->req.SetOutput(outBlobs);
988
989 InferenceEngine::IInferRequest::Ptr infRequestPtr = reqWrapper->req;
990 infRequestPtr->SetUserData(reqWrapper.get(), 0);
991
992 infRequestPtr->SetCompletionCallback(
993 [](InferenceEngine::IInferRequest::Ptr request, InferenceEngine::StatusCode status)
994 {
995 CV_LOG_DEBUG(NULL, "DNN(IE): completionCallback(" << (int)status << ")");
996
997 InfEngineReqWrapper* wrapper;
998 request->GetUserData((void**)&wrapper, 0);
999 CV_Assert(wrapper && "Internal error");
1000
1001 size_t processedOutputs = 0;
1002 try
1003 {
1004 for (; processedOutputs < wrapper->outProms.size(); ++processedOutputs)
1005 {
1006 const std::string& name = wrapper->outsNames[processedOutputs];
1007 Mat m = infEngineBlobToMat(wrapper->req.GetBlob(name));
1008
1009 try
1010 {
1011 CV_Assert(status == InferenceEngine::StatusCode::OK);
1012 wrapper->outProms[processedOutputs].setValue(m.clone());
1013 }
1014 catch (...)
1015 {
1016 try {
1017 wrapper->outProms[processedOutputs].setException(std::current_exception());
1018 } catch(...) {
1019 CV_LOG_ERROR(NULL, "DNN: Exception occurred during async inference exception propagation");
1020 }
1021 }
1022 }
1023 }
1024 catch (...)
1025 {
1026 std::exception_ptr e = std::current_exception();
1027 for (; processedOutputs < wrapper->outProms.size(); ++processedOutputs)
1028 {
1029 try {
1030 wrapper->outProms[processedOutputs].setException(e);
1031 } catch(...) {
1032 CV_LOG_ERROR(NULL, "DNN: Exception occurred during async inference exception propagation");
1033 }
1034 }
1035 }
1036 wrapper->isReady = true;
1037 }
1038 );
1039 }
1040 if (isAsync)
1041 {
1042 // Copy actual data to infer request's input blobs.
1043 for (const auto& it : cnn.getInputsInfo())
1044 {
1045 const std::string& name = it.first;
1046 auto blobIt = allBlobs.find(name);
1047 Mat srcMat = infEngineBlobToMat(blobIt->second);
1048 Mat dstMat = infEngineBlobToMat(reqWrapper->req.GetBlob(name));
1049 srcMat.copyTo(dstMat);
1050 }
1051
1052 // Set promises to output blobs wrappers.
1053 reqWrapper->makePromises(outBlobsWrappers);
1054
1055 reqWrapper->isReady = false;
1056 reqWrapper->req.StartAsync();
1057 }
1058 else
1059 {
1060 reqWrapper->req.Infer();
1061 }
1062 }
1063
1064 bool InfEngineBackendLayer::getMemoryShapes(const std::vector<MatShape> &inputs,
1065 const int requiredOutputs,
1066 std::vector<MatShape> &outputs,
1067 std::vector<MatShape> &internals) const
1068 {
1069 InferenceEngine::ICNNNetwork::InputShapes inShapes = t_net.getInputShapes();
1070 InferenceEngine::ICNNNetwork::InputShapes::iterator itr;
1071 bool equal_flag = true;
1072 size_t i = 0;
1073 for (itr = inShapes.begin(); itr != inShapes.end(); ++itr)
1074 {
1075 InferenceEngine::SizeVector currentInShape(inputs[i].begin(), inputs[i].end());
1076 if (itr->second != currentInShape)
1077 {
1078 itr->second = currentInShape;
1079 equal_flag = false;
1080 }
1081 i++;
1082 }
1083
1084 if (!equal_flag)
1085 {
1086 InferenceEngine::CNNNetwork curr_t_net(t_net);
1087 curr_t_net.reshape(inShapes);
1088 }
1089 std::vector<size_t> dims = t_net.getOutputsInfo()[name]->getDims();
1090 outputs.push_back(MatShape(dims.begin(), dims.end()));
1091 return false;
1092 }
1093
1094 bool InfEngineBackendLayer::supportBackend(int backendId)
1095 {
1096 CV_LOG_DEBUG(NULL, "InfEngineBackendLayer::supportBackend(" << backendId << ")");
1097 return backendId == DNN_BACKEND_DEFAULT ||
1098 (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019);
1099 }
1100
1101 void InfEngineBackendLayer::forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs,
1102 OutputArrayOfArrays internals)
1103 {
1104 CV_Error(Error::StsInternal, "Choose Inference Engine as a preferable backend.");
1105 }
1106
1107 InferenceEngine::Blob::Ptr convertFp16(const InferenceEngine::Blob::Ptr& blob)
1108 {
1109 auto halfs = InferenceEngine::make_shared_blob<int16_t>({
1110 InferenceEngine::Precision::FP16, blob->getTensorDesc().getDims(),
1111 blob->getTensorDesc().getLayout()
1112 });
1113 halfs->allocate();
1114 Mat floatsData(1, blob->size(), CV_32F, blob->buffer());
1115 Mat halfsData(1, blob->size(), CV_16SC1, halfs->buffer());
1116 convertFp16(floatsData, halfsData);
1117 return halfs;
1118 }
1119
1120 void addConstantData(const std::string& name, InferenceEngine::Blob::Ptr data,
1121 InferenceEngine::Builder::Layer& l)
1122 {
1123 #if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2019R1)
1124 l.getParameters()[name] = data;
1125 #else
1126 l.addConstantData(name, data);
1127 #endif
1128 }
1129
1130 #endif // HAVE_DNN_IE_NN_BUILDER_2019
1131
1132 #endif // HAVE_INF_ENGINE
1133
1134 bool haveInfEngine()
1135 {
1136 #ifdef HAVE_INF_ENGINE
1137 return true;
1138 #else
1139 return false;
1140 #endif // HAVE_INF_ENGINE
1141 }
1142
1143 void forwardInfEngine(const std::vector<Ptr<BackendWrapper> >& outBlobsWrappers,
1144 Ptr<BackendNode>& node, bool isAsync)
1145 {
1146 CV_Assert(haveInfEngine());
1147 #ifdef HAVE_DNN_IE_NN_BUILDER_2019
1148 CV_Assert(!node.empty());
1149 Ptr<InfEngineBackendNode> ieNode = node.dynamicCast<InfEngineBackendNode>();
1150 CV_Assert(!ieNode.empty());
1151 ieNode->net->forward(outBlobsWrappers, isAsync);
1152 #else
1153 CV_Error(Error::StsNotImplemented, "This OpenCV version is built without Inference Engine NN Builder API support");
1154 #endif // HAVE_INF_ENGINE
1155 }
1156
1157 CV__DNN_INLINE_NS_BEGIN
1158
1159 void resetMyriadDevice()
1160 {
1161 #ifdef HAVE_INF_ENGINE
1162 AutoLock lock(getInitializationMutex());
1163 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1)
1164 getSharedPlugins().erase("MYRIAD");
1165 #else
1166 // Unregister both "MYRIAD" and "HETERO:MYRIAD,CPU" plugins
1167 InferenceEngine::Core& ie = getCore("MYRIAD");
1168 try
1169 {
1170 ie.UnregisterPlugin("MYRIAD");
1171 ie.UnregisterPlugin("HETERO");
1172 }
1173 catch (...) {}
1174 #endif
1175 #endif // HAVE_INF_ENGINE
1176 }
1177
1178 void releaseHDDLPlugin()
1179 {
1180 #ifdef HAVE_INF_ENGINE
1181 AutoLock lock(getInitializationMutex());
1182 #if INF_ENGINE_VER_MAJOR_LE(INF_ENGINE_RELEASE_2019R1)
1183 getSharedPlugins().erase("HDDL");
1184 #else
1185 // Unregister both "HDDL" and "HETERO:HDDL,CPU" plugins
1186 InferenceEngine::Core& ie = getCore("HDDL");
1187 try
1188 {
1189 ie.UnregisterPlugin("HDDL");
1190 ie.UnregisterPlugin("HETERO");
1191 }
1192 catch (...) {}
1193 #endif
1194 #endif // HAVE_INF_ENGINE
1195 }
1196
1197 #ifdef HAVE_INF_ENGINE
1198 bool isMyriadX()
1199 {
1200 static bool myriadX = getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X;
1201 return myriadX;
1202 }
1203
1204 bool isArmComputePlugin()
1205 {
1206 static bool armPlugin = getInferenceEngineCPUType() == CV_DNN_INFERENCE_ENGINE_CPU_TYPE_ARM_COMPUTE;
1207 return armPlugin;
1208 }
1209
1210 static std::string getInferenceEngineVPUType_()
1211 {
1212 static std::string param_vpu_type = utils::getConfigurationParameterString("OPENCV_DNN_IE_VPU_TYPE", "");
1213 if (param_vpu_type == "")
1214 {
1215 #if defined(OPENCV_DNN_IE_VPU_TYPE_DEFAULT)
1216 param_vpu_type = OPENCV_DNN_IE_VPU_TYPE_DEFAULT;
1217 #else
1218 CV_LOG_INFO(NULL, "OpenCV-DNN: running Inference Engine VPU autodetection: Myriad2/X or HDDL. In case of other accelerator types specify 'OPENCV_DNN_IE_VPU_TYPE' parameter");
1219 try {
1220 bool isMyriadX_ = detectMyriadX_("MYRIAD");
1221 bool isHDDL_ = detectMyriadX_("HDDL");
1222 if (isMyriadX_ || isHDDL_)
1223 {
1224 param_vpu_type = CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X;
1225 }
1226 else
1227 {
1228 param_vpu_type = CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2;
1229 }
1230 }
1231 catch (...)
1232 {
1233 CV_LOG_WARNING(NULL, "OpenCV-DNN: Failed Inference Engine VPU autodetection. Specify 'OPENCV_DNN_IE_VPU_TYPE' parameter.");
1234 param_vpu_type.clear();
1235 }
1236 #endif
1237 }
1238 CV_LOG_INFO(NULL, "OpenCV-DNN: Inference Engine VPU type='" << param_vpu_type << "'");
1239 return param_vpu_type;
1240 }
1241
1242 cv::String getInferenceEngineVPUType()
1243 {
1244 static cv::String vpu_type = getInferenceEngineVPUType_();
1245 return vpu_type;
1246 }
1247
1248 cv::String getInferenceEngineCPUType()
1249 {
1250 static cv::String cpu_type = detectArmPlugin_() ?
1251 CV_DNN_INFERENCE_ENGINE_CPU_TYPE_ARM_COMPUTE :
1252 CV_DNN_INFERENCE_ENGINE_CPU_TYPE_X86;
1253 return cpu_type;
1254 }
1255
1256 #else // HAVE_INF_ENGINE
1257
1258 cv::String getInferenceEngineBackendType()
1259 {
1260 CV_Error(Error::StsNotImplemented, "This OpenCV build doesn't include InferenceEngine support");
1261 }
1262 cv::String setInferenceEngineBackendType(const cv::String& newBackendType)
1263 {
1264 CV_UNUSED(newBackendType);
1265 CV_Error(Error::StsNotImplemented, "This OpenCV build doesn't include InferenceEngine support");
1266 }
1267 cv::String getInferenceEngineVPUType()
1268 {
1269 CV_Error(Error::StsNotImplemented, "This OpenCV build doesn't include InferenceEngine support");
1270 }
1271
1272 cv::String getInferenceEngineCPUType()
1273 {
1274 CV_Error(Error::StsNotImplemented, "This OpenCV build doesn't include InferenceEngine support");
1275 }
1276 #endif // HAVE_INF_ENGINE
1277
1278
1279 CV__DNN_INLINE_NS_END
1280 }} // namespace dnn, namespace cv
1281