1 /*M///////////////////////////////////////////////////////////////////////////////////////
2 //
3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
4 //
5 // By downloading, copying, installing or using the software you agree to this license.
6 // If you do not agree to this license, do not download, install,
7 // copy or use the software.
8 //
9 //
10 // License Agreement
11 // For Open Source Computer Vision Library
12 // (3-clause BSD License)
13 //
14 // Copyright (C) 2017, Intel Corporation, all rights reserved.
15 // Third party copyrights are property of their respective owners.
16 //
17 // Redistribution and use in source and binary forms, with or without modification,
18 // are permitted provided that the following conditions are met:
19 //
20 // * Redistributions of source code must retain the above copyright notice,
21 // this list of conditions and the following disclaimer.
22 //
23 // * Redistributions in binary form must reproduce the above copyright notice,
24 // this list of conditions and the following disclaimer in the documentation
25 // and/or other materials provided with the distribution.
26 //
27 // * Neither the names of the copyright holders nor the names of the contributors
28 // may be used to endorse or promote products derived from this software
29 // without specific prior written permission.
30 //
31 // This software is provided by the copyright holders and contributors "as is" and
32 // any express or implied warranties, including, but not limited to, the implied
33 // warranties of merchantability and fitness for a particular purpose are disclaimed.
34 // In no event shall copyright holders or contributors be liable for any direct,
35 // indirect, incidental, special, exemplary, or consequential damages
36 // (including, but not limited to, procurement of substitute goods or services;
37 // loss of use, data, or profits; or business interruption) however caused
38 // and on any theory of liability, whether in contract, strict liability,
39 // or tort (including negligence or otherwise) arising in any way out of
40 // the use of this software, even if advised of the possibility of such damage.
41 //
42 //M*/
43
44 #include "test_precomp.hpp"
45 #include "npy_blob.hpp"
46 #include <opencv2/dnn/shape_utils.hpp>
47
48 namespace opencv_test { namespace {
49
50 template<typename TString>
_tf(TString filename)51 static std::string _tf(TString filename)
52 {
53 return (getOpenCVExtraDir() + "/dnn/") + filename;
54 }
55
TEST(Test_Darknet,read_tiny_yolo_voc)56 TEST(Test_Darknet, read_tiny_yolo_voc)
57 {
58 Net net = readNetFromDarknet(_tf("tiny-yolo-voc.cfg"));
59 ASSERT_FALSE(net.empty());
60 }
61
TEST(Test_Darknet,read_yolo_voc)62 TEST(Test_Darknet, read_yolo_voc)
63 {
64 Net net = readNetFromDarknet(_tf("yolo-voc.cfg"));
65 ASSERT_FALSE(net.empty());
66 }
67
TEST(Test_Darknet,read_yolo_voc_stream)68 TEST(Test_Darknet, read_yolo_voc_stream)
69 {
70 applyTestTag(CV_TEST_TAG_MEMORY_1GB);
71 Mat ref;
72 Mat sample = imread(_tf("dog416.png"));
73 Mat inp = blobFromImage(sample, 1.0/255, Size(416, 416), Scalar(), true, false);
74 const std::string cfgFile = findDataFile("dnn/yolo-voc.cfg");
75 const std::string weightsFile = findDataFile("dnn/yolo-voc.weights", false);
76 // Import by paths.
77 {
78 Net net = readNetFromDarknet(cfgFile, weightsFile);
79 net.setInput(inp);
80 net.setPreferableBackend(DNN_BACKEND_OPENCV);
81 ref = net.forward();
82 }
83 // Import from bytes array.
84 {
85 std::vector<char> cfg, weights;
86 readFileContent(cfgFile, cfg);
87 readFileContent(weightsFile, weights);
88
89 Net net = readNetFromDarknet(cfg.data(), cfg.size(), weights.data(), weights.size());
90 net.setInput(inp);
91 net.setPreferableBackend(DNN_BACKEND_OPENCV);
92 Mat out = net.forward();
93 normAssert(ref, out);
94 }
95 }
96
97 class Test_Darknet_layers : public DNNTestLayer
98 {
99 public:
testDarknetLayer(const std::string & name,bool hasWeights=false,bool testBatchProcessing=true)100 void testDarknetLayer(const std::string& name, bool hasWeights = false, bool testBatchProcessing = true)
101 {
102 SCOPED_TRACE(name);
103 Mat inp = blobFromNPY(findDataFile("dnn/darknet/" + name + "_in.npy"));
104 Mat ref = blobFromNPY(findDataFile("dnn/darknet/" + name + "_out.npy"));
105
106 std::string cfg = findDataFile("dnn/darknet/" + name + ".cfg");
107 std::string model = "";
108 if (hasWeights)
109 model = findDataFile("dnn/darknet/" + name + ".weights");
110
111 checkBackend(&inp, &ref);
112
113 Net net = readNet(cfg, model);
114 net.setPreferableBackend(backend);
115 net.setPreferableTarget(target);
116 net.setInput(inp);
117 Mat out = net.forward();
118 normAssert(out, ref, "", default_l1, default_lInf);
119
120 if (inp.size[0] == 1 && testBatchProcessing) // test handling of batch size
121 {
122 SCOPED_TRACE("batch size 2");
123
124 #if defined(INF_ENGINE_RELEASE)
125 if (target == DNN_TARGET_MYRIAD && name == "shortcut")
126 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
127 #endif
128
129 std::vector<int> sz2 = shape(inp);
130 sz2[0] = 2;
131
132 Net net2 = readNet(cfg, model);
133 net2.setPreferableBackend(backend);
134 net2.setPreferableTarget(target);
135 Range ranges0[4] = { Range(0, 1), Range::all(), Range::all(), Range::all() };
136 Range ranges1[4] = { Range(1, 2), Range::all(), Range::all(), Range::all() };
137 Mat inp2(sz2, inp.type(), Scalar::all(0));
138 inp.copyTo(inp2(ranges0));
139 inp.copyTo(inp2(ranges1));
140 net2.setInput(inp2);
141 Mat out2 = net2.forward();
142 EXPECT_EQ(0, cv::norm(out2(ranges0), out2(ranges1), NORM_INF)) << "Batch result is not equal: " << name;
143
144 Mat ref2 = ref;
145 if (ref.dims == 2 && out2.dims == 3)
146 {
147 int ref_3d_sizes[3] = {1, ref.rows, ref.cols};
148 ref2 = Mat(3, ref_3d_sizes, ref.type(), (void*)ref.data);
149 }
150 /*else if (ref.dims == 3 && out2.dims == 4)
151 {
152 int ref_4d_sizes[4] = {1, ref.size[0], ref.size[1], ref.size[2]};
153 ref2 = Mat(4, ref_4d_sizes, ref.type(), (void*)ref.data);
154 }*/
155 ASSERT_EQ(out2.dims, ref2.dims) << ref.dims;
156
157 normAssert(out2(ranges0), ref2, "", default_l1, default_lInf);
158 normAssert(out2(ranges1), ref2, "", default_l1, default_lInf);
159 }
160 }
161 };
162
163 class Test_Darknet_nets : public DNNTestLayer
164 {
165 public:
166 // Test object detection network from Darknet framework.
testDarknetModel(const std::string & cfg,const std::string & weights,const std::vector<std::vector<int>> & refClassIds,const std::vector<std::vector<float>> & refConfidences,const std::vector<std::vector<Rect2d>> & refBoxes,double scoreDiff,double iouDiff,float confThreshold=0.24,float nmsThreshold=0.4)167 void testDarknetModel(const std::string& cfg, const std::string& weights,
168 const std::vector<std::vector<int> >& refClassIds,
169 const std::vector<std::vector<float> >& refConfidences,
170 const std::vector<std::vector<Rect2d> >& refBoxes,
171 double scoreDiff, double iouDiff, float confThreshold = 0.24, float nmsThreshold = 0.4)
172 {
173 checkBackend();
174
175 Mat img1 = imread(_tf("dog416.png"));
176 Mat img2 = imread(_tf("street.png"));
177 std::vector<Mat> samples(2);
178 samples[0] = img1; samples[1] = img2;
179
180 // determine test type, whether batch or single img
181 int batch_size = refClassIds.size();
182 CV_Assert(batch_size == 1 || batch_size == 2);
183 samples.resize(batch_size);
184
185 Mat inp = blobFromImages(samples, 1.0/255, Size(416, 416), Scalar(), true, false);
186
187 Net net = readNet(findDataFile("dnn/" + cfg),
188 findDataFile("dnn/" + weights, false));
189 net.setPreferableBackend(backend);
190 net.setPreferableTarget(target);
191 net.setInput(inp);
192 std::vector<Mat> outs;
193 net.forward(outs, net.getUnconnectedOutLayersNames());
194
195 for (int b = 0; b < batch_size; ++b)
196 {
197 std::vector<int> classIds;
198 std::vector<float> confidences;
199 std::vector<Rect2d> boxes;
200 for (int i = 0; i < outs.size(); ++i)
201 {
202 Mat out;
203 if (batch_size > 1){
204 // get the sample slice from 3D matrix (batch, box, classes+5)
205 Range ranges[3] = {Range(b, b+1), Range::all(), Range::all()};
206 out = outs[i](ranges).reshape(1, outs[i].size[1]);
207 }else{
208 out = outs[i];
209 }
210 for (int j = 0; j < out.rows; ++j)
211 {
212 Mat scores = out.row(j).colRange(5, out.cols);
213 double confidence;
214 Point maxLoc;
215 minMaxLoc(scores, 0, &confidence, 0, &maxLoc);
216
217 if (confidence > confThreshold) {
218 float* detection = out.ptr<float>(j);
219 double centerX = detection[0];
220 double centerY = detection[1];
221 double width = detection[2];
222 double height = detection[3];
223 boxes.push_back(Rect2d(centerX - 0.5 * width, centerY - 0.5 * height,
224 width, height));
225 confidences.push_back(confidence);
226 classIds.push_back(maxLoc.x);
227 }
228 }
229 }
230
231 // here we need NMS of boxes
232 std::vector<int> indices;
233 NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
234
235 std::vector<int> nms_classIds;
236 std::vector<float> nms_confidences;
237 std::vector<Rect2d> nms_boxes;
238
239 for (size_t i = 0; i < indices.size(); ++i)
240 {
241 int idx = indices[i];
242 Rect2d box = boxes[idx];
243 float conf = confidences[idx];
244 int class_id = classIds[idx];
245 nms_boxes.push_back(box);
246 nms_confidences.push_back(conf);
247 nms_classIds.push_back(class_id);
248 #if 0 // use to update test reference data
249 std::cout << b << ", " << class_id << ", " << conf << "f, "
250 << box.x << "f, " << box.y << "f, "
251 << box.x + box.width << "f, " << box.y + box.height << "f,"
252 << std::endl;
253 #endif
254
255 }
256
257 if (cvIsNaN(iouDiff))
258 {
259 if (b == 0)
260 std::cout << "Skip accuracy checks" << std::endl;
261 continue;
262 }
263
264 normAssertDetections(refClassIds[b], refConfidences[b], refBoxes[b], nms_classIds,
265 nms_confidences, nms_boxes, format("batch size %d, sample %d\n", batch_size, b).c_str(), confThreshold, scoreDiff, iouDiff);
266 }
267 }
268
testDarknetModel(const std::string & cfg,const std::string & weights,const std::vector<int> & refClassIds,const std::vector<float> & refConfidences,const std::vector<Rect2d> & refBoxes,double scoreDiff,double iouDiff,float confThreshold=0.24,float nmsThreshold=0.4)269 void testDarknetModel(const std::string& cfg, const std::string& weights,
270 const std::vector<int>& refClassIds,
271 const std::vector<float>& refConfidences,
272 const std::vector<Rect2d>& refBoxes,
273 double scoreDiff, double iouDiff, float confThreshold = 0.24, float nmsThreshold = 0.4)
274 {
275 testDarknetModel(cfg, weights,
276 std::vector<std::vector<int> >(1, refClassIds),
277 std::vector<std::vector<float> >(1, refConfidences),
278 std::vector<std::vector<Rect2d> >(1, refBoxes),
279 scoreDiff, iouDiff, confThreshold, nmsThreshold);
280 }
281
testDarknetModel(const std::string & cfg,const std::string & weights,const cv::Mat & ref,double scoreDiff,double iouDiff,float confThreshold=0.24,float nmsThreshold=0.4)282 void testDarknetModel(const std::string& cfg, const std::string& weights,
283 const cv::Mat& ref, double scoreDiff, double iouDiff,
284 float confThreshold = 0.24, float nmsThreshold = 0.4)
285 {
286 CV_Assert(ref.cols == 7);
287 std::vector<std::vector<int> > refClassIds;
288 std::vector<std::vector<float> > refScores;
289 std::vector<std::vector<Rect2d> > refBoxes;
290 for (int i = 0; i < ref.rows; ++i)
291 {
292 int batchId = static_cast<int>(ref.at<float>(i, 0));
293 int classId = static_cast<int>(ref.at<float>(i, 1));
294 float score = ref.at<float>(i, 2);
295 float left = ref.at<float>(i, 3);
296 float top = ref.at<float>(i, 4);
297 float right = ref.at<float>(i, 5);
298 float bottom = ref.at<float>(i, 6);
299 Rect2d box(left, top, right - left, bottom - top);
300 if (batchId >= refClassIds.size())
301 {
302 refClassIds.resize(batchId + 1);
303 refScores.resize(batchId + 1);
304 refBoxes.resize(batchId + 1);
305 }
306 refClassIds[batchId].push_back(classId);
307 refScores[batchId].push_back(score);
308 refBoxes[batchId].push_back(box);
309 }
310 testDarknetModel(cfg, weights, refClassIds, refScores, refBoxes,
311 scoreDiff, iouDiff, confThreshold, nmsThreshold);
312 }
313 };
314
TEST_P(Test_Darknet_nets,YoloVoc)315 TEST_P(Test_Darknet_nets, YoloVoc)
316 {
317 applyTestTag(
318 #if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL)
319 CV_TEST_TAG_MEMORY_2GB,
320 #else
321 CV_TEST_TAG_MEMORY_1GB,
322 #endif
323 CV_TEST_TAG_LONG
324 );
325
326 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
327 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
328 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
329 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
330 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
331 #endif
332 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
333 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL_FP16)
334 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
335 #endif
336 #if defined(INF_ENGINE_RELEASE)
337 if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) &&
338 target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
339 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); // need to update check function
340 #endif
341
342 // batchId, classId, confidence, left, top, right, bottom
343 Mat ref = (Mat_<float>(6, 7) << 0, 6, 0.750469f, 0.577374f, 0.127391f, 0.902949f, 0.300809f, // a car
344 0, 1, 0.780879f, 0.270762f, 0.264102f, 0.732475f, 0.745412f, // a bicycle
345 0, 11, 0.901615f, 0.1386f, 0.338509f, 0.421337f, 0.938789f, // a dog
346 1, 14, 0.623813f, 0.183179f, 0.381921f, 0.247726f, 0.625847f, // a person
347 1, 6, 0.667770f, 0.446555f, 0.453578f, 0.499986f, 0.519167f, // a car
348 1, 6, 0.844947f, 0.637058f, 0.460398f, 0.828508f, 0.66427f); // a car
349
350 double nmsThreshold = (target == DNN_TARGET_MYRIAD) ? 0.397 : 0.4;
351 double scoreDiff = 8e-5, iouDiff = 3e-4;
352 if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
353 {
354 scoreDiff = 1e-2;
355 iouDiff = 0.018;
356 }
357 else if (target == DNN_TARGET_CUDA_FP16)
358 {
359 scoreDiff = 0.03;
360 iouDiff = 0.018;
361 }
362
363 std::string config_file = "yolo-voc.cfg";
364 std::string weights_file = "yolo-voc.weights";
365
366 {
367 SCOPED_TRACE("batch size 1");
368 testDarknetModel(config_file, weights_file, ref.rowRange(0, 3), scoreDiff, iouDiff);
369 }
370
371 {
372 SCOPED_TRACE("batch size 2");
373 testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, 0.24, nmsThreshold);
374 }
375 }
376
TEST_P(Test_Darknet_nets,TinyYoloVoc)377 TEST_P(Test_Darknet_nets, TinyYoloVoc)
378 {
379 applyTestTag(CV_TEST_TAG_MEMORY_512MB);
380
381 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
382 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
383 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
384 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
385 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
386 #endif
387 #if defined(INF_ENGINE_RELEASE)
388 if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) &&
389 target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
390 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); // need to update check function
391 #endif
392 // batchId, classId, confidence, left, top, right, bottom
393 Mat ref = (Mat_<float>(4, 7) << 0, 6, 0.761967f, 0.579042f, 0.159161f, 0.894482f, 0.31994f, // a car
394 0, 11, 0.780595f, 0.129696f, 0.386467f, 0.445275f, 0.920994f, // a dog
395 1, 6, 0.651450f, 0.460526f, 0.458019f, 0.522527f, 0.5341f, // a car
396 1, 6, 0.928758f, 0.651024f, 0.463539f, 0.823784f, 0.654998f); // a car
397
398 double scoreDiff = 8e-5, iouDiff = 3e-4;
399 if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
400 {
401 scoreDiff = 8e-3;
402 iouDiff = 0.018;
403 }
404 else if(target == DNN_TARGET_CUDA_FP16)
405 {
406 scoreDiff = 0.008;
407 iouDiff = 0.02;
408 }
409
410 std::string config_file = "tiny-yolo-voc.cfg";
411 std::string weights_file = "tiny-yolo-voc.weights";
412
413 {
414 SCOPED_TRACE("batch size 1");
415 testDarknetModel(config_file, weights_file, ref.rowRange(0, 2), scoreDiff, iouDiff);
416 }
417
418 {
419 SCOPED_TRACE("batch size 2");
420 testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff);
421 }
422 }
423
424 #ifdef HAVE_INF_ENGINE
425 static const std::chrono::milliseconds async_timeout(10000);
426
427 typedef testing::TestWithParam<tuple<std::string, tuple<Backend, Target> > > Test_Darknet_nets_async;
TEST_P(Test_Darknet_nets_async,Accuracy)428 TEST_P(Test_Darknet_nets_async, Accuracy)
429 {
430 Backend backendId = get<0>(get<1>(GetParam()));
431 Target targetId = get<1>(get<1>(GetParam()));
432
433 if (INF_ENGINE_VER_MAJOR_LT(2019020000) && backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
434 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER);
435 applyTestTag(CV_TEST_TAG_MEMORY_512MB);
436
437 if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
438 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH);
439
440 std::string prefix = get<0>(GetParam());
441
442 if (targetId == DNN_TARGET_MYRIAD && prefix == "yolov4") // NC_OUT_OF_MEMORY
443 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
444
445 if (backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
446 throw SkipTestException("No support for async forward");
447
448 const int numInputs = 2;
449 std::vector<Mat> inputs(numInputs);
450 int blobSize[] = {1, 3, 416, 416};
451 for (int i = 0; i < numInputs; ++i)
452 {
453 inputs[i].create(4, &blobSize[0], CV_32F);
454 randu(inputs[i], 0, 1);
455 }
456
457 Net netSync = readNet(findDataFile("dnn/" + prefix + ".cfg"),
458 findDataFile("dnn/" + prefix + ".weights", false));
459 netSync.setPreferableBackend(backendId);
460 netSync.setPreferableTarget(targetId);
461
462 // Run synchronously.
463 std::vector<Mat> refs(numInputs);
464 for (int i = 0; i < numInputs; ++i)
465 {
466 netSync.setInput(inputs[i]);
467 refs[i] = netSync.forward().clone();
468 }
469
470 Net netAsync = readNet(findDataFile("dnn/" + prefix + ".cfg"),
471 findDataFile("dnn/" + prefix + ".weights", false));
472 netAsync.setPreferableBackend(backendId);
473 netAsync.setPreferableTarget(targetId);
474
475 // Run asynchronously. To make test more robust, process inputs in the reversed order.
476 for (int i = numInputs - 1; i >= 0; --i)
477 {
478 netAsync.setInput(inputs[i]);
479
480 AsyncArray out = netAsync.forwardAsync();
481 ASSERT_TRUE(out.valid());
482 Mat result;
483 EXPECT_TRUE(out.get(result, async_timeout));
484 normAssert(refs[i], result, format("Index: %d", i).c_str(), 0, 0);
485 }
486 }
487
488 INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_nets_async, Combine(
489 Values("yolo-voc", "tiny-yolo-voc", "yolov3", "yolov4", "yolov4-tiny"),
490 dnnBackendsAndTargets()
491 ));
492
493 #endif
494
TEST_P(Test_Darknet_nets,YOLOv3)495 TEST_P(Test_Darknet_nets, YOLOv3)
496 {
497 applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB));
498
499 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
500 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
501 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
502 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
503 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
504 #endif
505
506 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
507 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH);
508
509 // batchId, classId, confidence, left, top, right, bottom
510 const int N0 = 3;
511 const int N1 = 6;
512 static const float ref_[/* (N0 + N1) * 7 */] = {
513 0, 16, 0.998836f, 0.160024f, 0.389964f, 0.417885f, 0.943716f,
514 0, 1, 0.987908f, 0.150913f, 0.221933f, 0.742255f, 0.746261f,
515 0, 7, 0.952983f, 0.614621f, 0.150257f, 0.901368f, 0.289251f,
516
517 1, 2, 0.997412f, 0.647584f, 0.459939f, 0.821037f, 0.663947f,
518 1, 2, 0.989633f, 0.450719f, 0.463353f, 0.496306f, 0.522258f,
519 1, 0, 0.980053f, 0.195856f, 0.378454f, 0.258626f, 0.629257f,
520 1, 9, 0.785341f, 0.665503f, 0.373543f, 0.688893f, 0.439244f,
521 1, 9, 0.733275f, 0.376029f, 0.315694f, 0.401776f, 0.395165f,
522 1, 9, 0.384815f, 0.659824f, 0.372389f, 0.673927f, 0.429412f,
523 };
524 Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_);
525
526 double scoreDiff = 8e-5, iouDiff = 3e-4;
527 if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
528 {
529 scoreDiff = 0.006;
530 iouDiff = 0.042;
531 }
532 else if (target == DNN_TARGET_CUDA_FP16)
533 {
534 scoreDiff = 0.04;
535 iouDiff = 0.03;
536 }
537 std::string config_file = "yolov3.cfg";
538 std::string weights_file = "yolov3.weights";
539
540 #if defined(INF_ENGINE_RELEASE)
541 if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ||
542 backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD &&
543 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
544 {
545 scoreDiff = 0.04;
546 iouDiff = 0.2;
547 }
548 #endif
549
550 {
551 SCOPED_TRACE("batch size 1");
552 testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff);
553 }
554
555 #if defined(INF_ENGINE_RELEASE)
556 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
557 {
558 if (target == DNN_TARGET_OPENCL)
559 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
560 else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000))
561 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
562 else if (target == DNN_TARGET_MYRIAD &&
563 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
564 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
565 }
566 #endif
567
568 {
569 SCOPED_TRACE("batch size 2");
570 testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff);
571 }
572 }
573
TEST_P(Test_Darknet_nets,YOLOv4)574 TEST_P(Test_Darknet_nets, YOLOv4)
575 {
576 applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB));
577
578 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
579 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
580 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
581 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
582 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
583 #endif
584 #if defined(INF_ENGINE_RELEASE)
585 if (target == DNN_TARGET_MYRIAD) // NC_OUT_OF_MEMORY
586 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
587 #endif
588
589 // batchId, classId, confidence, left, top, right, bottom
590 const int N0 = 3;
591 const int N1 = 7;
592 static const float ref_[/* (N0 + N1) * 7 */] = {
593 0, 16, 0.992194f, 0.172375f, 0.402458f, 0.403918f, 0.932801f,
594 0, 1, 0.988326f, 0.166708f, 0.228236f, 0.737208f, 0.735803f,
595 0, 7, 0.94639f, 0.602523f, 0.130399f, 0.901623f, 0.298452f,
596
597 1, 2, 0.99761f, 0.646556f, 0.45985f, 0.816041f, 0.659067f,
598 1, 0, 0.988913f, 0.201726f, 0.360282f, 0.266181f, 0.631728f,
599 1, 2, 0.98233f, 0.452007f, 0.462217f, 0.495612f, 0.521687f,
600 1, 9, 0.919195f, 0.374642f, 0.316524f, 0.398126f, 0.393714f,
601 1, 9, 0.856303f, 0.666842f, 0.372215f, 0.685539f, 0.44141f,
602 1, 9, 0.313516f, 0.656791f, 0.374734f, 0.671959f, 0.438371f,
603 1, 9, 0.256625f, 0.940232f, 0.326931f, 0.967586f, 0.374002f,
604 };
605 Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_);
606
607 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.006 : 8e-5;
608 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.042 : 3e-4;
609 if (target == DNN_TARGET_CUDA_FP16)
610 {
611 scoreDiff = 0.008;
612 iouDiff = 0.03;
613 }
614
615 std::string config_file = "yolov4.cfg";
616 std::string weights_file = "yolov4.weights";
617
618 #if defined(INF_ENGINE_RELEASE)
619 if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ||
620 backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD &&
621 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
622 {
623 scoreDiff = 0.04;
624 iouDiff = 0.2;
625 }
626 #endif
627
628 {
629 SCOPED_TRACE("batch size 1");
630 testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff);
631 }
632
633 {
634 SCOPED_TRACE("batch size 2");
635
636 #if defined(INF_ENGINE_RELEASE)
637 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
638 {
639 if (target == DNN_TARGET_OPENCL)
640 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
641 else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000))
642 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
643 else if (target == DNN_TARGET_MYRIAD &&
644 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
645 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
646 }
647 #endif
648
649 testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff);
650 }
651 }
652
TEST_P(Test_Darknet_nets,YOLOv4_tiny)653 TEST_P(Test_Darknet_nets, YOLOv4_tiny)
654 {
655 applyTestTag(
656 target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB
657 );
658
659 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000) // nGraph compilation failure
660 if (target == DNN_TARGET_MYRIAD)
661 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
662 #endif
663
664 const double confThreshold = 0.5;
665 // batchId, classId, confidence, left, top, right, bottom
666 const int N0 = 2;
667 const int N1 = 3;
668 static const float ref_[/* (N0 + N1) * 7 */] = {
669 0, 7, 0.85935f, 0.593484f, 0.141211f, 0.920356f, 0.291593f,
670 0, 16, 0.795188f, 0.169207f, 0.386886f, 0.423753f, 0.933004f,
671
672 1, 2, 0.996832f, 0.653802f, 0.464573f, 0.815193f, 0.653292f,
673 1, 2, 0.963325f, 0.451151f, 0.458915f, 0.496255f, 0.52241f,
674 1, 0, 0.926244f, 0.194851f, 0.361743f, 0.260277f, 0.632364f,
675 };
676 Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_);
677
678 double scoreDiff = 0.01f;
679 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.15 : 0.01f;
680 if (target == DNN_TARGET_CUDA_FP16)
681 iouDiff = 0.02;
682
683 std::string config_file = "yolov4-tiny.cfg";
684 std::string weights_file = "yolov4-tiny.weights";
685
686 #if defined(INF_ENGINE_RELEASE)
687 if (target == DNN_TARGET_MYRIAD) // bad accuracy
688 iouDiff = std::numeric_limits<double>::quiet_NaN();
689 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL)
690 iouDiff = std::numeric_limits<double>::quiet_NaN();
691 if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ||
692 backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16)
693 iouDiff = std::numeric_limits<double>::quiet_NaN();
694 #endif
695
696 {
697 SCOPED_TRACE("batch size 1");
698 testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold);
699 }
700
701 {
702 SCOPED_TRACE("batch size 2");
703 testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold);
704 }
705
706 #if defined(INF_ENGINE_RELEASE)
707 if (target == DNN_TARGET_MYRIAD) // bad accuracy
708 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
709 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL)
710 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
711 if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ||
712 backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16)
713 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
714 #endif
715 }
716
717
718 INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_nets, dnnBackendsAndTargets());
719
TEST_P(Test_Darknet_layers,shortcut)720 TEST_P(Test_Darknet_layers, shortcut)
721 {
722 testDarknetLayer("shortcut");
723 testDarknetLayer("shortcut_leaky");
724 testDarknetLayer("shortcut_unequal");
725 testDarknetLayer("shortcut_unequal_2");
726 }
727
TEST_P(Test_Darknet_layers,upsample)728 TEST_P(Test_Darknet_layers, upsample)
729 {
730 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021030000)
731 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
732 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // exception
733 #endif
734 testDarknetLayer("upsample");
735 }
736
TEST_P(Test_Darknet_layers,mish)737 TEST_P(Test_Darknet_layers, mish)
738 {
739 testDarknetLayer("mish", true);
740 }
741
TEST_P(Test_Darknet_layers,tanh)742 TEST_P(Test_Darknet_layers, tanh)
743 {
744 testDarknetLayer("tanh");
745 }
746
TEST_P(Test_Darknet_layers,avgpool_softmax)747 TEST_P(Test_Darknet_layers, avgpool_softmax)
748 {
749 testDarknetLayer("avgpool_softmax");
750 }
751
TEST_P(Test_Darknet_layers,region)752 TEST_P(Test_Darknet_layers, region)
753 {
754 #if defined(INF_ENGINE_RELEASE)
755 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && INF_ENGINE_VER_MAJOR_GE(2020020000))
756 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
757 #endif
758 testDarknetLayer("region");
759 }
760
TEST_P(Test_Darknet_layers,reorg)761 TEST_P(Test_Darknet_layers, reorg)
762 {
763 testDarknetLayer("reorg");
764 }
765
TEST_P(Test_Darknet_layers,route)766 TEST_P(Test_Darknet_layers, route)
767 {
768 testDarknetLayer("route");
769 testDarknetLayer("route_multi");
770 }
771
TEST_P(Test_Darknet_layers,maxpool)772 TEST_P(Test_Darknet_layers, maxpool)
773 {
774 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020020000)
775 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
776 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
777 #endif
778 testDarknetLayer("maxpool");
779 }
780
TEST_P(Test_Darknet_layers,convolutional)781 TEST_P(Test_Darknet_layers, convolutional)
782 {
783 if (target == DNN_TARGET_MYRIAD)
784 {
785 default_l1 = 0.01f;
786 }
787 testDarknetLayer("convolutional", true);
788 }
789
TEST_P(Test_Darknet_layers,scale_channels)790 TEST_P(Test_Darknet_layers, scale_channels)
791 {
792 bool testBatches = backend == DNN_BACKEND_CUDA;
793 testDarknetLayer("scale_channels", false, testBatches);
794 }
795
TEST_P(Test_Darknet_layers,connected)796 TEST_P(Test_Darknet_layers, connected)
797 {
798 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
799 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
800 testDarknetLayer("connected", true);
801 }
802
TEST_P(Test_Darknet_layers,relu)803 TEST_P(Test_Darknet_layers, relu)
804 {
805 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD)
806 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
807 testDarknetLayer("relu");
808 }
809
TEST_P(Test_Darknet_layers,sam)810 TEST_P(Test_Darknet_layers, sam)
811 {
812 testDarknetLayer("sam", true);
813 }
814
815 INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_layers, dnnBackendsAndTargets());
816
817 }} // namespace
818