1#!/usr/bin/env python
2import os
3import cv2 as cv
4import numpy as np
5
6from tests_common import NewOpenCVTests, unittest
7
8def normAssert(test, a, b, msg=None, lInf=1e-5):
9    test.assertLess(np.max(np.abs(a - b)), lInf, msg)
10
11def inter_area(box1, box2):
12    x_min, x_max = max(box1[0], box2[0]), min(box1[2], box2[2])
13    y_min, y_max = max(box1[1], box2[1]), min(box1[3], box2[3])
14    return (x_max - x_min) * (y_max - y_min)
15
16def area(box):
17    return (box[2] - box[0]) * (box[3] - box[1])
18
19def box2str(box):
20    left, top = box[0], box[1]
21    width, height = box[2] - left, box[3] - top
22    return '[%f x %f from (%f, %f)]' % (width, height, left, top)
23
24def normAssertDetections(test, refClassIds, refScores, refBoxes, testClassIds, testScores, testBoxes,
25                 confThreshold=0.0, scores_diff=1e-5, boxes_iou_diff=1e-4):
26    matchedRefBoxes = [False] * len(refBoxes)
27    errMsg = ''
28    for i in range(len(testBoxes)):
29        testScore = testScores[i]
30        if testScore < confThreshold:
31            continue
32
33        testClassId, testBox = testClassIds[i], testBoxes[i]
34        matched = False
35        for j in range(len(refBoxes)):
36            if (not matchedRefBoxes[j]) and testClassId == refClassIds[j] and \
37               abs(testScore - refScores[j]) < scores_diff:
38                interArea = inter_area(testBox, refBoxes[j])
39                iou = interArea / (area(testBox) + area(refBoxes[j]) - interArea)
40                if abs(iou - 1.0) < boxes_iou_diff:
41                    matched = True
42                    matchedRefBoxes[j] = True
43        if not matched:
44            errMsg += '\nUnmatched prediction: class %d score %f box %s' % (testClassId, testScore, box2str(testBox))
45
46    for i in range(len(refBoxes)):
47        if (not matchedRefBoxes[i]) and refScores[i] > confThreshold:
48            errMsg += '\nUnmatched reference: class %d score %f box %s' % (refClassIds[i], refScores[i], box2str(refBoxes[i]))
49    if errMsg:
50        test.fail(errMsg)
51
52def printParams(backend, target):
53    backendNames = {
54        cv.dnn.DNN_BACKEND_OPENCV: 'OCV',
55        cv.dnn.DNN_BACKEND_INFERENCE_ENGINE: 'DLIE'
56    }
57    targetNames = {
58        cv.dnn.DNN_TARGET_CPU: 'CPU',
59        cv.dnn.DNN_TARGET_OPENCL: 'OCL',
60        cv.dnn.DNN_TARGET_OPENCL_FP16: 'OCL_FP16',
61        cv.dnn.DNN_TARGET_MYRIAD: 'MYRIAD'
62    }
63    print('%s/%s' % (backendNames[backend], targetNames[target]))
64
65testdata_required = bool(os.environ.get('OPENCV_DNN_TEST_REQUIRE_TESTDATA', False))
66
67g_dnnBackendsAndTargets = None
68
69class dnn_test(NewOpenCVTests):
70
71    def setUp(self):
72        super(dnn_test, self).setUp()
73
74        global g_dnnBackendsAndTargets
75        if g_dnnBackendsAndTargets is None:
76            g_dnnBackendsAndTargets = self.initBackendsAndTargets()
77        self.dnnBackendsAndTargets = g_dnnBackendsAndTargets
78
79    def initBackendsAndTargets(self):
80        self.dnnBackendsAndTargets = [
81            [cv.dnn.DNN_BACKEND_OPENCV, cv.dnn.DNN_TARGET_CPU],
82        ]
83
84        if self.checkIETarget(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_TARGET_CPU):
85            self.dnnBackendsAndTargets.append([cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_TARGET_CPU])
86        if self.checkIETarget(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_TARGET_MYRIAD):
87            self.dnnBackendsAndTargets.append([cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_TARGET_MYRIAD])
88
89        if cv.ocl.haveOpenCL() and cv.ocl.useOpenCL():
90            self.dnnBackendsAndTargets.append([cv.dnn.DNN_BACKEND_OPENCV, cv.dnn.DNN_TARGET_OPENCL])
91            self.dnnBackendsAndTargets.append([cv.dnn.DNN_BACKEND_OPENCV, cv.dnn.DNN_TARGET_OPENCL_FP16])
92            if cv.ocl_Device.getDefault().isIntel():
93                if self.checkIETarget(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_TARGET_OPENCL):
94                    self.dnnBackendsAndTargets.append([cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_TARGET_OPENCL])
95                if self.checkIETarget(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_TARGET_OPENCL_FP16):
96                    self.dnnBackendsAndTargets.append([cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_TARGET_OPENCL_FP16])
97        return self.dnnBackendsAndTargets
98
99    def find_dnn_file(self, filename, required=True):
100        if not required:
101            required = testdata_required
102        return self.find_file(filename, [os.environ.get('OPENCV_DNN_TEST_DATA_PATH', os.getcwd()),
103                                         os.environ['OPENCV_TEST_DATA_PATH']],
104                              required=required)
105
106    def checkIETarget(self, backend, target):
107        proto = self.find_dnn_file('dnn/layers/layer_convolution.prototxt')
108        model = self.find_dnn_file('dnn/layers/layer_convolution.caffemodel')
109        net = cv.dnn.readNet(proto, model)
110        net.setPreferableBackend(backend)
111        net.setPreferableTarget(target)
112        inp = np.random.standard_normal([1, 2, 10, 11]).astype(np.float32)
113        try:
114            net.setInput(inp)
115            net.forward()
116        except BaseException as e:
117            return False
118        return True
119
120    def test_getAvailableTargets(self):
121        targets = cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_OPENCV)
122        self.assertTrue(cv.dnn.DNN_TARGET_CPU in targets)
123
124    def test_blobFromImage(self):
125        np.random.seed(324)
126
127        width = 6
128        height = 7
129        scale = 1.0/127.5
130        mean = (10, 20, 30)
131
132        # Test arguments names.
133        img = np.random.randint(0, 255, [4, 5, 3]).astype(np.uint8)
134        blob = cv.dnn.blobFromImage(img, scale, (width, height), mean, True, False)
135        blob_args = cv.dnn.blobFromImage(img, scalefactor=scale, size=(width, height),
136                                         mean=mean, swapRB=True, crop=False)
137        normAssert(self, blob, blob_args)
138
139        # Test values.
140        target = cv.resize(img, (width, height), interpolation=cv.INTER_LINEAR)
141        target = target.astype(np.float32)
142        target = target[:,:,[2, 1, 0]]  # BGR2RGB
143        target[:,:,0] -= mean[0]
144        target[:,:,1] -= mean[1]
145        target[:,:,2] -= mean[2]
146        target *= scale
147        target = target.transpose(2, 0, 1).reshape(1, 3, height, width)  # to NCHW
148        normAssert(self, blob, target)
149
150
151    def test_model(self):
152        img_path = self.find_dnn_file("dnn/street.png")
153        weights = self.find_dnn_file("dnn/MobileNetSSD_deploy.caffemodel", required=False)
154        config = self.find_dnn_file("dnn/MobileNetSSD_deploy.prototxt", required=False)
155        if weights is None or config is None:
156            raise unittest.SkipTest("Missing DNN test files (dnn/MobileNetSSD_deploy.{prototxt/caffemodel}). Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.")
157
158        frame = cv.imread(img_path)
159        model = cv.dnn_DetectionModel(weights, config)
160        model.setInputParams(size=(300, 300), mean=(127.5, 127.5, 127.5), scale=1.0/127.5)
161
162        iouDiff = 0.05
163        confThreshold = 0.0001
164        nmsThreshold = 0
165        scoreDiff = 1e-3
166
167        classIds, confidences, boxes = model.detect(frame, confThreshold, nmsThreshold)
168
169        refClassIds = (7, 15)
170        refConfidences = (0.9998, 0.8793)
171        refBoxes = ((328, 238, 85, 102), (101, 188, 34, 138))
172
173        normAssertDetections(self, refClassIds, refConfidences, refBoxes,
174                             classIds, confidences, boxes,confThreshold, scoreDiff, iouDiff)
175
176        for box in boxes:
177            cv.rectangle(frame, box, (0, 255, 0))
178            cv.rectangle(frame, np.array(box), (0, 255, 0))
179            cv.rectangle(frame, tuple(box), (0, 255, 0))
180            cv.rectangle(frame, list(box), (0, 255, 0))
181
182
183    def test_classification_model(self):
184        img_path = self.find_dnn_file("dnn/googlenet_0.png")
185        weights = self.find_dnn_file("dnn/squeezenet_v1.1.caffemodel", required=False)
186        config = self.find_dnn_file("dnn/squeezenet_v1.1.prototxt")
187        ref = np.load(self.find_dnn_file("dnn/squeezenet_v1.1_prob.npy"))
188        if weights is None or config is None:
189            raise unittest.SkipTest("Missing DNN test files (dnn/squeezenet_v1.1.{prototxt/caffemodel}). Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.")
190
191        frame = cv.imread(img_path)
192        model = cv.dnn_ClassificationModel(config, weights)
193        model.setInputSize(227, 227)
194        model.setInputCrop(True)
195
196        out = model.predict(frame)
197        normAssert(self, out, ref)
198
199
200    def test_textdetection_model(self):
201        img_path = self.find_dnn_file("dnn/text_det_test1.png")
202        weights = self.find_dnn_file("dnn/onnx/models/DB_TD500_resnet50.onnx", required=False)
203        if weights is None:
204            raise unittest.SkipTest("Missing DNN test files (onnx/models/DB_TD500_resnet50.onnx). Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.")
205
206        frame = cv.imread(img_path)
207        scale = 1.0 / 255.0
208        size = (736, 736)
209        mean = (122.67891434, 116.66876762, 104.00698793)
210
211        model = cv.dnn_TextDetectionModel_DB(weights)
212        model.setInputParams(scale, size, mean)
213        out, _ = model.detect(frame)
214
215        self.assertTrue(type(out) == list)
216        self.assertTrue(np.array(out).shape == (2, 4, 2))
217
218
219    def test_face_detection(self):
220        proto = self.find_dnn_file('dnn/opencv_face_detector.prototxt')
221        model = self.find_dnn_file('dnn/opencv_face_detector.caffemodel', required=False)
222        if proto is None or model is None:
223            raise unittest.SkipTest("Missing DNN test files (dnn/opencv_face_detector.{prototxt/caffemodel}). Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.")
224
225        img = self.get_sample('gpu/lbpcascade/er.png')
226        blob = cv.dnn.blobFromImage(img, mean=(104, 177, 123), swapRB=False, crop=False)
227
228        ref = [[0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631],
229               [0, 1, 0.9934696,  0.2831718,  0.50738752, 0.345781,   0.5985168],
230               [0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290],
231               [0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477],
232               [0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494],
233               [0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427,  0.5347801]]
234
235        print('\n')
236        for backend, target in self.dnnBackendsAndTargets:
237            printParams(backend, target)
238
239            net = cv.dnn.readNet(proto, model)
240            net.setPreferableBackend(backend)
241            net.setPreferableTarget(target)
242            net.setInput(blob)
243            out = net.forward().reshape(-1, 7)
244
245            scoresDiff = 4e-3 if target in [cv.dnn.DNN_TARGET_OPENCL_FP16, cv.dnn.DNN_TARGET_MYRIAD] else 1e-5
246            iouDiff = 2e-2 if target in [cv.dnn.DNN_TARGET_OPENCL_FP16, cv.dnn.DNN_TARGET_MYRIAD] else 1e-4
247
248            ref = np.array(ref, np.float32)
249            refClassIds, testClassIds = ref[:, 1], out[:, 1]
250            refScores, testScores = ref[:, 2], out[:, 2]
251            refBoxes, testBoxes = ref[:, 3:], out[:, 3:]
252
253            normAssertDetections(self, refClassIds, refScores, refBoxes, testClassIds,
254                                 testScores, testBoxes, 0.5, scoresDiff, iouDiff)
255
256    def test_async(self):
257        timeout = 10*1000*10**6  # in nanoseconds (10 sec)
258        proto = self.find_dnn_file('dnn/layers/layer_convolution.prototxt')
259        model = self.find_dnn_file('dnn/layers/layer_convolution.caffemodel')
260        if proto is None or model is None:
261            raise unittest.SkipTest("Missing DNN test files (dnn/layers/layer_convolution.{prototxt/caffemodel}). Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.")
262
263        print('\n')
264        for backend, target in self.dnnBackendsAndTargets:
265            if backend != cv.dnn.DNN_BACKEND_INFERENCE_ENGINE:
266                continue
267
268            printParams(backend, target)
269
270            netSync = cv.dnn.readNet(proto, model)
271            netSync.setPreferableBackend(backend)
272            netSync.setPreferableTarget(target)
273
274            netAsync = cv.dnn.readNet(proto, model)
275            netAsync.setPreferableBackend(backend)
276            netAsync.setPreferableTarget(target)
277
278            # Generate inputs
279            numInputs = 10
280            inputs = []
281            for _ in range(numInputs):
282                inputs.append(np.random.standard_normal([2, 6, 75, 113]).astype(np.float32))
283
284            # Run synchronously
285            refs = []
286            for i in range(numInputs):
287                netSync.setInput(inputs[i])
288                refs.append(netSync.forward())
289
290            # Run asynchronously. To make test more robust, process inputs in the reversed order.
291            outs = []
292            for i in reversed(range(numInputs)):
293                netAsync.setInput(inputs[i])
294                outs.insert(0, netAsync.forwardAsync())
295
296            for i in reversed(range(numInputs)):
297                ret, result = outs[i].get(timeoutNs=float(timeout))
298                self.assertTrue(ret)
299                normAssert(self, refs[i], result, 'Index: %d' % i, 1e-10)
300
301    def test_nms(self):
302        confs = (1, 1)
303        rects = ((0, 0, 0.4, 0.4), (0, 0, 0.2, 0.4)) # 0.5 overlap
304
305        self.assertTrue(all(cv.dnn.NMSBoxes(rects, confs, 0, 0.6).ravel() == (0, 1)))
306
307    def test_custom_layer(self):
308        class CropLayer(object):
309            def __init__(self, params, blobs):
310                self.xstart = 0
311                self.xend = 0
312                self.ystart = 0
313                self.yend = 0
314            # Our layer receives two inputs. We need to crop the first input blob
315            # to match a shape of the second one (keeping batch size and number of channels)
316            def getMemoryShapes(self, inputs):
317                inputShape, targetShape = inputs[0], inputs[1]
318                batchSize, numChannels = inputShape[0], inputShape[1]
319                height, width = targetShape[2], targetShape[3]
320                self.ystart = (inputShape[2] - targetShape[2]) // 2
321                self.xstart = (inputShape[3] - targetShape[3]) // 2
322                self.yend = self.ystart + height
323                self.xend = self.xstart + width
324                return [[batchSize, numChannels, height, width]]
325            def forward(self, inputs):
326                return [inputs[0][:,:,self.ystart:self.yend,self.xstart:self.xend]]
327
328        cv.dnn_registerLayer('CropCaffe', CropLayer)
329        proto = '''
330        name: "TestCrop"
331        input: "input"
332        input_shape
333        {
334            dim: 1
335            dim: 2
336            dim: 5
337            dim: 5
338        }
339        input: "roi"
340        input_shape
341        {
342            dim: 1
343            dim: 2
344            dim: 3
345            dim: 3
346        }
347        layer {
348          name: "Crop"
349          type: "CropCaffe"
350          bottom: "input"
351          bottom: "roi"
352          top: "Crop"
353        }'''
354
355        net = cv.dnn.readNetFromCaffe(bytearray(proto.encode()))
356        for backend, target in self.dnnBackendsAndTargets:
357            if backend != cv.dnn.DNN_BACKEND_OPENCV:
358                continue
359
360            printParams(backend, target)
361
362            net.setPreferableBackend(backend)
363            net.setPreferableTarget(target)
364            src_shape = [1, 2, 5, 5]
365            dst_shape = [1, 2, 3, 3]
366            inp = np.arange(0, np.prod(src_shape), dtype=np.float32).reshape(src_shape)
367            roi = np.empty(dst_shape, dtype=np.float32)
368            net.setInput(inp, "input")
369            net.setInput(roi, "roi")
370            out = net.forward()
371            ref = inp[:, :, 1:4, 1:4]
372            normAssert(self, out, ref)
373
374        cv.dnn_unregisterLayer('CropCaffe')
375
376if __name__ == '__main__':
377    NewOpenCVTests.bootstrap()
378