1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * License); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing,
13 * software distributed under the License is distributed on an
14 * AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 * KIND, either express or implied. See the License for the
16 * specific language governing permissions and limitations
17 * under the License.
18 */
19
20 /*
21 * Copyright (c) 2020, OPEN AI LAB
22 * Author: qtang@openailab.com
23 */
24
25 #include "../../precomp.hpp"
26 #include <iostream>
27 #include <vector>
28
29 #include <opencv2/core/utils/configuration.private.hpp>
30 #include <opencv2/core/utils/logger.hpp>
31
32 #include "../include/tengine_graph_convolution.hpp"
33
34 #ifdef HAVE_TENGINE
35
36 #include "tengine_c_api.h"
37
38
39 namespace cv
40 {
41 namespace dnn
42 {
create_input_node(teng_graph_t graph,const char * node_name,int inch,int in_h,int in_w)43 static int create_input_node(teng_graph_t graph, const char* node_name, int inch, int in_h, int in_w)
44 {
45 node_t node = teng_create_graph_node(graph, node_name, "InputOp");
46 tensor_t tensor = teng_create_graph_tensor(graph, node_name, TENGINE_DT_FP32);
47 teng_set_node_output_tensor(node, 0, tensor, TENSOR_TYPE_INPUT);
48
49 int dims[4] = {1, inch, in_h, in_w};
50 teng_set_tensor_shape(tensor, dims, 4);
51
52 teng_release_graph_tensor(tensor);
53 teng_release_graph_node(node);
54
55 return 0;
56 }
57
create_conv_node(teng_graph_t graph,const char * node_name,const char * input_name,int in_h,int in_w,int out_h,int out_w,int kernel_h,int kernel_w,int stride_h,int stride_w,int pad_h,int pad_w,int inch,int outch,int group,int dilation_h,int dilation_w,int activation,std::string padMode)58 static int create_conv_node(teng_graph_t graph, const char* node_name, const char* input_name, int in_h, int in_w, int out_h, int out_w,
59 int kernel_h, int kernel_w, int stride_h, int stride_w, int pad_h, int pad_w, int inch, int outch, int group,
60 int dilation_h, int dilation_w, int activation, std::string padMode)
61 {
62 node_t conv_node = teng_create_graph_node(graph, node_name, "Convolution");
63 tensor_t input_tensor = teng_get_graph_tensor(graph, input_name);
64
65 if (input_tensor == NULL)
66 {
67 CV_LOG_WARNING(NULL,"Tengine: input_tensor is NULL." );
68 return -1;
69 }
70
71 teng_set_node_input_tensor(conv_node, 0, input_tensor);
72 teng_release_graph_tensor(input_tensor);
73
74 /* output */
75 tensor_t output_tensor = teng_create_graph_tensor(graph, node_name, TENGINE_DT_FP32);
76
77 teng_set_node_output_tensor(conv_node, 0, output_tensor, TENSOR_TYPE_VAR);
78 teng_release_graph_tensor(output_tensor);
79
80 /* weight */
81 std::string weight_name(node_name);
82 weight_name += "/weight";
83
84 node_t w_node = teng_create_graph_node(graph, weight_name.c_str(), "Const");
85 tensor_t w_tensor = teng_create_graph_tensor(graph, weight_name.c_str(), TENGINE_DT_FP32);
86 teng_set_node_output_tensor(w_node, 0, w_tensor, TENSOR_TYPE_CONST);
87 teng_set_node_input_tensor(conv_node, 1, w_tensor);
88 int w_dims[] = {outch, inch / group, kernel_h, kernel_w};
89
90 teng_set_tensor_shape(w_tensor, w_dims, 4);
91
92 teng_release_graph_node(w_node);
93 teng_release_graph_tensor(w_tensor);
94
95 /* bias */
96 std::string bias_name(node_name);
97 bias_name += "/bias";
98
99 node_t b_node = teng_create_graph_node(graph, bias_name.c_str(), "Const");
100 tensor_t b_tensor = teng_create_graph_tensor(graph, bias_name.c_str(), TENGINE_DT_FP32);
101 teng_set_node_output_tensor(b_node, 0, b_tensor, TENSOR_TYPE_CONST);
102 int b_dims[] = {outch};
103
104 teng_set_tensor_shape(b_tensor, b_dims, 1);
105
106 teng_set_node_input_tensor(conv_node, 2, b_tensor);
107 teng_release_graph_node(b_node);
108 teng_release_graph_tensor(b_tensor);
109
110 int pad_h1 = pad_h;
111 int pad_w1 = pad_w;
112
113 if (!padMode.empty())
114 {
115 if (padMode == "SAME")
116 {
117 int out_h_temp = (in_h-kernel_h + 2*pad_h)/stride_h + 1;
118 int out_w_temp = (in_w-kernel_w + 2*pad_w)/stride_w + 1;
119
120 if (out_h_temp < out_h)
121 pad_h1 += 1;
122 if (out_w_temp < out_w)
123 pad_w1 += 1;
124 }
125 }
126
127 /* attr */
128 teng_set_node_attr_int(conv_node, "kernel_h", &kernel_h);
129 teng_set_node_attr_int(conv_node, "kernel_w", &kernel_w);
130 teng_set_node_attr_int(conv_node, "stride_h", &stride_h);
131 teng_set_node_attr_int(conv_node, "stride_w", &stride_w);
132 teng_set_node_attr_int(conv_node, "pad_h0", &pad_h);
133 teng_set_node_attr_int(conv_node, "pad_w0", &pad_w);
134 teng_set_node_attr_int(conv_node, "pad_h1", &pad_h1);
135 teng_set_node_attr_int(conv_node, "pad_w1", &pad_w1);
136 teng_set_node_attr_int(conv_node, "output_channel", &outch);
137 teng_set_node_attr_int(conv_node, "input_channel", &inch);
138 teng_set_node_attr_int(conv_node, "group", &group);
139 teng_set_node_attr_int(conv_node, "dilation_h", &dilation_h);
140 teng_set_node_attr_int(conv_node, "dilation_w", &dilation_w);
141 // set_node_attr_int(conv_node, "activation", &activation);
142
143 teng_release_graph_node(conv_node);
144
145 return 0;
146 }
147
create_conv_graph(const char * layer_name,float * input_data,int inch,int group,int in_h,int in_w,float * output_data,int outch,int out_h,int out_w,int kernel_h,int kernel_w,int stride_h,int stride_w,int pad_h,int pad_w,int dilation_h,int dilation_w,int activation,float * teg_weight,float * teg_bias,std::string padMode,int nstripes)148 static teng_graph_t create_conv_graph(const char* layer_name, float* input_data, int inch, int group, int in_h, int in_w,
149 float* output_data, int outch, int out_h, int out_w,
150 int kernel_h, int kernel_w,
151 int stride_h,int stride_w,
152 int pad_h, int pad_w, int dilation_h, int dilation_w, int activation,
153 float* teg_weight, float* teg_bias, std::string padMode, int nstripes)
154 {
155 node_t conv_node = NULL;
156
157 tensor_t input_tensor = NULL;
158 tensor_t output_tensor = NULL;
159 tensor_t weight_tensor = NULL;
160 tensor_t bias_tensor = NULL;
161
162 /* create graph for convolution */
163 int in_size = in_h * in_w * inch;
164 int out_size = out_h * out_w * outch;
165 int weight_size = outch * (inch / group) * kernel_w * kernel_h;
166 int bias_size = outch;
167
168 int buf_size = 0;
169 int input_num = 0;
170
171 /* create graph */
172 teng_graph_t graph = teng_create_graph(NULL, NULL, NULL);
173 bool ok = true;
174
175 if(graph == NULL)
176 {
177 CV_LOG_WARNING(NULL,"Tengine: create_graph failed." );
178 ok = false;
179 }
180
181 const char* input_name = "data";
182 const char* conv_name = layer_name;
183
184 if (ok && create_input_node(graph, input_name, inch, in_h, in_w) < 0)
185 {
186 CV_LOG_WARNING(NULL,"Tengine: create_input_node failed." );
187 ok = false;
188 }
189
190 if (ok && create_conv_node(graph, conv_name, input_name, in_h, in_w, out_h, out_w, kernel_h, kernel_w,
191 stride_h, stride_w, pad_h, pad_w, inch, outch, group, dilation_h, dilation_w, activation, padMode) < 0)
192 {
193 CV_LOG_WARNING(NULL,"Tengine: create conv node failed." );
194 ok = false;
195 }
196
197 /* set input/output node */
198 const char* inputs_name[] = {input_name};
199 const char* outputs_name[] = {conv_name};
200
201 if (ok && teng_set_graph_input_node(graph, inputs_name, sizeof(inputs_name) / sizeof(char*)) < 0)
202 {
203 CV_LOG_WARNING(NULL,"Tengine: set inputs failed." );
204 ok = false;
205 }
206
207 if (ok && teng_set_graph_output_node(graph, outputs_name, sizeof(outputs_name) / sizeof(char*)) < 0)
208 {
209 CV_LOG_WARNING(NULL,"Tengine: set outputs failed." );
210 ok = false;
211 }
212
213 /* set input data */
214 if (ok)
215 {
216 input_tensor = teng_get_graph_input_tensor(graph, 0, 0);
217 buf_size = teng_get_tensor_buffer_size(input_tensor);
218 if (buf_size != in_size * FLOAT_TO_REALSIZE)
219 {
220 CV_LOG_WARNING(NULL,"Tengine: Input data size check failed.");
221 ok = false;
222 }
223 }
224
225 if (ok)
226 {
227 teng_set_tensor_buffer(input_tensor, (float *)input_data, buf_size);
228 teng_release_graph_tensor(input_tensor);
229
230 /* create convolution node */
231 /* set weight node */
232 conv_node = teng_get_graph_node(graph, conv_name);
233 weight_tensor = teng_get_node_input_tensor(conv_node, 1);
234 buf_size = teng_get_tensor_buffer_size(weight_tensor);
235
236 if (buf_size != weight_size * FLOAT_TO_REALSIZE)
237 {
238 CV_LOG_WARNING(NULL,"Tengine: Input weight size check failed.");
239 ok = false;
240 }
241 }
242
243 if (ok)
244 {
245 teng_set_tensor_buffer(weight_tensor, teg_weight, buf_size);
246
247 /* set bias node */
248 input_num = teng_get_node_input_number(conv_node);
249 if (input_num > 2)
250 {
251 bias_tensor = teng_get_node_input_tensor(conv_node, 2);
252 buf_size = teng_get_tensor_buffer_size(bias_tensor);
253 if (buf_size != bias_size * FLOAT_TO_REALSIZE)
254 {
255 CV_LOG_WARNING(NULL,"Tengine: Input bias size check failed.");
256 ok = false;
257 }
258 else teng_set_tensor_buffer(bias_tensor, teg_bias, buf_size);
259 }
260 }
261
262 /* prerun */
263 if (ok && teng_prerun_graph_multithread(graph, TENGINE_CLUSTER_BIG, nstripes) < 0)
264 {
265 CV_LOG_WARNING(NULL, "Tengine: prerun_graph failed.");
266 ok = false;
267 }
268
269 if (ok)
270 {
271 /* set output data */
272 output_tensor = teng_get_node_output_tensor(conv_node, 0);
273 int ret = teng_set_tensor_buffer(output_tensor, output_data, out_size * FLOAT_TO_REALSIZE);
274 if(ret)
275 {
276 CV_LOG_WARNING(NULL,"Tengine: Set output tensor buffer failed." );
277 ok = false;
278 }
279 }
280
281 if (false == ok)
282 {
283 teng_destroy_graph(graph) ;
284 return NULL ;
285 }
286 return graph;
287 }
288 static bool tengine_init_flag = false;
tengine_init(const char * layer_name,float * input_,int inch,int group,int in_h,int in_w,float * output_,int out_b,int outch,int out_h,int out_w,float * kernel_,int kernel_s,int kernel_h,int kernel_w,float * teg_bias,int stride_h,int stride_w,int pad_h,int pad_w,int dilation_h,int dilation_w,size_t wstep,const std::string padMode,teng_graph_t & graph,int nstripes)289 teng_graph_t tengine_init(const char* layer_name, float* input_, int inch, int group, int in_h, int in_w,
290 float *output_, int out_b, int outch, int out_h, int out_w,
291 float *kernel_, int kernel_s ,int kernel_h, int kernel_w,
292 float *teg_bias, int stride_h,int stride_w,
293 int pad_h, int pad_w, int dilation_h, int dilation_w,
294 size_t wstep, const std::string padMode, teng_graph_t &graph, int nstripes)
295 {
296 std::vector<float> teg_weight_vec;
297 float *teg_weight = NULL;
298 int kernel_inwh = (inch / group) * kernel_w * kernel_h;
299 // Do not using the activation fuse mode, just convolution only.
300 int activation = -1;
301
302 if (!(kernel_s == 2 && kernel_h == kernel_w && pad_h == pad_w
303 && dilation_h == dilation_w && stride_h == stride_w
304 && out_b == 1 && pad_h < 10)) // just for Conv2D
305 {
306 // printf("return : just for Conv2D\n");
307 return NULL;
308 }
309
310 {
311 /* printf("Tengine(%s): input (1 x %d x %d x %d),output (%d x %d x %d x %d), kernel (%d x %d), stride (%d x %d), dilation (%d x %d), pad (%d x %d).\n",
312 layer_name, inch, in_h, in_w,
313 out_b, outch, out_h, out_w,
314 kernel_w, kernel_h,
315 stride_w, stride_h,
316 dilation_w, dilation_h,
317 pad_w, pad_h);
318 */
319 // weight
320 if (kernel_inwh != wstep)
321 {
322 teg_weight_vec.resize(kernel_inwh * outch);
323 teg_weight = &teg_weight_vec[0];
324 for (int i=0; i<outch; i++)
325 {
326 memcpy(teg_weight+i*kernel_inwh, kernel_+i*wstep, kernel_inwh*FLOAT_TO_REALSIZE);
327 }
328 }
329 else
330 {
331 teg_weight = kernel_;
332 }
333
334 /* initial the resoruce of tengine */
335 if(false == tengine_init_flag)
336 {
337 init_tengine();
338 tengine_init_flag = true;
339 }
340
341 /* create the convolution graph */
342 graph = create_conv_graph(layer_name, input_, inch, group, in_h, in_w,
343 output_, outch, out_h, out_w,
344 kernel_h, kernel_w, stride_h,stride_w,
345 pad_h, pad_w, dilation_h, dilation_w, activation,
346 teg_weight, teg_bias, padMode, nstripes);
347 if(NULL == graph )
348 {
349 return NULL;
350 }
351 }
352 return graph ;
353 }
354
tengine_forward(teng_graph_t & graph)355 bool tengine_forward(teng_graph_t &graph)
356 {
357 /* run */
358 if(teng_run_graph(graph, 1) < 0)
359 {
360 CV_LOG_WARNING(NULL,"Tengine: run_graph failed.");
361 return false ;
362 }
363 return true;
364 }
tengine_release(teng_graph_t & graph)365 bool tengine_release(teng_graph_t &graph)
366 {
367 teng_postrun_graph(graph);
368 teng_destroy_graph(graph);
369 return true;
370 }
371 }
372 }
373 #endif
374