1 //
2 // ImageSampler.cpp
3 // MNN
4 //
5 // Created by MNN on 2018/12/24.
6 // Copyright © 2018, Alibaba Group Holding Limited
7 //
8
9 #include "cv/ImageSampler.hpp"
10 #include <algorithm>
11 #include "core/Macro.h"
12 #ifdef MNN_USE_NEON
13 #include <arm_neon.h>
14 #endif
15 extern "C" {
16 void MNNSamplerC4BilinearOpt(const unsigned char* source, unsigned char* dest, float* points, size_t count, size_t xMax,
17 size_t yMax, size_t yStride);
18 void MNNSamplerC1BilinearOpt(const unsigned char* source, unsigned char* dest, float* points, size_t count, size_t xMax,
19 size_t yMax, size_t yStride);
20
21 void MNNSamplerC4NearestOpt(const unsigned char* source, unsigned char* dest, float* points, size_t count, size_t iw,
22 size_t ih, size_t yStride);
23 void MNNSamplerC1NearestOpt(const unsigned char* source, unsigned char* dest, float* points, size_t count, size_t iw,
24 size_t ih, size_t yStride);
25 }
26
27 namespace MNN {
28 namespace CV {
29
__clamp(float v,float minV,float maxV)30 static inline float __clamp(float v, float minV, float maxV) {
31 return std::max(std::min(v, maxV), minV);
32 }
33
_sampleBilinearCommon(const unsigned char * source,unsigned char * dest,Point * points,size_t count,size_t iw,size_t ih,size_t yStride,size_t bpp)34 static void _sampleBilinearCommon(const unsigned char* source, unsigned char* dest, Point* points, size_t count,
35 size_t iw, size_t ih, size_t yStride, size_t bpp) {
36 float dy = points[1].fY;
37 float dx = points[1].fX;
38 float xMax = iw - 1;
39 float yMax = ih - 1;
40
41 Point curPoints;
42 curPoints.fX = points[0].fX;
43 curPoints.fY = points[0].fY;
44 for (int i = 0; i < count; ++i) {
45 float y = __clamp(curPoints.fY, 0, yMax);
46 float x = __clamp(curPoints.fX, 0, xMax);
47 int y0 = (int)y;
48 int x0 = (int)x;
49 int y1 = (int)ceilf(y);
50 int x1 = (int)ceilf(x);
51 float xF = x - (float)x0;
52 float yF = y - (float)y0;
53
54 for (int b = 0; b < bpp; ++b) {
55 unsigned char c00 = source[y0 * yStride + bpp * x0 + b];
56 unsigned char c01 = source[y0 * yStride + bpp * x1 + b];
57 unsigned char c10 = source[y1 * yStride + bpp * x0 + b];
58 unsigned char c11 = source[y1 * yStride + bpp * x1 + b];
59
60 float v =
61 (1.0f - xF) * (1.0f - yF) * c00 + xF * (1.0f - yF) * c01 + yF * (1.0 - xF) * c10 + xF * yF * (c11);
62 v = std::min(std::max(v, 0.0f), 255.0f);
63 dest[bpp * i + b] = (unsigned char)v;
64 }
65 curPoints.fY += dy;
66 curPoints.fX += dx;
67 }
68 }
69
MNNSamplerC4Bilinear(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)70 static void MNNSamplerC4Bilinear(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
71 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
72 #ifdef MNN_USE_NEON
73 MNNSamplerC4BilinearOpt(source, dest + 4 * sta, reinterpret_cast<float*>(points), count, iw - 1, ih - 1, yStride);
74 #else
75 _sampleBilinearCommon(source, dest + 4 * sta, points, count, iw, ih, yStride, 4);
76 #endif
77 }
MNNSamplerC3Bilinear(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)78 static void MNNSamplerC3Bilinear(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
79 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
80 _sampleBilinearCommon(source, dest + 3 * sta, points, count, iw, ih, yStride, 3);
81 }
MNNSamplerC1Bilinear(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)82 static void MNNSamplerC1Bilinear(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
83 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
84 #ifdef MNN_USE_NEON
85 MNNSamplerC1BilinearOpt(source, dest + sta, reinterpret_cast<float*>(points), count, iw - 1, ih - 1, yStride);
86 #else
87 _sampleBilinearCommon(source, dest + sta, points, count, iw, ih, yStride, 1);
88 #endif
89 }
MNNSamplerNearest(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t iw,size_t ih,size_t yStride,int bpp)90 static void MNNSamplerNearest(const unsigned char* source, unsigned char* dest, Point* points, size_t sta, size_t count,
91 size_t iw, size_t ih, size_t yStride, int bpp) {
92 dest = dest + bpp * sta;
93 Point curPoints;
94 curPoints.fX = points[0].fX;
95 curPoints.fY = points[0].fY;
96 float dy = points[1].fY;
97 float dx = points[1].fX;
98 float xMax = iw - 1;
99 float yMax = ih - 1;
100
101 for (int i = 0; i < count; ++i) {
102 int y = (int)roundf(__clamp(curPoints.fY, 0, yMax));
103 int x = (int)roundf(__clamp(curPoints.fX, 0, xMax));
104 curPoints.fY += dy;
105 curPoints.fX += dx;
106 auto sourcePos = y * yStride + bpp * x;
107 for (int j = 0; j < bpp; ++j) {
108 dest[bpp * i + j] = source[sourcePos + j];
109 }
110 }
111 }
112
MNNSamplerC4Nearest(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)113 static void MNNSamplerC4Nearest(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
114 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
115 #ifdef MNN_USE_NEON
116 MNNSamplerC4NearestOpt(source, dest + 4 * sta, (float*)points, count, iw - 1, ih - 1, yStride);
117 #else
118 MNNSamplerNearest(source, dest, points, sta, count, iw, ih, yStride, 4);
119 #endif
120 }
121
MNNSamplerC1Nearest(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)122 static void MNNSamplerC1Nearest(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
123 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
124 #ifdef MNN_USE_NEON
125 MNNSamplerC1NearestOpt(source, dest + sta, (float*)points, count, iw - 1, ih - 1, yStride);
126 #else
127 MNNSamplerNearest(source, dest, points, sta, count, iw, ih, yStride, 1);
128 #endif
129 }
130
MNNSamplerC3Nearest(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)131 static void MNNSamplerC3Nearest(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
132 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
133 MNNSamplerNearest(source, dest, points, sta, count, iw, ih, yStride, 3);
134 }
MNNSamplerCopyCommon(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t iw,size_t ih,size_t yStride,int bpp)135 static void MNNSamplerCopyCommon(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
136 size_t count, size_t iw, size_t ih, size_t yStride, int bpp) {
137 dest = dest + bpp * sta;
138 Point curPoints;
139 curPoints.fX = points[0].fX;
140 curPoints.fY = points[0].fY;
141 float xMax = iw - 1;
142 float yMax = ih - 1;
143 int y = (int)roundf(__clamp(curPoints.fY, 0, yMax));
144 int x = (int)roundf(__clamp(curPoints.fX, 0, xMax));
145 auto sourcePos = y * yStride + bpp * x;
146 ::memcpy(dest, source + sourcePos, bpp * count);
147 }
148
MNNSamplerC1Copy(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)149 static void MNNSamplerC1Copy(const unsigned char* source, unsigned char* dest, Point* points, size_t sta, size_t count,
150 size_t capacity, size_t iw, size_t ih, size_t yStride) {
151 MNNSamplerCopyCommon(source, dest, points, sta, count, iw, ih, yStride, 1);
152 }
MNNSamplerC3Copy(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)153 static void MNNSamplerC3Copy(const unsigned char* source, unsigned char* dest, Point* points, size_t sta, size_t count,
154 size_t capacity, size_t iw, size_t ih, size_t yStride) {
155 MNNSamplerCopyCommon(source, dest, points, sta, count, iw, ih, yStride, 3);
156 }
157
MNNSamplerC4Copy(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)158 static void MNNSamplerC4Copy(const unsigned char* source, unsigned char* dest, Point* points, size_t sta, size_t count,
159 size_t capacity, size_t iw, size_t ih, size_t yStride) {
160 MNNSamplerCopyCommon(source, dest, points, sta, count, iw, ih, yStride, 4);
161 }
MNNSamplerI420Copy(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)162 static void MNNSamplerI420Copy(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
163 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
164 Point curPoints;
165 curPoints.fX = points[0].fX;
166 curPoints.fY = points[0].fY;
167 float xMax = iw - 1;
168 float yMax = ih - 1;
169 int y = (int)roundf(__clamp(curPoints.fY, 0, yMax));
170 int x = (int)roundf(__clamp(curPoints.fX, 0, xMax));
171 auto uvPlane = (((int)iw + 1) / 2) * ((int(ih) + 1) / 2);
172 int sourcePosY = y * (int)iw + x;
173 auto sourcePosU = source + (int)iw * (int)ih + (y / 2) * (((int)iw + 1) / 2) + (x / 2);
174 auto sourcePosV = source + (int)iw * (int)ih + (y / 2) * (((int)iw + 1) / 2) + (x / 2) + uvPlane;
175 auto uvCount = (count + 1) / 2;
176 ::memcpy(dest + sta, source + sourcePosY, count);
177 auto uDest = dest + (capacity) + (sta / 2) * 2;
178 for (int i=0; i<uvCount; ++i) {
179 uDest[2 * i + 0] = sourcePosV[i];
180 uDest[2 * i + 1] = sourcePosU[i];
181 }
182 }
MNNSamplerI420Nearest(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)183 static void MNNSamplerI420Nearest(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
184 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
185 auto srcY = source;
186
187 auto dstY = dest + sta;
188 auto dstUV = dest + (capacity) + (sta / 2) * 2;
189 auto stride = yStride;
190 if (yStride == 0) {
191 stride = iw;
192 }
193 auto srcU = source + stride * ih;
194 MNNSamplerC1Nearest(srcY, dstY, points, 0, count, capacity, iw, ih, stride);
195
196 Point uvPoints[2];
197 uvPoints[0].fX = (points[0].fX - 0.01f) / 2.0f;
198 uvPoints[0].fY = (points[0].fY - 0.01f) / 2.0f;
199 uvPoints[1].fX = points[1].fX;
200 uvPoints[1].fY = points[1].fY;
201 if (yStride == 0) {
202 stride = ((iw + 1) / 2);
203 }
204 auto srcV = srcU + stride * ((ih + 1) / 2);
205 auto uvCount = (count + 1) / 2;
206 {
207 Point curPoints;
208 curPoints.fX = uvPoints[0].fX;
209 curPoints.fY = uvPoints[0].fY;
210 float dy = uvPoints[1].fY;
211 float dx = uvPoints[1].fX;
212 float xMax = (iw + 1 / 2) - 1;
213 float yMax = (ih + 1 / 2) - 1;
214
215 for (int i = 0; i < uvCount; ++i) {
216 int y = (int)roundf(__clamp(curPoints.fY, 0, yMax));
217 int x = (int)roundf(__clamp(curPoints.fX, 0, xMax));
218 curPoints.fY += dy;
219 curPoints.fX += dx;
220 auto offset = y * stride + x;
221 dstUV[2 * i + 0] = srcV[offset];
222 dstUV[2 * i + 1] = srcU[offset];
223 }
224 }
225 }
226
MNNSamplerNV21Copy(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)227 static void MNNSamplerNV21Copy(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
228 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
229 Point curPoints;
230 curPoints.fX = points[0].fX;
231 curPoints.fY = points[0].fY;
232 float xMax = iw - 1;
233 float yMax = ih - 1;
234 int y = (int)roundf(__clamp(curPoints.fY, 0, yMax));
235 int x = (int)roundf(__clamp(curPoints.fX, 0, xMax));
236 int sourcePosY = y * (int)iw + x;
237 int sourcePosUV = (int)iw * (int)ih + (y / 2) * (((int)iw + 1) / 2) * 2 + (x / 2) * 2;
238
239 ::memcpy(dest + sta, source + sourcePosY, count);
240 ::memcpy(dest + (capacity) + (sta / 2) * 2, source + sourcePosUV, ((count + 1) / 2) * 2);
241 }
242
MNNSamplerNV21Nearest(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)243 static void MNNSamplerNV21Nearest(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
244 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
245 auto srcY = source;
246
247 auto dstY = dest + sta;
248 auto dstUV = dest + (capacity) + (sta / 2) * 2;
249 auto stride = yStride;
250 if (yStride == 0) {
251 stride = iw;
252 }
253 auto srcUV = source + stride * ih;
254 MNNSamplerC1Nearest(srcY, dstY, points, 0, count, capacity, iw, ih, stride);
255
256 Point uvPoints[2];
257 uvPoints[0].fX = (points[0].fX - 0.01f) / 2.0f;
258 uvPoints[0].fY = (points[0].fY - 0.01f) / 2.0f;
259 uvPoints[1].fX = points[1].fX;
260 uvPoints[1].fY = points[1].fY;
261 if (yStride == 0) {
262 stride = ((iw + 1) / 2) * 2;
263 }
264 MNNSamplerNearest(srcUV, dstUV, uvPoints, 0, (count + 1) / 2, (iw + 1) / 2, (ih + 1) / 2, stride, 2);
265 }
266
_swapUV(const unsigned char * source,unsigned char * dest,size_t countC2)267 static void _swapUV(const unsigned char* source, unsigned char* dest, size_t countC2) {
268 int sta = 0;
269 #ifdef MNN_USE_NEON
270 int countC2C16 = (int)countC2 / 16;
271 sta = countC2C16 * 16;
272 for (int i=0; i<countC2C16; ++i) {
273 auto src = vld2q_u8(source + i * 32);
274 auto temp = src.val[0];
275 src.val[0] = src.val[1];
276 src.val[1] = temp;
277 vst2q_u8(dest + i * 32, src);
278 }
279 #endif
280 for (int i=sta; i < countC2; ++i) {
281 auto temp = source[2*i];
282 dest[2*i] = source[2*i+1];
283 dest[2*i+1] = temp;
284 }
285 }
286
MNNSamplerNV12Copy(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)287 static void MNNSamplerNV12Copy(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
288 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
289 MNNSamplerNV21Copy(source, dest, points, sta, count, capacity, iw, ih, yStride);
290 auto destUV = dest + (capacity) + (sta / 2) * 2;
291 auto countC2 = ((count + 1) / 2);
292 _swapUV(destUV, destUV, countC2);
293 }
294
MNNSamplerNV12Nearest(const unsigned char * source,unsigned char * dest,Point * points,size_t sta,size_t count,size_t capacity,size_t iw,size_t ih,size_t yStride)295 static void MNNSamplerNV12Nearest(const unsigned char* source, unsigned char* dest, Point* points, size_t sta,
296 size_t count, size_t capacity, size_t iw, size_t ih, size_t yStride) {
297 MNNSamplerNV21Nearest(source, dest, points, sta, count, capacity, iw, ih, yStride);
298 auto destUV = dest + (capacity) + (sta / 2) * 2;
299 auto countC2 = ((count + 1) / 2);
300 _swapUV(destUV, destUV, countC2);
301 }
302
303
choose(ImageFormat format,Filter type,bool identity)304 ImageSampler::PROC ImageSampler::choose(ImageFormat format, Filter type, bool identity) {
305 if (identity) {
306 switch (format) {
307 case RGBA:
308 case BGRA:
309 return MNNSamplerC4Copy;
310 case GRAY:
311 return MNNSamplerC1Copy;
312
313 case RGB:
314 case BGR:
315 return MNNSamplerC3Copy;
316 case YUV_NV21:
317 return MNNSamplerNV21Copy;
318 case YUV_NV12:
319 return MNNSamplerNV12Copy;
320 case YUV_I420:
321 return MNNSamplerI420Copy;
322 default:
323 break;
324 }
325 }
326 if (BILINEAR == type) {
327 switch (format) {
328 case RGBA:
329 case BGRA:
330 return MNNSamplerC4Bilinear;
331 case GRAY:
332 return MNNSamplerC1Bilinear;
333
334 case RGB:
335 case BGR:
336 return MNNSamplerC3Bilinear;
337 default:
338 break;
339 }
340 }
341
342 // Nearest
343 switch (format) {
344 case RGBA:
345 case BGRA:
346 return MNNSamplerC4Nearest;
347 case GRAY:
348 return MNNSamplerC1Nearest;
349
350 case RGB:
351 case BGR:
352 return MNNSamplerC3Nearest;
353 case YUV_NV12:
354 return MNNSamplerNV12Nearest;
355 case YUV_NV21:
356 return MNNSamplerNV21Nearest;
357 case YUV_I420:
358 return MNNSamplerI420Nearest;
359 default:
360 break;
361 }
362 MNN_PRINT("Don't support sampler for format:%d, type:%d", format, type);
363 return nullptr;
364 }
365
366 } // namespace CV
367 } // namespace MNN
368