1 /* 2 * Copyright 2012 The LibYuv Project Authors. All rights reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "libyuv/row.h" 12 13 #include <string.h> // For memset. 14 15 #include "libyuv/basic_types.h" 16 17 #ifdef __cplusplus 18 namespace libyuv { 19 extern "C" { 20 #endif 21 22 // memset for temp is meant to clear the source buffer (not dest) so that 23 // SIMD that reads full multiple of 16 bytes will not trigger msan errors. 24 // memset is not needed for production, as the garbage values are processed but 25 // not used, although there may be edge cases for subsampling. 26 // The size of the buffer is based on the largest read, which can be inferred 27 // by the source type (e.g. ARGB) and the mask (last parameter), or by examining 28 // the source code for how much the source pointers are advanced. 29 30 // Subsampled source needs to be increase by 1 of not even. 31 #define SS(width, shift) (((width) + (1 << (shift)) - 1) >> (shift)) 32 33 // Any 4 planes to 1 with yuvconstants 34 #define ANY41C(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, BPP, MASK) \ 35 void NAMEANY(const uint8_t* y_buf, const uint8_t* u_buf, \ 36 const uint8_t* v_buf, const uint8_t* a_buf, uint8_t* dst_ptr, \ 37 const struct YuvConstants* yuvconstants, int width) { \ 38 SIMD_ALIGNED(uint8_t temp[64 * 5]); \ 39 memset(temp, 0, 64 * 4); /* for msan */ \ 40 int r = width & MASK; \ 41 int n = width & ~MASK; \ 42 if (n > 0) { \ 43 ANY_SIMD(y_buf, u_buf, v_buf, a_buf, dst_ptr, yuvconstants, n); \ 44 } \ 45 memcpy(temp, y_buf + n, r); \ 46 memcpy(temp + 64, u_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \ 47 memcpy(temp + 128, v_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \ 48 memcpy(temp + 192, a_buf + n, r); \ 49 ANY_SIMD(temp, temp + 64, temp + 128, temp + 192, temp + 256, \ 50 yuvconstants, MASK + 1); \ 51 memcpy(dst_ptr + (n >> DUVSHIFT) * BPP, temp + 256, \ 52 SS(r, DUVSHIFT) * BPP); \ 53 } 54 55 #ifdef HAS_I422ALPHATOARGBROW_SSSE3 56 ANY41C(I422AlphaToARGBRow_Any_SSSE3, I422AlphaToARGBRow_SSSE3, 1, 0, 4, 7) 57 #endif 58 #ifdef HAS_I422ALPHATOARGBROW_AVX2 59 ANY41C(I422AlphaToARGBRow_Any_AVX2, I422AlphaToARGBRow_AVX2, 1, 0, 4, 15) 60 #endif 61 #ifdef HAS_I422ALPHATOARGBROW_NEON 62 ANY41C(I422AlphaToARGBRow_Any_NEON, I422AlphaToARGBRow_NEON, 1, 0, 4, 7) 63 #endif 64 #ifdef HAS_I422ALPHATOARGBROW_MSA 65 ANY41C(I422AlphaToARGBRow_Any_MSA, I422AlphaToARGBRow_MSA, 1, 0, 4, 7) 66 #endif 67 #ifdef HAS_I422ALPHATOARGBROW_MMI 68 ANY41C(I422AlphaToARGBRow_Any_MMI, I422AlphaToARGBRow_MMI, 1, 0, 4, 7) 69 #endif 70 #undef ANY41C 71 72 // Any 3 planes to 1. 73 #define ANY31(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, BPP, MASK) \ 74 void NAMEANY(const uint8_t* y_buf, const uint8_t* u_buf, \ 75 const uint8_t* v_buf, uint8_t* dst_ptr, int width) { \ 76 SIMD_ALIGNED(uint8_t temp[64 * 4]); \ 77 memset(temp, 0, 64 * 3); /* for YUY2 and msan */ \ 78 int r = width & MASK; \ 79 int n = width & ~MASK; \ 80 if (n > 0) { \ 81 ANY_SIMD(y_buf, u_buf, v_buf, dst_ptr, n); \ 82 } \ 83 memcpy(temp, y_buf + n, r); \ 84 memcpy(temp + 64, u_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \ 85 memcpy(temp + 128, v_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \ 86 ANY_SIMD(temp, temp + 64, temp + 128, temp + 192, MASK + 1); \ 87 memcpy(dst_ptr + (n >> DUVSHIFT) * BPP, temp + 192, \ 88 SS(r, DUVSHIFT) * BPP); \ 89 } 90 91 // Merge functions. 92 #ifdef HAS_MERGERGBROW_SSSE3 93 ANY31(MergeRGBRow_Any_SSSE3, MergeRGBRow_SSSE3, 0, 0, 3, 15) 94 #endif 95 #ifdef HAS_MERGERGBROW_NEON 96 ANY31(MergeRGBRow_Any_NEON, MergeRGBRow_NEON, 0, 0, 3, 15) 97 #endif 98 #ifdef HAS_MERGERGBROW_MMI 99 ANY31(MergeRGBRow_Any_MMI, MergeRGBRow_MMI, 0, 0, 3, 7) 100 #endif 101 #ifdef HAS_I422TOYUY2ROW_SSE2 102 ANY31(I422ToYUY2Row_Any_SSE2, I422ToYUY2Row_SSE2, 1, 1, 4, 15) 103 ANY31(I422ToUYVYRow_Any_SSE2, I422ToUYVYRow_SSE2, 1, 1, 4, 15) 104 #endif 105 #ifdef HAS_I422TOYUY2ROW_AVX2 106 ANY31(I422ToYUY2Row_Any_AVX2, I422ToYUY2Row_AVX2, 1, 1, 4, 31) 107 ANY31(I422ToUYVYRow_Any_AVX2, I422ToUYVYRow_AVX2, 1, 1, 4, 31) 108 #endif 109 #ifdef HAS_I422TOYUY2ROW_NEON 110 ANY31(I422ToYUY2Row_Any_NEON, I422ToYUY2Row_NEON, 1, 1, 4, 15) 111 #endif 112 #ifdef HAS_I422TOYUY2ROW_MSA 113 ANY31(I422ToYUY2Row_Any_MSA, I422ToYUY2Row_MSA, 1, 1, 4, 31) 114 #endif 115 #ifdef HAS_I422TOYUY2ROW_MMI 116 ANY31(I422ToYUY2Row_Any_MMI, I422ToYUY2Row_MMI, 1, 1, 4, 7) 117 #endif 118 #ifdef HAS_I422TOUYVYROW_NEON 119 ANY31(I422ToUYVYRow_Any_NEON, I422ToUYVYRow_NEON, 1, 1, 4, 15) 120 #endif 121 #ifdef HAS_I422TOUYVYROW_MSA 122 ANY31(I422ToUYVYRow_Any_MSA, I422ToUYVYRow_MSA, 1, 1, 4, 31) 123 #endif 124 #ifdef HAS_I422TOUYVYROW_MMI 125 ANY31(I422ToUYVYRow_Any_MMI, I422ToUYVYRow_MMI, 1, 1, 4, 7) 126 #endif 127 #ifdef HAS_BLENDPLANEROW_AVX2 128 ANY31(BlendPlaneRow_Any_AVX2, BlendPlaneRow_AVX2, 0, 0, 1, 31) 129 #endif 130 #ifdef HAS_BLENDPLANEROW_SSSE3 131 ANY31(BlendPlaneRow_Any_SSSE3, BlendPlaneRow_SSSE3, 0, 0, 1, 7) 132 #endif 133 #ifdef HAS_BLENDPLANEROW_MMI 134 ANY31(BlendPlaneRow_Any_MMI, BlendPlaneRow_MMI, 0, 0, 1, 7) 135 #endif 136 #undef ANY31 137 138 // Note that odd width replication includes 444 due to implementation 139 // on arm that subsamples 444 to 422 internally. 140 // Any 3 planes to 1 with yuvconstants 141 #define ANY31C(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, BPP, MASK) \ 142 void NAMEANY(const uint8_t* y_buf, const uint8_t* u_buf, \ 143 const uint8_t* v_buf, uint8_t* dst_ptr, \ 144 const struct YuvConstants* yuvconstants, int width) { \ 145 SIMD_ALIGNED(uint8_t temp[128 * 4]); \ 146 memset(temp, 0, 128 * 3); /* for YUY2 and msan */ \ 147 int r = width & MASK; \ 148 int n = width & ~MASK; \ 149 if (n > 0) { \ 150 ANY_SIMD(y_buf, u_buf, v_buf, dst_ptr, yuvconstants, n); \ 151 } \ 152 memcpy(temp, y_buf + n, r); \ 153 memcpy(temp + 128, u_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \ 154 memcpy(temp + 256, v_buf + (n >> UVSHIFT), SS(r, UVSHIFT)); \ 155 if (width & 1) { \ 156 temp[128 + SS(r, UVSHIFT)] = temp[128 + SS(r, UVSHIFT) - 1]; \ 157 temp[256 + SS(r, UVSHIFT)] = temp[256 + SS(r, UVSHIFT) - 1]; \ 158 } \ 159 ANY_SIMD(temp, temp + 128, temp + 256, temp + 384, yuvconstants, \ 160 MASK + 1); \ 161 memcpy(dst_ptr + (n >> DUVSHIFT) * BPP, temp + 384, \ 162 SS(r, DUVSHIFT) * BPP); \ 163 } 164 165 #ifdef HAS_I422TOARGBROW_SSSE3 166 ANY31C(I422ToARGBRow_Any_SSSE3, I422ToARGBRow_SSSE3, 1, 0, 4, 7) 167 #endif 168 #ifdef HAS_I422TOAR30ROW_SSSE3 169 ANY31C(I422ToAR30Row_Any_SSSE3, I422ToAR30Row_SSSE3, 1, 0, 4, 7) 170 #endif 171 #ifdef HAS_I422TOAR30ROW_AVX2 172 ANY31C(I422ToAR30Row_Any_AVX2, I422ToAR30Row_AVX2, 1, 0, 4, 15) 173 #endif 174 #ifdef HAS_I444TOARGBROW_SSSE3 175 ANY31C(I444ToARGBRow_Any_SSSE3, I444ToARGBRow_SSSE3, 0, 0, 4, 7) 176 ANY31C(I422ToRGBARow_Any_SSSE3, I422ToRGBARow_SSSE3, 1, 0, 4, 7) 177 ANY31C(I422ToARGB4444Row_Any_SSSE3, I422ToARGB4444Row_SSSE3, 1, 0, 2, 7) 178 ANY31C(I422ToARGB1555Row_Any_SSSE3, I422ToARGB1555Row_SSSE3, 1, 0, 2, 7) 179 ANY31C(I422ToRGB565Row_Any_SSSE3, I422ToRGB565Row_SSSE3, 1, 0, 2, 7) 180 ANY31C(I422ToRGB24Row_Any_SSSE3, I422ToRGB24Row_SSSE3, 1, 0, 3, 15) 181 #endif // HAS_I444TOARGBROW_SSSE3 182 #ifdef HAS_I422TORGB24ROW_AVX2 183 ANY31C(I422ToRGB24Row_Any_AVX2, I422ToRGB24Row_AVX2, 1, 0, 3, 31) 184 #endif 185 #ifdef HAS_I422TOARGBROW_AVX2 186 ANY31C(I422ToARGBRow_Any_AVX2, I422ToARGBRow_AVX2, 1, 0, 4, 15) 187 #endif 188 #ifdef HAS_I422TORGBAROW_AVX2 189 ANY31C(I422ToRGBARow_Any_AVX2, I422ToRGBARow_AVX2, 1, 0, 4, 15) 190 #endif 191 #ifdef HAS_I444TOARGBROW_AVX2 192 ANY31C(I444ToARGBRow_Any_AVX2, I444ToARGBRow_AVX2, 0, 0, 4, 15) 193 #endif 194 #ifdef HAS_I422TOARGB4444ROW_AVX2 195 ANY31C(I422ToARGB4444Row_Any_AVX2, I422ToARGB4444Row_AVX2, 1, 0, 2, 15) 196 #endif 197 #ifdef HAS_I422TOARGB1555ROW_AVX2 198 ANY31C(I422ToARGB1555Row_Any_AVX2, I422ToARGB1555Row_AVX2, 1, 0, 2, 15) 199 #endif 200 #ifdef HAS_I422TORGB565ROW_AVX2 201 ANY31C(I422ToRGB565Row_Any_AVX2, I422ToRGB565Row_AVX2, 1, 0, 2, 15) 202 #endif 203 #ifdef HAS_I422TOARGBROW_NEON 204 ANY31C(I444ToARGBRow_Any_NEON, I444ToARGBRow_NEON, 0, 0, 4, 7) 205 ANY31C(I422ToARGBRow_Any_NEON, I422ToARGBRow_NEON, 1, 0, 4, 7) 206 ANY31C(I422ToRGBARow_Any_NEON, I422ToRGBARow_NEON, 1, 0, 4, 7) 207 ANY31C(I422ToRGB24Row_Any_NEON, I422ToRGB24Row_NEON, 1, 0, 3, 7) 208 ANY31C(I422ToARGB4444Row_Any_NEON, I422ToARGB4444Row_NEON, 1, 0, 2, 7) 209 ANY31C(I422ToARGB1555Row_Any_NEON, I422ToARGB1555Row_NEON, 1, 0, 2, 7) 210 ANY31C(I422ToRGB565Row_Any_NEON, I422ToRGB565Row_NEON, 1, 0, 2, 7) 211 #endif 212 #ifdef HAS_I422TOARGBROW_MSA 213 ANY31C(I444ToARGBRow_Any_MSA, I444ToARGBRow_MSA, 0, 0, 4, 7) 214 ANY31C(I422ToARGBRow_Any_MSA, I422ToARGBRow_MSA, 1, 0, 4, 7) 215 ANY31C(I422ToRGBARow_Any_MSA, I422ToRGBARow_MSA, 1, 0, 4, 7) 216 ANY31C(I422ToRGB24Row_Any_MSA, I422ToRGB24Row_MSA, 1, 0, 3, 15) 217 ANY31C(I422ToARGB4444Row_Any_MSA, I422ToARGB4444Row_MSA, 1, 0, 2, 7) 218 ANY31C(I422ToARGB1555Row_Any_MSA, I422ToARGB1555Row_MSA, 1, 0, 2, 7) 219 ANY31C(I422ToRGB565Row_Any_MSA, I422ToRGB565Row_MSA, 1, 0, 2, 7) 220 #endif 221 #ifdef HAS_I422TOARGBROW_MMI 222 ANY31C(I444ToARGBRow_Any_MMI, I444ToARGBRow_MMI, 0, 0, 4, 7) 223 ANY31C(I422ToARGBRow_Any_MMI, I422ToARGBRow_MMI, 1, 0, 4, 7) 224 ANY31C(I422ToRGB24Row_Any_MMI, I422ToRGB24Row_MMI, 1, 0, 3, 15) 225 ANY31C(I422ToARGB4444Row_Any_MMI, I422ToARGB4444Row_MMI, 1, 0, 2, 7) 226 ANY31C(I422ToARGB1555Row_Any_MMI, I422ToARGB1555Row_MMI, 1, 0, 2, 7) 227 ANY31C(I422ToRGB565Row_Any_MMI, I422ToRGB565Row_MMI, 1, 0, 2, 7) 228 ANY31C(I422ToRGBARow_Any_MMI, I422ToRGBARow_MMI, 1, 0, 4, 7) 229 #endif 230 #undef ANY31C 231 232 // Any 3 planes of 16 bit to 1 with yuvconstants 233 // TODO(fbarchard): consider sharing this code with ANY31C 234 #define ANY31CT(NAMEANY, ANY_SIMD, UVSHIFT, DUVSHIFT, T, SBPP, BPP, MASK) \ 235 void NAMEANY(const T* y_buf, const T* u_buf, const T* v_buf, \ 236 uint8_t* dst_ptr, const struct YuvConstants* yuvconstants, \ 237 int width) { \ 238 SIMD_ALIGNED(T temp[16 * 3]); \ 239 SIMD_ALIGNED(uint8_t out[64]); \ 240 memset(temp, 0, 16 * 3 * SBPP); /* for YUY2 and msan */ \ 241 int r = width & MASK; \ 242 int n = width & ~MASK; \ 243 if (n > 0) { \ 244 ANY_SIMD(y_buf, u_buf, v_buf, dst_ptr, yuvconstants, n); \ 245 } \ 246 memcpy(temp, y_buf + n, r * SBPP); \ 247 memcpy(temp + 16, u_buf + (n >> UVSHIFT), SS(r, UVSHIFT) * SBPP); \ 248 memcpy(temp + 32, v_buf + (n >> UVSHIFT), SS(r, UVSHIFT) * SBPP); \ 249 ANY_SIMD(temp, temp + 16, temp + 32, out, yuvconstants, MASK + 1); \ 250 memcpy(dst_ptr + (n >> DUVSHIFT) * BPP, out, SS(r, DUVSHIFT) * BPP); \ 251 } 252 253 #ifdef HAS_I210TOAR30ROW_SSSE3 254 ANY31CT(I210ToAR30Row_Any_SSSE3, I210ToAR30Row_SSSE3, 1, 0, uint16_t, 2, 4, 7) 255 #endif 256 #ifdef HAS_I210TOARGBROW_SSSE3 257 ANY31CT(I210ToARGBRow_Any_SSSE3, I210ToARGBRow_SSSE3, 1, 0, uint16_t, 2, 4, 7) 258 #endif 259 #ifdef HAS_I210TOARGBROW_AVX2 260 ANY31CT(I210ToARGBRow_Any_AVX2, I210ToARGBRow_AVX2, 1, 0, uint16_t, 2, 4, 15) 261 #endif 262 #ifdef HAS_I210TOAR30ROW_AVX2 263 ANY31CT(I210ToAR30Row_Any_AVX2, I210ToAR30Row_AVX2, 1, 0, uint16_t, 2, 4, 15) 264 #endif 265 #ifdef HAS_I210TOARGBROW_MMI 266 ANY31CT(I210ToARGBRow_Any_MMI, I210ToARGBRow_MMI, 1, 0, uint16_t, 2, 4, 7) 267 #endif 268 #undef ANY31CT 269 270 // Any 2 planes to 1. 271 #define ANY21(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, SBPP2, BPP, MASK) \ 272 void NAMEANY(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, \ 273 int width) { \ 274 SIMD_ALIGNED(uint8_t temp[64 * 3]); \ 275 memset(temp, 0, 64 * 2); /* for msan */ \ 276 int r = width & MASK; \ 277 int n = width & ~MASK; \ 278 if (n > 0) { \ 279 ANY_SIMD(y_buf, uv_buf, dst_ptr, n); \ 280 } \ 281 memcpy(temp, y_buf + n * SBPP, r * SBPP); \ 282 memcpy(temp + 64, uv_buf + (n >> UVSHIFT) * SBPP2, \ 283 SS(r, UVSHIFT) * SBPP2); \ 284 ANY_SIMD(temp, temp + 64, temp + 128, MASK + 1); \ 285 memcpy(dst_ptr + n * BPP, temp + 128, r * BPP); \ 286 } 287 288 // Merge functions. 289 #ifdef HAS_MERGEUVROW_SSE2 290 ANY21(MergeUVRow_Any_SSE2, MergeUVRow_SSE2, 0, 1, 1, 2, 15) 291 #endif 292 #ifdef HAS_MERGEUVROW_AVX2 293 ANY21(MergeUVRow_Any_AVX2, MergeUVRow_AVX2, 0, 1, 1, 2, 31) 294 #endif 295 #ifdef HAS_MERGEUVROW_NEON 296 ANY21(MergeUVRow_Any_NEON, MergeUVRow_NEON, 0, 1, 1, 2, 15) 297 #endif 298 #ifdef HAS_MERGEUVROW_MSA 299 ANY21(MergeUVRow_Any_MSA, MergeUVRow_MSA, 0, 1, 1, 2, 15) 300 #endif 301 #ifdef HAS_MERGEUVROW_MMI 302 ANY21(MergeUVRow_Any_MMI, MergeUVRow_MMI, 0, 1, 1, 2, 7) 303 #endif 304 #ifdef HAS_NV21TOYUV24ROW_NEON 305 ANY21(NV21ToYUV24Row_Any_NEON, NV21ToYUV24Row_NEON, 1, 1, 2, 3, 15) 306 #endif 307 #ifdef HAS_NV21TOYUV24ROW_AVX2 308 ANY21(NV21ToYUV24Row_Any_AVX2, NV21ToYUV24Row_AVX2, 1, 1, 2, 3, 31) 309 #endif 310 // Math functions. 311 #ifdef HAS_ARGBMULTIPLYROW_SSE2 312 ANY21(ARGBMultiplyRow_Any_SSE2, ARGBMultiplyRow_SSE2, 0, 4, 4, 4, 3) 313 #endif 314 #ifdef HAS_ARGBADDROW_SSE2 315 ANY21(ARGBAddRow_Any_SSE2, ARGBAddRow_SSE2, 0, 4, 4, 4, 3) 316 #endif 317 #ifdef HAS_ARGBSUBTRACTROW_SSE2 318 ANY21(ARGBSubtractRow_Any_SSE2, ARGBSubtractRow_SSE2, 0, 4, 4, 4, 3) 319 #endif 320 #ifdef HAS_ARGBMULTIPLYROW_AVX2 321 ANY21(ARGBMultiplyRow_Any_AVX2, ARGBMultiplyRow_AVX2, 0, 4, 4, 4, 7) 322 #endif 323 #ifdef HAS_ARGBADDROW_AVX2 324 ANY21(ARGBAddRow_Any_AVX2, ARGBAddRow_AVX2, 0, 4, 4, 4, 7) 325 #endif 326 #ifdef HAS_ARGBSUBTRACTROW_AVX2 327 ANY21(ARGBSubtractRow_Any_AVX2, ARGBSubtractRow_AVX2, 0, 4, 4, 4, 7) 328 #endif 329 #ifdef HAS_ARGBMULTIPLYROW_NEON 330 ANY21(ARGBMultiplyRow_Any_NEON, ARGBMultiplyRow_NEON, 0, 4, 4, 4, 7) 331 #endif 332 #ifdef HAS_ARGBADDROW_NEON 333 ANY21(ARGBAddRow_Any_NEON, ARGBAddRow_NEON, 0, 4, 4, 4, 7) 334 #endif 335 #ifdef HAS_ARGBSUBTRACTROW_NEON 336 ANY21(ARGBSubtractRow_Any_NEON, ARGBSubtractRow_NEON, 0, 4, 4, 4, 7) 337 #endif 338 #ifdef HAS_ARGBMULTIPLYROW_MSA 339 ANY21(ARGBMultiplyRow_Any_MSA, ARGBMultiplyRow_MSA, 0, 4, 4, 4, 3) 340 #endif 341 #ifdef HAS_ARGBMULTIPLYROW_MMI 342 ANY21(ARGBMultiplyRow_Any_MMI, ARGBMultiplyRow_MMI, 0, 4, 4, 4, 1) 343 #endif 344 #ifdef HAS_ARGBADDROW_MSA 345 ANY21(ARGBAddRow_Any_MSA, ARGBAddRow_MSA, 0, 4, 4, 4, 7) 346 #endif 347 #ifdef HAS_ARGBADDROW_MMI 348 ANY21(ARGBAddRow_Any_MMI, ARGBAddRow_MMI, 0, 4, 4, 4, 1) 349 #endif 350 #ifdef HAS_ARGBSUBTRACTROW_MSA 351 ANY21(ARGBSubtractRow_Any_MSA, ARGBSubtractRow_MSA, 0, 4, 4, 4, 7) 352 #endif 353 #ifdef HAS_ARGBSUBTRACTROW_MMI 354 ANY21(ARGBSubtractRow_Any_MMI, ARGBSubtractRow_MMI, 0, 4, 4, 4, 1) 355 #endif 356 #ifdef HAS_SOBELROW_SSE2 357 ANY21(SobelRow_Any_SSE2, SobelRow_SSE2, 0, 1, 1, 4, 15) 358 #endif 359 #ifdef HAS_SOBELROW_NEON 360 ANY21(SobelRow_Any_NEON, SobelRow_NEON, 0, 1, 1, 4, 7) 361 #endif 362 #ifdef HAS_SOBELROW_MSA 363 ANY21(SobelRow_Any_MSA, SobelRow_MSA, 0, 1, 1, 4, 15) 364 #endif 365 #ifdef HAS_SOBELROW_MMI 366 ANY21(SobelRow_Any_MMI, SobelRow_MMI, 0, 1, 1, 4, 7) 367 #endif 368 #ifdef HAS_SOBELTOPLANEROW_SSE2 369 ANY21(SobelToPlaneRow_Any_SSE2, SobelToPlaneRow_SSE2, 0, 1, 1, 1, 15) 370 #endif 371 #ifdef HAS_SOBELTOPLANEROW_NEON 372 ANY21(SobelToPlaneRow_Any_NEON, SobelToPlaneRow_NEON, 0, 1, 1, 1, 15) 373 #endif 374 #ifdef HAS_SOBELTOPLANEROW_MSA 375 ANY21(SobelToPlaneRow_Any_MSA, SobelToPlaneRow_MSA, 0, 1, 1, 1, 31) 376 #endif 377 #ifdef HAS_SOBELTOPLANEROW_MMI 378 ANY21(SobelToPlaneRow_Any_MMI, SobelToPlaneRow_MMI, 0, 1, 1, 1, 7) 379 #endif 380 #ifdef HAS_SOBELXYROW_SSE2 381 ANY21(SobelXYRow_Any_SSE2, SobelXYRow_SSE2, 0, 1, 1, 4, 15) 382 #endif 383 #ifdef HAS_SOBELXYROW_NEON 384 ANY21(SobelXYRow_Any_NEON, SobelXYRow_NEON, 0, 1, 1, 4, 7) 385 #endif 386 #ifdef HAS_SOBELXYROW_MSA 387 ANY21(SobelXYRow_Any_MSA, SobelXYRow_MSA, 0, 1, 1, 4, 15) 388 #endif 389 #ifdef HAS_SOBELXYROW_MMI 390 ANY21(SobelXYRow_Any_MMI, SobelXYRow_MMI, 0, 1, 1, 4, 7) 391 #endif 392 #undef ANY21 393 394 // Any 2 planes to 1 with yuvconstants 395 #define ANY21C(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, SBPP2, BPP, MASK) \ 396 void NAMEANY(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, \ 397 const struct YuvConstants* yuvconstants, int width) { \ 398 SIMD_ALIGNED(uint8_t temp[128 * 3]); \ 399 memset(temp, 0, 128 * 2); /* for msan */ \ 400 int r = width & MASK; \ 401 int n = width & ~MASK; \ 402 if (n > 0) { \ 403 ANY_SIMD(y_buf, uv_buf, dst_ptr, yuvconstants, n); \ 404 } \ 405 memcpy(temp, y_buf + n * SBPP, r * SBPP); \ 406 memcpy(temp + 128, uv_buf + (n >> UVSHIFT) * SBPP2, \ 407 SS(r, UVSHIFT) * SBPP2); \ 408 ANY_SIMD(temp, temp + 128, temp + 256, yuvconstants, MASK + 1); \ 409 memcpy(dst_ptr + n * BPP, temp + 256, r * BPP); \ 410 } 411 412 // Biplanar to RGB. 413 #ifdef HAS_NV12TOARGBROW_SSSE3 414 ANY21C(NV12ToARGBRow_Any_SSSE3, NV12ToARGBRow_SSSE3, 1, 1, 2, 4, 7) 415 #endif 416 #ifdef HAS_NV12TOARGBROW_AVX2 417 ANY21C(NV12ToARGBRow_Any_AVX2, NV12ToARGBRow_AVX2, 1, 1, 2, 4, 15) 418 #endif 419 #ifdef HAS_NV12TOARGBROW_NEON 420 ANY21C(NV12ToARGBRow_Any_NEON, NV12ToARGBRow_NEON, 1, 1, 2, 4, 7) 421 #endif 422 #ifdef HAS_NV12TOARGBROW_MSA 423 ANY21C(NV12ToARGBRow_Any_MSA, NV12ToARGBRow_MSA, 1, 1, 2, 4, 7) 424 #endif 425 #ifdef HAS_NV12TOARGBROW_MMI 426 ANY21C(NV12ToARGBRow_Any_MMI, NV12ToARGBRow_MMI, 1, 1, 2, 4, 7) 427 #endif 428 #ifdef HAS_NV21TOARGBROW_SSSE3 429 ANY21C(NV21ToARGBRow_Any_SSSE3, NV21ToARGBRow_SSSE3, 1, 1, 2, 4, 7) 430 #endif 431 #ifdef HAS_NV21TOARGBROW_AVX2 432 ANY21C(NV21ToARGBRow_Any_AVX2, NV21ToARGBRow_AVX2, 1, 1, 2, 4, 15) 433 #endif 434 #ifdef HAS_NV21TOARGBROW_NEON 435 ANY21C(NV21ToARGBRow_Any_NEON, NV21ToARGBRow_NEON, 1, 1, 2, 4, 7) 436 #endif 437 #ifdef HAS_NV21TOARGBROW_MSA 438 ANY21C(NV21ToARGBRow_Any_MSA, NV21ToARGBRow_MSA, 1, 1, 2, 4, 7) 439 #endif 440 #ifdef HAS_NV21TOARGBROW_MMI 441 ANY21C(NV21ToARGBRow_Any_MMI, NV21ToARGBRow_MMI, 1, 1, 2, 4, 7) 442 #endif 443 #ifdef HAS_NV12TORGB24ROW_NEON 444 ANY21C(NV12ToRGB24Row_Any_NEON, NV12ToRGB24Row_NEON, 1, 1, 2, 3, 7) 445 #endif 446 #ifdef HAS_NV21TORGB24ROW_NEON 447 ANY21C(NV21ToRGB24Row_Any_NEON, NV21ToRGB24Row_NEON, 1, 1, 2, 3, 7) 448 #endif 449 #ifdef HAS_NV12TORGB24ROW_SSSE3 450 ANY21C(NV12ToRGB24Row_Any_SSSE3, NV12ToRGB24Row_SSSE3, 1, 1, 2, 3, 15) 451 #endif 452 #ifdef HAS_NV12TORGB24ROW_MMI 453 ANY21C(NV12ToRGB24Row_Any_MMI, NV12ToRGB24Row_MMI, 1, 1, 2, 3, 7) 454 #endif 455 #ifdef HAS_NV21TORGB24ROW_SSSE3 456 ANY21C(NV21ToRGB24Row_Any_SSSE3, NV21ToRGB24Row_SSSE3, 1, 1, 2, 3, 15) 457 #endif 458 #ifdef HAS_NV12TORGB24ROW_AVX2 459 ANY21C(NV12ToRGB24Row_Any_AVX2, NV12ToRGB24Row_AVX2, 1, 1, 2, 3, 31) 460 #endif 461 #ifdef HAS_NV21TORGB24ROW_AVX2 462 ANY21C(NV21ToRGB24Row_Any_AVX2, NV21ToRGB24Row_AVX2, 1, 1, 2, 3, 31) 463 #endif 464 #ifdef HAS_NV21TORGB24ROW_MMI 465 ANY21C(NV21ToRGB24Row_Any_MMI, NV21ToRGB24Row_MMI, 1, 1, 2, 3, 7) 466 #endif 467 #ifdef HAS_NV12TORGB565ROW_SSSE3 468 ANY21C(NV12ToRGB565Row_Any_SSSE3, NV12ToRGB565Row_SSSE3, 1, 1, 2, 2, 7) 469 #endif 470 #ifdef HAS_NV12TORGB565ROW_AVX2 471 ANY21C(NV12ToRGB565Row_Any_AVX2, NV12ToRGB565Row_AVX2, 1, 1, 2, 2, 15) 472 #endif 473 #ifdef HAS_NV12TORGB565ROW_NEON 474 ANY21C(NV12ToRGB565Row_Any_NEON, NV12ToRGB565Row_NEON, 1, 1, 2, 2, 7) 475 #endif 476 #ifdef HAS_NV12TORGB565ROW_MSA 477 ANY21C(NV12ToRGB565Row_Any_MSA, NV12ToRGB565Row_MSA, 1, 1, 2, 2, 7) 478 #endif 479 #ifdef HAS_NV12TORGB565ROW_MMI 480 ANY21C(NV12ToRGB565Row_Any_MMI, NV12ToRGB565Row_MMI, 1, 1, 2, 2, 7) 481 #endif 482 #undef ANY21C 483 484 // Any 1 to 1. 485 #define ANY11(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, BPP, MASK) \ 486 void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width) { \ 487 SIMD_ALIGNED(uint8_t temp[128 * 2]); \ 488 memset(temp, 0, 128); /* for YUY2 and msan */ \ 489 int r = width & MASK; \ 490 int n = width & ~MASK; \ 491 if (n > 0) { \ 492 ANY_SIMD(src_ptr, dst_ptr, n); \ 493 } \ 494 memcpy(temp, src_ptr + (n >> UVSHIFT) * SBPP, SS(r, UVSHIFT) * SBPP); \ 495 ANY_SIMD(temp, temp + 128, MASK + 1); \ 496 memcpy(dst_ptr + n * BPP, temp + 128, r * BPP); \ 497 } 498 499 #ifdef HAS_COPYROW_AVX 500 ANY11(CopyRow_Any_AVX, CopyRow_AVX, 0, 1, 1, 63) 501 #endif 502 #ifdef HAS_COPYROW_SSE2 503 ANY11(CopyRow_Any_SSE2, CopyRow_SSE2, 0, 1, 1, 31) 504 #endif 505 #ifdef HAS_COPYROW_NEON 506 ANY11(CopyRow_Any_NEON, CopyRow_NEON, 0, 1, 1, 31) 507 #endif 508 #if defined(HAS_ARGBTORGB24ROW_SSSE3) 509 ANY11(ARGBToRGB24Row_Any_SSSE3, ARGBToRGB24Row_SSSE3, 0, 4, 3, 15) 510 ANY11(ARGBToRAWRow_Any_SSSE3, ARGBToRAWRow_SSSE3, 0, 4, 3, 15) 511 ANY11(ARGBToRGB565Row_Any_SSE2, ARGBToRGB565Row_SSE2, 0, 4, 2, 3) 512 ANY11(ARGBToARGB1555Row_Any_SSE2, ARGBToARGB1555Row_SSE2, 0, 4, 2, 3) 513 ANY11(ARGBToARGB4444Row_Any_SSE2, ARGBToARGB4444Row_SSE2, 0, 4, 2, 3) 514 #endif 515 #if defined(HAS_ARGBTORGB24ROW_AVX2) 516 ANY11(ARGBToRGB24Row_Any_AVX2, ARGBToRGB24Row_AVX2, 0, 4, 3, 31) 517 #endif 518 #if defined(HAS_ARGBTORGB24ROW_AVX512VBMI) 519 ANY11(ARGBToRGB24Row_Any_AVX512VBMI, ARGBToRGB24Row_AVX512VBMI, 0, 4, 3, 31) 520 #endif 521 #if defined(HAS_ARGBTORAWROW_AVX2) 522 ANY11(ARGBToRAWRow_Any_AVX2, ARGBToRAWRow_AVX2, 0, 4, 3, 31) 523 #endif 524 #if defined(HAS_ARGBTORGB565ROW_AVX2) 525 ANY11(ARGBToRGB565Row_Any_AVX2, ARGBToRGB565Row_AVX2, 0, 4, 2, 7) 526 #endif 527 #if defined(HAS_ARGBTOARGB4444ROW_AVX2) 528 ANY11(ARGBToARGB1555Row_Any_AVX2, ARGBToARGB1555Row_AVX2, 0, 4, 2, 7) 529 ANY11(ARGBToARGB4444Row_Any_AVX2, ARGBToARGB4444Row_AVX2, 0, 4, 2, 7) 530 #endif 531 #if defined(HAS_ABGRTOAR30ROW_SSSE3) 532 ANY11(ABGRToAR30Row_Any_SSSE3, ABGRToAR30Row_SSSE3, 0, 4, 4, 3) 533 #endif 534 #if defined(HAS_ARGBTOAR30ROW_SSSE3) 535 ANY11(ARGBToAR30Row_Any_SSSE3, ARGBToAR30Row_SSSE3, 0, 4, 4, 3) 536 #endif 537 #if defined(HAS_ABGRTOAR30ROW_AVX2) 538 ANY11(ABGRToAR30Row_Any_AVX2, ABGRToAR30Row_AVX2, 0, 4, 4, 7) 539 #endif 540 #if defined(HAS_ARGBTOAR30ROW_AVX2) 541 ANY11(ARGBToAR30Row_Any_AVX2, ARGBToAR30Row_AVX2, 0, 4, 4, 7) 542 #endif 543 #if defined(HAS_J400TOARGBROW_SSE2) 544 ANY11(J400ToARGBRow_Any_SSE2, J400ToARGBRow_SSE2, 0, 1, 4, 7) 545 #endif 546 #if defined(HAS_J400TOARGBROW_AVX2) 547 ANY11(J400ToARGBRow_Any_AVX2, J400ToARGBRow_AVX2, 0, 1, 4, 15) 548 #endif 549 #if defined(HAS_I400TOARGBROW_SSE2) 550 ANY11(I400ToARGBRow_Any_SSE2, I400ToARGBRow_SSE2, 0, 1, 4, 7) 551 #endif 552 #if defined(HAS_I400TOARGBROW_AVX2) 553 ANY11(I400ToARGBRow_Any_AVX2, I400ToARGBRow_AVX2, 0, 1, 4, 15) 554 #endif 555 #if defined(HAS_RGB24TOARGBROW_SSSE3) 556 ANY11(RGB24ToARGBRow_Any_SSSE3, RGB24ToARGBRow_SSSE3, 0, 3, 4, 15) 557 ANY11(RAWToARGBRow_Any_SSSE3, RAWToARGBRow_SSSE3, 0, 3, 4, 15) 558 ANY11(RGB565ToARGBRow_Any_SSE2, RGB565ToARGBRow_SSE2, 0, 2, 4, 7) 559 ANY11(ARGB1555ToARGBRow_Any_SSE2, ARGB1555ToARGBRow_SSE2, 0, 2, 4, 7) 560 ANY11(ARGB4444ToARGBRow_Any_SSE2, ARGB4444ToARGBRow_SSE2, 0, 2, 4, 7) 561 #endif 562 #if defined(HAS_RAWTORGBAROW_SSSE3) 563 ANY11(RAWToRGBARow_Any_SSSE3, RAWToRGBARow_SSSE3, 0, 3, 4, 15) 564 #endif 565 #if defined(HAS_RAWTORGB24ROW_SSSE3) 566 ANY11(RAWToRGB24Row_Any_SSSE3, RAWToRGB24Row_SSSE3, 0, 3, 3, 7) 567 #endif 568 #if defined(HAS_RGB565TOARGBROW_AVX2) 569 ANY11(RGB565ToARGBRow_Any_AVX2, RGB565ToARGBRow_AVX2, 0, 2, 4, 15) 570 #endif 571 #if defined(HAS_ARGB1555TOARGBROW_AVX2) 572 ANY11(ARGB1555ToARGBRow_Any_AVX2, ARGB1555ToARGBRow_AVX2, 0, 2, 4, 15) 573 #endif 574 #if defined(HAS_ARGB4444TOARGBROW_AVX2) 575 ANY11(ARGB4444ToARGBRow_Any_AVX2, ARGB4444ToARGBRow_AVX2, 0, 2, 4, 15) 576 #endif 577 #if defined(HAS_ARGBTORGB24ROW_NEON) 578 ANY11(ARGBToRGB24Row_Any_NEON, ARGBToRGB24Row_NEON, 0, 4, 3, 7) 579 ANY11(ARGBToRAWRow_Any_NEON, ARGBToRAWRow_NEON, 0, 4, 3, 7) 580 ANY11(ARGBToRGB565Row_Any_NEON, ARGBToRGB565Row_NEON, 0, 4, 2, 7) 581 ANY11(ARGBToARGB1555Row_Any_NEON, ARGBToARGB1555Row_NEON, 0, 4, 2, 7) 582 ANY11(ARGBToARGB4444Row_Any_NEON, ARGBToARGB4444Row_NEON, 0, 4, 2, 7) 583 ANY11(J400ToARGBRow_Any_NEON, J400ToARGBRow_NEON, 0, 1, 4, 7) 584 ANY11(I400ToARGBRow_Any_NEON, I400ToARGBRow_NEON, 0, 1, 4, 7) 585 #endif 586 #if defined(HAS_ARGBTORGB24ROW_MSA) 587 ANY11(ARGBToRGB24Row_Any_MSA, ARGBToRGB24Row_MSA, 0, 4, 3, 15) 588 ANY11(ARGBToRAWRow_Any_MSA, ARGBToRAWRow_MSA, 0, 4, 3, 15) 589 ANY11(ARGBToRGB565Row_Any_MSA, ARGBToRGB565Row_MSA, 0, 4, 2, 7) 590 ANY11(ARGBToARGB1555Row_Any_MSA, ARGBToARGB1555Row_MSA, 0, 4, 2, 7) 591 ANY11(ARGBToARGB4444Row_Any_MSA, ARGBToARGB4444Row_MSA, 0, 4, 2, 7) 592 ANY11(J400ToARGBRow_Any_MSA, J400ToARGBRow_MSA, 0, 1, 4, 15) 593 ANY11(I400ToARGBRow_Any_MSA, I400ToARGBRow_MSA, 0, 1, 4, 15) 594 #endif 595 #if defined(HAS_ARGBTORGB24ROW_MMI) 596 ANY11(ARGBToRGB24Row_Any_MMI, ARGBToRGB24Row_MMI, 0, 4, 3, 3) 597 ANY11(ARGBToRAWRow_Any_MMI, ARGBToRAWRow_MMI, 0, 4, 3, 3) 598 ANY11(ARGBToRGB565Row_Any_MMI, ARGBToRGB565Row_MMI, 0, 4, 2, 3) 599 ANY11(ARGBToARGB1555Row_Any_MMI, ARGBToARGB1555Row_MMI, 0, 4, 2, 3) 600 ANY11(ARGBToARGB4444Row_Any_MMI, ARGBToARGB4444Row_MMI, 0, 4, 2, 3) 601 ANY11(J400ToARGBRow_Any_MMI, J400ToARGBRow_MMI, 0, 1, 4, 3) 602 ANY11(I400ToARGBRow_Any_MMI, I400ToARGBRow_MMI, 0, 1, 4, 7) 603 #endif 604 #if defined(HAS_RAWTORGB24ROW_NEON) 605 ANY11(RAWToRGB24Row_Any_NEON, RAWToRGB24Row_NEON, 0, 3, 3, 7) 606 #endif 607 #if defined(HAS_RAWTORGB24ROW_MSA) 608 ANY11(RAWToRGB24Row_Any_MSA, RAWToRGB24Row_MSA, 0, 3, 3, 15) 609 #endif 610 #if defined(HAS_RAWTORGB24ROW_MMI) 611 ANY11(RAWToRGB24Row_Any_MMI, RAWToRGB24Row_MMI, 0, 3, 3, 3) 612 #endif 613 #ifdef HAS_ARGBTOYROW_AVX2 614 ANY11(ARGBToYRow_Any_AVX2, ARGBToYRow_AVX2, 0, 4, 1, 31) 615 #endif 616 #ifdef HAS_ABGRTOYROW_AVX2 617 ANY11(ABGRToYRow_Any_AVX2, ABGRToYRow_AVX2, 0, 4, 1, 31) 618 #endif 619 #ifdef HAS_ARGBTOYJROW_AVX2 620 ANY11(ARGBToYJRow_Any_AVX2, ARGBToYJRow_AVX2, 0, 4, 1, 31) 621 #endif 622 #ifdef HAS_RGBATOYJROW_AVX2 623 ANY11(RGBAToYJRow_Any_AVX2, RGBAToYJRow_AVX2, 0, 4, 1, 31) 624 #endif 625 #ifdef HAS_UYVYTOYROW_AVX2 626 ANY11(UYVYToYRow_Any_AVX2, UYVYToYRow_AVX2, 0, 2, 1, 31) 627 #endif 628 #ifdef HAS_YUY2TOYROW_AVX2 629 ANY11(YUY2ToYRow_Any_AVX2, YUY2ToYRow_AVX2, 1, 4, 1, 31) 630 #endif 631 #ifdef HAS_ARGBTOYROW_SSSE3 632 ANY11(ARGBToYRow_Any_SSSE3, ARGBToYRow_SSSE3, 0, 4, 1, 15) 633 #endif 634 #ifdef HAS_BGRATOYROW_SSSE3 635 ANY11(BGRAToYRow_Any_SSSE3, BGRAToYRow_SSSE3, 0, 4, 1, 15) 636 ANY11(ABGRToYRow_Any_SSSE3, ABGRToYRow_SSSE3, 0, 4, 1, 15) 637 ANY11(RGBAToYRow_Any_SSSE3, RGBAToYRow_SSSE3, 0, 4, 1, 15) 638 ANY11(YUY2ToYRow_Any_SSE2, YUY2ToYRow_SSE2, 1, 4, 1, 15) 639 ANY11(UYVYToYRow_Any_SSE2, UYVYToYRow_SSE2, 1, 4, 1, 15) 640 #endif 641 #ifdef HAS_ARGBTOYJROW_SSSE3 642 ANY11(ARGBToYJRow_Any_SSSE3, ARGBToYJRow_SSSE3, 0, 4, 1, 15) 643 #endif 644 #ifdef HAS_RGBATOYJROW_SSSE3 645 ANY11(RGBAToYJRow_Any_SSSE3, RGBAToYJRow_SSSE3, 0, 4, 1, 15) 646 #endif 647 #ifdef HAS_ARGBTOYROW_NEON 648 ANY11(ARGBToYRow_Any_NEON, ARGBToYRow_NEON, 0, 4, 1, 7) 649 #endif 650 #ifdef HAS_ARGBTOYROW_MSA 651 ANY11(ARGBToYRow_Any_MSA, ARGBToYRow_MSA, 0, 4, 1, 15) 652 #endif 653 #ifdef HAS_ARGBTOYROW_MMI 654 ANY11(ARGBToYRow_Any_MMI, ARGBToYRow_MMI, 0, 4, 1, 7) 655 #endif 656 #ifdef HAS_ARGBTOYJROW_NEON 657 ANY11(ARGBToYJRow_Any_NEON, ARGBToYJRow_NEON, 0, 4, 1, 7) 658 #endif 659 #ifdef HAS_RGBATOYJROW_NEON 660 ANY11(RGBAToYJRow_Any_NEON, RGBAToYJRow_NEON, 0, 4, 1, 7) 661 #endif 662 #ifdef HAS_ARGBTOYJROW_MSA 663 ANY11(ARGBToYJRow_Any_MSA, ARGBToYJRow_MSA, 0, 4, 1, 15) 664 #endif 665 #ifdef HAS_ARGBTOYJROW_MMI 666 ANY11(ARGBToYJRow_Any_MMI, ARGBToYJRow_MMI, 0, 4, 1, 7) 667 #endif 668 #ifdef HAS_BGRATOYROW_NEON 669 ANY11(BGRAToYRow_Any_NEON, BGRAToYRow_NEON, 0, 4, 1, 7) 670 #endif 671 #ifdef HAS_BGRATOYROW_MSA 672 ANY11(BGRAToYRow_Any_MSA, BGRAToYRow_MSA, 0, 4, 1, 15) 673 #endif 674 #ifdef HAS_BGRATOYROW_MMI 675 ANY11(BGRAToYRow_Any_MMI, BGRAToYRow_MMI, 0, 4, 1, 7) 676 #endif 677 #ifdef HAS_ABGRTOYROW_NEON 678 ANY11(ABGRToYRow_Any_NEON, ABGRToYRow_NEON, 0, 4, 1, 7) 679 #endif 680 #ifdef HAS_ABGRTOYROW_MSA 681 ANY11(ABGRToYRow_Any_MSA, ABGRToYRow_MSA, 0, 4, 1, 7) 682 #endif 683 #ifdef HAS_ABGRTOYROW_MMI 684 ANY11(ABGRToYRow_Any_MMI, ABGRToYRow_MMI, 0, 4, 1, 7) 685 #endif 686 #ifdef HAS_RGBATOYROW_NEON 687 ANY11(RGBAToYRow_Any_NEON, RGBAToYRow_NEON, 0, 4, 1, 7) 688 #endif 689 #ifdef HAS_RGBATOYROW_MSA 690 ANY11(RGBAToYRow_Any_MSA, RGBAToYRow_MSA, 0, 4, 1, 15) 691 #endif 692 #ifdef HAS_RGBATOYROW_MMI 693 ANY11(RGBAToYRow_Any_MMI, RGBAToYRow_MMI, 0, 4, 1, 7) 694 #endif 695 #ifdef HAS_RGB24TOYROW_NEON 696 ANY11(RGB24ToYRow_Any_NEON, RGB24ToYRow_NEON, 0, 3, 1, 7) 697 #endif 698 #ifdef HAS_RGB24TOYROW_MSA 699 ANY11(RGB24ToYRow_Any_MSA, RGB24ToYRow_MSA, 0, 3, 1, 15) 700 #endif 701 #ifdef HAS_RGB24TOYROW_MMI 702 ANY11(RGB24ToYRow_Any_MMI, RGB24ToYRow_MMI, 0, 3, 1, 7) 703 #endif 704 #ifdef HAS_RAWTOYROW_NEON 705 ANY11(RAWToYRow_Any_NEON, RAWToYRow_NEON, 0, 3, 1, 7) 706 #endif 707 #ifdef HAS_RAWTOYROW_MSA 708 ANY11(RAWToYRow_Any_MSA, RAWToYRow_MSA, 0, 3, 1, 15) 709 #endif 710 #ifdef HAS_RAWTOYROW_MMI 711 ANY11(RAWToYRow_Any_MMI, RAWToYRow_MMI, 0, 3, 1, 7) 712 #endif 713 #ifdef HAS_RGB565TOYROW_NEON 714 ANY11(RGB565ToYRow_Any_NEON, RGB565ToYRow_NEON, 0, 2, 1, 7) 715 #endif 716 #ifdef HAS_RGB565TOYROW_MSA 717 ANY11(RGB565ToYRow_Any_MSA, RGB565ToYRow_MSA, 0, 2, 1, 15) 718 #endif 719 #ifdef HAS_RGB565TOYROW_MMI 720 ANY11(RGB565ToYRow_Any_MMI, RGB565ToYRow_MMI, 0, 2, 1, 7) 721 #endif 722 #ifdef HAS_ARGB1555TOYROW_NEON 723 ANY11(ARGB1555ToYRow_Any_NEON, ARGB1555ToYRow_NEON, 0, 2, 1, 7) 724 #endif 725 #ifdef HAS_ARGB1555TOYROW_MSA 726 ANY11(ARGB1555ToYRow_Any_MSA, ARGB1555ToYRow_MSA, 0, 2, 1, 15) 727 #endif 728 #ifdef HAS_ARGB1555TOYROW_MMI 729 ANY11(ARGB1555ToYRow_Any_MMI, ARGB1555ToYRow_MMI, 0, 2, 1, 7) 730 #endif 731 #ifdef HAS_ARGB4444TOYROW_NEON 732 ANY11(ARGB4444ToYRow_Any_NEON, ARGB4444ToYRow_NEON, 0, 2, 1, 7) 733 #endif 734 #ifdef HAS_ARGB4444TOYROW_MMI 735 ANY11(ARGB4444ToYRow_Any_MMI, ARGB4444ToYRow_MMI, 0, 2, 1, 7) 736 #endif 737 #ifdef HAS_YUY2TOYROW_NEON 738 ANY11(YUY2ToYRow_Any_NEON, YUY2ToYRow_NEON, 1, 4, 1, 15) 739 #endif 740 #ifdef HAS_UYVYTOYROW_NEON 741 ANY11(UYVYToYRow_Any_NEON, UYVYToYRow_NEON, 1, 4, 1, 15) 742 #endif 743 #ifdef HAS_YUY2TOYROW_MSA 744 ANY11(YUY2ToYRow_Any_MSA, YUY2ToYRow_MSA, 1, 4, 1, 31) 745 #endif 746 #ifdef HAS_YUY2TOYROW_MMI 747 ANY11(YUY2ToYRow_Any_MMI, YUY2ToYRow_MMI, 1, 4, 1, 7) 748 #endif 749 #ifdef HAS_UYVYTOYROW_MSA 750 ANY11(UYVYToYRow_Any_MSA, UYVYToYRow_MSA, 1, 4, 1, 31) 751 #endif 752 #ifdef HAS_UYVYTOYROW_MMI 753 ANY11(UYVYToYRow_Any_MMI, UYVYToYRow_MMI, 1, 4, 1, 15) 754 #endif 755 #ifdef HAS_AYUVTOYROW_NEON 756 ANY11(AYUVToYRow_Any_NEON, AYUVToYRow_NEON, 0, 4, 1, 15) 757 #endif 758 #ifdef HAS_SWAPUVROW_SSSE3 759 ANY11(SwapUVRow_Any_SSSE3, SwapUVRow_SSSE3, 0, 2, 2, 15) 760 #endif 761 #ifdef HAS_SWAPUVROW_AVX2 762 ANY11(SwapUVRow_Any_AVX2, SwapUVRow_AVX2, 0, 2, 2, 31) 763 #endif 764 #ifdef HAS_SWAPUVROW_NEON 765 ANY11(SwapUVRow_Any_NEON, SwapUVRow_NEON, 0, 2, 2, 15) 766 #endif 767 #ifdef HAS_RGB24TOARGBROW_NEON 768 ANY11(RGB24ToARGBRow_Any_NEON, RGB24ToARGBRow_NEON, 0, 3, 4, 7) 769 #endif 770 #ifdef HAS_RGB24TOARGBROW_MSA 771 ANY11(RGB24ToARGBRow_Any_MSA, RGB24ToARGBRow_MSA, 0, 3, 4, 15) 772 #endif 773 #ifdef HAS_RGB24TOARGBROW_MMI 774 ANY11(RGB24ToARGBRow_Any_MMI, RGB24ToARGBRow_MMI, 0, 3, 4, 3) 775 #endif 776 #ifdef HAS_RAWTOARGBROW_NEON 777 ANY11(RAWToARGBRow_Any_NEON, RAWToARGBRow_NEON, 0, 3, 4, 7) 778 #endif 779 #ifdef HAS_RAWTORGBAROW_NEON 780 ANY11(RAWToRGBARow_Any_NEON, RAWToRGBARow_NEON, 0, 3, 4, 7) 781 #endif 782 #ifdef HAS_RAWTOARGBROW_MSA 783 ANY11(RAWToARGBRow_Any_MSA, RAWToARGBRow_MSA, 0, 3, 4, 15) 784 #endif 785 #ifdef HAS_RAWTOARGBROW_MMI 786 ANY11(RAWToARGBRow_Any_MMI, RAWToARGBRow_MMI, 0, 3, 4, 3) 787 #endif 788 #ifdef HAS_RGB565TOARGBROW_NEON 789 ANY11(RGB565ToARGBRow_Any_NEON, RGB565ToARGBRow_NEON, 0, 2, 4, 7) 790 #endif 791 #ifdef HAS_RGB565TOARGBROW_MSA 792 ANY11(RGB565ToARGBRow_Any_MSA, RGB565ToARGBRow_MSA, 0, 2, 4, 15) 793 #endif 794 #ifdef HAS_RGB565TOARGBROW_MMI 795 ANY11(RGB565ToARGBRow_Any_MMI, RGB565ToARGBRow_MMI, 0, 2, 4, 3) 796 #endif 797 #ifdef HAS_ARGB1555TOARGBROW_NEON 798 ANY11(ARGB1555ToARGBRow_Any_NEON, ARGB1555ToARGBRow_NEON, 0, 2, 4, 7) 799 #endif 800 #ifdef HAS_ARGB1555TOARGBROW_MSA 801 ANY11(ARGB1555ToARGBRow_Any_MSA, ARGB1555ToARGBRow_MSA, 0, 2, 4, 15) 802 #endif 803 #ifdef HAS_ARGB1555TOARGBROW_MMI 804 ANY11(ARGB1555ToARGBRow_Any_MMI, ARGB1555ToARGBRow_MMI, 0, 2, 4, 3) 805 #endif 806 #ifdef HAS_ARGB4444TOARGBROW_NEON 807 ANY11(ARGB4444ToARGBRow_Any_NEON, ARGB4444ToARGBRow_NEON, 0, 2, 4, 7) 808 #endif 809 #ifdef HAS_ARGB4444TOARGBROW_MSA 810 ANY11(ARGB4444ToARGBRow_Any_MSA, ARGB4444ToARGBRow_MSA, 0, 2, 4, 15) 811 #endif 812 #ifdef HAS_ARGB4444TOARGBROW_MMI 813 ANY11(ARGB4444ToARGBRow_Any_MMI, ARGB4444ToARGBRow_MMI, 0, 2, 4, 3) 814 #endif 815 #ifdef HAS_ARGBATTENUATEROW_SSSE3 816 ANY11(ARGBAttenuateRow_Any_SSSE3, ARGBAttenuateRow_SSSE3, 0, 4, 4, 3) 817 #endif 818 #ifdef HAS_ARGBUNATTENUATEROW_SSE2 819 ANY11(ARGBUnattenuateRow_Any_SSE2, ARGBUnattenuateRow_SSE2, 0, 4, 4, 3) 820 #endif 821 #ifdef HAS_ARGBATTENUATEROW_AVX2 822 ANY11(ARGBAttenuateRow_Any_AVX2, ARGBAttenuateRow_AVX2, 0, 4, 4, 7) 823 #endif 824 #ifdef HAS_ARGBUNATTENUATEROW_AVX2 825 ANY11(ARGBUnattenuateRow_Any_AVX2, ARGBUnattenuateRow_AVX2, 0, 4, 4, 7) 826 #endif 827 #ifdef HAS_ARGBATTENUATEROW_NEON 828 ANY11(ARGBAttenuateRow_Any_NEON, ARGBAttenuateRow_NEON, 0, 4, 4, 7) 829 #endif 830 #ifdef HAS_ARGBATTENUATEROW_MSA 831 ANY11(ARGBAttenuateRow_Any_MSA, ARGBAttenuateRow_MSA, 0, 4, 4, 7) 832 #endif 833 #ifdef HAS_ARGBATTENUATEROW_MMI 834 ANY11(ARGBAttenuateRow_Any_MMI, ARGBAttenuateRow_MMI, 0, 4, 4, 1) 835 #endif 836 #ifdef HAS_ARGBEXTRACTALPHAROW_SSE2 837 ANY11(ARGBExtractAlphaRow_Any_SSE2, ARGBExtractAlphaRow_SSE2, 0, 4, 1, 7) 838 #endif 839 #ifdef HAS_ARGBEXTRACTALPHAROW_AVX2 840 ANY11(ARGBExtractAlphaRow_Any_AVX2, ARGBExtractAlphaRow_AVX2, 0, 4, 1, 31) 841 #endif 842 #ifdef HAS_ARGBEXTRACTALPHAROW_NEON 843 ANY11(ARGBExtractAlphaRow_Any_NEON, ARGBExtractAlphaRow_NEON, 0, 4, 1, 15) 844 #endif 845 #ifdef HAS_ARGBEXTRACTALPHAROW_MSA 846 ANY11(ARGBExtractAlphaRow_Any_MSA, ARGBExtractAlphaRow_MSA, 0, 4, 1, 15) 847 #endif 848 #ifdef HAS_ARGBEXTRACTALPHAROW_MMI 849 ANY11(ARGBExtractAlphaRow_Any_MMI, ARGBExtractAlphaRow_MMI, 0, 4, 1, 7) 850 #endif 851 #undef ANY11 852 853 // Any 1 to 1 blended. Destination is read, modify, write. 854 #define ANY11B(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, BPP, MASK) \ 855 void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width) { \ 856 SIMD_ALIGNED(uint8_t temp[64 * 2]); \ 857 memset(temp, 0, 64 * 2); /* for msan */ \ 858 int r = width & MASK; \ 859 int n = width & ~MASK; \ 860 if (n > 0) { \ 861 ANY_SIMD(src_ptr, dst_ptr, n); \ 862 } \ 863 memcpy(temp, src_ptr + (n >> UVSHIFT) * SBPP, SS(r, UVSHIFT) * SBPP); \ 864 memcpy(temp + 64, dst_ptr + n * BPP, r * BPP); \ 865 ANY_SIMD(temp, temp + 64, MASK + 1); \ 866 memcpy(dst_ptr + n * BPP, temp + 64, r * BPP); \ 867 } 868 869 #ifdef HAS_ARGBCOPYALPHAROW_AVX2 870 ANY11B(ARGBCopyAlphaRow_Any_AVX2, ARGBCopyAlphaRow_AVX2, 0, 4, 4, 15) 871 #endif 872 #ifdef HAS_ARGBCOPYALPHAROW_SSE2 873 ANY11B(ARGBCopyAlphaRow_Any_SSE2, ARGBCopyAlphaRow_SSE2, 0, 4, 4, 7) 874 #endif 875 #ifdef HAS_ARGBCOPYALPHAROW_MMI 876 ANY11B(ARGBCopyAlphaRow_Any_MMI, ARGBCopyAlphaRow_MMI, 0, 4, 4, 1) 877 #endif 878 #ifdef HAS_ARGBCOPYYTOALPHAROW_AVX2 879 ANY11B(ARGBCopyYToAlphaRow_Any_AVX2, ARGBCopyYToAlphaRow_AVX2, 0, 1, 4, 15) 880 #endif 881 #ifdef HAS_ARGBCOPYYTOALPHAROW_SSE2 882 ANY11B(ARGBCopyYToAlphaRow_Any_SSE2, ARGBCopyYToAlphaRow_SSE2, 0, 1, 4, 7) 883 #endif 884 #ifdef HAS_ARGBCOPYYTOALPHAROW_MMI 885 ANY11B(ARGBCopyYToAlphaRow_Any_MMI, ARGBCopyYToAlphaRow_MMI, 0, 1, 4, 7) 886 #endif 887 #undef ANY11B 888 889 // Any 1 to 1 with parameter. 890 #define ANY11P(NAMEANY, ANY_SIMD, T, SBPP, BPP, MASK) \ 891 void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, T param, int width) { \ 892 SIMD_ALIGNED(uint8_t temp[64 * 2]); \ 893 memset(temp, 0, 64); /* for msan */ \ 894 int r = width & MASK; \ 895 int n = width & ~MASK; \ 896 if (n > 0) { \ 897 ANY_SIMD(src_ptr, dst_ptr, param, n); \ 898 } \ 899 memcpy(temp, src_ptr + n * SBPP, r * SBPP); \ 900 ANY_SIMD(temp, temp + 64, param, MASK + 1); \ 901 memcpy(dst_ptr + n * BPP, temp + 64, r * BPP); \ 902 } 903 904 #if defined(HAS_ARGBTORGB565DITHERROW_SSE2) 905 ANY11P(ARGBToRGB565DitherRow_Any_SSE2, 906 ARGBToRGB565DitherRow_SSE2, 907 const uint32_t, 908 4, 909 2, 910 3) 911 #endif 912 #if defined(HAS_ARGBTORGB565DITHERROW_AVX2) 913 ANY11P(ARGBToRGB565DitherRow_Any_AVX2, 914 ARGBToRGB565DitherRow_AVX2, 915 const uint32_t, 916 4, 917 2, 918 7) 919 #endif 920 #if defined(HAS_ARGBTORGB565DITHERROW_NEON) 921 ANY11P(ARGBToRGB565DitherRow_Any_NEON, 922 ARGBToRGB565DitherRow_NEON, 923 const uint32_t, 924 4, 925 2, 926 7) 927 #endif 928 #if defined(HAS_ARGBTORGB565DITHERROW_MSA) 929 ANY11P(ARGBToRGB565DitherRow_Any_MSA, 930 ARGBToRGB565DitherRow_MSA, 931 const uint32_t, 932 4, 933 2, 934 7) 935 #endif 936 #if defined(HAS_ARGBTORGB565DITHERROW_MMI) 937 ANY11P(ARGBToRGB565DitherRow_Any_MMI, 938 ARGBToRGB565DitherRow_MMI, 939 const uint32_t, 940 4, 941 2, 942 3) 943 #endif 944 #ifdef HAS_ARGBSHUFFLEROW_SSSE3 945 ANY11P(ARGBShuffleRow_Any_SSSE3, ARGBShuffleRow_SSSE3, const uint8_t*, 4, 4, 7) 946 #endif 947 #ifdef HAS_ARGBSHUFFLEROW_AVX2 948 ANY11P(ARGBShuffleRow_Any_AVX2, ARGBShuffleRow_AVX2, const uint8_t*, 4, 4, 15) 949 #endif 950 #ifdef HAS_ARGBSHUFFLEROW_NEON 951 ANY11P(ARGBShuffleRow_Any_NEON, ARGBShuffleRow_NEON, const uint8_t*, 4, 4, 3) 952 #endif 953 #ifdef HAS_ARGBSHUFFLEROW_MSA 954 ANY11P(ARGBShuffleRow_Any_MSA, ARGBShuffleRow_MSA, const uint8_t*, 4, 4, 7) 955 #endif 956 #ifdef HAS_ARGBSHUFFLEROW_MMI 957 ANY11P(ARGBShuffleRow_Any_MMI, ARGBShuffleRow_MMI, const uint8_t*, 4, 4, 1) 958 #endif 959 #undef ANY11P 960 #undef ANY11P 961 962 // Any 1 to 1 with parameter and shorts. BPP measures in shorts. 963 #define ANY11C(NAMEANY, ANY_SIMD, SBPP, BPP, STYPE, DTYPE, MASK) \ 964 void NAMEANY(const STYPE* src_ptr, DTYPE* dst_ptr, int scale, int width) { \ 965 SIMD_ALIGNED(STYPE temp[32]); \ 966 SIMD_ALIGNED(DTYPE out[32]); \ 967 memset(temp, 0, 32 * SBPP); /* for msan */ \ 968 int r = width & MASK; \ 969 int n = width & ~MASK; \ 970 if (n > 0) { \ 971 ANY_SIMD(src_ptr, dst_ptr, scale, n); \ 972 } \ 973 memcpy(temp, src_ptr + n, r * SBPP); \ 974 ANY_SIMD(temp, out, scale, MASK + 1); \ 975 memcpy(dst_ptr + n, out, r * BPP); \ 976 } 977 978 #ifdef HAS_CONVERT16TO8ROW_SSSE3 979 ANY11C(Convert16To8Row_Any_SSSE3, 980 Convert16To8Row_SSSE3, 981 2, 982 1, 983 uint16_t, 984 uint8_t, 985 15) 986 #endif 987 #ifdef HAS_CONVERT16TO8ROW_AVX2 988 ANY11C(Convert16To8Row_Any_AVX2, 989 Convert16To8Row_AVX2, 990 2, 991 1, 992 uint16_t, 993 uint8_t, 994 31) 995 #endif 996 #ifdef HAS_CONVERT8TO16ROW_SSE2 997 ANY11C(Convert8To16Row_Any_SSE2, 998 Convert8To16Row_SSE2, 999 1, 1000 2, 1001 uint8_t, 1002 uint16_t, 1003 15) 1004 #endif 1005 #ifdef HAS_CONVERT8TO16ROW_AVX2 1006 ANY11C(Convert8To16Row_Any_AVX2, 1007 Convert8To16Row_AVX2, 1008 1, 1009 2, 1010 uint8_t, 1011 uint16_t, 1012 31) 1013 #endif 1014 #undef ANY11C 1015 1016 // Any 1 to 1 with parameter and shorts to byte. BPP measures in shorts. 1017 #define ANY11P16(NAMEANY, ANY_SIMD, ST, T, SBPP, BPP, MASK) \ 1018 void NAMEANY(const ST* src_ptr, T* dst_ptr, float param, int width) { \ 1019 SIMD_ALIGNED(ST temp[32]); \ 1020 SIMD_ALIGNED(T out[32]); \ 1021 memset(temp, 0, SBPP * 32); /* for msan */ \ 1022 int r = width & MASK; \ 1023 int n = width & ~MASK; \ 1024 if (n > 0) { \ 1025 ANY_SIMD(src_ptr, dst_ptr, param, n); \ 1026 } \ 1027 memcpy(temp, src_ptr + n, r * SBPP); \ 1028 ANY_SIMD(temp, out, param, MASK + 1); \ 1029 memcpy(dst_ptr + n, out, r * BPP); \ 1030 } 1031 1032 #ifdef HAS_HALFFLOATROW_SSE2 1033 ANY11P16(HalfFloatRow_Any_SSE2, HalfFloatRow_SSE2, uint16_t, uint16_t, 2, 2, 7) 1034 #endif 1035 #ifdef HAS_HALFFLOATROW_AVX2 1036 ANY11P16(HalfFloatRow_Any_AVX2, HalfFloatRow_AVX2, uint16_t, uint16_t, 2, 2, 15) 1037 #endif 1038 #ifdef HAS_HALFFLOATROW_F16C 1039 ANY11P16(HalfFloatRow_Any_F16C, HalfFloatRow_F16C, uint16_t, uint16_t, 2, 2, 15) 1040 ANY11P16(HalfFloat1Row_Any_F16C, 1041 HalfFloat1Row_F16C, 1042 uint16_t, 1043 uint16_t, 1044 2, 1045 2, 1046 15) 1047 #endif 1048 #ifdef HAS_HALFFLOATROW_NEON 1049 ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, uint16_t, uint16_t, 2, 2, 7) 1050 ANY11P16(HalfFloat1Row_Any_NEON, 1051 HalfFloat1Row_NEON, 1052 uint16_t, 1053 uint16_t, 1054 2, 1055 2, 1056 7) 1057 #endif 1058 #ifdef HAS_HALFFLOATROW_MSA 1059 ANY11P16(HalfFloatRow_Any_MSA, HalfFloatRow_MSA, uint16_t, uint16_t, 2, 2, 31) 1060 #endif 1061 #ifdef HAS_BYTETOFLOATROW_NEON 1062 ANY11P16(ByteToFloatRow_Any_NEON, ByteToFloatRow_NEON, uint8_t, float, 1, 3, 7) 1063 #endif 1064 #undef ANY11P16 1065 1066 // Any 1 to 1 with yuvconstants 1067 #define ANY11C(NAMEANY, ANY_SIMD, UVSHIFT, SBPP, BPP, MASK) \ 1068 void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, \ 1069 const struct YuvConstants* yuvconstants, int width) { \ 1070 SIMD_ALIGNED(uint8_t temp[128 * 2]); \ 1071 memset(temp, 0, 128); /* for YUY2 and msan */ \ 1072 int r = width & MASK; \ 1073 int n = width & ~MASK; \ 1074 if (n > 0) { \ 1075 ANY_SIMD(src_ptr, dst_ptr, yuvconstants, n); \ 1076 } \ 1077 memcpy(temp, src_ptr + (n >> UVSHIFT) * SBPP, SS(r, UVSHIFT) * SBPP); \ 1078 ANY_SIMD(temp, temp + 128, yuvconstants, MASK + 1); \ 1079 memcpy(dst_ptr + n * BPP, temp + 128, r * BPP); \ 1080 } 1081 #if defined(HAS_YUY2TOARGBROW_SSSE3) 1082 ANY11C(YUY2ToARGBRow_Any_SSSE3, YUY2ToARGBRow_SSSE3, 1, 4, 4, 15) 1083 ANY11C(UYVYToARGBRow_Any_SSSE3, UYVYToARGBRow_SSSE3, 1, 4, 4, 15) 1084 #endif 1085 #if defined(HAS_YUY2TOARGBROW_AVX2) 1086 ANY11C(YUY2ToARGBRow_Any_AVX2, YUY2ToARGBRow_AVX2, 1, 4, 4, 31) 1087 ANY11C(UYVYToARGBRow_Any_AVX2, UYVYToARGBRow_AVX2, 1, 4, 4, 31) 1088 #endif 1089 #if defined(HAS_YUY2TOARGBROW_NEON) 1090 ANY11C(YUY2ToARGBRow_Any_NEON, YUY2ToARGBRow_NEON, 1, 4, 4, 7) 1091 ANY11C(UYVYToARGBRow_Any_NEON, UYVYToARGBRow_NEON, 1, 4, 4, 7) 1092 #endif 1093 #if defined(HAS_YUY2TOARGBROW_MSA) 1094 ANY11C(YUY2ToARGBRow_Any_MSA, YUY2ToARGBRow_MSA, 1, 4, 4, 7) 1095 ANY11C(UYVYToARGBRow_Any_MSA, UYVYToARGBRow_MSA, 1, 4, 4, 7) 1096 #endif 1097 #if defined(HAS_YUY2TOARGBROW_MMI) 1098 ANY11C(YUY2ToARGBRow_Any_MMI, YUY2ToARGBRow_MMI, 1, 4, 4, 7) 1099 ANY11C(UYVYToARGBRow_Any_MMI, UYVYToARGBRow_MMI, 1, 4, 4, 7) 1100 #endif 1101 #undef ANY11C 1102 1103 // Any 1 to 1 interpolate. Takes 2 rows of source via stride. 1104 #define ANY11T(NAMEANY, ANY_SIMD, SBPP, BPP, MASK) \ 1105 void NAMEANY(uint8_t* dst_ptr, const uint8_t* src_ptr, \ 1106 ptrdiff_t src_stride_ptr, int width, int source_y_fraction) { \ 1107 SIMD_ALIGNED(uint8_t temp[64 * 3]); \ 1108 memset(temp, 0, 64 * 2); /* for msan */ \ 1109 int r = width & MASK; \ 1110 int n = width & ~MASK; \ 1111 if (n > 0) { \ 1112 ANY_SIMD(dst_ptr, src_ptr, src_stride_ptr, n, source_y_fraction); \ 1113 } \ 1114 memcpy(temp, src_ptr + n * SBPP, r * SBPP); \ 1115 memcpy(temp + 64, src_ptr + src_stride_ptr + n * SBPP, r * SBPP); \ 1116 ANY_SIMD(temp + 128, temp, 64, MASK + 1, source_y_fraction); \ 1117 memcpy(dst_ptr + n * BPP, temp + 128, r * BPP); \ 1118 } 1119 1120 #ifdef HAS_INTERPOLATEROW_AVX2 1121 ANY11T(InterpolateRow_Any_AVX2, InterpolateRow_AVX2, 1, 1, 31) 1122 #endif 1123 #ifdef HAS_INTERPOLATEROW_SSSE3 1124 ANY11T(InterpolateRow_Any_SSSE3, InterpolateRow_SSSE3, 1, 1, 15) 1125 #endif 1126 #ifdef HAS_INTERPOLATEROW_NEON 1127 ANY11T(InterpolateRow_Any_NEON, InterpolateRow_NEON, 1, 1, 15) 1128 #endif 1129 #ifdef HAS_INTERPOLATEROW_MSA 1130 ANY11T(InterpolateRow_Any_MSA, InterpolateRow_MSA, 1, 1, 31) 1131 #endif 1132 #ifdef HAS_INTERPOLATEROW_MMI 1133 ANY11T(InterpolateRow_Any_MMI, InterpolateRow_MMI, 1, 1, 7) 1134 #endif 1135 #undef ANY11T 1136 1137 // Any 1 to 1 mirror. 1138 #define ANY11M(NAMEANY, ANY_SIMD, BPP, MASK) \ 1139 void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_ptr, int width) { \ 1140 SIMD_ALIGNED(uint8_t temp[64 * 2]); \ 1141 memset(temp, 0, 64); /* for msan */ \ 1142 int r = width & MASK; \ 1143 int n = width & ~MASK; \ 1144 if (n > 0) { \ 1145 ANY_SIMD(src_ptr + r * BPP, dst_ptr, n); \ 1146 } \ 1147 memcpy(temp, src_ptr, r* BPP); \ 1148 ANY_SIMD(temp, temp + 64, MASK + 1); \ 1149 memcpy(dst_ptr + n * BPP, temp + 64 + (MASK + 1 - r) * BPP, r * BPP); \ 1150 } 1151 1152 #ifdef HAS_MIRRORROW_AVX2 1153 ANY11M(MirrorRow_Any_AVX2, MirrorRow_AVX2, 1, 31) 1154 #endif 1155 #ifdef HAS_MIRRORROW_SSSE3 1156 ANY11M(MirrorRow_Any_SSSE3, MirrorRow_SSSE3, 1, 15) 1157 #endif 1158 #ifdef HAS_MIRRORROW_NEON 1159 ANY11M(MirrorRow_Any_NEON, MirrorRow_NEON, 1, 15) 1160 #endif 1161 #ifdef HAS_MIRRORROW_MSA 1162 ANY11M(MirrorRow_Any_MSA, MirrorRow_MSA, 1, 63) 1163 #endif 1164 #ifdef HAS_MIRRORROW_MMI 1165 ANY11M(MirrorRow_Any_MMI, MirrorRow_MMI, 1, 7) 1166 #endif 1167 #ifdef HAS_ARGBMIRRORROW_AVX2 1168 ANY11M(ARGBMirrorRow_Any_AVX2, ARGBMirrorRow_AVX2, 4, 7) 1169 #endif 1170 #ifdef HAS_ARGBMIRRORROW_SSE2 1171 ANY11M(ARGBMirrorRow_Any_SSE2, ARGBMirrorRow_SSE2, 4, 3) 1172 #endif 1173 #ifdef HAS_ARGBMIRRORROW_NEON 1174 ANY11M(ARGBMirrorRow_Any_NEON, ARGBMirrorRow_NEON, 4, 3) 1175 #endif 1176 #ifdef HAS_ARGBMIRRORROW_MSA 1177 ANY11M(ARGBMirrorRow_Any_MSA, ARGBMirrorRow_MSA, 4, 15) 1178 #endif 1179 #ifdef HAS_ARGBMIRRORROW_MMI 1180 ANY11M(ARGBMirrorRow_Any_MMI, ARGBMirrorRow_MMI, 4, 1) 1181 #endif 1182 #undef ANY11M 1183 1184 // Any 1 plane. (memset) 1185 #define ANY1(NAMEANY, ANY_SIMD, T, BPP, MASK) \ 1186 void NAMEANY(uint8_t* dst_ptr, T v32, int width) { \ 1187 SIMD_ALIGNED(uint8_t temp[64]); \ 1188 int r = width & MASK; \ 1189 int n = width & ~MASK; \ 1190 if (n > 0) { \ 1191 ANY_SIMD(dst_ptr, v32, n); \ 1192 } \ 1193 ANY_SIMD(temp, v32, MASK + 1); \ 1194 memcpy(dst_ptr + n * BPP, temp, r * BPP); \ 1195 } 1196 1197 #ifdef HAS_SETROW_X86 1198 ANY1(SetRow_Any_X86, SetRow_X86, uint8_t, 1, 3) 1199 #endif 1200 #ifdef HAS_SETROW_NEON 1201 ANY1(SetRow_Any_NEON, SetRow_NEON, uint8_t, 1, 15) 1202 #endif 1203 #ifdef HAS_ARGBSETROW_NEON 1204 ANY1(ARGBSetRow_Any_NEON, ARGBSetRow_NEON, uint32_t, 4, 3) 1205 #endif 1206 #ifdef HAS_ARGBSETROW_MSA 1207 ANY1(ARGBSetRow_Any_MSA, ARGBSetRow_MSA, uint32_t, 4, 3) 1208 #endif 1209 #ifdef HAS_ARGBSETROW_MMI 1210 ANY1(ARGBSetRow_Any_MMI, ARGBSetRow_MMI, uint32_t, 4, 3) 1211 #endif 1212 #undef ANY1 1213 1214 // Any 1 to 2. Outputs UV planes. 1215 #define ANY12(NAMEANY, ANY_SIMD, UVSHIFT, BPP, DUVSHIFT, MASK) \ 1216 void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_u, uint8_t* dst_v, \ 1217 int width) { \ 1218 SIMD_ALIGNED(uint8_t temp[128 * 3]); \ 1219 memset(temp, 0, 128); /* for msan */ \ 1220 int r = width & MASK; \ 1221 int n = width & ~MASK; \ 1222 if (n > 0) { \ 1223 ANY_SIMD(src_ptr, dst_u, dst_v, n); \ 1224 } \ 1225 memcpy(temp, src_ptr + (n >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \ 1226 ANY_SIMD(temp, temp + 128, temp + 256, MASK + 1); \ 1227 memcpy(dst_u + (n >> DUVSHIFT), temp + 128, SS(r, DUVSHIFT)); \ 1228 memcpy(dst_v + (n >> DUVSHIFT), temp + 256, SS(r, DUVSHIFT)); \ 1229 } 1230 1231 #ifdef HAS_SPLITUVROW_SSE2 1232 ANY12(SplitUVRow_Any_SSE2, SplitUVRow_SSE2, 0, 2, 0, 15) 1233 #endif 1234 #ifdef HAS_SPLITUVROW_AVX2 1235 ANY12(SplitUVRow_Any_AVX2, SplitUVRow_AVX2, 0, 2, 0, 31) 1236 #endif 1237 #ifdef HAS_SPLITUVROW_NEON 1238 ANY12(SplitUVRow_Any_NEON, SplitUVRow_NEON, 0, 2, 0, 15) 1239 #endif 1240 #ifdef HAS_SPLITUVROW_MSA 1241 ANY12(SplitUVRow_Any_MSA, SplitUVRow_MSA, 0, 2, 0, 31) 1242 #endif 1243 #ifdef HAS_SPLITUVROW_MMI 1244 ANY12(SplitUVRow_Any_MMI, SplitUVRow_MMI, 0, 2, 0, 7) 1245 #endif 1246 #ifdef HAS_ARGBTOUV444ROW_SSSE3 1247 ANY12(ARGBToUV444Row_Any_SSSE3, ARGBToUV444Row_SSSE3, 0, 4, 0, 15) 1248 #endif 1249 #ifdef HAS_YUY2TOUV422ROW_AVX2 1250 ANY12(YUY2ToUV422Row_Any_AVX2, YUY2ToUV422Row_AVX2, 1, 4, 1, 31) 1251 ANY12(UYVYToUV422Row_Any_AVX2, UYVYToUV422Row_AVX2, 1, 4, 1, 31) 1252 #endif 1253 #ifdef HAS_YUY2TOUV422ROW_SSE2 1254 ANY12(YUY2ToUV422Row_Any_SSE2, YUY2ToUV422Row_SSE2, 1, 4, 1, 15) 1255 ANY12(UYVYToUV422Row_Any_SSE2, UYVYToUV422Row_SSE2, 1, 4, 1, 15) 1256 #endif 1257 #ifdef HAS_YUY2TOUV422ROW_NEON 1258 ANY12(ARGBToUV444Row_Any_NEON, ARGBToUV444Row_NEON, 0, 4, 0, 7) 1259 ANY12(YUY2ToUV422Row_Any_NEON, YUY2ToUV422Row_NEON, 1, 4, 1, 15) 1260 ANY12(UYVYToUV422Row_Any_NEON, UYVYToUV422Row_NEON, 1, 4, 1, 15) 1261 #endif 1262 #ifdef HAS_YUY2TOUV422ROW_MSA 1263 ANY12(ARGBToUV444Row_Any_MSA, ARGBToUV444Row_MSA, 0, 4, 0, 15) 1264 ANY12(YUY2ToUV422Row_Any_MSA, YUY2ToUV422Row_MSA, 1, 4, 1, 31) 1265 ANY12(UYVYToUV422Row_Any_MSA, UYVYToUV422Row_MSA, 1, 4, 1, 31) 1266 #endif 1267 #ifdef HAS_YUY2TOUV422ROW_MMI 1268 ANY12(ARGBToUV444Row_Any_MMI, ARGBToUV444Row_MMI, 0, 4, 0, 7) 1269 ANY12(UYVYToUV422Row_Any_MMI, UYVYToUV422Row_MMI, 1, 4, 1, 15) 1270 ANY12(YUY2ToUV422Row_Any_MMI, YUY2ToUV422Row_MMI, 1, 4, 1, 15) 1271 #endif 1272 #undef ANY12 1273 1274 // Any 1 to 3. Outputs RGB planes. 1275 #define ANY13(NAMEANY, ANY_SIMD, BPP, MASK) \ 1276 void NAMEANY(const uint8_t* src_ptr, uint8_t* dst_r, uint8_t* dst_g, \ 1277 uint8_t* dst_b, int width) { \ 1278 SIMD_ALIGNED(uint8_t temp[16 * 6]); \ 1279 memset(temp, 0, 16 * 3); /* for msan */ \ 1280 int r = width & MASK; \ 1281 int n = width & ~MASK; \ 1282 if (n > 0) { \ 1283 ANY_SIMD(src_ptr, dst_r, dst_g, dst_b, n); \ 1284 } \ 1285 memcpy(temp, src_ptr + n * BPP, r * BPP); \ 1286 ANY_SIMD(temp, temp + 16 * 3, temp + 16 * 4, temp + 16 * 5, MASK + 1); \ 1287 memcpy(dst_r + n, temp + 16 * 3, r); \ 1288 memcpy(dst_g + n, temp + 16 * 4, r); \ 1289 memcpy(dst_b + n, temp + 16 * 5, r); \ 1290 } 1291 1292 #ifdef HAS_SPLITRGBROW_SSSE3 1293 ANY13(SplitRGBRow_Any_SSSE3, SplitRGBRow_SSSE3, 3, 15) 1294 #endif 1295 #ifdef HAS_SPLITRGBROW_NEON 1296 ANY13(SplitRGBRow_Any_NEON, SplitRGBRow_NEON, 3, 15) 1297 #endif 1298 #ifdef HAS_SPLITRGBROW_MMI 1299 ANY13(SplitRGBRow_Any_MMI, SplitRGBRow_MMI, 3, 3) 1300 #endif 1301 1302 // Any 1 to 2 with source stride (2 rows of source). Outputs UV planes. 1303 // 128 byte row allows for 32 avx ARGB pixels. 1304 #define ANY12S(NAMEANY, ANY_SIMD, UVSHIFT, BPP, MASK) \ 1305 void NAMEANY(const uint8_t* src_ptr, int src_stride_ptr, uint8_t* dst_u, \ 1306 uint8_t* dst_v, int width) { \ 1307 SIMD_ALIGNED(uint8_t temp[128 * 4]); \ 1308 memset(temp, 0, 128 * 2); /* for msan */ \ 1309 int r = width & MASK; \ 1310 int n = width & ~MASK; \ 1311 if (n > 0) { \ 1312 ANY_SIMD(src_ptr, src_stride_ptr, dst_u, dst_v, n); \ 1313 } \ 1314 memcpy(temp, src_ptr + (n >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \ 1315 memcpy(temp + 128, src_ptr + src_stride_ptr + (n >> UVSHIFT) * BPP, \ 1316 SS(r, UVSHIFT) * BPP); \ 1317 if ((width & 1) && UVSHIFT == 0) { /* repeat last pixel for subsample */ \ 1318 memcpy(temp + SS(r, UVSHIFT) * BPP, temp + SS(r, UVSHIFT) * BPP - BPP, \ 1319 BPP); \ 1320 memcpy(temp + 128 + SS(r, UVSHIFT) * BPP, \ 1321 temp + 128 + SS(r, UVSHIFT) * BPP - BPP, BPP); \ 1322 } \ 1323 ANY_SIMD(temp, 128, temp + 256, temp + 384, MASK + 1); \ 1324 memcpy(dst_u + (n >> 1), temp + 256, SS(r, 1)); \ 1325 memcpy(dst_v + (n >> 1), temp + 384, SS(r, 1)); \ 1326 } 1327 1328 #ifdef HAS_ARGBTOUVROW_AVX2 1329 ANY12S(ARGBToUVRow_Any_AVX2, ARGBToUVRow_AVX2, 0, 4, 31) 1330 #endif 1331 #ifdef HAS_ABGRTOUVROW_AVX2 1332 ANY12S(ABGRToUVRow_Any_AVX2, ABGRToUVRow_AVX2, 0, 4, 31) 1333 #endif 1334 #ifdef HAS_ARGBTOUVJROW_AVX2 1335 ANY12S(ARGBToUVJRow_Any_AVX2, ARGBToUVJRow_AVX2, 0, 4, 31) 1336 #endif 1337 #ifdef HAS_ARGBTOUVROW_SSSE3 1338 ANY12S(ARGBToUVRow_Any_SSSE3, ARGBToUVRow_SSSE3, 0, 4, 15) 1339 ANY12S(ARGBToUVJRow_Any_SSSE3, ARGBToUVJRow_SSSE3, 0, 4, 15) 1340 ANY12S(BGRAToUVRow_Any_SSSE3, BGRAToUVRow_SSSE3, 0, 4, 15) 1341 ANY12S(ABGRToUVRow_Any_SSSE3, ABGRToUVRow_SSSE3, 0, 4, 15) 1342 ANY12S(RGBAToUVRow_Any_SSSE3, RGBAToUVRow_SSSE3, 0, 4, 15) 1343 #endif 1344 #ifdef HAS_YUY2TOUVROW_AVX2 1345 ANY12S(YUY2ToUVRow_Any_AVX2, YUY2ToUVRow_AVX2, 1, 4, 31) 1346 ANY12S(UYVYToUVRow_Any_AVX2, UYVYToUVRow_AVX2, 1, 4, 31) 1347 #endif 1348 #ifdef HAS_YUY2TOUVROW_SSE2 1349 ANY12S(YUY2ToUVRow_Any_SSE2, YUY2ToUVRow_SSE2, 1, 4, 15) 1350 ANY12S(UYVYToUVRow_Any_SSE2, UYVYToUVRow_SSE2, 1, 4, 15) 1351 #endif 1352 #ifdef HAS_ARGBTOUVROW_NEON 1353 ANY12S(ARGBToUVRow_Any_NEON, ARGBToUVRow_NEON, 0, 4, 15) 1354 #endif 1355 #ifdef HAS_ARGBTOUVROW_MSA 1356 ANY12S(ARGBToUVRow_Any_MSA, ARGBToUVRow_MSA, 0, 4, 31) 1357 #endif 1358 #ifdef HAS_ARGBTOUVROW_MMI 1359 ANY12S(ARGBToUVRow_Any_MMI, ARGBToUVRow_MMI, 0, 4, 15) 1360 #endif 1361 #ifdef HAS_ARGBTOUVJROW_NEON 1362 ANY12S(ARGBToUVJRow_Any_NEON, ARGBToUVJRow_NEON, 0, 4, 15) 1363 #endif 1364 #ifdef HAS_ARGBTOUVJROW_MSA 1365 ANY12S(ARGBToUVJRow_Any_MSA, ARGBToUVJRow_MSA, 0, 4, 31) 1366 #endif 1367 #ifdef HAS_ARGBTOUVJROW_MMI 1368 ANY12S(ARGBToUVJRow_Any_MMI, ARGBToUVJRow_MMI, 0, 4, 15) 1369 #endif 1370 #ifdef HAS_BGRATOUVROW_NEON 1371 ANY12S(BGRAToUVRow_Any_NEON, BGRAToUVRow_NEON, 0, 4, 15) 1372 #endif 1373 #ifdef HAS_BGRATOUVROW_MSA 1374 ANY12S(BGRAToUVRow_Any_MSA, BGRAToUVRow_MSA, 0, 4, 31) 1375 #endif 1376 #ifdef HAS_BGRATOUVROW_MMI 1377 ANY12S(BGRAToUVRow_Any_MMI, BGRAToUVRow_MMI, 0, 4, 15) 1378 #endif 1379 #ifdef HAS_ABGRTOUVROW_NEON 1380 ANY12S(ABGRToUVRow_Any_NEON, ABGRToUVRow_NEON, 0, 4, 15) 1381 #endif 1382 #ifdef HAS_ABGRTOUVROW_MSA 1383 ANY12S(ABGRToUVRow_Any_MSA, ABGRToUVRow_MSA, 0, 4, 31) 1384 #endif 1385 #ifdef HAS_ABGRTOUVROW_MMI 1386 ANY12S(ABGRToUVRow_Any_MMI, ABGRToUVRow_MMI, 0, 4, 15) 1387 #endif 1388 #ifdef HAS_RGBATOUVROW_NEON 1389 ANY12S(RGBAToUVRow_Any_NEON, RGBAToUVRow_NEON, 0, 4, 15) 1390 #endif 1391 #ifdef HAS_RGBATOUVROW_MSA 1392 ANY12S(RGBAToUVRow_Any_MSA, RGBAToUVRow_MSA, 0, 4, 31) 1393 #endif 1394 #ifdef HAS_RGBATOUVROW_MMI 1395 ANY12S(RGBAToUVRow_Any_MMI, RGBAToUVRow_MMI, 0, 4, 15) 1396 #endif 1397 #ifdef HAS_RGB24TOUVROW_NEON 1398 ANY12S(RGB24ToUVRow_Any_NEON, RGB24ToUVRow_NEON, 0, 3, 15) 1399 #endif 1400 #ifdef HAS_RGB24TOUVROW_MSA 1401 ANY12S(RGB24ToUVRow_Any_MSA, RGB24ToUVRow_MSA, 0, 3, 15) 1402 #endif 1403 #ifdef HAS_RGB24TOUVROW_MMI 1404 ANY12S(RGB24ToUVRow_Any_MMI, RGB24ToUVRow_MMI, 0, 3, 15) 1405 #endif 1406 #ifdef HAS_RAWTOUVROW_NEON 1407 ANY12S(RAWToUVRow_Any_NEON, RAWToUVRow_NEON, 0, 3, 15) 1408 #endif 1409 #ifdef HAS_RAWTOUVROW_MSA 1410 ANY12S(RAWToUVRow_Any_MSA, RAWToUVRow_MSA, 0, 3, 15) 1411 #endif 1412 #ifdef HAS_RAWTOUVROW_MMI 1413 ANY12S(RAWToUVRow_Any_MMI, RAWToUVRow_MMI, 0, 3, 15) 1414 #endif 1415 #ifdef HAS_RGB565TOUVROW_NEON 1416 ANY12S(RGB565ToUVRow_Any_NEON, RGB565ToUVRow_NEON, 0, 2, 15) 1417 #endif 1418 #ifdef HAS_RGB565TOUVROW_MSA 1419 ANY12S(RGB565ToUVRow_Any_MSA, RGB565ToUVRow_MSA, 0, 2, 15) 1420 #endif 1421 #ifdef HAS_RGB565TOUVROW_MMI 1422 ANY12S(RGB565ToUVRow_Any_MMI, RGB565ToUVRow_MMI, 0, 2, 15) 1423 #endif 1424 #ifdef HAS_ARGB1555TOUVROW_NEON 1425 ANY12S(ARGB1555ToUVRow_Any_NEON, ARGB1555ToUVRow_NEON, 0, 2, 15) 1426 #endif 1427 #ifdef HAS_ARGB1555TOUVROW_MSA 1428 ANY12S(ARGB1555ToUVRow_Any_MSA, ARGB1555ToUVRow_MSA, 0, 2, 15) 1429 #endif 1430 #ifdef HAS_ARGB1555TOUVROW_MMI 1431 ANY12S(ARGB1555ToUVRow_Any_MMI, ARGB1555ToUVRow_MMI, 0, 2, 15) 1432 #endif 1433 #ifdef HAS_ARGB4444TOUVROW_NEON 1434 ANY12S(ARGB4444ToUVRow_Any_NEON, ARGB4444ToUVRow_NEON, 0, 2, 15) 1435 #endif 1436 #ifdef HAS_ARGB4444TOUVROW_MMI 1437 ANY12S(ARGB4444ToUVRow_Any_MMI, ARGB4444ToUVRow_MMI, 0, 2, 15) 1438 #endif 1439 #ifdef HAS_YUY2TOUVROW_NEON 1440 ANY12S(YUY2ToUVRow_Any_NEON, YUY2ToUVRow_NEON, 1, 4, 15) 1441 #endif 1442 #ifdef HAS_UYVYTOUVROW_NEON 1443 ANY12S(UYVYToUVRow_Any_NEON, UYVYToUVRow_NEON, 1, 4, 15) 1444 #endif 1445 #ifdef HAS_YUY2TOUVROW_MSA 1446 ANY12S(YUY2ToUVRow_Any_MSA, YUY2ToUVRow_MSA, 1, 4, 31) 1447 #endif 1448 #ifdef HAS_YUY2TOUVROW_MMI 1449 ANY12S(YUY2ToUVRow_Any_MMI, YUY2ToUVRow_MMI, 1, 4, 15) 1450 #endif 1451 #ifdef HAS_UYVYTOUVROW_MSA 1452 ANY12S(UYVYToUVRow_Any_MSA, UYVYToUVRow_MSA, 1, 4, 31) 1453 #endif 1454 #ifdef HAS_UYVYTOUVROW_MMI 1455 ANY12S(UYVYToUVRow_Any_MMI, UYVYToUVRow_MMI, 1, 4, 15) 1456 #endif 1457 #undef ANY12S 1458 1459 // Any 1 to 1 with source stride (2 rows of source). Outputs UV plane. 1460 // 128 byte row allows for 32 avx ARGB pixels. 1461 #define ANY11S(NAMEANY, ANY_SIMD, UVSHIFT, BPP, MASK) \ 1462 void NAMEANY(const uint8_t* src_ptr, int src_stride_ptr, uint8_t* dst_vu, \ 1463 int width) { \ 1464 SIMD_ALIGNED(uint8_t temp[128 * 3]); \ 1465 memset(temp, 0, 128 * 2); /* for msan */ \ 1466 int r = width & MASK; \ 1467 int n = width & ~MASK; \ 1468 if (n > 0) { \ 1469 ANY_SIMD(src_ptr, src_stride_ptr, dst_vu, n); \ 1470 } \ 1471 memcpy(temp, src_ptr + (n >> UVSHIFT) * BPP, SS(r, UVSHIFT) * BPP); \ 1472 memcpy(temp + 128, src_ptr + src_stride_ptr + (n >> UVSHIFT) * BPP, \ 1473 SS(r, UVSHIFT) * BPP); \ 1474 if ((width & 1) && UVSHIFT == 0) { /* repeat last pixel for subsample */ \ 1475 memcpy(temp + SS(r, UVSHIFT) * BPP, temp + SS(r, UVSHIFT) * BPP - BPP, \ 1476 BPP); \ 1477 memcpy(temp + 128 + SS(r, UVSHIFT) * BPP, \ 1478 temp + 128 + SS(r, UVSHIFT) * BPP - BPP, BPP); \ 1479 } \ 1480 ANY_SIMD(temp, 128, temp + 256, MASK + 1); \ 1481 memcpy(dst_vu + (n >> 1) * 2, temp + 256, SS(r, 1) * 2); \ 1482 } 1483 1484 #ifdef HAS_AYUVTOVUROW_NEON 1485 ANY11S(AYUVToUVRow_Any_NEON, AYUVToUVRow_NEON, 0, 4, 15) 1486 ANY11S(AYUVToVURow_Any_NEON, AYUVToVURow_NEON, 0, 4, 15) 1487 #endif 1488 #undef ANY11S 1489 1490 #ifdef __cplusplus 1491 } // extern "C" 1492 } // namespace libyuv 1493 #endif 1494