1 // Copyright 2016 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // MSA common macros
11 //
12 // Author(s): Prashant Patil (prashant.patil@imgtec.com)
13
14 #ifndef WEBP_DSP_MSA_MACRO_H_
15 #define WEBP_DSP_MSA_MACRO_H_
16
17 #include <stdint.h>
18 #include <msa.h>
19
20 #if defined(__clang__)
21 #define CLANG_BUILD
22 #endif
23
24 #ifdef CLANG_BUILD
25 #define ALPHAVAL (-1)
26 #define ADDVI_H(a, b) __msa_addvi_h((v8i16)a, b)
27 #define ADDVI_W(a, b) __msa_addvi_w((v4i32)a, b)
28 #define SRAI_B(a, b) __msa_srai_b((v16i8)a, b)
29 #define SRAI_H(a, b) __msa_srai_h((v8i16)a, b)
30 #define SRAI_W(a, b) __msa_srai_w((v4i32)a, b)
31 #define SRLI_H(a, b) __msa_srli_h((v8i16)a, b)
32 #define SLLI_B(a, b) __msa_slli_b((v4i32)a, b)
33 #define ANDI_B(a, b) __msa_andi_b((v16u8)a, b)
34 #define ORI_B(a, b) __msa_ori_b((v16u8)a, b)
35 #else
36 #define ALPHAVAL (0xff)
37 #define ADDVI_H(a, b) (a + b)
38 #define ADDVI_W(a, b) (a + b)
39 #define SRAI_B(a, b) (a >> b)
40 #define SRAI_H(a, b) (a >> b)
41 #define SRAI_W(a, b) (a >> b)
42 #define SRLI_H(a, b) (a << b)
43 #define SLLI_B(a, b) (a << b)
44 #define ANDI_B(a, b) (a & b)
45 #define ORI_B(a, b) (a | b)
46 #endif
47
48 #define LD_B(RTYPE, psrc) *((RTYPE*)(psrc))
49 #define LD_UB(...) LD_B(v16u8, __VA_ARGS__)
50 #define LD_SB(...) LD_B(v16i8, __VA_ARGS__)
51
52 #define LD_H(RTYPE, psrc) *((RTYPE*)(psrc))
53 #define LD_UH(...) LD_H(v8u16, __VA_ARGS__)
54 #define LD_SH(...) LD_H(v8i16, __VA_ARGS__)
55
56 #define LD_W(RTYPE, psrc) *((RTYPE*)(psrc))
57 #define LD_UW(...) LD_W(v4u32, __VA_ARGS__)
58 #define LD_SW(...) LD_W(v4i32, __VA_ARGS__)
59
60 #define ST_B(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in
61 #define ST_UB(...) ST_B(v16u8, __VA_ARGS__)
62 #define ST_SB(...) ST_B(v16i8, __VA_ARGS__)
63
64 #define ST_H(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in
65 #define ST_UH(...) ST_H(v8u16, __VA_ARGS__)
66 #define ST_SH(...) ST_H(v8i16, __VA_ARGS__)
67
68 #define ST_W(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in
69 #define ST_UW(...) ST_W(v4u32, __VA_ARGS__)
70 #define ST_SW(...) ST_W(v4i32, __VA_ARGS__)
71
72 #define MSA_LOAD_FUNC(TYPE, INSTR, FUNC_NAME) \
73 static inline TYPE FUNC_NAME(const void* const psrc) { \
74 const uint8_t* const psrc_m = (const uint8_t*)psrc; \
75 TYPE val_m; \
76 asm volatile ( \
77 "" #INSTR " %[val_m], %[psrc_m] \n\t" \
78 : [val_m] "=r" (val_m) \
79 : [psrc_m] "m" (*psrc_m)); \
80 return val_m; \
81 }
82
83 #define MSA_LOAD(psrc, FUNC_NAME) FUNC_NAME(psrc)
84
85 #define MSA_STORE_FUNC(TYPE, INSTR, FUNC_NAME) \
86 static inline void FUNC_NAME(TYPE val, void* const pdst) { \
87 uint8_t* const pdst_m = (uint8_t*)pdst; \
88 TYPE val_m = val; \
89 asm volatile ( \
90 " " #INSTR " %[val_m], %[pdst_m] \n\t" \
91 : [pdst_m] "=m" (*pdst_m) \
92 : [val_m] "r" (val_m)); \
93 }
94
95 #define MSA_STORE(val, pdst, FUNC_NAME) FUNC_NAME(val, pdst)
96
97 #if (__mips_isa_rev >= 6)
98 MSA_LOAD_FUNC(uint16_t, lh, msa_lh);
99 #define LH(psrc) MSA_LOAD(psrc, msa_lh)
100 MSA_LOAD_FUNC(uint32_t, lw, msa_lw);
101 #define LW(psrc) MSA_LOAD(psrc, msa_lw)
102 #if (__mips == 64)
103 MSA_LOAD_FUNC(uint64_t, ld, msa_ld);
104 #define LD(psrc) MSA_LOAD(psrc, msa_ld)
105 #else // !(__mips == 64)
106 #define LD(psrc) ((((uint64_t)MSA_LOAD(psrc + 4, msa_lw)) << 32) | \
107 MSA_LOAD(psrc, msa_lw))
108 #endif // (__mips == 64)
109
110 MSA_STORE_FUNC(uint16_t, sh, msa_sh);
111 #define SH(val, pdst) MSA_STORE(val, pdst, msa_sh)
112 MSA_STORE_FUNC(uint32_t, sw, msa_sw);
113 #define SW(val, pdst) MSA_STORE(val, pdst, msa_sw)
114 MSA_STORE_FUNC(uint64_t, sd, msa_sd);
115 #define SD(val, pdst) MSA_STORE(val, pdst, msa_sd)
116 #else // !(__mips_isa_rev >= 6)
117 MSA_LOAD_FUNC(uint16_t, ulh, msa_ulh);
118 #define LH(psrc) MSA_LOAD(psrc, msa_ulh)
119 MSA_LOAD_FUNC(uint32_t, ulw, msa_ulw);
120 #define LW(psrc) MSA_LOAD(psrc, msa_ulw)
121 #if (__mips == 64)
122 MSA_LOAD_FUNC(uint64_t, uld, msa_uld);
123 #define LD(psrc) MSA_LOAD(psrc, msa_uld)
124 #else // !(__mips == 64)
125 #define LD(psrc) ((((uint64_t)MSA_LOAD(psrc + 4, msa_ulw)) << 32) | \
126 MSA_LOAD(psrc, msa_ulw))
127 #endif // (__mips == 64)
128
129 MSA_STORE_FUNC(uint16_t, ush, msa_ush);
130 #define SH(val, pdst) MSA_STORE(val, pdst, msa_ush)
131 MSA_STORE_FUNC(uint32_t, usw, msa_usw);
132 #define SW(val, pdst) MSA_STORE(val, pdst, msa_usw)
133 #define SD(val, pdst) do { \
134 uint8_t* const pdst_sd_m = (uint8_t*)(pdst); \
135 const uint32_t val0_m = (uint32_t)(val & 0x00000000FFFFFFFF); \
136 const uint32_t val1_m = (uint32_t)((val >> 32) & 0x00000000FFFFFFFF); \
137 SW(val0_m, pdst_sd_m); \
138 SW(val1_m, pdst_sd_m + 4); \
139 } while (0)
140 #endif // (__mips_isa_rev >= 6)
141
142 /* Description : Load 4 words with stride
143 * Arguments : Inputs - psrc, stride
144 * Outputs - out0, out1, out2, out3
145 * Details : Load word in 'out0' from (psrc)
146 * Load word in 'out1' from (psrc + stride)
147 * Load word in 'out2' from (psrc + 2 * stride)
148 * Load word in 'out3' from (psrc + 3 * stride)
149 */
150 #define LW4(psrc, stride, out0, out1, out2, out3) do { \
151 const uint8_t* ptmp = (const uint8_t*)psrc; \
152 out0 = LW(ptmp); \
153 ptmp += stride; \
154 out1 = LW(ptmp); \
155 ptmp += stride; \
156 out2 = LW(ptmp); \
157 ptmp += stride; \
158 out3 = LW(ptmp); \
159 } while (0)
160
161 /* Description : Store words with stride
162 * Arguments : Inputs - in0, in1, in2, in3, pdst, stride
163 * Details : Store word from 'in0' to (pdst)
164 * Store word from 'in1' to (pdst + stride)
165 * Store word from 'in2' to (pdst + 2 * stride)
166 * Store word from 'in3' to (pdst + 3 * stride)
167 */
168 #define SW4(in0, in1, in2, in3, pdst, stride) do { \
169 uint8_t* ptmp = (uint8_t*)pdst; \
170 SW(in0, ptmp); \
171 ptmp += stride; \
172 SW(in1, ptmp); \
173 ptmp += stride; \
174 SW(in2, ptmp); \
175 ptmp += stride; \
176 SW(in3, ptmp); \
177 } while (0)
178
179 #define SW3(in0, in1, in2, pdst, stride) do { \
180 uint8_t* ptmp = (uint8_t*)pdst; \
181 SW(in0, ptmp); \
182 ptmp += stride; \
183 SW(in1, ptmp); \
184 ptmp += stride; \
185 SW(in2, ptmp); \
186 } while (0)
187
188 #define SW2(in0, in1, pdst, stride) do { \
189 uint8_t* ptmp = (uint8_t*)pdst; \
190 SW(in0, ptmp); \
191 ptmp += stride; \
192 SW(in1, ptmp); \
193 } while (0)
194
195 /* Description : Store 4 double words with stride
196 * Arguments : Inputs - in0, in1, in2, in3, pdst, stride
197 * Details : Store double word from 'in0' to (pdst)
198 * Store double word from 'in1' to (pdst + stride)
199 * Store double word from 'in2' to (pdst + 2 * stride)
200 * Store double word from 'in3' to (pdst + 3 * stride)
201 */
202 #define SD4(in0, in1, in2, in3, pdst, stride) do { \
203 uint8_t* ptmp = (uint8_t*)pdst; \
204 SD(in0, ptmp); \
205 ptmp += stride; \
206 SD(in1, ptmp); \
207 ptmp += stride; \
208 SD(in2, ptmp); \
209 ptmp += stride; \
210 SD(in3, ptmp); \
211 } while (0)
212
213 /* Description : Load vectors with 16 byte elements with stride
214 * Arguments : Inputs - psrc, stride
215 * Outputs - out0, out1
216 * Return Type - as per RTYPE
217 * Details : Load 16 byte elements in 'out0' from (psrc)
218 * Load 16 byte elements in 'out1' from (psrc + stride)
219 */
220 #define LD_B2(RTYPE, psrc, stride, out0, out1) do { \
221 out0 = LD_B(RTYPE, psrc); \
222 out1 = LD_B(RTYPE, psrc + stride); \
223 } while (0)
224 #define LD_UB2(...) LD_B2(v16u8, __VA_ARGS__)
225 #define LD_SB2(...) LD_B2(v16i8, __VA_ARGS__)
226
227 #define LD_B3(RTYPE, psrc, stride, out0, out1, out2) do { \
228 LD_B2(RTYPE, psrc, stride, out0, out1); \
229 out2 = LD_B(RTYPE, psrc + 2 * stride); \
230 } while (0)
231 #define LD_UB3(...) LD_B3(v16u8, __VA_ARGS__)
232 #define LD_SB3(...) LD_B3(v16i8, __VA_ARGS__)
233
234 #define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) do { \
235 LD_B2(RTYPE, psrc, stride, out0, out1); \
236 LD_B2(RTYPE, psrc + 2 * stride , stride, out2, out3); \
237 } while (0)
238 #define LD_UB4(...) LD_B4(v16u8, __VA_ARGS__)
239 #define LD_SB4(...) LD_B4(v16i8, __VA_ARGS__)
240
241 #define LD_B8(RTYPE, psrc, stride, \
242 out0, out1, out2, out3, out4, out5, out6, out7) do { \
243 LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3); \
244 LD_B4(RTYPE, psrc + 4 * stride, stride, out4, out5, out6, out7); \
245 } while (0)
246 #define LD_UB8(...) LD_B8(v16u8, __VA_ARGS__)
247 #define LD_SB8(...) LD_B8(v16i8, __VA_ARGS__)
248
249 /* Description : Load vectors with 8 halfword elements with stride
250 * Arguments : Inputs - psrc, stride
251 * Outputs - out0, out1
252 * Details : Load 8 halfword elements in 'out0' from (psrc)
253 * Load 8 halfword elements in 'out1' from (psrc + stride)
254 */
255 #define LD_H2(RTYPE, psrc, stride, out0, out1) do { \
256 out0 = LD_H(RTYPE, psrc); \
257 out1 = LD_H(RTYPE, psrc + stride); \
258 } while (0)
259 #define LD_UH2(...) LD_H2(v8u16, __VA_ARGS__)
260 #define LD_SH2(...) LD_H2(v8i16, __VA_ARGS__)
261
262 /* Description : Load vectors with 4 word elements with stride
263 * Arguments : Inputs - psrc, stride
264 * Outputs - out0, out1, out2, out3
265 * Details : Load 4 word elements in 'out0' from (psrc + 0 * stride)
266 * Load 4 word elements in 'out1' from (psrc + 1 * stride)
267 * Load 4 word elements in 'out2' from (psrc + 2 * stride)
268 * Load 4 word elements in 'out3' from (psrc + 3 * stride)
269 */
270 #define LD_W2(RTYPE, psrc, stride, out0, out1) do { \
271 out0 = LD_W(RTYPE, psrc); \
272 out1 = LD_W(RTYPE, psrc + stride); \
273 } while (0)
274 #define LD_UW2(...) LD_W2(v4u32, __VA_ARGS__)
275 #define LD_SW2(...) LD_W2(v4i32, __VA_ARGS__)
276
277 #define LD_W3(RTYPE, psrc, stride, out0, out1, out2) do { \
278 LD_W2(RTYPE, psrc, stride, out0, out1); \
279 out2 = LD_W(RTYPE, psrc + 2 * stride); \
280 } while (0)
281 #define LD_UW3(...) LD_W3(v4u32, __VA_ARGS__)
282 #define LD_SW3(...) LD_W3(v4i32, __VA_ARGS__)
283
284 #define LD_W4(RTYPE, psrc, stride, out0, out1, out2, out3) do { \
285 LD_W2(RTYPE, psrc, stride, out0, out1); \
286 LD_W2(RTYPE, psrc + 2 * stride, stride, out2, out3); \
287 } while (0)
288 #define LD_UW4(...) LD_W4(v4u32, __VA_ARGS__)
289 #define LD_SW4(...) LD_W4(v4i32, __VA_ARGS__)
290
291 /* Description : Store vectors of 16 byte elements with stride
292 * Arguments : Inputs - in0, in1, pdst, stride
293 * Details : Store 16 byte elements from 'in0' to (pdst)
294 * Store 16 byte elements from 'in1' to (pdst + stride)
295 */
296 #define ST_B2(RTYPE, in0, in1, pdst, stride) do { \
297 ST_B(RTYPE, in0, pdst); \
298 ST_B(RTYPE, in1, pdst + stride); \
299 } while (0)
300 #define ST_UB2(...) ST_B2(v16u8, __VA_ARGS__)
301 #define ST_SB2(...) ST_B2(v16i8, __VA_ARGS__)
302
303 #define ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride) do { \
304 ST_B2(RTYPE, in0, in1, pdst, stride); \
305 ST_B2(RTYPE, in2, in3, pdst + 2 * stride, stride); \
306 } while (0)
307 #define ST_UB4(...) ST_B4(v16u8, __VA_ARGS__)
308 #define ST_SB4(...) ST_B4(v16i8, __VA_ARGS__)
309
310 #define ST_B8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \
311 pdst, stride) do { \
312 ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride); \
313 ST_B4(RTYPE, in4, in5, in6, in7, pdst + 4 * stride, stride); \
314 } while (0)
315 #define ST_UB8(...) ST_B8(v16u8, __VA_ARGS__)
316
317 /* Description : Store vectors of 4 word elements with stride
318 * Arguments : Inputs - in0, in1, in2, in3, pdst, stride
319 * Details : Store 4 word elements from 'in0' to (pdst + 0 * stride)
320 * Store 4 word elements from 'in1' to (pdst + 1 * stride)
321 * Store 4 word elements from 'in2' to (pdst + 2 * stride)
322 * Store 4 word elements from 'in3' to (pdst + 3 * stride)
323 */
324 #define ST_W2(RTYPE, in0, in1, pdst, stride) do { \
325 ST_W(RTYPE, in0, pdst); \
326 ST_W(RTYPE, in1, pdst + stride); \
327 } while (0)
328 #define ST_UW2(...) ST_W2(v4u32, __VA_ARGS__)
329 #define ST_SW2(...) ST_W2(v4i32, __VA_ARGS__)
330
331 #define ST_W3(RTYPE, in0, in1, in2, pdst, stride) do { \
332 ST_W2(RTYPE, in0, in1, pdst, stride); \
333 ST_W(RTYPE, in2, pdst + 2 * stride); \
334 } while (0)
335 #define ST_UW3(...) ST_W3(v4u32, __VA_ARGS__)
336 #define ST_SW3(...) ST_W3(v4i32, __VA_ARGS__)
337
338 #define ST_W4(RTYPE, in0, in1, in2, in3, pdst, stride) do { \
339 ST_W2(RTYPE, in0, in1, pdst, stride); \
340 ST_W2(RTYPE, in2, in3, pdst + 2 * stride, stride); \
341 } while (0)
342 #define ST_UW4(...) ST_W4(v4u32, __VA_ARGS__)
343 #define ST_SW4(...) ST_W4(v4i32, __VA_ARGS__)
344
345 /* Description : Store vectors of 8 halfword elements with stride
346 * Arguments : Inputs - in0, in1, pdst, stride
347 * Details : Store 8 halfword elements from 'in0' to (pdst)
348 * Store 8 halfword elements from 'in1' to (pdst + stride)
349 */
350 #define ST_H2(RTYPE, in0, in1, pdst, stride) do { \
351 ST_H(RTYPE, in0, pdst); \
352 ST_H(RTYPE, in1, pdst + stride); \
353 } while (0)
354 #define ST_UH2(...) ST_H2(v8u16, __VA_ARGS__)
355 #define ST_SH2(...) ST_H2(v8i16, __VA_ARGS__)
356
357 /* Description : Store 2x4 byte block to destination memory from input vector
358 * Arguments : Inputs - in, stidx, pdst, stride
359 * Details : Index 'stidx' halfword element from 'in' vector is copied to
360 * the GP register and stored to (pdst)
361 * Index 'stidx+1' halfword element from 'in' vector is copied to
362 * the GP register and stored to (pdst + stride)
363 * Index 'stidx+2' halfword element from 'in' vector is copied to
364 * the GP register and stored to (pdst + 2 * stride)
365 * Index 'stidx+3' halfword element from 'in' vector is copied to
366 * the GP register and stored to (pdst + 3 * stride)
367 */
368 #define ST2x4_UB(in, stidx, pdst, stride) do { \
369 uint8_t* pblk_2x4_m = (uint8_t*)pdst; \
370 const uint16_t out0_m = __msa_copy_s_h((v8i16)in, stidx); \
371 const uint16_t out1_m = __msa_copy_s_h((v8i16)in, stidx + 1); \
372 const uint16_t out2_m = __msa_copy_s_h((v8i16)in, stidx + 2); \
373 const uint16_t out3_m = __msa_copy_s_h((v8i16)in, stidx + 3); \
374 SH(out0_m, pblk_2x4_m); \
375 pblk_2x4_m += stride; \
376 SH(out1_m, pblk_2x4_m); \
377 pblk_2x4_m += stride; \
378 SH(out2_m, pblk_2x4_m); \
379 pblk_2x4_m += stride; \
380 SH(out3_m, pblk_2x4_m); \
381 } while (0)
382
383 /* Description : Store 4x4 byte block to destination memory from input vector
384 * Arguments : Inputs - in0, in1, pdst, stride
385 * Details : 'Idx0' word element from input vector 'in0' is copied to the
386 * GP register and stored to (pdst)
387 * 'Idx1' word element from input vector 'in0' is copied to the
388 * GP register and stored to (pdst + stride)
389 * 'Idx2' word element from input vector 'in0' is copied to the
390 * GP register and stored to (pdst + 2 * stride)
391 * 'Idx3' word element from input vector 'in0' is copied to the
392 * GP register and stored to (pdst + 3 * stride)
393 */
394 #define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) do { \
395 uint8_t* const pblk_4x4_m = (uint8_t*)pdst; \
396 const uint32_t out0_m = __msa_copy_s_w((v4i32)in0, idx0); \
397 const uint32_t out1_m = __msa_copy_s_w((v4i32)in0, idx1); \
398 const uint32_t out2_m = __msa_copy_s_w((v4i32)in1, idx2); \
399 const uint32_t out3_m = __msa_copy_s_w((v4i32)in1, idx3); \
400 SW4(out0_m, out1_m, out2_m, out3_m, pblk_4x4_m, stride); \
401 } while (0)
402
403 #define ST4x8_UB(in0, in1, pdst, stride) do { \
404 uint8_t* const pblk_4x8 = (uint8_t*)pdst; \
405 ST4x4_UB(in0, in0, 0, 1, 2, 3, pblk_4x8, stride); \
406 ST4x4_UB(in1, in1, 0, 1, 2, 3, pblk_4x8 + 4 * stride, stride); \
407 } while (0)
408
409 /* Description : Immediate number of elements to slide
410 * Arguments : Inputs - in0, in1, slide_val
411 * Outputs - out
412 * Return Type - as per RTYPE
413 * Details : Byte elements from 'in1' vector are slid into 'in0' by
414 * value specified in the 'slide_val'
415 */
416 #define SLDI_B(RTYPE, in0, in1, slide_val) \
417 (RTYPE)__msa_sldi_b((v16i8)in0, (v16i8)in1, slide_val) \
418
419 #define SLDI_UB(...) SLDI_B(v16u8, __VA_ARGS__)
420 #define SLDI_SB(...) SLDI_B(v16i8, __VA_ARGS__)
421 #define SLDI_SH(...) SLDI_B(v8i16, __VA_ARGS__)
422
423 /* Description : Shuffle byte vector elements as per mask vector
424 * Arguments : Inputs - in0, in1, in2, in3, mask0, mask1
425 * Outputs - out0, out1
426 * Return Type - as per RTYPE
427 * Details : Byte elements from 'in0' & 'in1' are copied selectively to
428 * 'out0' as per control vector 'mask0'
429 */
430 #define VSHF_B(RTYPE, in0, in1, mask) \
431 (RTYPE)__msa_vshf_b((v16i8)mask, (v16i8)in1, (v16i8)in0)
432
433 #define VSHF_UB(...) VSHF_B(v16u8, __VA_ARGS__)
434 #define VSHF_SB(...) VSHF_B(v16i8, __VA_ARGS__)
435 #define VSHF_UH(...) VSHF_B(v8u16, __VA_ARGS__)
436 #define VSHF_SH(...) VSHF_B(v8i16, __VA_ARGS__)
437
438 #define VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do { \
439 out0 = VSHF_B(RTYPE, in0, in1, mask0); \
440 out1 = VSHF_B(RTYPE, in2, in3, mask1); \
441 } while (0)
442 #define VSHF_B2_UB(...) VSHF_B2(v16u8, __VA_ARGS__)
443 #define VSHF_B2_SB(...) VSHF_B2(v16i8, __VA_ARGS__)
444 #define VSHF_B2_UH(...) VSHF_B2(v8u16, __VA_ARGS__)
445 #define VSHF_B2_SH(...) VSHF_B2(v8i16, __VA_ARGS__)
446
447 /* Description : Shuffle halfword vector elements as per mask vector
448 * Arguments : Inputs - in0, in1, in2, in3, mask0, mask1
449 * Outputs - out0, out1
450 * Return Type - as per RTYPE
451 * Details : halfword elements from 'in0' & 'in1' are copied selectively to
452 * 'out0' as per control vector 'mask0'
453 */
454 #define VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do { \
455 out0 = (RTYPE)__msa_vshf_h((v8i16)mask0, (v8i16)in1, (v8i16)in0); \
456 out1 = (RTYPE)__msa_vshf_h((v8i16)mask1, (v8i16)in3, (v8i16)in2); \
457 } while (0)
458 #define VSHF_H2_UH(...) VSHF_H2(v8u16, __VA_ARGS__)
459 #define VSHF_H2_SH(...) VSHF_H2(v8i16, __VA_ARGS__)
460
461 /* Description : Dot product of byte vector elements
462 * Arguments : Inputs - mult0, mult1, cnst0, cnst1
463 * Outputs - out0, out1
464 * Return Type - as per RTYPE
465 * Details : Signed byte elements from 'mult0' are multiplied with
466 * signed byte elements from 'cnst0' producing a result
467 * twice the size of input i.e. signed halfword.
468 * The multiplication result of adjacent odd-even elements
469 * are added together and written to the 'out0' vector
470 */
471 #define DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \
472 out0 = (RTYPE)__msa_dotp_s_h((v16i8)mult0, (v16i8)cnst0); \
473 out1 = (RTYPE)__msa_dotp_s_h((v16i8)mult1, (v16i8)cnst1); \
474 } while (0)
475 #define DOTP_SB2_SH(...) DOTP_SB2(v8i16, __VA_ARGS__)
476
477 /* Description : Dot product of halfword vector elements
478 * Arguments : Inputs - mult0, mult1, cnst0, cnst1
479 * Outputs - out0, out1
480 * Return Type - as per RTYPE
481 * Details : Signed halfword elements from 'mult0' are multiplied with
482 * signed halfword elements from 'cnst0' producing a result
483 * twice the size of input i.e. signed word.
484 * The multiplication result of adjacent odd-even elements
485 * are added together and written to the 'out0' vector
486 */
487 #define DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \
488 out0 = (RTYPE)__msa_dotp_s_w((v8i16)mult0, (v8i16)cnst0); \
489 out1 = (RTYPE)__msa_dotp_s_w((v8i16)mult1, (v8i16)cnst1); \
490 } while (0)
491 #define DOTP_SH2_SW(...) DOTP_SH2(v4i32, __VA_ARGS__)
492
493 /* Description : Dot product of unsigned word vector elements
494 * Arguments : Inputs - mult0, mult1, cnst0, cnst1
495 * Outputs - out0, out1
496 * Return Type - as per RTYPE
497 * Details : Unsigned word elements from 'mult0' are multiplied with
498 * unsigned word elements from 'cnst0' producing a result
499 * twice the size of input i.e. unsigned double word.
500 * The multiplication result of adjacent odd-even elements
501 * are added together and written to the 'out0' vector
502 */
503 #define DOTP_UW2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \
504 out0 = (RTYPE)__msa_dotp_u_d((v4u32)mult0, (v4u32)cnst0); \
505 out1 = (RTYPE)__msa_dotp_u_d((v4u32)mult1, (v4u32)cnst1); \
506 } while (0)
507 #define DOTP_UW2_UD(...) DOTP_UW2(v2u64, __VA_ARGS__)
508
509 /* Description : Dot product & addition of halfword vector elements
510 * Arguments : Inputs - mult0, mult1, cnst0, cnst1
511 * Outputs - out0, out1
512 * Return Type - as per RTYPE
513 * Details : Signed halfword elements from 'mult0' are multiplied with
514 * signed halfword elements from 'cnst0' producing a result
515 * twice the size of input i.e. signed word.
516 * The multiplication result of adjacent odd-even elements
517 * are added to the 'out0' vector
518 */
519 #define DPADD_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \
520 out0 = (RTYPE)__msa_dpadd_s_w((v4i32)out0, (v8i16)mult0, (v8i16)cnst0); \
521 out1 = (RTYPE)__msa_dpadd_s_w((v4i32)out1, (v8i16)mult1, (v8i16)cnst1); \
522 } while (0)
523 #define DPADD_SH2_SW(...) DPADD_SH2(v4i32, __VA_ARGS__)
524
525 /* Description : Clips all signed halfword elements of input vector
526 * between 0 & 255
527 * Arguments : Input/output - val
528 * Return Type - signed halfword
529 */
530 #define CLIP_SH_0_255(val) do { \
531 const v8i16 max_m = __msa_ldi_h(255); \
532 val = __msa_maxi_s_h((v8i16)val, 0); \
533 val = __msa_min_s_h(max_m, (v8i16)val); \
534 } while (0)
535
536 #define CLIP_SH2_0_255(in0, in1) do { \
537 CLIP_SH_0_255(in0); \
538 CLIP_SH_0_255(in1); \
539 } while (0)
540
541 #define CLIP_SH4_0_255(in0, in1, in2, in3) do { \
542 CLIP_SH2_0_255(in0, in1); \
543 CLIP_SH2_0_255(in2, in3); \
544 } while (0)
545
546 /* Description : Clips all unsigned halfword elements of input vector
547 * between 0 & 255
548 * Arguments : Input - in
549 * Output - out_m
550 * Return Type - unsigned halfword
551 */
552 #define CLIP_UH_0_255(in) do { \
553 const v8u16 max_m = (v8u16)__msa_ldi_h(255); \
554 in = __msa_maxi_u_h((v8u16) in, 0); \
555 in = __msa_min_u_h((v8u16) max_m, (v8u16) in); \
556 } while (0)
557
558 #define CLIP_UH2_0_255(in0, in1) do { \
559 CLIP_UH_0_255(in0); \
560 CLIP_UH_0_255(in1); \
561 } while (0)
562
563 /* Description : Clips all signed word elements of input vector
564 * between 0 & 255
565 * Arguments : Input/output - val
566 * Return Type - signed word
567 */
568 #define CLIP_SW_0_255(val) do { \
569 const v4i32 max_m = __msa_ldi_w(255); \
570 val = __msa_maxi_s_w((v4i32)val, 0); \
571 val = __msa_min_s_w(max_m, (v4i32)val); \
572 } while (0)
573
574 #define CLIP_SW4_0_255(in0, in1, in2, in3) do { \
575 CLIP_SW_0_255(in0); \
576 CLIP_SW_0_255(in1); \
577 CLIP_SW_0_255(in2); \
578 CLIP_SW_0_255(in3); \
579 } while (0)
580
581 /* Description : Horizontal addition of 4 signed word elements of input vector
582 * Arguments : Input - in (signed word vector)
583 * Output - sum_m (i32 sum)
584 * Return Type - signed word (GP)
585 * Details : 4 signed word elements of 'in' vector are added together and
586 * the resulting integer sum is returned
587 */
func_hadd_sw_s32(v4i32 in)588 static WEBP_INLINE int32_t func_hadd_sw_s32(v4i32 in) {
589 const v2i64 res0_m = __msa_hadd_s_d((v4i32)in, (v4i32)in);
590 const v2i64 res1_m = __msa_splati_d(res0_m, 1);
591 const v2i64 out = res0_m + res1_m;
592 int32_t sum_m = __msa_copy_s_w((v4i32)out, 0);
593 return sum_m;
594 }
595 #define HADD_SW_S32(in) func_hadd_sw_s32(in)
596
597 /* Description : Horizontal addition of 8 signed halfword elements
598 * Arguments : Input - in (signed halfword vector)
599 * Output - sum_m (s32 sum)
600 * Return Type - signed word
601 * Details : 8 signed halfword elements of input vector are added
602 * together and the resulting integer sum is returned
603 */
func_hadd_sh_s32(v8i16 in)604 static WEBP_INLINE int32_t func_hadd_sh_s32(v8i16 in) {
605 const v4i32 res = __msa_hadd_s_w(in, in);
606 const v2i64 res0 = __msa_hadd_s_d(res, res);
607 const v2i64 res1 = __msa_splati_d(res0, 1);
608 const v2i64 res2 = res0 + res1;
609 const int32_t sum_m = __msa_copy_s_w((v4i32)res2, 0);
610 return sum_m;
611 }
612 #define HADD_SH_S32(in) func_hadd_sh_s32(in)
613
614 /* Description : Horizontal addition of 8 unsigned halfword elements
615 * Arguments : Input - in (unsigned halfword vector)
616 * Output - sum_m (u32 sum)
617 * Return Type - unsigned word
618 * Details : 8 unsigned halfword elements of input vector are added
619 * together and the resulting integer sum is returned
620 */
func_hadd_uh_u32(v8u16 in)621 static WEBP_INLINE uint32_t func_hadd_uh_u32(v8u16 in) {
622 uint32_t sum_m;
623 const v4u32 res_m = __msa_hadd_u_w(in, in);
624 v2u64 res0_m = __msa_hadd_u_d(res_m, res_m);
625 v2u64 res1_m = (v2u64)__msa_splati_d((v2i64)res0_m, 1);
626 res0_m = res0_m + res1_m;
627 sum_m = __msa_copy_s_w((v4i32)res0_m, 0);
628 return sum_m;
629 }
630 #define HADD_UH_U32(in) func_hadd_uh_u32(in)
631
632 /* Description : Horizontal addition of signed half word vector elements
633 Arguments : Inputs - in0, in1
634 Outputs - out0, out1
635 Return Type - as per RTYPE
636 Details : Each signed odd half word element from 'in0' is added to
637 even signed half word element from 'in0' (pairwise) and the
638 halfword result is written in 'out0'
639 */
640 #define HADD_SH2(RTYPE, in0, in1, out0, out1) do { \
641 out0 = (RTYPE)__msa_hadd_s_w((v8i16)in0, (v8i16)in0); \
642 out1 = (RTYPE)__msa_hadd_s_w((v8i16)in1, (v8i16)in1); \
643 } while (0)
644 #define HADD_SH2_SW(...) HADD_SH2(v4i32, __VA_ARGS__)
645
646 #define HADD_SH4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) do { \
647 HADD_SH2(RTYPE, in0, in1, out0, out1); \
648 HADD_SH2(RTYPE, in2, in3, out2, out3); \
649 } while (0)
650 #define HADD_SH4_SW(...) HADD_SH4(v4i32, __VA_ARGS__)
651
652 /* Description : Horizontal subtraction of unsigned byte vector elements
653 * Arguments : Inputs - in0, in1
654 * Outputs - out0, out1
655 * Return Type - as per RTYPE
656 * Details : Each unsigned odd byte element from 'in0' is subtracted from
657 * even unsigned byte element from 'in0' (pairwise) and the
658 * halfword result is written to 'out0'
659 */
660 #define HSUB_UB2(RTYPE, in0, in1, out0, out1) do { \
661 out0 = (RTYPE)__msa_hsub_u_h((v16u8)in0, (v16u8)in0); \
662 out1 = (RTYPE)__msa_hsub_u_h((v16u8)in1, (v16u8)in1); \
663 } while (0)
664 #define HSUB_UB2_UH(...) HSUB_UB2(v8u16, __VA_ARGS__)
665 #define HSUB_UB2_SH(...) HSUB_UB2(v8i16, __VA_ARGS__)
666 #define HSUB_UB2_SW(...) HSUB_UB2(v4i32, __VA_ARGS__)
667
668 /* Description : Set element n input vector to GPR value
669 * Arguments : Inputs - in0, in1, in2, in3
670 * Output - out
671 * Return Type - as per RTYPE
672 * Details : Set element 0 in vector 'out' to value specified in 'in0'
673 */
674 #define INSERT_W2(RTYPE, in0, in1, out) do { \
675 out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \
676 out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \
677 } while (0)
678 #define INSERT_W2_UB(...) INSERT_W2(v16u8, __VA_ARGS__)
679 #define INSERT_W2_SB(...) INSERT_W2(v16i8, __VA_ARGS__)
680
681 #define INSERT_W4(RTYPE, in0, in1, in2, in3, out) do { \
682 out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \
683 out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \
684 out = (RTYPE)__msa_insert_w((v4i32)out, 2, in2); \
685 out = (RTYPE)__msa_insert_w((v4i32)out, 3, in3); \
686 } while (0)
687 #define INSERT_W4_UB(...) INSERT_W4(v16u8, __VA_ARGS__)
688 #define INSERT_W4_SB(...) INSERT_W4(v16i8, __VA_ARGS__)
689 #define INSERT_W4_SW(...) INSERT_W4(v4i32, __VA_ARGS__)
690
691 /* Description : Set element n of double word input vector to GPR value
692 * Arguments : Inputs - in0, in1
693 * Output - out
694 * Return Type - as per RTYPE
695 * Details : Set element 0 in vector 'out' to GPR value specified in 'in0'
696 * Set element 1 in vector 'out' to GPR value specified in 'in1'
697 */
698 #define INSERT_D2(RTYPE, in0, in1, out) do { \
699 out = (RTYPE)__msa_insert_d((v2i64)out, 0, in0); \
700 out = (RTYPE)__msa_insert_d((v2i64)out, 1, in1); \
701 } while (0)
702 #define INSERT_D2_UB(...) INSERT_D2(v16u8, __VA_ARGS__)
703 #define INSERT_D2_SB(...) INSERT_D2(v16i8, __VA_ARGS__)
704
705 /* Description : Interleave even byte elements from vectors
706 * Arguments : Inputs - in0, in1, in2, in3
707 * Outputs - out0, out1
708 * Return Type - as per RTYPE
709 * Details : Even byte elements of 'in0' and 'in1' are interleaved
710 * and written to 'out0'
711 */
712 #define ILVEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
713 out0 = (RTYPE)__msa_ilvev_b((v16i8)in1, (v16i8)in0); \
714 out1 = (RTYPE)__msa_ilvev_b((v16i8)in3, (v16i8)in2); \
715 } while (0)
716 #define ILVEV_B2_UB(...) ILVEV_B2(v16u8, __VA_ARGS__)
717 #define ILVEV_B2_SB(...) ILVEV_B2(v16i8, __VA_ARGS__)
718 #define ILVEV_B2_UH(...) ILVEV_B2(v8u16, __VA_ARGS__)
719 #define ILVEV_B2_SH(...) ILVEV_B2(v8i16, __VA_ARGS__)
720 #define ILVEV_B2_SD(...) ILVEV_B2(v2i64, __VA_ARGS__)
721
722 /* Description : Interleave odd byte elements from vectors
723 * Arguments : Inputs - in0, in1, in2, in3
724 * Outputs - out0, out1
725 * Return Type - as per RTYPE
726 * Details : Odd byte elements of 'in0' and 'in1' are interleaved
727 * and written to 'out0'
728 */
729 #define ILVOD_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
730 out0 = (RTYPE)__msa_ilvod_b((v16i8)in1, (v16i8)in0); \
731 out1 = (RTYPE)__msa_ilvod_b((v16i8)in3, (v16i8)in2); \
732 } while (0)
733 #define ILVOD_B2_UB(...) ILVOD_B2(v16u8, __VA_ARGS__)
734 #define ILVOD_B2_SB(...) ILVOD_B2(v16i8, __VA_ARGS__)
735 #define ILVOD_B2_UH(...) ILVOD_B2(v8u16, __VA_ARGS__)
736 #define ILVOD_B2_SH(...) ILVOD_B2(v8i16, __VA_ARGS__)
737 #define ILVOD_B2_SD(...) ILVOD_B2(v2i64, __VA_ARGS__)
738
739 /* Description : Interleave even halfword elements from vectors
740 * Arguments : Inputs - in0, in1, in2, in3
741 * Outputs - out0, out1
742 * Return Type - as per RTYPE
743 * Details : Even halfword elements of 'in0' and 'in1' are interleaved
744 * and written to 'out0'
745 */
746 #define ILVEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
747 out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0); \
748 out1 = (RTYPE)__msa_ilvev_h((v8i16)in3, (v8i16)in2); \
749 } while (0)
750 #define ILVEV_H2_UB(...) ILVEV_H2(v16u8, __VA_ARGS__)
751 #define ILVEV_H2_UH(...) ILVEV_H2(v8u16, __VA_ARGS__)
752 #define ILVEV_H2_SH(...) ILVEV_H2(v8i16, __VA_ARGS__)
753 #define ILVEV_H2_SW(...) ILVEV_H2(v4i32, __VA_ARGS__)
754
755 /* Description : Interleave odd halfword elements from vectors
756 * Arguments : Inputs - in0, in1, in2, in3
757 * Outputs - out0, out1
758 * Return Type - as per RTYPE
759 * Details : Odd halfword elements of 'in0' and 'in1' are interleaved
760 * and written to 'out0'
761 */
762 #define ILVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
763 out0 = (RTYPE)__msa_ilvod_h((v8i16)in1, (v8i16)in0); \
764 out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2); \
765 } while (0)
766 #define ILVOD_H2_UB(...) ILVOD_H2(v16u8, __VA_ARGS__)
767 #define ILVOD_H2_UH(...) ILVOD_H2(v8u16, __VA_ARGS__)
768 #define ILVOD_H2_SH(...) ILVOD_H2(v8i16, __VA_ARGS__)
769 #define ILVOD_H2_SW(...) ILVOD_H2(v4i32, __VA_ARGS__)
770
771 /* Description : Interleave even word elements from vectors
772 * Arguments : Inputs - in0, in1, in2, in3
773 * Outputs - out0, out1
774 * Return Type - as per RTYPE
775 * Details : Even word elements of 'in0' and 'in1' are interleaved
776 * and written to 'out0'
777 */
778 #define ILVEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
779 out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0); \
780 out1 = (RTYPE)__msa_ilvev_w((v4i32)in3, (v4i32)in2); \
781 } while (0)
782 #define ILVEV_W2_UB(...) ILVEV_W2(v16u8, __VA_ARGS__)
783 #define ILVEV_W2_SB(...) ILVEV_W2(v16i8, __VA_ARGS__)
784 #define ILVEV_W2_UH(...) ILVEV_W2(v8u16, __VA_ARGS__)
785 #define ILVEV_W2_SD(...) ILVEV_W2(v2i64, __VA_ARGS__)
786
787 /* Description : Interleave even-odd word elements from vectors
788 * Arguments : Inputs - in0, in1, in2, in3
789 * Outputs - out0, out1
790 * Return Type - as per RTYPE
791 * Details : Even word elements of 'in0' and 'in1' are interleaved
792 * and written to 'out0'
793 * Odd word elements of 'in2' and 'in3' are interleaved
794 * and written to 'out1'
795 */
796 #define ILVEVOD_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
797 out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0); \
798 out1 = (RTYPE)__msa_ilvod_w((v4i32)in3, (v4i32)in2); \
799 } while (0)
800 #define ILVEVOD_W2_UB(...) ILVEVOD_W2(v16u8, __VA_ARGS__)
801 #define ILVEVOD_W2_UH(...) ILVEVOD_W2(v8u16, __VA_ARGS__)
802 #define ILVEVOD_W2_SH(...) ILVEVOD_W2(v8i16, __VA_ARGS__)
803 #define ILVEVOD_W2_SW(...) ILVEVOD_W2(v4i32, __VA_ARGS__)
804
805 /* Description : Interleave even-odd half-word elements from vectors
806 * Arguments : Inputs - in0, in1, in2, in3
807 * Outputs - out0, out1
808 * Return Type - as per RTYPE
809 * Details : Even half-word elements of 'in0' and 'in1' are interleaved
810 * and written to 'out0'
811 * Odd half-word elements of 'in2' and 'in3' are interleaved
812 * and written to 'out1'
813 */
814 #define ILVEVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
815 out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0); \
816 out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2); \
817 } while (0)
818 #define ILVEVOD_H2_UB(...) ILVEVOD_H2(v16u8, __VA_ARGS__)
819 #define ILVEVOD_H2_UH(...) ILVEVOD_H2(v8u16, __VA_ARGS__)
820 #define ILVEVOD_H2_SH(...) ILVEVOD_H2(v8i16, __VA_ARGS__)
821 #define ILVEVOD_H2_SW(...) ILVEVOD_H2(v4i32, __VA_ARGS__)
822
823 /* Description : Interleave even double word elements from vectors
824 * Arguments : Inputs - in0, in1, in2, in3
825 * Outputs - out0, out1
826 * Return Type - as per RTYPE
827 * Details : Even double word elements of 'in0' and 'in1' are interleaved
828 * and written to 'out0'
829 */
830 #define ILVEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
831 out0 = (RTYPE)__msa_ilvev_d((v2i64)in1, (v2i64)in0); \
832 out1 = (RTYPE)__msa_ilvev_d((v2i64)in3, (v2i64)in2); \
833 } while (0)
834 #define ILVEV_D2_UB(...) ILVEV_D2(v16u8, __VA_ARGS__)
835 #define ILVEV_D2_SB(...) ILVEV_D2(v16i8, __VA_ARGS__)
836 #define ILVEV_D2_SW(...) ILVEV_D2(v4i32, __VA_ARGS__)
837 #define ILVEV_D2_SD(...) ILVEV_D2(v2i64, __VA_ARGS__)
838
839 /* Description : Interleave left half of byte elements from vectors
840 * Arguments : Inputs - in0, in1, in2, in3
841 * Outputs - out0, out1
842 * Return Type - as per RTYPE
843 * Details : Left half of byte elements of 'in0' and 'in1' are interleaved
844 * and written to 'out0'.
845 */
846 #define ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
847 out0 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1); \
848 out1 = (RTYPE)__msa_ilvl_b((v16i8)in2, (v16i8)in3); \
849 } while (0)
850 #define ILVL_B2_UB(...) ILVL_B2(v16u8, __VA_ARGS__)
851 #define ILVL_B2_SB(...) ILVL_B2(v16i8, __VA_ARGS__)
852 #define ILVL_B2_UH(...) ILVL_B2(v8u16, __VA_ARGS__)
853 #define ILVL_B2_SH(...) ILVL_B2(v8i16, __VA_ARGS__)
854 #define ILVL_B2_SW(...) ILVL_B2(v4i32, __VA_ARGS__)
855
856 /* Description : Interleave right half of byte elements from vectors
857 * Arguments : Inputs - in0, in1, in2, in3
858 * Outputs - out0, out1
859 * Return Type - as per RTYPE
860 * Details : Right half of byte elements of 'in0' and 'in1' are interleaved
861 * and written to out0.
862 */
863 #define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
864 out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \
865 out1 = (RTYPE)__msa_ilvr_b((v16i8)in2, (v16i8)in3); \
866 } while (0)
867 #define ILVR_B2_UB(...) ILVR_B2(v16u8, __VA_ARGS__)
868 #define ILVR_B2_SB(...) ILVR_B2(v16i8, __VA_ARGS__)
869 #define ILVR_B2_UH(...) ILVR_B2(v8u16, __VA_ARGS__)
870 #define ILVR_B2_SH(...) ILVR_B2(v8i16, __VA_ARGS__)
871 #define ILVR_B2_SW(...) ILVR_B2(v4i32, __VA_ARGS__)
872
873 #define ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \
874 out0, out1, out2, out3) do { \
875 ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1); \
876 ILVR_B2(RTYPE, in4, in5, in6, in7, out2, out3); \
877 } while (0)
878 #define ILVR_B4_UB(...) ILVR_B4(v16u8, __VA_ARGS__)
879 #define ILVR_B4_SB(...) ILVR_B4(v16i8, __VA_ARGS__)
880 #define ILVR_B4_UH(...) ILVR_B4(v8u16, __VA_ARGS__)
881 #define ILVR_B4_SH(...) ILVR_B4(v8i16, __VA_ARGS__)
882 #define ILVR_B4_SW(...) ILVR_B4(v4i32, __VA_ARGS__)
883
884 /* Description : Interleave right half of halfword elements from vectors
885 * Arguments : Inputs - in0, in1, in2, in3
886 * Outputs - out0, out1
887 * Return Type - as per RTYPE
888 * Details : Right half of halfword elements of 'in0' and 'in1' are
889 * interleaved and written to 'out0'.
890 */
891 #define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
892 out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \
893 out1 = (RTYPE)__msa_ilvr_h((v8i16)in2, (v8i16)in3); \
894 } while (0)
895 #define ILVR_H2_UB(...) ILVR_H2(v16u8, __VA_ARGS__)
896 #define ILVR_H2_SH(...) ILVR_H2(v8i16, __VA_ARGS__)
897 #define ILVR_H2_SW(...) ILVR_H2(v4i32, __VA_ARGS__)
898
899 #define ILVR_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \
900 out0, out1, out2, out3) do { \
901 ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1); \
902 ILVR_H2(RTYPE, in4, in5, in6, in7, out2, out3); \
903 } while (0)
904 #define ILVR_H4_UB(...) ILVR_H4(v16u8, __VA_ARGS__)
905 #define ILVR_H4_SH(...) ILVR_H4(v8i16, __VA_ARGS__)
906 #define ILVR_H4_SW(...) ILVR_H4(v4i32, __VA_ARGS__)
907
908 /* Description : Interleave right half of double word elements from vectors
909 * Arguments : Inputs - in0, in1, in2, in3
910 * Outputs - out0, out1
911 * Return Type - as per RTYPE
912 * Details : Right half of double word elements of 'in0' and 'in1' are
913 * interleaved and written to 'out0'.
914 */
915 #define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
916 out0 = (RTYPE)__msa_ilvr_d((v2i64)in0, (v2i64)in1); \
917 out1 = (RTYPE)__msa_ilvr_d((v2i64)in2, (v2i64)in3); \
918 } while (0)
919 #define ILVR_D2_UB(...) ILVR_D2(v16u8, __VA_ARGS__)
920 #define ILVR_D2_SB(...) ILVR_D2(v16i8, __VA_ARGS__)
921 #define ILVR_D2_SH(...) ILVR_D2(v8i16, __VA_ARGS__)
922
923 #define ILVR_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \
924 out0, out1, out2, out3) do { \
925 ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1); \
926 ILVR_D2(RTYPE, in4, in5, in6, in7, out2, out3); \
927 } while (0)
928 #define ILVR_D4_SB(...) ILVR_D4(v16i8, __VA_ARGS__)
929 #define ILVR_D4_UB(...) ILVR_D4(v16u8, __VA_ARGS__)
930
931 /* Description : Interleave both left and right half of input vectors
932 * Arguments : Inputs - in0, in1
933 * Outputs - out0, out1
934 * Return Type - as per RTYPE
935 * Details : Right half of byte elements from 'in0' and 'in1' are
936 * interleaved and written to 'out0'
937 */
938 #define ILVRL_B2(RTYPE, in0, in1, out0, out1) do { \
939 out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \
940 out1 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1); \
941 } while (0)
942 #define ILVRL_B2_UB(...) ILVRL_B2(v16u8, __VA_ARGS__)
943 #define ILVRL_B2_SB(...) ILVRL_B2(v16i8, __VA_ARGS__)
944 #define ILVRL_B2_UH(...) ILVRL_B2(v8u16, __VA_ARGS__)
945 #define ILVRL_B2_SH(...) ILVRL_B2(v8i16, __VA_ARGS__)
946 #define ILVRL_B2_SW(...) ILVRL_B2(v4i32, __VA_ARGS__)
947
948 #define ILVRL_H2(RTYPE, in0, in1, out0, out1) do { \
949 out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \
950 out1 = (RTYPE)__msa_ilvl_h((v8i16)in0, (v8i16)in1); \
951 } while (0)
952 #define ILVRL_H2_UB(...) ILVRL_H2(v16u8, __VA_ARGS__)
953 #define ILVRL_H2_SB(...) ILVRL_H2(v16i8, __VA_ARGS__)
954 #define ILVRL_H2_SH(...) ILVRL_H2(v8i16, __VA_ARGS__)
955 #define ILVRL_H2_SW(...) ILVRL_H2(v4i32, __VA_ARGS__)
956 #define ILVRL_H2_UW(...) ILVRL_H2(v4u32, __VA_ARGS__)
957
958 #define ILVRL_W2(RTYPE, in0, in1, out0, out1) do { \
959 out0 = (RTYPE)__msa_ilvr_w((v4i32)in0, (v4i32)in1); \
960 out1 = (RTYPE)__msa_ilvl_w((v4i32)in0, (v4i32)in1); \
961 } while (0)
962 #define ILVRL_W2_UB(...) ILVRL_W2(v16u8, __VA_ARGS__)
963 #define ILVRL_W2_SH(...) ILVRL_W2(v8i16, __VA_ARGS__)
964 #define ILVRL_W2_SW(...) ILVRL_W2(v4i32, __VA_ARGS__)
965 #define ILVRL_W2_UW(...) ILVRL_W2(v4u32, __VA_ARGS__)
966
967 /* Description : Pack even byte elements of vector pairs
968 * Arguments : Inputs - in0, in1, in2, in3
969 * Outputs - out0, out1
970 * Return Type - as per RTYPE
971 * Details : Even byte elements of 'in0' are copied to the left half of
972 * 'out0' & even byte elements of 'in1' are copied to the right
973 * half of 'out0'.
974 */
975 #define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
976 out0 = (RTYPE)__msa_pckev_b((v16i8)in0, (v16i8)in1); \
977 out1 = (RTYPE)__msa_pckev_b((v16i8)in2, (v16i8)in3); \
978 } while (0)
979 #define PCKEV_B2_SB(...) PCKEV_B2(v16i8, __VA_ARGS__)
980 #define PCKEV_B2_UB(...) PCKEV_B2(v16u8, __VA_ARGS__)
981 #define PCKEV_B2_SH(...) PCKEV_B2(v8i16, __VA_ARGS__)
982 #define PCKEV_B2_SW(...) PCKEV_B2(v4i32, __VA_ARGS__)
983
984 #define PCKEV_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \
985 out0, out1, out2, out3) do { \
986 PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1); \
987 PCKEV_B2(RTYPE, in4, in5, in6, in7, out2, out3); \
988 } while (0)
989 #define PCKEV_B4_SB(...) PCKEV_B4(v16i8, __VA_ARGS__)
990 #define PCKEV_B4_UB(...) PCKEV_B4(v16u8, __VA_ARGS__)
991 #define PCKEV_B4_SH(...) PCKEV_B4(v8i16, __VA_ARGS__)
992 #define PCKEV_B4_SW(...) PCKEV_B4(v4i32, __VA_ARGS__)
993
994 /* Description : Pack even halfword elements of vector pairs
995 * Arguments : Inputs - in0, in1, in2, in3
996 * Outputs - out0, out1
997 * Return Type - as per RTYPE
998 * Details : Even halfword elements of 'in0' are copied to the left half of
999 * 'out0' & even halfword elements of 'in1' are copied to the
1000 * right half of 'out0'.
1001 */
1002 #define PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
1003 out0 = (RTYPE)__msa_pckev_h((v8i16)in0, (v8i16)in1); \
1004 out1 = (RTYPE)__msa_pckev_h((v8i16)in2, (v8i16)in3); \
1005 } while (0)
1006 #define PCKEV_H2_UH(...) PCKEV_H2(v8u16, __VA_ARGS__)
1007 #define PCKEV_H2_SH(...) PCKEV_H2(v8i16, __VA_ARGS__)
1008 #define PCKEV_H2_SW(...) PCKEV_H2(v4i32, __VA_ARGS__)
1009 #define PCKEV_H2_UW(...) PCKEV_H2(v4u32, __VA_ARGS__)
1010
1011 /* Description : Pack even word elements of vector pairs
1012 * Arguments : Inputs - in0, in1, in2, in3
1013 * Outputs - out0, out1
1014 * Return Type - as per RTYPE
1015 * Details : Even word elements of 'in0' are copied to the left half of
1016 * 'out0' & even word elements of 'in1' are copied to the
1017 * right half of 'out0'.
1018 */
1019 #define PCKEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
1020 out0 = (RTYPE)__msa_pckev_w((v4i32)in0, (v4i32)in1); \
1021 out1 = (RTYPE)__msa_pckev_w((v4i32)in2, (v4i32)in3); \
1022 } while (0)
1023 #define PCKEV_W2_UH(...) PCKEV_W2(v8u16, __VA_ARGS__)
1024 #define PCKEV_W2_SH(...) PCKEV_W2(v8i16, __VA_ARGS__)
1025 #define PCKEV_W2_SW(...) PCKEV_W2(v4i32, __VA_ARGS__)
1026 #define PCKEV_W2_UW(...) PCKEV_W2(v4u32, __VA_ARGS__)
1027
1028 /* Description : Pack odd halfword elements of vector pairs
1029 * Arguments : Inputs - in0, in1, in2, in3
1030 * Outputs - out0, out1
1031 * Return Type - as per RTYPE
1032 * Details : Odd halfword elements of 'in0' are copied to the left half of
1033 * 'out0' & odd halfword elements of 'in1' are copied to the
1034 * right half of 'out0'.
1035 */
1036 #define PCKOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
1037 out0 = (RTYPE)__msa_pckod_h((v8i16)in0, (v8i16)in1); \
1038 out1 = (RTYPE)__msa_pckod_h((v8i16)in2, (v8i16)in3); \
1039 } while (0)
1040 #define PCKOD_H2_UH(...) PCKOD_H2(v8u16, __VA_ARGS__)
1041 #define PCKOD_H2_SH(...) PCKOD_H2(v8i16, __VA_ARGS__)
1042 #define PCKOD_H2_SW(...) PCKOD_H2(v4i32, __VA_ARGS__)
1043 #define PCKOD_H2_UW(...) PCKOD_H2(v4u32, __VA_ARGS__)
1044
1045 /* Description : Arithmetic immediate shift right all elements of word vector
1046 * Arguments : Inputs - in0, in1, shift
1047 * Outputs - in place operation
1048 * Return Type - as per input vector RTYPE
1049 * Details : Each element of vector 'in0' is right shifted by 'shift' and
1050 * the result is written in-place. 'shift' is a GP variable.
1051 */
1052 #define SRAI_W2(RTYPE, in0, in1, shift_val) do { \
1053 in0 = (RTYPE)SRAI_W(in0, shift_val); \
1054 in1 = (RTYPE)SRAI_W(in1, shift_val); \
1055 } while (0)
1056 #define SRAI_W2_SW(...) SRAI_W2(v4i32, __VA_ARGS__)
1057 #define SRAI_W2_UW(...) SRAI_W2(v4u32, __VA_ARGS__)
1058
1059 #define SRAI_W4(RTYPE, in0, in1, in2, in3, shift_val) do { \
1060 SRAI_W2(RTYPE, in0, in1, shift_val); \
1061 SRAI_W2(RTYPE, in2, in3, shift_val); \
1062 } while (0)
1063 #define SRAI_W4_SW(...) SRAI_W4(v4i32, __VA_ARGS__)
1064 #define SRAI_W4_UW(...) SRAI_W4(v4u32, __VA_ARGS__)
1065
1066 /* Description : Arithmetic shift right all elements of half-word vector
1067 * Arguments : Inputs - in0, in1, shift
1068 * Outputs - in place operation
1069 * Return Type - as per input vector RTYPE
1070 * Details : Each element of vector 'in0' is right shifted by 'shift' and
1071 * the result is written in-place. 'shift' is a GP variable.
1072 */
1073 #define SRAI_H2(RTYPE, in0, in1, shift_val) do { \
1074 in0 = (RTYPE)SRAI_H(in0, shift_val); \
1075 in1 = (RTYPE)SRAI_H(in1, shift_val); \
1076 } while (0)
1077 #define SRAI_H2_SH(...) SRAI_H2(v8i16, __VA_ARGS__)
1078 #define SRAI_H2_UH(...) SRAI_H2(v8u16, __VA_ARGS__)
1079
1080 /* Description : Arithmetic rounded shift right all elements of word vector
1081 * Arguments : Inputs - in0, in1, shift
1082 * Outputs - in place operation
1083 * Return Type - as per input vector RTYPE
1084 * Details : Each element of vector 'in0' is right shifted by 'shift' and
1085 * the result is written in-place. 'shift' is a GP variable.
1086 */
1087 #define SRARI_W2(RTYPE, in0, in1, shift) do { \
1088 in0 = (RTYPE)__msa_srari_w((v4i32)in0, shift); \
1089 in1 = (RTYPE)__msa_srari_w((v4i32)in1, shift); \
1090 } while (0)
1091 #define SRARI_W2_SW(...) SRARI_W2(v4i32, __VA_ARGS__)
1092
1093 #define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) do { \
1094 SRARI_W2(RTYPE, in0, in1, shift); \
1095 SRARI_W2(RTYPE, in2, in3, shift); \
1096 } while (0)
1097 #define SRARI_W4_SH(...) SRARI_W4(v8i16, __VA_ARGS__)
1098 #define SRARI_W4_UW(...) SRARI_W4(v4u32, __VA_ARGS__)
1099 #define SRARI_W4_SW(...) SRARI_W4(v4i32, __VA_ARGS__)
1100
1101 /* Description : Shift right arithmetic rounded double words
1102 * Arguments : Inputs - in0, in1, shift
1103 * Outputs - in place operation
1104 * Return Type - as per RTYPE
1105 * Details : Each element of vector 'in0' is shifted right arithmetically by
1106 * the number of bits in the corresponding element in the vector
1107 * 'shift'. The last discarded bit is added to shifted value for
1108 * rounding and the result is written in-place.
1109 * 'shift' is a vector.
1110 */
1111 #define SRAR_D2(RTYPE, in0, in1, shift) do { \
1112 in0 = (RTYPE)__msa_srar_d((v2i64)in0, (v2i64)shift); \
1113 in1 = (RTYPE)__msa_srar_d((v2i64)in1, (v2i64)shift); \
1114 } while (0)
1115 #define SRAR_D2_SW(...) SRAR_D2(v4i32, __VA_ARGS__)
1116 #define SRAR_D2_SD(...) SRAR_D2(v2i64, __VA_ARGS__)
1117 #define SRAR_D2_UD(...) SRAR_D2(v2u64, __VA_ARGS__)
1118
1119 #define SRAR_D4(RTYPE, in0, in1, in2, in3, shift) do { \
1120 SRAR_D2(RTYPE, in0, in1, shift); \
1121 SRAR_D2(RTYPE, in2, in3, shift); \
1122 } while (0)
1123 #define SRAR_D4_SD(...) SRAR_D4(v2i64, __VA_ARGS__)
1124 #define SRAR_D4_UD(...) SRAR_D4(v2u64, __VA_ARGS__)
1125
1126 /* Description : Addition of 2 pairs of half-word vectors
1127 * Arguments : Inputs - in0, in1, in2, in3
1128 * Outputs - out0, out1
1129 * Details : Each element in 'in0' is added to 'in1' and result is written
1130 * to 'out0'.
1131 */
1132 #define ADDVI_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
1133 out0 = (RTYPE)ADDVI_H(in0, in1); \
1134 out1 = (RTYPE)ADDVI_H(in2, in3); \
1135 } while (0)
1136 #define ADDVI_H2_SH(...) ADDVI_H2(v8i16, __VA_ARGS__)
1137 #define ADDVI_H2_UH(...) ADDVI_H2(v8u16, __VA_ARGS__)
1138
1139 /* Description : Addition of 2 pairs of word vectors
1140 * Arguments : Inputs - in0, in1, in2, in3
1141 * Outputs - out0, out1
1142 * Details : Each element in 'in0' is added to 'in1' and result is written
1143 * to 'out0'.
1144 */
1145 #define ADDVI_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
1146 out0 = (RTYPE)ADDVI_W(in0, in1); \
1147 out1 = (RTYPE)ADDVI_W(in2, in3); \
1148 } while (0)
1149 #define ADDVI_W2_SW(...) ADDVI_W2(v4i32, __VA_ARGS__)
1150
1151 /* Description : Fill 2 pairs of word vectors with GP registers
1152 * Arguments : Inputs - in0, in1
1153 * Outputs - out0, out1
1154 * Details : GP register in0 is replicated in each word element of out0
1155 * GP register in1 is replicated in each word element of out1
1156 */
1157 #define FILL_W2(RTYPE, in0, in1, out0, out1) do { \
1158 out0 = (RTYPE)__msa_fill_w(in0); \
1159 out1 = (RTYPE)__msa_fill_w(in1); \
1160 } while (0)
1161 #define FILL_W2_SW(...) FILL_W2(v4i32, __VA_ARGS__)
1162
1163 /* Description : Addition of 2 pairs of vectors
1164 * Arguments : Inputs - in0, in1, in2, in3
1165 * Outputs - out0, out1
1166 * Details : Each element in 'in0' is added to 'in1' and result is written
1167 * to 'out0'.
1168 */
1169 #define ADD2(in0, in1, in2, in3, out0, out1) do { \
1170 out0 = in0 + in1; \
1171 out1 = in2 + in3; \
1172 } while (0)
1173
1174 #define ADD4(in0, in1, in2, in3, in4, in5, in6, in7, \
1175 out0, out1, out2, out3) do { \
1176 ADD2(in0, in1, in2, in3, out0, out1); \
1177 ADD2(in4, in5, in6, in7, out2, out3); \
1178 } while (0)
1179
1180 /* Description : Subtraction of 2 pairs of vectors
1181 * Arguments : Inputs - in0, in1, in2, in3
1182 * Outputs - out0, out1
1183 * Details : Each element in 'in1' is subtracted from 'in0' and result is
1184 * written to 'out0'.
1185 */
1186 #define SUB2(in0, in1, in2, in3, out0, out1) do { \
1187 out0 = in0 - in1; \
1188 out1 = in2 - in3; \
1189 } while (0)
1190
1191 #define SUB3(in0, in1, in2, in3, in4, in5, out0, out1, out2) do { \
1192 out0 = in0 - in1; \
1193 out1 = in2 - in3; \
1194 out2 = in4 - in5; \
1195 } while (0)
1196
1197 #define SUB4(in0, in1, in2, in3, in4, in5, in6, in7, \
1198 out0, out1, out2, out3) do { \
1199 out0 = in0 - in1; \
1200 out1 = in2 - in3; \
1201 out2 = in4 - in5; \
1202 out3 = in6 - in7; \
1203 } while (0)
1204
1205 /* Description : Addition - Subtraction of input vectors
1206 * Arguments : Inputs - in0, in1
1207 * Outputs - out0, out1
1208 * Details : Each element in 'in1' is added to 'in0' and result is
1209 * written to 'out0'.
1210 * Each element in 'in1' is subtracted from 'in0' and result is
1211 * written to 'out1'.
1212 */
1213 #define ADDSUB2(in0, in1, out0, out1) do { \
1214 out0 = in0 + in1; \
1215 out1 = in0 - in1; \
1216 } while (0)
1217
1218 /* Description : Multiplication of pairs of vectors
1219 * Arguments : Inputs - in0, in1, in2, in3
1220 * Outputs - out0, out1
1221 * Details : Each element from 'in0' is multiplied with elements from 'in1'
1222 * and the result is written to 'out0'
1223 */
1224 #define MUL2(in0, in1, in2, in3, out0, out1) do { \
1225 out0 = in0 * in1; \
1226 out1 = in2 * in3; \
1227 } while (0)
1228
1229 #define MUL4(in0, in1, in2, in3, in4, in5, in6, in7, \
1230 out0, out1, out2, out3) do { \
1231 MUL2(in0, in1, in2, in3, out0, out1); \
1232 MUL2(in4, in5, in6, in7, out2, out3); \
1233 } while (0)
1234
1235 /* Description : Sign extend halfword elements from right half of the vector
1236 * Arguments : Input - in (halfword vector)
1237 * Output - out (sign extended word vector)
1238 * Return Type - signed word
1239 * Details : Sign bit of halfword elements from input vector 'in' is
1240 * extracted and interleaved with same vector 'in0' to generate
1241 * 4 word elements keeping sign intact
1242 */
1243 #define UNPCK_R_SH_SW(in, out) do { \
1244 const v8i16 sign_m = __msa_clti_s_h((v8i16)in, 0); \
1245 out = (v4i32)__msa_ilvr_h(sign_m, (v8i16)in); \
1246 } while (0)
1247
1248 /* Description : Sign extend halfword elements from input vector and return
1249 * the result in pair of vectors
1250 * Arguments : Input - in (halfword vector)
1251 * Outputs - out0, out1 (sign extended word vectors)
1252 * Return Type - signed word
1253 * Details : Sign bit of halfword elements from input vector 'in' is
1254 * extracted and interleaved right with same vector 'in0' to
1255 * generate 4 signed word elements in 'out0'
1256 * Then interleaved left with same vector 'in0' to
1257 * generate 4 signed word elements in 'out1'
1258 */
1259 #define UNPCK_SH_SW(in, out0, out1) do { \
1260 const v8i16 tmp_m = __msa_clti_s_h((v8i16)in, 0); \
1261 ILVRL_H2_SW(tmp_m, in, out0, out1); \
1262 } while (0)
1263
1264 /* Description : Butterfly of 4 input vectors
1265 * Arguments : Inputs - in0, in1, in2, in3
1266 * Outputs - out0, out1, out2, out3
1267 * Details : Butterfly operation
1268 */
1269 #define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) do { \
1270 out0 = in0 + in3; \
1271 out1 = in1 + in2; \
1272 out2 = in1 - in2; \
1273 out3 = in0 - in3; \
1274 } while (0)
1275
1276 /* Description : Transpose 16x4 block into 4x16 with byte elements in vectors
1277 * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7,
1278 * in8, in9, in10, in11, in12, in13, in14, in15
1279 * Outputs - out0, out1, out2, out3
1280 * Return Type - unsigned byte
1281 */
1282 #define TRANSPOSE16x4_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \
1283 in8, in9, in10, in11, in12, in13, in14, in15, \
1284 out0, out1, out2, out3) do { \
1285 v2i64 tmp0_m, tmp1_m, tmp2_m, tmp3_m, tmp4_m, tmp5_m; \
1286 ILVEV_W2_SD(in0, in4, in8, in12, tmp2_m, tmp3_m); \
1287 ILVEV_W2_SD(in1, in5, in9, in13, tmp0_m, tmp1_m); \
1288 ILVEV_D2_UB(tmp2_m, tmp3_m, tmp0_m, tmp1_m, out1, out3); \
1289 ILVEV_W2_SD(in2, in6, in10, in14, tmp4_m, tmp5_m); \
1290 ILVEV_W2_SD(in3, in7, in11, in15, tmp0_m, tmp1_m); \
1291 ILVEV_D2_SD(tmp4_m, tmp5_m, tmp0_m, tmp1_m, tmp2_m, tmp3_m); \
1292 ILVEV_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m); \
1293 ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out0, out2); \
1294 ILVOD_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m); \
1295 ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out1, out3); \
1296 } while (0)
1297
1298 /* Description : Transpose 16x8 block into 8x16 with byte elements in vectors
1299 * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7,
1300 * in8, in9, in10, in11, in12, in13, in14, in15
1301 * Outputs - out0, out1, out2, out3, out4, out5, out6, out7
1302 * Return Type - unsigned byte
1303 */
1304 #define TRANSPOSE16x8_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \
1305 in8, in9, in10, in11, in12, in13, in14, in15, \
1306 out0, out1, out2, out3, out4, out5, \
1307 out6, out7) do { \
1308 v8i16 tmp0_m, tmp1_m, tmp4_m, tmp5_m, tmp6_m, tmp7_m; \
1309 v4i32 tmp2_m, tmp3_m; \
1310 ILVEV_D2_UB(in0, in8, in1, in9, out7, out6); \
1311 ILVEV_D2_UB(in2, in10, in3, in11, out5, out4); \
1312 ILVEV_D2_UB(in4, in12, in5, in13, out3, out2); \
1313 ILVEV_D2_UB(in6, in14, in7, in15, out1, out0); \
1314 ILVEV_B2_SH(out7, out6, out5, out4, tmp0_m, tmp1_m); \
1315 ILVOD_B2_SH(out7, out6, out5, out4, tmp4_m, tmp5_m); \
1316 ILVEV_B2_UB(out3, out2, out1, out0, out5, out7); \
1317 ILVOD_B2_SH(out3, out2, out1, out0, tmp6_m, tmp7_m); \
1318 ILVEV_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m); \
1319 ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out0, out4); \
1320 ILVOD_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m); \
1321 ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out2, out6); \
1322 ILVEV_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m); \
1323 ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out1, out5); \
1324 ILVOD_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m); \
1325 ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out3, out7); \
1326 } while (0)
1327
1328 /* Description : Transpose 4x4 block with word elements in vectors
1329 * Arguments : Inputs - in0, in1, in2, in3
1330 * Outputs - out0, out1, out2, out3
1331 * Return Type - as per RTYPE
1332 */
1333 #define TRANSPOSE4x4_W(RTYPE, in0, in1, in2, in3, \
1334 out0, out1, out2, out3) do { \
1335 v4i32 s0_m, s1_m, s2_m, s3_m; \
1336 ILVRL_W2_SW(in1, in0, s0_m, s1_m); \
1337 ILVRL_W2_SW(in3, in2, s2_m, s3_m); \
1338 out0 = (RTYPE)__msa_ilvr_d((v2i64)s2_m, (v2i64)s0_m); \
1339 out1 = (RTYPE)__msa_ilvl_d((v2i64)s2_m, (v2i64)s0_m); \
1340 out2 = (RTYPE)__msa_ilvr_d((v2i64)s3_m, (v2i64)s1_m); \
1341 out3 = (RTYPE)__msa_ilvl_d((v2i64)s3_m, (v2i64)s1_m); \
1342 } while (0)
1343 #define TRANSPOSE4x4_SW_SW(...) TRANSPOSE4x4_W(v4i32, __VA_ARGS__)
1344
1345 /* Description : Add block 4x4
1346 * Arguments : Inputs - in0, in1, in2, in3, pdst, stride
1347 * Details : Least significant 4 bytes from each input vector are added to
1348 * the destination bytes, clipped between 0-255 and stored.
1349 */
1350 #define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do { \
1351 uint32_t src0_m, src1_m, src2_m, src3_m; \
1352 v8i16 inp0_m, inp1_m, res0_m, res1_m; \
1353 v16i8 dst0_m = { 0 }; \
1354 v16i8 dst1_m = { 0 }; \
1355 const v16i8 zero_m = { 0 }; \
1356 ILVR_D2_SH(in1, in0, in3, in2, inp0_m, inp1_m); \
1357 LW4(pdst, stride, src0_m, src1_m, src2_m, src3_m); \
1358 INSERT_W2_SB(src0_m, src1_m, dst0_m); \
1359 INSERT_W2_SB(src2_m, src3_m, dst1_m); \
1360 ILVR_B2_SH(zero_m, dst0_m, zero_m, dst1_m, res0_m, res1_m); \
1361 ADD2(res0_m, inp0_m, res1_m, inp1_m, res0_m, res1_m); \
1362 CLIP_SH2_0_255(res0_m, res1_m); \
1363 PCKEV_B2_SB(res0_m, res0_m, res1_m, res1_m, dst0_m, dst1_m); \
1364 ST4x4_UB(dst0_m, dst1_m, 0, 1, 0, 1, pdst, stride); \
1365 } while (0)
1366
1367 /* Description : Pack even byte elements, extract 0 & 2 index words from pair
1368 * of results and store 4 words in destination memory as per
1369 * stride
1370 * Arguments : Inputs - in0, in1, in2, in3, pdst, stride
1371 */
1372 #define PCKEV_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do { \
1373 v16i8 tmp0_m, tmp1_m; \
1374 PCKEV_B2_SB(in1, in0, in3, in2, tmp0_m, tmp1_m); \
1375 ST4x4_UB(tmp0_m, tmp1_m, 0, 2, 0, 2, pdst, stride); \
1376 } while (0)
1377
1378 /* Description : average with rounding (in0 + in1 + 1) / 2.
1379 * Arguments : Inputs - in0, in1, in2, in3,
1380 * Outputs - out0, out1
1381 * Return Type - as per RTYPE
1382 * Details : Each unsigned byte element from 'in0' vector is added with
1383 * each unsigned byte element from 'in1' vector. Then the average
1384 * with rounding is calculated and written to 'out0'
1385 */
1386 #define AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) do { \
1387 out0 = (RTYPE)__msa_aver_u_b((v16u8)in0, (v16u8)in1); \
1388 out1 = (RTYPE)__msa_aver_u_b((v16u8)in2, (v16u8)in3); \
1389 } while (0)
1390 #define AVER_UB2_UB(...) AVER_UB2(v16u8, __VA_ARGS__)
1391
1392 #endif // WEBP_DSP_MSA_MACRO_H_
1393