1 /*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include <emmintrin.h> // SSE2
13
14 #include "config/aom_dsp_rtcd.h"
15
16 #include "aom_dsp/txfm_common.h"
17 #include "aom_dsp/x86/fwd_txfm_sse2.h"
18 #include "aom_dsp/x86/txfm_common_sse2.h"
19 #include "aom_ports/mem.h"
20
21 // TODO(jingning) The high bit-depth functions need rework for performance.
22 // After we properly fix the high bit-depth function implementations, this
23 // file's dependency should be substantially simplified.
24 #if DCT_HIGH_BIT_DEPTH
25 #define ADD_EPI16 _mm_adds_epi16
26 #define SUB_EPI16 _mm_subs_epi16
27
28 #else
29 #define ADD_EPI16 _mm_add_epi16
30 #define SUB_EPI16 _mm_sub_epi16
31 #endif
32
FDCT8x8_2D(const int16_t * input,tran_low_t * output,int stride)33 void FDCT8x8_2D(const int16_t *input, tran_low_t *output, int stride) {
34 int pass;
35 // Constants
36 // When we use them, in one case, they are all the same. In all others
37 // it's a pair of them that we need to repeat four times. This is done
38 // by constructing the 32 bit constant corresponding to that pair.
39 const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64);
40 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
41 const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
42 const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
43 const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64);
44 const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64);
45 const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64);
46 const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
47 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
48 #if DCT_HIGH_BIT_DEPTH
49 int overflow;
50 #endif
51 // Load input
52 __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride));
53 __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride));
54 __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride));
55 __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride));
56 __m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride));
57 __m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride));
58 __m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride));
59 __m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride));
60 // Pre-condition input (shift by two)
61 in0 = _mm_slli_epi16(in0, 2);
62 in1 = _mm_slli_epi16(in1, 2);
63 in2 = _mm_slli_epi16(in2, 2);
64 in3 = _mm_slli_epi16(in3, 2);
65 in4 = _mm_slli_epi16(in4, 2);
66 in5 = _mm_slli_epi16(in5, 2);
67 in6 = _mm_slli_epi16(in6, 2);
68 in7 = _mm_slli_epi16(in7, 2);
69
70 // We do two passes, first the columns, then the rows. The results of the
71 // first pass are transposed so that the same column code can be reused. The
72 // results of the second pass are also transposed so that the rows (processed
73 // as columns) are put back in row positions.
74 for (pass = 0; pass < 2; pass++) {
75 // To store results of each pass before the transpose.
76 __m128i res0, res1, res2, res3, res4, res5, res6, res7;
77 // Add/subtract
78 const __m128i q0 = ADD_EPI16(in0, in7);
79 const __m128i q1 = ADD_EPI16(in1, in6);
80 const __m128i q2 = ADD_EPI16(in2, in5);
81 const __m128i q3 = ADD_EPI16(in3, in4);
82 const __m128i q4 = SUB_EPI16(in3, in4);
83 const __m128i q5 = SUB_EPI16(in2, in5);
84 const __m128i q6 = SUB_EPI16(in1, in6);
85 const __m128i q7 = SUB_EPI16(in0, in7);
86 #if DCT_HIGH_BIT_DEPTH
87 if (pass == 1) {
88 overflow =
89 check_epi16_overflow_x8(&q0, &q1, &q2, &q3, &q4, &q5, &q6, &q7);
90 if (overflow) {
91 aom_highbd_fdct8x8_c(input, output, stride);
92 return;
93 }
94 }
95 #endif // DCT_HIGH_BIT_DEPTH
96 // Work on first four results
97 {
98 // Add/subtract
99 const __m128i r0 = ADD_EPI16(q0, q3);
100 const __m128i r1 = ADD_EPI16(q1, q2);
101 const __m128i r2 = SUB_EPI16(q1, q2);
102 const __m128i r3 = SUB_EPI16(q0, q3);
103 #if DCT_HIGH_BIT_DEPTH
104 overflow = check_epi16_overflow_x4(&r0, &r1, &r2, &r3);
105 if (overflow) {
106 aom_highbd_fdct8x8_c(input, output, stride);
107 return;
108 }
109 #endif // DCT_HIGH_BIT_DEPTH
110 // Interleave to do the multiply by constants which gets us into 32bits
111 {
112 const __m128i t0 = _mm_unpacklo_epi16(r0, r1);
113 const __m128i t1 = _mm_unpackhi_epi16(r0, r1);
114 const __m128i t2 = _mm_unpacklo_epi16(r2, r3);
115 const __m128i t3 = _mm_unpackhi_epi16(r2, r3);
116 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16);
117 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16);
118 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16);
119 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16);
120 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08);
121 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08);
122 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24);
123 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24);
124 // dct_const_round_shift
125 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
126 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
127 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
128 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
129 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
130 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
131 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
132 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
133 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
134 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
135 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
136 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
137 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
138 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
139 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
140 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
141 // Combine
142 res0 = _mm_packs_epi32(w0, w1);
143 res4 = _mm_packs_epi32(w2, w3);
144 res2 = _mm_packs_epi32(w4, w5);
145 res6 = _mm_packs_epi32(w6, w7);
146 #if DCT_HIGH_BIT_DEPTH
147 overflow = check_epi16_overflow_x4(&res0, &res4, &res2, &res6);
148 if (overflow) {
149 aom_highbd_fdct8x8_c(input, output, stride);
150 return;
151 }
152 #endif // DCT_HIGH_BIT_DEPTH
153 }
154 }
155 // Work on next four results
156 {
157 // Interleave to do the multiply by constants which gets us into 32bits
158 const __m128i d0 = _mm_unpacklo_epi16(q6, q5);
159 const __m128i d1 = _mm_unpackhi_epi16(q6, q5);
160 const __m128i e0 = _mm_madd_epi16(d0, k__cospi_p16_m16);
161 const __m128i e1 = _mm_madd_epi16(d1, k__cospi_p16_m16);
162 const __m128i e2 = _mm_madd_epi16(d0, k__cospi_p16_p16);
163 const __m128i e3 = _mm_madd_epi16(d1, k__cospi_p16_p16);
164 // dct_const_round_shift
165 const __m128i f0 = _mm_add_epi32(e0, k__DCT_CONST_ROUNDING);
166 const __m128i f1 = _mm_add_epi32(e1, k__DCT_CONST_ROUNDING);
167 const __m128i f2 = _mm_add_epi32(e2, k__DCT_CONST_ROUNDING);
168 const __m128i f3 = _mm_add_epi32(e3, k__DCT_CONST_ROUNDING);
169 const __m128i s0 = _mm_srai_epi32(f0, DCT_CONST_BITS);
170 const __m128i s1 = _mm_srai_epi32(f1, DCT_CONST_BITS);
171 const __m128i s2 = _mm_srai_epi32(f2, DCT_CONST_BITS);
172 const __m128i s3 = _mm_srai_epi32(f3, DCT_CONST_BITS);
173 // Combine
174 const __m128i r0 = _mm_packs_epi32(s0, s1);
175 const __m128i r1 = _mm_packs_epi32(s2, s3);
176 #if DCT_HIGH_BIT_DEPTH
177 overflow = check_epi16_overflow_x2(&r0, &r1);
178 if (overflow) {
179 aom_highbd_fdct8x8_c(input, output, stride);
180 return;
181 }
182 #endif // DCT_HIGH_BIT_DEPTH
183 {
184 // Add/subtract
185 const __m128i x0 = ADD_EPI16(q4, r0);
186 const __m128i x1 = SUB_EPI16(q4, r0);
187 const __m128i x2 = SUB_EPI16(q7, r1);
188 const __m128i x3 = ADD_EPI16(q7, r1);
189 #if DCT_HIGH_BIT_DEPTH
190 overflow = check_epi16_overflow_x4(&x0, &x1, &x2, &x3);
191 if (overflow) {
192 aom_highbd_fdct8x8_c(input, output, stride);
193 return;
194 }
195 #endif // DCT_HIGH_BIT_DEPTH
196 // Interleave to do the multiply by constants which gets us into 32bits
197 {
198 const __m128i t0 = _mm_unpacklo_epi16(x0, x3);
199 const __m128i t1 = _mm_unpackhi_epi16(x0, x3);
200 const __m128i t2 = _mm_unpacklo_epi16(x1, x2);
201 const __m128i t3 = _mm_unpackhi_epi16(x1, x2);
202 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04);
203 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04);
204 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28);
205 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28);
206 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20);
207 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20);
208 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12);
209 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12);
210 // dct_const_round_shift
211 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
212 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
213 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
214 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
215 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
216 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
217 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
218 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
219 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
220 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
221 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
222 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
223 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
224 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
225 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
226 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
227 // Combine
228 res1 = _mm_packs_epi32(w0, w1);
229 res7 = _mm_packs_epi32(w2, w3);
230 res5 = _mm_packs_epi32(w4, w5);
231 res3 = _mm_packs_epi32(w6, w7);
232 #if DCT_HIGH_BIT_DEPTH
233 overflow = check_epi16_overflow_x4(&res1, &res7, &res5, &res3);
234 if (overflow) {
235 aom_highbd_fdct8x8_c(input, output, stride);
236 return;
237 }
238 #endif // DCT_HIGH_BIT_DEPTH
239 }
240 }
241 }
242 // Transpose the 8x8.
243 {
244 // 00 01 02 03 04 05 06 07
245 // 10 11 12 13 14 15 16 17
246 // 20 21 22 23 24 25 26 27
247 // 30 31 32 33 34 35 36 37
248 // 40 41 42 43 44 45 46 47
249 // 50 51 52 53 54 55 56 57
250 // 60 61 62 63 64 65 66 67
251 // 70 71 72 73 74 75 76 77
252 const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1);
253 const __m128i tr0_1 = _mm_unpacklo_epi16(res2, res3);
254 const __m128i tr0_2 = _mm_unpackhi_epi16(res0, res1);
255 const __m128i tr0_3 = _mm_unpackhi_epi16(res2, res3);
256 const __m128i tr0_4 = _mm_unpacklo_epi16(res4, res5);
257 const __m128i tr0_5 = _mm_unpacklo_epi16(res6, res7);
258 const __m128i tr0_6 = _mm_unpackhi_epi16(res4, res5);
259 const __m128i tr0_7 = _mm_unpackhi_epi16(res6, res7);
260 // 00 10 01 11 02 12 03 13
261 // 20 30 21 31 22 32 23 33
262 // 04 14 05 15 06 16 07 17
263 // 24 34 25 35 26 36 27 37
264 // 40 50 41 51 42 52 43 53
265 // 60 70 61 71 62 72 63 73
266 // 54 54 55 55 56 56 57 57
267 // 64 74 65 75 66 76 67 77
268 const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1);
269 const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3);
270 const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1);
271 const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3);
272 const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5);
273 const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7);
274 const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5);
275 const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7);
276 // 00 10 20 30 01 11 21 31
277 // 40 50 60 70 41 51 61 71
278 // 02 12 22 32 03 13 23 33
279 // 42 52 62 72 43 53 63 73
280 // 04 14 24 34 05 15 21 36
281 // 44 54 64 74 45 55 61 76
282 // 06 16 26 36 07 17 27 37
283 // 46 56 66 76 47 57 67 77
284 in0 = _mm_unpacklo_epi64(tr1_0, tr1_4);
285 in1 = _mm_unpackhi_epi64(tr1_0, tr1_4);
286 in2 = _mm_unpacklo_epi64(tr1_2, tr1_6);
287 in3 = _mm_unpackhi_epi64(tr1_2, tr1_6);
288 in4 = _mm_unpacklo_epi64(tr1_1, tr1_5);
289 in5 = _mm_unpackhi_epi64(tr1_1, tr1_5);
290 in6 = _mm_unpacklo_epi64(tr1_3, tr1_7);
291 in7 = _mm_unpackhi_epi64(tr1_3, tr1_7);
292 // 00 10 20 30 40 50 60 70
293 // 01 11 21 31 41 51 61 71
294 // 02 12 22 32 42 52 62 72
295 // 03 13 23 33 43 53 63 73
296 // 04 14 24 34 44 54 64 74
297 // 05 15 25 35 45 55 65 75
298 // 06 16 26 36 46 56 66 76
299 // 07 17 27 37 47 57 67 77
300 }
301 }
302 // Post-condition output and store it
303 {
304 // Post-condition (division by two)
305 // division of two 16 bits signed numbers using shifts
306 // n / 2 = (n - (n >> 15)) >> 1
307 const __m128i sign_in0 = _mm_srai_epi16(in0, 15);
308 const __m128i sign_in1 = _mm_srai_epi16(in1, 15);
309 const __m128i sign_in2 = _mm_srai_epi16(in2, 15);
310 const __m128i sign_in3 = _mm_srai_epi16(in3, 15);
311 const __m128i sign_in4 = _mm_srai_epi16(in4, 15);
312 const __m128i sign_in5 = _mm_srai_epi16(in5, 15);
313 const __m128i sign_in6 = _mm_srai_epi16(in6, 15);
314 const __m128i sign_in7 = _mm_srai_epi16(in7, 15);
315 in0 = _mm_sub_epi16(in0, sign_in0);
316 in1 = _mm_sub_epi16(in1, sign_in1);
317 in2 = _mm_sub_epi16(in2, sign_in2);
318 in3 = _mm_sub_epi16(in3, sign_in3);
319 in4 = _mm_sub_epi16(in4, sign_in4);
320 in5 = _mm_sub_epi16(in5, sign_in5);
321 in6 = _mm_sub_epi16(in6, sign_in6);
322 in7 = _mm_sub_epi16(in7, sign_in7);
323 in0 = _mm_srai_epi16(in0, 1);
324 in1 = _mm_srai_epi16(in1, 1);
325 in2 = _mm_srai_epi16(in2, 1);
326 in3 = _mm_srai_epi16(in3, 1);
327 in4 = _mm_srai_epi16(in4, 1);
328 in5 = _mm_srai_epi16(in5, 1);
329 in6 = _mm_srai_epi16(in6, 1);
330 in7 = _mm_srai_epi16(in7, 1);
331 // store results
332 store_output(&in0, (output + 0 * 8));
333 store_output(&in1, (output + 1 * 8));
334 store_output(&in2, (output + 2 * 8));
335 store_output(&in3, (output + 3 * 8));
336 store_output(&in4, (output + 4 * 8));
337 store_output(&in5, (output + 5 * 8));
338 store_output(&in6, (output + 6 * 8));
339 store_output(&in7, (output + 7 * 8));
340 }
341 }
342
343 #undef ADD_EPI16
344 #undef SUB_EPI16
345