1 /*
2 * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <arm_neon.h>
12 #include <assert.h>
13
14 #include "./vp9_rtcd.h"
15 #include "./vpx_config.h"
16 #include "vp9/common/vp9_common.h"
17
18 static int16_t sinpi_1_9 = 0x14a3;
19 static int16_t sinpi_2_9 = 0x26c9;
20 static int16_t sinpi_3_9 = 0x3441;
21 static int16_t sinpi_4_9 = 0x3b6c;
22 static int16_t cospi_8_64 = 0x3b21;
23 static int16_t cospi_16_64 = 0x2d41;
24 static int16_t cospi_24_64 = 0x187e;
25
TRANSPOSE4X4(int16x8_t * q8s16,int16x8_t * q9s16)26 static INLINE void TRANSPOSE4X4(
27 int16x8_t *q8s16,
28 int16x8_t *q9s16) {
29 int32x4_t q8s32, q9s32;
30 int16x4x2_t d0x2s16, d1x2s16;
31 int32x4x2_t q0x2s32;
32
33 d0x2s16 = vtrn_s16(vget_low_s16(*q8s16), vget_high_s16(*q8s16));
34 d1x2s16 = vtrn_s16(vget_low_s16(*q9s16), vget_high_s16(*q9s16));
35
36 q8s32 = vreinterpretq_s32_s16(vcombine_s16(d0x2s16.val[0], d0x2s16.val[1]));
37 q9s32 = vreinterpretq_s32_s16(vcombine_s16(d1x2s16.val[0], d1x2s16.val[1]));
38 q0x2s32 = vtrnq_s32(q8s32, q9s32);
39
40 *q8s16 = vreinterpretq_s16_s32(q0x2s32.val[0]);
41 *q9s16 = vreinterpretq_s16_s32(q0x2s32.val[1]);
42 return;
43 }
44
GENERATE_COSINE_CONSTANTS(int16x4_t * d0s16,int16x4_t * d1s16,int16x4_t * d2s16)45 static INLINE void GENERATE_COSINE_CONSTANTS(
46 int16x4_t *d0s16,
47 int16x4_t *d1s16,
48 int16x4_t *d2s16) {
49 *d0s16 = vdup_n_s16(cospi_8_64);
50 *d1s16 = vdup_n_s16(cospi_16_64);
51 *d2s16 = vdup_n_s16(cospi_24_64);
52 return;
53 }
54
GENERATE_SINE_CONSTANTS(int16x4_t * d3s16,int16x4_t * d4s16,int16x4_t * d5s16,int16x8_t * q3s16)55 static INLINE void GENERATE_SINE_CONSTANTS(
56 int16x4_t *d3s16,
57 int16x4_t *d4s16,
58 int16x4_t *d5s16,
59 int16x8_t *q3s16) {
60 *d3s16 = vdup_n_s16(sinpi_1_9);
61 *d4s16 = vdup_n_s16(sinpi_2_9);
62 *q3s16 = vdupq_n_s16(sinpi_3_9);
63 *d5s16 = vdup_n_s16(sinpi_4_9);
64 return;
65 }
66
IDCT4x4_1D(int16x4_t * d0s16,int16x4_t * d1s16,int16x4_t * d2s16,int16x8_t * q8s16,int16x8_t * q9s16)67 static INLINE void IDCT4x4_1D(
68 int16x4_t *d0s16,
69 int16x4_t *d1s16,
70 int16x4_t *d2s16,
71 int16x8_t *q8s16,
72 int16x8_t *q9s16) {
73 int16x4_t d16s16, d17s16, d18s16, d19s16, d23s16, d24s16;
74 int16x4_t d26s16, d27s16, d28s16, d29s16;
75 int32x4_t q10s32, q13s32, q14s32, q15s32;
76 int16x8_t q13s16, q14s16;
77
78 d16s16 = vget_low_s16(*q8s16);
79 d17s16 = vget_high_s16(*q8s16);
80 d18s16 = vget_low_s16(*q9s16);
81 d19s16 = vget_high_s16(*q9s16);
82
83 d23s16 = vadd_s16(d16s16, d18s16);
84 d24s16 = vsub_s16(d16s16, d18s16);
85
86 q15s32 = vmull_s16(d17s16, *d2s16);
87 q10s32 = vmull_s16(d17s16, *d0s16);
88 q13s32 = vmull_s16(d23s16, *d1s16);
89 q14s32 = vmull_s16(d24s16, *d1s16);
90 q15s32 = vmlsl_s16(q15s32, d19s16, *d0s16);
91 q10s32 = vmlal_s16(q10s32, d19s16, *d2s16);
92
93 d26s16 = vqrshrn_n_s32(q13s32, 14);
94 d27s16 = vqrshrn_n_s32(q14s32, 14);
95 d29s16 = vqrshrn_n_s32(q15s32, 14);
96 d28s16 = vqrshrn_n_s32(q10s32, 14);
97
98 q13s16 = vcombine_s16(d26s16, d27s16);
99 q14s16 = vcombine_s16(d28s16, d29s16);
100 *q8s16 = vaddq_s16(q13s16, q14s16);
101 *q9s16 = vsubq_s16(q13s16, q14s16);
102 *q9s16 = vcombine_s16(vget_high_s16(*q9s16),
103 vget_low_s16(*q9s16)); // vswp
104 return;
105 }
106
IADST4x4_1D(int16x4_t * d3s16,int16x4_t * d4s16,int16x4_t * d5s16,int16x8_t * q3s16,int16x8_t * q8s16,int16x8_t * q9s16)107 static INLINE void IADST4x4_1D(
108 int16x4_t *d3s16,
109 int16x4_t *d4s16,
110 int16x4_t *d5s16,
111 int16x8_t *q3s16,
112 int16x8_t *q8s16,
113 int16x8_t *q9s16) {
114 int16x4_t d6s16, d16s16, d17s16, d18s16, d19s16;
115 int32x4_t q8s32, q9s32, q10s32, q11s32, q12s32, q13s32, q14s32, q15s32;
116
117 d6s16 = vget_low_s16(*q3s16);
118
119 d16s16 = vget_low_s16(*q8s16);
120 d17s16 = vget_high_s16(*q8s16);
121 d18s16 = vget_low_s16(*q9s16);
122 d19s16 = vget_high_s16(*q9s16);
123
124 q10s32 = vmull_s16(*d3s16, d16s16);
125 q11s32 = vmull_s16(*d4s16, d16s16);
126 q12s32 = vmull_s16(d6s16, d17s16);
127 q13s32 = vmull_s16(*d5s16, d18s16);
128 q14s32 = vmull_s16(*d3s16, d18s16);
129 q15s32 = vmovl_s16(d16s16);
130 q15s32 = vaddw_s16(q15s32, d19s16);
131 q8s32 = vmull_s16(*d4s16, d19s16);
132 q15s32 = vsubw_s16(q15s32, d18s16);
133 q9s32 = vmull_s16(*d5s16, d19s16);
134
135 q10s32 = vaddq_s32(q10s32, q13s32);
136 q10s32 = vaddq_s32(q10s32, q8s32);
137 q11s32 = vsubq_s32(q11s32, q14s32);
138 q8s32 = vdupq_n_s32(sinpi_3_9);
139 q11s32 = vsubq_s32(q11s32, q9s32);
140 q15s32 = vmulq_s32(q15s32, q8s32);
141
142 q13s32 = vaddq_s32(q10s32, q12s32);
143 q10s32 = vaddq_s32(q10s32, q11s32);
144 q14s32 = vaddq_s32(q11s32, q12s32);
145 q10s32 = vsubq_s32(q10s32, q12s32);
146
147 d16s16 = vqrshrn_n_s32(q13s32, 14);
148 d17s16 = vqrshrn_n_s32(q14s32, 14);
149 d18s16 = vqrshrn_n_s32(q15s32, 14);
150 d19s16 = vqrshrn_n_s32(q10s32, 14);
151
152 *q8s16 = vcombine_s16(d16s16, d17s16);
153 *q9s16 = vcombine_s16(d18s16, d19s16);
154 return;
155 }
156
vp9_iht4x4_16_add_neon(const tran_low_t * input,uint8_t * dest,int dest_stride,int tx_type)157 void vp9_iht4x4_16_add_neon(const tran_low_t *input, uint8_t *dest,
158 int dest_stride, int tx_type) {
159 uint8x8_t d26u8, d27u8;
160 int16x4_t d0s16, d1s16, d2s16, d3s16, d4s16, d5s16;
161 uint32x2_t d26u32, d27u32;
162 int16x8_t q3s16, q8s16, q9s16;
163 uint16x8_t q8u16, q9u16;
164
165 d26u32 = d27u32 = vdup_n_u32(0);
166
167 q8s16 = vld1q_s16(input);
168 q9s16 = vld1q_s16(input + 8);
169
170 TRANSPOSE4X4(&q8s16, &q9s16);
171
172 switch (tx_type) {
173 case 0: // idct_idct is not supported. Fall back to C
174 vp9_iht4x4_16_add_c(input, dest, dest_stride, tx_type);
175 return;
176 break;
177 case 1: // iadst_idct
178 // generate constants
179 GENERATE_COSINE_CONSTANTS(&d0s16, &d1s16, &d2s16);
180 GENERATE_SINE_CONSTANTS(&d3s16, &d4s16, &d5s16, &q3s16);
181
182 // first transform rows
183 IDCT4x4_1D(&d0s16, &d1s16, &d2s16, &q8s16, &q9s16);
184
185 // transpose the matrix
186 TRANSPOSE4X4(&q8s16, &q9s16);
187
188 // then transform columns
189 IADST4x4_1D(&d3s16, &d4s16, &d5s16, &q3s16, &q8s16, &q9s16);
190 break;
191 case 2: // idct_iadst
192 // generate constantsyy
193 GENERATE_COSINE_CONSTANTS(&d0s16, &d1s16, &d2s16);
194 GENERATE_SINE_CONSTANTS(&d3s16, &d4s16, &d5s16, &q3s16);
195
196 // first transform rows
197 IADST4x4_1D(&d3s16, &d4s16, &d5s16, &q3s16, &q8s16, &q9s16);
198
199 // transpose the matrix
200 TRANSPOSE4X4(&q8s16, &q9s16);
201
202 // then transform columns
203 IDCT4x4_1D(&d0s16, &d1s16, &d2s16, &q8s16, &q9s16);
204 break;
205 case 3: // iadst_iadst
206 // generate constants
207 GENERATE_SINE_CONSTANTS(&d3s16, &d4s16, &d5s16, &q3s16);
208
209 // first transform rows
210 IADST4x4_1D(&d3s16, &d4s16, &d5s16, &q3s16, &q8s16, &q9s16);
211
212 // transpose the matrix
213 TRANSPOSE4X4(&q8s16, &q9s16);
214
215 // then transform columns
216 IADST4x4_1D(&d3s16, &d4s16, &d5s16, &q3s16, &q8s16, &q9s16);
217 break;
218 default: // iadst_idct
219 assert(0);
220 break;
221 }
222
223 q8s16 = vrshrq_n_s16(q8s16, 4);
224 q9s16 = vrshrq_n_s16(q9s16, 4);
225
226 d26u32 = vld1_lane_u32((const uint32_t *)dest, d26u32, 0);
227 dest += dest_stride;
228 d26u32 = vld1_lane_u32((const uint32_t *)dest, d26u32, 1);
229 dest += dest_stride;
230 d27u32 = vld1_lane_u32((const uint32_t *)dest, d27u32, 0);
231 dest += dest_stride;
232 d27u32 = vld1_lane_u32((const uint32_t *)dest, d27u32, 1);
233
234 q8u16 = vaddw_u8(vreinterpretq_u16_s16(q8s16), vreinterpret_u8_u32(d26u32));
235 q9u16 = vaddw_u8(vreinterpretq_u16_s16(q9s16), vreinterpret_u8_u32(d27u32));
236
237 d26u8 = vqmovun_s16(vreinterpretq_s16_u16(q8u16));
238 d27u8 = vqmovun_s16(vreinterpretq_s16_u16(q9u16));
239
240 vst1_lane_u32((uint32_t *)dest, vreinterpret_u32_u8(d27u8), 1);
241 dest -= dest_stride;
242 vst1_lane_u32((uint32_t *)dest, vreinterpret_u32_u8(d27u8), 0);
243 dest -= dest_stride;
244 vst1_lane_u32((uint32_t *)dest, vreinterpret_u32_u8(d26u8), 1);
245 dest -= dest_stride;
246 vst1_lane_u32((uint32_t *)dest, vreinterpret_u32_u8(d26u8), 0);
247 return;
248 }
249