1 /*
2 * Copyright (c) 2013 Seppo Tomperi
3 * Copyright (c) 2013 - 2014 Pierre-Edouard Lepere
4 *
5 *
6 * This file is part of FFmpeg.
7 *
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23 #include "config.h"
24
25 #include "libavutil/cpu.h"
26 #include "libavutil/x86/asm.h"
27 #include "libavutil/x86/cpu.h"
28 #include "libavcodec/get_bits.h" /* required for hevcdsp.h GetBitContext */
29 #include "libavcodec/hevcdsp.h"
30 #include "libavcodec/x86/hevcdsp.h"
31
32 #define LFC_FUNC(DIR, DEPTH, OPT) \
33 void ff_hevc_ ## DIR ## _loop_filter_chroma_ ## DEPTH ## _ ## OPT(uint8_t *pix, ptrdiff_t stride, int *tc, uint8_t *no_p, uint8_t *no_q);
34
35 #define LFL_FUNC(DIR, DEPTH, OPT) \
36 void ff_hevc_ ## DIR ## _loop_filter_luma_ ## DEPTH ## _ ## OPT(uint8_t *pix, ptrdiff_t stride, int beta, int *tc, uint8_t *no_p, uint8_t *no_q);
37
38 #define LFC_FUNCS(type, depth, opt) \
39 LFC_FUNC(h, depth, opt) \
40 LFC_FUNC(v, depth, opt)
41
42 #define LFL_FUNCS(type, depth, opt) \
43 LFL_FUNC(h, depth, opt) \
44 LFL_FUNC(v, depth, opt)
45
46 LFC_FUNCS(uint8_t, 8, sse2)
47 LFC_FUNCS(uint8_t, 10, sse2)
48 LFC_FUNCS(uint8_t, 12, sse2)
49 LFC_FUNCS(uint8_t, 8, avx)
50 LFC_FUNCS(uint8_t, 10, avx)
51 LFC_FUNCS(uint8_t, 12, avx)
52 LFL_FUNCS(uint8_t, 8, sse2)
53 LFL_FUNCS(uint8_t, 10, sse2)
54 LFL_FUNCS(uint8_t, 12, sse2)
55 LFL_FUNCS(uint8_t, 8, ssse3)
56 LFL_FUNCS(uint8_t, 10, ssse3)
57 LFL_FUNCS(uint8_t, 12, ssse3)
58 LFL_FUNCS(uint8_t, 8, avx)
59 LFL_FUNCS(uint8_t, 10, avx)
60 LFL_FUNCS(uint8_t, 12, avx)
61
62 #define IDCT_FUNCS(W, opt) \
63 void ff_hevc_idct##W##_dc_8_##opt(int16_t *coeffs); \
64 void ff_hevc_idct##W##_dc_10_##opt(int16_t *coeffs); \
65 void ff_hevc_idct##W##_dc_12_##opt(int16_t *coeffs)
66
67 IDCT_FUNCS(4x4, mmxext);
68 IDCT_FUNCS(8x8, mmxext);
69 IDCT_FUNCS(8x8, sse2);
70 IDCT_FUNCS(16x16, sse2);
71 IDCT_FUNCS(32x32, sse2);
72 IDCT_FUNCS(16x16, avx2);
73 IDCT_FUNCS(32x32, avx2);
74
75 #define mc_rep_func(name, bitd, step, W, opt) \
76 void ff_hevc_put_hevc_##name##W##_##bitd##_##opt(int16_t *_dst, \
77 uint8_t *_src, ptrdiff_t _srcstride, int height, \
78 intptr_t mx, intptr_t my, int width) \
79 { \
80 int i; \
81 uint8_t *src; \
82 int16_t *dst; \
83 for (i = 0; i < W; i += step) { \
84 src = _src + (i * ((bitd + 7) / 8)); \
85 dst = _dst + i; \
86 ff_hevc_put_hevc_##name##step##_##bitd##_##opt(dst, src, _srcstride, height, mx, my, width); \
87 } \
88 }
89 #define mc_rep_uni_func(name, bitd, step, W, opt) \
90 void ff_hevc_put_hevc_uni_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, \
91 uint8_t *_src, ptrdiff_t _srcstride, int height, \
92 intptr_t mx, intptr_t my, int width) \
93 { \
94 int i; \
95 uint8_t *src; \
96 uint8_t *dst; \
97 for (i = 0; i < W; i += step) { \
98 src = _src + (i * ((bitd + 7) / 8)); \
99 dst = _dst + (i * ((bitd + 7) / 8)); \
100 ff_hevc_put_hevc_uni_##name##step##_##bitd##_##opt(dst, dststride, src, _srcstride, \
101 height, mx, my, width); \
102 } \
103 }
104 #define mc_rep_bi_func(name, bitd, step, W, opt) \
105 void ff_hevc_put_hevc_bi_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, uint8_t *_src, \
106 ptrdiff_t _srcstride, int16_t* _src2, \
107 int height, intptr_t mx, intptr_t my, int width) \
108 { \
109 int i; \
110 uint8_t *src; \
111 uint8_t *dst; \
112 int16_t *src2; \
113 for (i = 0; i < W ; i += step) { \
114 src = _src + (i * ((bitd + 7) / 8)); \
115 dst = _dst + (i * ((bitd + 7) / 8)); \
116 src2 = _src2 + i; \
117 ff_hevc_put_hevc_bi_##name##step##_##bitd##_##opt(dst, dststride, src, _srcstride, src2, \
118 height, mx, my, width); \
119 } \
120 }
121
122 #define mc_rep_funcs(name, bitd, step, W, opt) \
123 mc_rep_func(name, bitd, step, W, opt); \
124 mc_rep_uni_func(name, bitd, step, W, opt); \
125 mc_rep_bi_func(name, bitd, step, W, opt)
126
127 #define mc_rep_func2(name, bitd, step1, step2, W, opt) \
128 void ff_hevc_put_hevc_##name##W##_##bitd##_##opt(int16_t *dst, \
129 uint8_t *src, ptrdiff_t _srcstride, int height, \
130 intptr_t mx, intptr_t my, int width) \
131 { \
132 ff_hevc_put_hevc_##name##step1##_##bitd##_##opt(dst, src, _srcstride, height, mx, my, width); \
133 ff_hevc_put_hevc_##name##step2##_##bitd##_##opt(dst + step1, src + (step1 * ((bitd + 7) / 8)), \
134 _srcstride, height, mx, my, width); \
135 }
136 #define mc_rep_uni_func2(name, bitd, step1, step2, W, opt) \
137 void ff_hevc_put_hevc_uni_##name##W##_##bitd##_##opt(uint8_t *dst, ptrdiff_t dststride, \
138 uint8_t *src, ptrdiff_t _srcstride, int height, \
139 intptr_t mx, intptr_t my, int width) \
140 { \
141 ff_hevc_put_hevc_uni_##name##step1##_##bitd##_##opt(dst, dststride, src, _srcstride, height, mx, my, width);\
142 ff_hevc_put_hevc_uni_##name##step2##_##bitd##_##opt(dst + (step1 * ((bitd + 7) / 8)), dststride, \
143 src + (step1 * ((bitd + 7) / 8)), _srcstride, \
144 height, mx, my, width); \
145 }
146 #define mc_rep_bi_func2(name, bitd, step1, step2, W, opt) \
147 void ff_hevc_put_hevc_bi_##name##W##_##bitd##_##opt(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, \
148 ptrdiff_t _srcstride, int16_t* src2, \
149 int height, intptr_t mx, intptr_t my, int width) \
150 { \
151 ff_hevc_put_hevc_bi_##name##step1##_##bitd##_##opt(dst, dststride, src, _srcstride, src2, height, mx, my, width);\
152 ff_hevc_put_hevc_bi_##name##step2##_##bitd##_##opt(dst + (step1 * ((bitd + 7) / 8)), dststride, \
153 src + (step1 * ((bitd + 7) / 8)), _srcstride, \
154 src2 + step1, height, mx, my, width); \
155 }
156
157 #define mc_rep_funcs(name, bitd, step, W, opt) \
158 mc_rep_func(name, bitd, step, W, opt); \
159 mc_rep_uni_func(name, bitd, step, W, opt); \
160 mc_rep_bi_func(name, bitd, step, W, opt)
161
162 #define mc_rep_funcs2(name, bitd, step1, step2, W, opt) \
163 mc_rep_func2(name, bitd, step1, step2, W, opt); \
164 mc_rep_uni_func2(name, bitd, step1, step2, W, opt); \
165 mc_rep_bi_func2(name, bitd, step1, step2, W, opt)
166
167 #if ARCH_X86_64 && HAVE_SSE4_EXTERNAL
168
169 mc_rep_funcs(pel_pixels, 8, 16, 64, sse4);
170 mc_rep_funcs(pel_pixels, 8, 16, 48, sse4);
171 mc_rep_funcs(pel_pixels, 8, 16, 32, sse4);
172 mc_rep_funcs(pel_pixels, 8, 8, 24, sse4);
173 mc_rep_funcs(pel_pixels,10, 8, 64, sse4);
174 mc_rep_funcs(pel_pixels,10, 8, 48, sse4);
175 mc_rep_funcs(pel_pixels,10, 8, 32, sse4);
176 mc_rep_funcs(pel_pixels,10, 8, 24, sse4);
177 mc_rep_funcs(pel_pixels,10, 8, 16, sse4);
178 mc_rep_funcs(pel_pixels,10, 4, 12, sse4);
179 mc_rep_funcs(pel_pixels,12, 8, 64, sse4);
180 mc_rep_funcs(pel_pixels,12, 8, 48, sse4);
181 mc_rep_funcs(pel_pixels,12, 8, 32, sse4);
182 mc_rep_funcs(pel_pixels,12, 8, 24, sse4);
183 mc_rep_funcs(pel_pixels,12, 8, 16, sse4);
184 mc_rep_funcs(pel_pixels,12, 4, 12, sse4);
185
186 mc_rep_funcs(epel_h, 8, 16, 64, sse4);
187 mc_rep_funcs(epel_h, 8, 16, 48, sse4);
188 mc_rep_funcs(epel_h, 8, 16, 32, sse4);
189 mc_rep_funcs(epel_h, 8, 8, 24, sse4);
190 mc_rep_funcs(epel_h,10, 8, 64, sse4);
191 mc_rep_funcs(epel_h,10, 8, 48, sse4);
192 mc_rep_funcs(epel_h,10, 8, 32, sse4);
193 mc_rep_funcs(epel_h,10, 8, 24, sse4);
194 mc_rep_funcs(epel_h,10, 8, 16, sse4);
195 mc_rep_funcs(epel_h,10, 4, 12, sse4);
196 mc_rep_funcs(epel_h,12, 8, 64, sse4);
197 mc_rep_funcs(epel_h,12, 8, 48, sse4);
198 mc_rep_funcs(epel_h,12, 8, 32, sse4);
199 mc_rep_funcs(epel_h,12, 8, 24, sse4);
200 mc_rep_funcs(epel_h,12, 8, 16, sse4);
201 mc_rep_funcs(epel_h,12, 4, 12, sse4);
202 mc_rep_funcs(epel_v, 8, 16, 64, sse4);
203 mc_rep_funcs(epel_v, 8, 16, 48, sse4);
204 mc_rep_funcs(epel_v, 8, 16, 32, sse4);
205 mc_rep_funcs(epel_v, 8, 8, 24, sse4);
206 mc_rep_funcs(epel_v,10, 8, 64, sse4);
207 mc_rep_funcs(epel_v,10, 8, 48, sse4);
208 mc_rep_funcs(epel_v,10, 8, 32, sse4);
209 mc_rep_funcs(epel_v,10, 8, 24, sse4);
210 mc_rep_funcs(epel_v,10, 8, 16, sse4);
211 mc_rep_funcs(epel_v,10, 4, 12, sse4);
212 mc_rep_funcs(epel_v,12, 8, 64, sse4);
213 mc_rep_funcs(epel_v,12, 8, 48, sse4);
214 mc_rep_funcs(epel_v,12, 8, 32, sse4);
215 mc_rep_funcs(epel_v,12, 8, 24, sse4);
216 mc_rep_funcs(epel_v,12, 8, 16, sse4);
217 mc_rep_funcs(epel_v,12, 4, 12, sse4);
218 mc_rep_funcs(epel_hv, 8, 8, 64, sse4);
219 mc_rep_funcs(epel_hv, 8, 8, 48, sse4);
220 mc_rep_funcs(epel_hv, 8, 8, 32, sse4);
221 mc_rep_funcs(epel_hv, 8, 8, 24, sse4);
222 mc_rep_funcs(epel_hv, 8, 8, 16, sse4);
223 mc_rep_funcs2(epel_hv,8, 8, 4, 12, sse4);
224 mc_rep_funcs(epel_hv,10, 8, 64, sse4);
225 mc_rep_funcs(epel_hv,10, 8, 48, sse4);
226 mc_rep_funcs(epel_hv,10, 8, 32, sse4);
227 mc_rep_funcs(epel_hv,10, 8, 24, sse4);
228 mc_rep_funcs(epel_hv,10, 8, 16, sse4);
229 mc_rep_funcs(epel_hv,10, 4, 12, sse4);
230 mc_rep_funcs(epel_hv,12, 8, 64, sse4);
231 mc_rep_funcs(epel_hv,12, 8, 48, sse4);
232 mc_rep_funcs(epel_hv,12, 8, 32, sse4);
233 mc_rep_funcs(epel_hv,12, 8, 24, sse4);
234 mc_rep_funcs(epel_hv,12, 8, 16, sse4);
235 mc_rep_funcs(epel_hv,12, 4, 12, sse4);
236
237 mc_rep_funcs(qpel_h, 8, 16, 64, sse4);
238 mc_rep_funcs(qpel_h, 8, 16, 48, sse4);
239 mc_rep_funcs(qpel_h, 8, 16, 32, sse4);
240 mc_rep_funcs(qpel_h, 8, 8, 24, sse4);
241 mc_rep_funcs(qpel_h,10, 8, 64, sse4);
242 mc_rep_funcs(qpel_h,10, 8, 48, sse4);
243 mc_rep_funcs(qpel_h,10, 8, 32, sse4);
244 mc_rep_funcs(qpel_h,10, 8, 24, sse4);
245 mc_rep_funcs(qpel_h,10, 8, 16, sse4);
246 mc_rep_funcs(qpel_h,10, 4, 12, sse4);
247 mc_rep_funcs(qpel_h,12, 8, 64, sse4);
248 mc_rep_funcs(qpel_h,12, 8, 48, sse4);
249 mc_rep_funcs(qpel_h,12, 8, 32, sse4);
250 mc_rep_funcs(qpel_h,12, 8, 24, sse4);
251 mc_rep_funcs(qpel_h,12, 8, 16, sse4);
252 mc_rep_funcs(qpel_h,12, 4, 12, sse4);
253 mc_rep_funcs(qpel_v, 8, 16, 64, sse4);
254 mc_rep_funcs(qpel_v, 8, 16, 48, sse4);
255 mc_rep_funcs(qpel_v, 8, 16, 32, sse4);
256 mc_rep_funcs(qpel_v, 8, 8, 24, sse4);
257 mc_rep_funcs(qpel_v,10, 8, 64, sse4);
258 mc_rep_funcs(qpel_v,10, 8, 48, sse4);
259 mc_rep_funcs(qpel_v,10, 8, 32, sse4);
260 mc_rep_funcs(qpel_v,10, 8, 24, sse4);
261 mc_rep_funcs(qpel_v,10, 8, 16, sse4);
262 mc_rep_funcs(qpel_v,10, 4, 12, sse4);
263 mc_rep_funcs(qpel_v,12, 8, 64, sse4);
264 mc_rep_funcs(qpel_v,12, 8, 48, sse4);
265 mc_rep_funcs(qpel_v,12, 8, 32, sse4);
266 mc_rep_funcs(qpel_v,12, 8, 24, sse4);
267 mc_rep_funcs(qpel_v,12, 8, 16, sse4);
268 mc_rep_funcs(qpel_v,12, 4, 12, sse4);
269 mc_rep_funcs(qpel_hv, 8, 8, 64, sse4);
270 mc_rep_funcs(qpel_hv, 8, 8, 48, sse4);
271 mc_rep_funcs(qpel_hv, 8, 8, 32, sse4);
272 mc_rep_funcs(qpel_hv, 8, 8, 24, sse4);
273 mc_rep_funcs(qpel_hv, 8, 8, 16, sse4);
274 mc_rep_funcs2(qpel_hv,8, 8, 4, 12, sse4);
275 mc_rep_funcs(qpel_hv,10, 8, 64, sse4);
276 mc_rep_funcs(qpel_hv,10, 8, 48, sse4);
277 mc_rep_funcs(qpel_hv,10, 8, 32, sse4);
278 mc_rep_funcs(qpel_hv,10, 8, 24, sse4);
279 mc_rep_funcs(qpel_hv,10, 8, 16, sse4);
280 mc_rep_funcs(qpel_hv,10, 4, 12, sse4);
281 mc_rep_funcs(qpel_hv,12, 8, 64, sse4);
282 mc_rep_funcs(qpel_hv,12, 8, 48, sse4);
283 mc_rep_funcs(qpel_hv,12, 8, 32, sse4);
284 mc_rep_funcs(qpel_hv,12, 8, 24, sse4);
285 mc_rep_funcs(qpel_hv,12, 8, 16, sse4);
286 mc_rep_funcs(qpel_hv,12, 4, 12, sse4);
287
288 #define mc_rep_uni_w(bitd, step, W, opt) \
289 void ff_hevc_put_hevc_uni_w##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, int16_t *_src, ptrdiff_t _srcstride,\
290 int height, int denom, int _wx, int _ox) \
291 { \
292 int i; \
293 int16_t *src; \
294 uint8_t *dst; \
295 for (i = 0; i < W; i += step) { \
296 src= _src + i; \
297 dst= _dst + (i * ((bitd + 7) / 8)); \
298 ff_hevc_put_hevc_uni_w##step##_##bitd##_##opt(dst, dststride, src, _srcstride, \
299 height, denom, _wx, _ox); \
300 } \
301 }
302
303 mc_rep_uni_w(8, 6, 12, sse4);
304 mc_rep_uni_w(8, 8, 16, sse4);
305 mc_rep_uni_w(8, 8, 24, sse4);
306 mc_rep_uni_w(8, 8, 32, sse4);
307 mc_rep_uni_w(8, 8, 48, sse4);
308 mc_rep_uni_w(8, 8, 64, sse4);
309
310 mc_rep_uni_w(10, 6, 12, sse4);
311 mc_rep_uni_w(10, 8, 16, sse4);
312 mc_rep_uni_w(10, 8, 24, sse4);
313 mc_rep_uni_w(10, 8, 32, sse4);
314 mc_rep_uni_w(10, 8, 48, sse4);
315 mc_rep_uni_w(10, 8, 64, sse4);
316
317 mc_rep_uni_w(12, 6, 12, sse4);
318 mc_rep_uni_w(12, 8, 16, sse4);
319 mc_rep_uni_w(12, 8, 24, sse4);
320 mc_rep_uni_w(12, 8, 32, sse4);
321 mc_rep_uni_w(12, 8, 48, sse4);
322 mc_rep_uni_w(12, 8, 64, sse4);
323
324 #define mc_rep_bi_w(bitd, step, W, opt) \
325 void ff_hevc_put_hevc_bi_w##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, int16_t *_src, ptrdiff_t _srcstride, \
326 int16_t *_src2, int height, \
327 int denom, int _wx0, int _wx1, int _ox0, int _ox1) \
328 { \
329 int i; \
330 int16_t *src; \
331 int16_t *src2; \
332 uint8_t *dst; \
333 for (i = 0; i < W; i += step) { \
334 src = _src + i; \
335 src2 = _src2 + i; \
336 dst = _dst + (i * ((bitd + 7) / 8)); \
337 ff_hevc_put_hevc_bi_w##step##_##bitd##_##opt(dst, dststride, src, _srcstride, src2, \
338 height, denom, _wx0, _wx1, _ox0, _ox1); \
339 } \
340 }
341
342 mc_rep_bi_w(8, 6, 12, sse4);
343 mc_rep_bi_w(8, 8, 16, sse4);
344 mc_rep_bi_w(8, 8, 24, sse4);
345 mc_rep_bi_w(8, 8, 32, sse4);
346 mc_rep_bi_w(8, 8, 48, sse4);
347 mc_rep_bi_w(8, 8, 64, sse4);
348
349 mc_rep_bi_w(10, 6, 12, sse4);
350 mc_rep_bi_w(10, 8, 16, sse4);
351 mc_rep_bi_w(10, 8, 24, sse4);
352 mc_rep_bi_w(10, 8, 32, sse4);
353 mc_rep_bi_w(10, 8, 48, sse4);
354 mc_rep_bi_w(10, 8, 64, sse4);
355
356 mc_rep_bi_w(12, 6, 12, sse4);
357 mc_rep_bi_w(12, 8, 16, sse4);
358 mc_rep_bi_w(12, 8, 24, sse4);
359 mc_rep_bi_w(12, 8, 32, sse4);
360 mc_rep_bi_w(12, 8, 48, sse4);
361 mc_rep_bi_w(12, 8, 64, sse4);
362
363 #define mc_uni_w_func(name, bitd, W, opt) \
364 void ff_hevc_put_hevc_uni_w_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t _dststride, \
365 uint8_t *_src, ptrdiff_t _srcstride, \
366 int height, int denom, \
367 int _wx, int _ox, \
368 intptr_t mx, intptr_t my, int width) \
369 { \
370 LOCAL_ALIGNED_16(int16_t, temp, [71 * MAX_PB_SIZE]); \
371 ff_hevc_put_hevc_##name##W##_##bitd##_##opt(temp, _src, _srcstride, height, mx, my, width); \
372 ff_hevc_put_hevc_uni_w##W##_##bitd##_##opt(_dst, _dststride, temp, MAX_PB_SIZE, height, denom, _wx, _ox);\
373 }
374
375 #define mc_uni_w_funcs(name, bitd, opt) \
376 mc_uni_w_func(name, bitd, 4, opt); \
377 mc_uni_w_func(name, bitd, 8, opt); \
378 mc_uni_w_func(name, bitd, 12, opt); \
379 mc_uni_w_func(name, bitd, 16, opt); \
380 mc_uni_w_func(name, bitd, 24, opt); \
381 mc_uni_w_func(name, bitd, 32, opt); \
382 mc_uni_w_func(name, bitd, 48, opt); \
383 mc_uni_w_func(name, bitd, 64, opt)
384
385 mc_uni_w_funcs(pel_pixels, 8, sse4);
386 mc_uni_w_func(pel_pixels, 8, 6, sse4);
387 mc_uni_w_funcs(epel_h, 8, sse4);
388 mc_uni_w_func(epel_h, 8, 6, sse4);
389 mc_uni_w_funcs(epel_v, 8, sse4);
390 mc_uni_w_func(epel_v, 8, 6, sse4);
391 mc_uni_w_funcs(epel_hv, 8, sse4);
392 mc_uni_w_func(epel_hv, 8, 6, sse4);
393 mc_uni_w_funcs(qpel_h, 8, sse4);
394 mc_uni_w_funcs(qpel_v, 8, sse4);
395 mc_uni_w_funcs(qpel_hv, 8, sse4);
396
397 mc_uni_w_funcs(pel_pixels, 10, sse4);
398 mc_uni_w_func(pel_pixels, 10, 6, sse4);
399 mc_uni_w_funcs(epel_h, 10, sse4);
400 mc_uni_w_func(epel_h, 10, 6, sse4);
401 mc_uni_w_funcs(epel_v, 10, sse4);
402 mc_uni_w_func(epel_v, 10, 6, sse4);
403 mc_uni_w_funcs(epel_hv, 10, sse4);
404 mc_uni_w_func(epel_hv, 10, 6, sse4);
405 mc_uni_w_funcs(qpel_h, 10, sse4);
406 mc_uni_w_funcs(qpel_v, 10, sse4);
407 mc_uni_w_funcs(qpel_hv, 10, sse4);
408
409 mc_uni_w_funcs(pel_pixels, 12, sse4);
410 mc_uni_w_func(pel_pixels, 12, 6, sse4);
411 mc_uni_w_funcs(epel_h, 12, sse4);
412 mc_uni_w_func(epel_h, 12, 6, sse4);
413 mc_uni_w_funcs(epel_v, 12, sse4);
414 mc_uni_w_func(epel_v, 12, 6, sse4);
415 mc_uni_w_funcs(epel_hv, 12, sse4);
416 mc_uni_w_func(epel_hv, 12, 6, sse4);
417 mc_uni_w_funcs(qpel_h, 12, sse4);
418 mc_uni_w_funcs(qpel_v, 12, sse4);
419 mc_uni_w_funcs(qpel_hv, 12, sse4);
420
421 #define mc_bi_w_func(name, bitd, W, opt) \
422 void ff_hevc_put_hevc_bi_w_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t _dststride, \
423 uint8_t *_src, ptrdiff_t _srcstride, \
424 int16_t *_src2, \
425 int height, int denom, \
426 int _wx0, int _wx1, int _ox0, int _ox1, \
427 intptr_t mx, intptr_t my, int width) \
428 { \
429 LOCAL_ALIGNED_16(int16_t, temp, [71 * MAX_PB_SIZE]); \
430 ff_hevc_put_hevc_##name##W##_##bitd##_##opt(temp, _src, _srcstride, height, mx, my, width); \
431 ff_hevc_put_hevc_bi_w##W##_##bitd##_##opt(_dst, _dststride, temp, MAX_PB_SIZE, _src2, \
432 height, denom, _wx0, _wx1, _ox0, _ox1); \
433 }
434
435 #define mc_bi_w_funcs(name, bitd, opt) \
436 mc_bi_w_func(name, bitd, 4, opt); \
437 mc_bi_w_func(name, bitd, 8, opt); \
438 mc_bi_w_func(name, bitd, 12, opt); \
439 mc_bi_w_func(name, bitd, 16, opt); \
440 mc_bi_w_func(name, bitd, 24, opt); \
441 mc_bi_w_func(name, bitd, 32, opt); \
442 mc_bi_w_func(name, bitd, 48, opt); \
443 mc_bi_w_func(name, bitd, 64, opt)
444
445 mc_bi_w_funcs(pel_pixels, 8, sse4);
446 mc_bi_w_func(pel_pixels, 8, 6, sse4);
447 mc_bi_w_funcs(epel_h, 8, sse4);
448 mc_bi_w_func(epel_h, 8, 6, sse4);
449 mc_bi_w_funcs(epel_v, 8, sse4);
450 mc_bi_w_func(epel_v, 8, 6, sse4);
451 mc_bi_w_funcs(epel_hv, 8, sse4);
452 mc_bi_w_func(epel_hv, 8, 6, sse4);
453 mc_bi_w_funcs(qpel_h, 8, sse4);
454 mc_bi_w_funcs(qpel_v, 8, sse4);
455 mc_bi_w_funcs(qpel_hv, 8, sse4);
456
457 mc_bi_w_funcs(pel_pixels, 10, sse4);
458 mc_bi_w_func(pel_pixels, 10, 6, sse4);
459 mc_bi_w_funcs(epel_h, 10, sse4);
460 mc_bi_w_func(epel_h, 10, 6, sse4);
461 mc_bi_w_funcs(epel_v, 10, sse4);
462 mc_bi_w_func(epel_v, 10, 6, sse4);
463 mc_bi_w_funcs(epel_hv, 10, sse4);
464 mc_bi_w_func(epel_hv, 10, 6, sse4);
465 mc_bi_w_funcs(qpel_h, 10, sse4);
466 mc_bi_w_funcs(qpel_v, 10, sse4);
467 mc_bi_w_funcs(qpel_hv, 10, sse4);
468
469 mc_bi_w_funcs(pel_pixels, 12, sse4);
470 mc_bi_w_func(pel_pixels, 12, 6, sse4);
471 mc_bi_w_funcs(epel_h, 12, sse4);
472 mc_bi_w_func(epel_h, 12, 6, sse4);
473 mc_bi_w_funcs(epel_v, 12, sse4);
474 mc_bi_w_func(epel_v, 12, 6, sse4);
475 mc_bi_w_funcs(epel_hv, 12, sse4);
476 mc_bi_w_func(epel_hv, 12, 6, sse4);
477 mc_bi_w_funcs(qpel_h, 12, sse4);
478 mc_bi_w_funcs(qpel_v, 12, sse4);
479 mc_bi_w_funcs(qpel_hv, 12, sse4);
480 #endif //ARCH_X86_64 && HAVE_SSE4_EXTERNAL
481
482
483 #define EPEL_LINKS(pointer, my, mx, fname, bitd, opt ) \
484 PEL_LINK(pointer, 1, my , mx , fname##4 , bitd, opt ); \
485 PEL_LINK(pointer, 2, my , mx , fname##6 , bitd, opt ); \
486 PEL_LINK(pointer, 3, my , mx , fname##8 , bitd, opt ); \
487 PEL_LINK(pointer, 4, my , mx , fname##12, bitd, opt ); \
488 PEL_LINK(pointer, 5, my , mx , fname##16, bitd, opt ); \
489 PEL_LINK(pointer, 6, my , mx , fname##24, bitd, opt ); \
490 PEL_LINK(pointer, 7, my , mx , fname##32, bitd, opt ); \
491 PEL_LINK(pointer, 8, my , mx , fname##48, bitd, opt ); \
492 PEL_LINK(pointer, 9, my , mx , fname##64, bitd, opt )
493 #define QPEL_LINKS(pointer, my, mx, fname, bitd, opt) \
494 PEL_LINK(pointer, 1, my , mx , fname##4 , bitd, opt ); \
495 PEL_LINK(pointer, 3, my , mx , fname##8 , bitd, opt ); \
496 PEL_LINK(pointer, 4, my , mx , fname##12, bitd, opt ); \
497 PEL_LINK(pointer, 5, my , mx , fname##16, bitd, opt ); \
498 PEL_LINK(pointer, 6, my , mx , fname##24, bitd, opt ); \
499 PEL_LINK(pointer, 7, my , mx , fname##32, bitd, opt ); \
500 PEL_LINK(pointer, 8, my , mx , fname##48, bitd, opt ); \
501 PEL_LINK(pointer, 9, my , mx , fname##64, bitd, opt )
502
503
ff_hevc_dsp_init_x86(HEVCDSPContext * c,const int bit_depth)504 void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
505 {
506 int cpu_flags = av_get_cpu_flags();
507
508 if (bit_depth == 8) {
509 #if (HAVE_MMXEXT_EXTERNAL == 1)
510 if (EXTERNAL_MMXEXT(cpu_flags)) {
511 c->idct_dc[0] = ff_hevc_idct4x4_dc_8_mmxext;
512 c->idct_dc[1] = ff_hevc_idct8x8_dc_8_mmxext;
513 c->transform_add[0] = ff_hevc_transform_add4_8_mmxext;
514 }
515 #endif
516 #if (HAVE_SSE2_EXTERNAL == 1)
517 if (EXTERNAL_SSE2(cpu_flags)) {
518 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_sse2;
519 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_sse2;
520 if (ARCH_X86_64) {
521 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_sse2;
522 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_sse2;
523 }
524 c->idct_dc[1] = ff_hevc_idct8x8_dc_8_sse2;
525 c->idct_dc[2] = ff_hevc_idct16x16_dc_8_sse2;
526 c->idct_dc[3] = ff_hevc_idct32x32_dc_8_sse2;
527
528 c->transform_add[1] = ff_hevc_transform_add8_8_sse2;
529 c->transform_add[2] = ff_hevc_transform_add16_8_sse2;
530 c->transform_add[3] = ff_hevc_transform_add32_8_sse2;
531 }
532 #endif
533 #if (HAVE_SSSE3_EXTERNAL == 1)
534 if (EXTERNAL_SSSE3(cpu_flags) && ARCH_X86_64) {
535 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_ssse3;
536 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_ssse3;
537 }
538 #endif
539 #if (HAVE_SSE4_EXTERNAL == 1)
540 if (EXTERNAL_SSE4(cpu_flags) && ARCH_X86_64) {
541 EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 8, sse4);
542 EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 8, sse4);
543 EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 8, sse4);
544 EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 8, sse4);
545
546 QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 8, sse4);
547 QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 8, sse4);
548 QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 8, sse4);
549 QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 8, sse4);
550 }
551 #endif
552 #if (HAVE_AVX_EXTERNAL == 1)
553 if (EXTERNAL_AVX(cpu_flags)) {
554 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_avx;
555 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_avx;
556 if (ARCH_X86_64) {
557 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_avx;
558 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_avx;
559 }
560 c->transform_add[1] = ff_hevc_transform_add8_8_avx;
561 c->transform_add[2] = ff_hevc_transform_add16_8_avx;
562 c->transform_add[3] = ff_hevc_transform_add32_8_avx;
563 }
564 #endif
565 #if (HAVE_AVX2_EXTERNAL == 1)
566 if (EXTERNAL_AVX2(cpu_flags)) {
567 c->idct_dc[2] = ff_hevc_idct16x16_dc_8_avx2;
568 c->idct_dc[3] = ff_hevc_idct32x32_dc_8_avx2;
569
570 c->transform_add[3] = ff_hevc_transform_add32_8_avx2;
571 }
572 #endif
573 } else if (bit_depth == 10) {
574 #if (HAVE_MMXEXT_EXTERNAL == 1)
575 if (EXTERNAL_MMXEXT(cpu_flags)) {
576 c->transform_add[0] = ff_hevc_transform_add4_10_mmxext;
577 c->idct_dc[0] = ff_hevc_idct4x4_dc_10_mmxext;
578 c->idct_dc[1] = ff_hevc_idct8x8_dc_10_mmxext;
579 }
580 #endif
581 #if (HAVE_SSE2_EXTERNAL == 1)
582 if (EXTERNAL_SSE2(cpu_flags)) {
583 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_sse2;
584 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_sse2;
585 if (ARCH_X86_64) {
586 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_sse2;
587 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_sse2;
588 }
589
590 c->idct_dc[1] = ff_hevc_idct8x8_dc_10_sse2;
591 c->idct_dc[2] = ff_hevc_idct16x16_dc_10_sse2;
592 c->idct_dc[3] = ff_hevc_idct32x32_dc_10_sse2;
593
594 c->transform_add[1] = ff_hevc_transform_add8_10_sse2;
595 c->transform_add[2] = ff_hevc_transform_add16_10_sse2;
596 c->transform_add[3] = ff_hevc_transform_add32_10_sse2;
597 }
598 #endif
599 #if (HAVE_SSSE3_EXTERNAL == 1)
600 if (EXTERNAL_SSSE3(cpu_flags) && ARCH_X86_64) {
601 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_ssse3;
602 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_ssse3;
603 }
604 #endif
605 #if (HAVE_SSE4_EXTERNAL == 1)
606 if (EXTERNAL_SSE4(cpu_flags) && ARCH_X86_64) {
607 EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 10, sse4);
608 EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 10, sse4);
609 EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 10, sse4);
610 EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 10, sse4);
611
612 QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 10, sse4);
613 QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 10, sse4);
614 QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 10, sse4);
615 QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 10, sse4);
616 }
617 #endif
618 #if (HAVE_AVX_EXTERNAL == 1)
619 if (EXTERNAL_AVX(cpu_flags)) {
620 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_avx;
621 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_avx;
622 if (ARCH_X86_64) {
623 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_avx;
624 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_avx;
625 }
626 }
627 #endif
628 #if (HAVE_AVX2_EXTERNAL == 1)
629 if (EXTERNAL_AVX2(cpu_flags)) {
630
631 c->idct_dc[2] = ff_hevc_idct16x16_dc_10_avx2;
632 c->idct_dc[3] = ff_hevc_idct32x32_dc_10_avx2;
633
634 c->transform_add[2] = ff_hevc_transform_add16_10_avx2;
635 c->transform_add[3] = ff_hevc_transform_add32_10_avx2;
636
637 }
638 #endif
639 } else if (bit_depth == 12) {
640 #if (HAVE_MMXEXT_EXTERNAL == 1)
641 if (EXTERNAL_MMXEXT(cpu_flags)) {
642 c->idct_dc[0] = ff_hevc_idct4x4_dc_12_mmxext;
643 c->idct_dc[1] = ff_hevc_idct8x8_dc_12_mmxext;
644 }
645 #endif
646 #if (HAVE_SSE2_EXTERNAL == 1)
647 if (EXTERNAL_SSE2(cpu_flags)) {
648 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_sse2;
649 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_sse2;
650 if (ARCH_X86_64) {
651 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_sse2;
652 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_sse2;
653 }
654
655 c->idct_dc[1] = ff_hevc_idct8x8_dc_12_sse2;
656 c->idct_dc[2] = ff_hevc_idct16x16_dc_12_sse2;
657 c->idct_dc[3] = ff_hevc_idct32x32_dc_12_sse2;
658 }
659 #endif
660 #if (HAVE_SSSE3_EXTERNAL == 1)
661 if (EXTERNAL_SSSE3(cpu_flags) && ARCH_X86_64) {
662 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_ssse3;
663 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_ssse3;
664 }
665 #endif
666 #if (HAVE_SSE4_EXTERNAL == 1)
667 if (EXTERNAL_SSE4(cpu_flags) && ARCH_X86_64) {
668 EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 12, sse4);
669 EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 12, sse4);
670 EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 12, sse4);
671 EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 12, sse4);
672
673 QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 12, sse4);
674 QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 12, sse4);
675 QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 12, sse4);
676 QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 12, sse4);
677 }
678 #endif
679 #if (HAVE_AVX_EXTERNAL == 1)
680 if (EXTERNAL_AVX(cpu_flags)) {
681 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_avx;
682 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_avx;
683 if (ARCH_X86_64) {
684 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_avx;
685 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_avx;
686 }
687 }
688 #endif
689 #if (HAVE_AVX2_EXTERNAL == 1)
690 if (EXTERNAL_AVX2(cpu_flags)) {
691 c->idct_dc[2] = ff_hevc_idct16x16_dc_12_avx2;
692 c->idct_dc[3] = ff_hevc_idct32x32_dc_12_avx2;
693 }
694 #endif
695 }
696 }
697