1 /*****************************************************************************
2 * pixel.c: pixel metrics
3 *****************************************************************************
4 * Copyright (C) 2003-2014 x264 project
5 *
6 * Authors: Loren Merritt <lorenm@u.washington.edu>
7 * Laurent Aimar <fenrir@via.ecp.fr>
8 * Fiona Glaser <fiona@x264.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
23 *
24 * This program is also available under a commercial proprietary license.
25 * For more information, contact us at licensing@x264.com.
26 *****************************************************************************/
27
28 #include "common.h"
29
30 #if HAVE_MMX
31 # include "x86/pixel.h"
32 # include "x86/predict.h"
33 #endif
34 #if ARCH_PPC
35 # include "ppc/pixel.h"
36 #endif
37 #if ARCH_ARM
38 # include "arm/pixel.h"
39 # include "arm/predict.h"
40 #endif
41 #if ARCH_AARCH64
42 # include "aarch64/pixel.h"
43 # include "aarch64/predict.h"
44 #endif
45
46
47 /****************************************************************************
48 * pixel_sad_WxH
49 ****************************************************************************/
50 #define PIXEL_SAD_C( name, lx, ly ) \
51 static int name( pixel *pix1, intptr_t i_stride_pix1, \
52 pixel *pix2, intptr_t i_stride_pix2 ) \
53 { \
54 int i_sum = 0; \
55 int y; \
56 int x; \
57 for( y = 0; y < ly; y++ ) \
58 { \
59 for( x = 0; x < lx; x++ ) \
60 { \
61 i_sum += abs( pix1[x] - pix2[x] ); \
62 } \
63 pix1 += i_stride_pix1; \
64 pix2 += i_stride_pix2; \
65 } \
66 return i_sum; \
67 }
68
69 PIXEL_SAD_C( x264_pixel_sad_16x16, 16, 16 )
70 PIXEL_SAD_C( x264_pixel_sad_16x8, 16, 8 )
71 PIXEL_SAD_C( x264_pixel_sad_8x16, 8, 16 )
72 PIXEL_SAD_C( x264_pixel_sad_8x8, 8, 8 )
73 PIXEL_SAD_C( x264_pixel_sad_8x4, 8, 4 )
74 PIXEL_SAD_C( x264_pixel_sad_4x16, 4, 16 )
75 PIXEL_SAD_C( x264_pixel_sad_4x8, 4, 8 )
76 PIXEL_SAD_C( x264_pixel_sad_4x4, 4, 4 )
77
78 /****************************************************************************
79 * pixel_ssd_WxH
80 ****************************************************************************/
81 #define PIXEL_SSD_C( name, lx, ly ) \
82 static int name( pixel *pix1, intptr_t i_stride_pix1, \
83 pixel *pix2, intptr_t i_stride_pix2 ) \
84 { \
85 int i_sum = 0; \
86 int y; \
87 int x; \
88 \
89 for( y = 0; y < ly; y++ ) \
90 { \
91 for( x = 0; x < lx; x++ ) \
92 { \
93 int d = pix1[x] - pix2[x]; \
94 i_sum += d*d; \
95 } \
96 pix1 += i_stride_pix1; \
97 pix2 += i_stride_pix2; \
98 } \
99 return i_sum; \
100 }
101
102 PIXEL_SSD_C( x264_pixel_ssd_16x16, 16, 16 )
103 PIXEL_SSD_C( x264_pixel_ssd_16x8, 16, 8 )
104 PIXEL_SSD_C( x264_pixel_ssd_8x16, 8, 16 )
105 PIXEL_SSD_C( x264_pixel_ssd_8x8, 8, 8 )
106 PIXEL_SSD_C( x264_pixel_ssd_8x4, 8, 4 )
107 PIXEL_SSD_C( x264_pixel_ssd_4x16, 4, 16 )
108 PIXEL_SSD_C( x264_pixel_ssd_4x8, 4, 8 )
109 PIXEL_SSD_C( x264_pixel_ssd_4x4, 4, 4 )
110
x264_pixel_ssd_wxh(x264_pixel_function_t * pf,pixel * pix1,intptr_t i_pix1,pixel * pix2,intptr_t i_pix2,int i_width,int i_height)111 uint64_t x264_pixel_ssd_wxh( x264_pixel_function_t *pf, pixel *pix1, intptr_t i_pix1,
112 pixel *pix2, intptr_t i_pix2, int i_width, int i_height )
113 {
114 uint64_t i_ssd = 0;
115 int y;
116 int align = !(((intptr_t)pix1 | (intptr_t)pix2 | i_pix1 | i_pix2) & 15);
117 int x;
118
119 #define SSD(size) i_ssd += pf->ssd[size]( pix1 + y*i_pix1 + x, i_pix1, \
120 pix2 + y*i_pix2 + x, i_pix2 );
121 for( y = 0; y < i_height-15; y += 16 )
122 {
123 x = 0;
124 if( align )
125 for( ; x < i_width-15; x += 16 )
126 SSD(PIXEL_16x16);
127 for( ; x < i_width-7; x += 8 )
128 SSD(PIXEL_8x16);
129 }
130 if( y < i_height-7 ) {
131 for( x = 0; x < i_width-7; x += 8 )
132 SSD(PIXEL_8x8);
133 }
134 #undef SSD
135
136 #define SSD1 { int d = pix1[y*i_pix1+x] - pix2[y*i_pix2+x]; i_ssd += d*d; }
137 if( i_width & 7 )
138 {
139 for( y = 0; y < (i_height & ~7); y++ )
140 for( x = i_width & ~7; x < i_width; x++ )
141 SSD1;
142 }
143 if( i_height & 7 )
144 {
145 for( y = i_height & ~7; y < i_height; y++ )
146 for( x = 0; x < i_width; x++ )
147 SSD1;
148 }
149 #undef SSD1
150
151 return i_ssd;
152 }
153
pixel_ssd_nv12_core(pixel * pixuv1,intptr_t stride1,pixel * pixuv2,intptr_t stride2,int width,int height,uint64_t * ssd_u,uint64_t * ssd_v)154 static void pixel_ssd_nv12_core( pixel *pixuv1, intptr_t stride1, pixel *pixuv2, intptr_t stride2,
155 int width, int height, uint64_t *ssd_u, uint64_t *ssd_v )
156 {
157 int y;
158 int x;
159 *ssd_u = 0, *ssd_v = 0;
160
161 for( y = 0; y < height; y++, pixuv1+=stride1, pixuv2+=stride2 )
162 for( x = 0; x < width; x++ )
163 {
164 int du = pixuv1[2*x] - pixuv2[2*x];
165 int dv = pixuv1[2*x+1] - pixuv2[2*x+1];
166 *ssd_u += du*du;
167 *ssd_v += dv*dv;
168 }
169 }
170
x264_pixel_ssd_nv12(x264_pixel_function_t * pf,pixel * pix1,intptr_t i_pix1,pixel * pix2,intptr_t i_pix2,int i_width,int i_height,uint64_t * ssd_u,uint64_t * ssd_v)171 void x264_pixel_ssd_nv12( x264_pixel_function_t *pf, pixel *pix1, intptr_t i_pix1, pixel *pix2, intptr_t i_pix2,
172 int i_width, int i_height, uint64_t *ssd_u, uint64_t *ssd_v )
173 {
174 pf->ssd_nv12_core( pix1, i_pix1, pix2, i_pix2, i_width&~7, i_height, ssd_u, ssd_v );
175 if( i_width&7 )
176 {
177 uint64_t tmp[2];
178 pixel_ssd_nv12_core( pix1+(i_width&~7), i_pix1, pix2+(i_width&~7), i_pix2, i_width&7, i_height, &tmp[0], &tmp[1] );
179 *ssd_u += tmp[0];
180 *ssd_v += tmp[1];
181 }
182 }
183
184 /****************************************************************************
185 * pixel_var_wxh
186 ****************************************************************************/
187 #define PIXEL_VAR_C( name, w, h ) \
188 static uint64_t name( pixel *pix, intptr_t i_stride ) \
189 { \
190 uint32_t sum = 0, sqr = 0; \
191 int y; \
192 int x; \
193 for( y = 0; y < h; y++ ) \
194 { \
195 for( x = 0; x < w; x++ ) \
196 { \
197 sum += pix[x]; \
198 sqr += pix[x] * pix[x]; \
199 } \
200 pix += i_stride; \
201 } \
202 return sum + ((uint64_t)sqr << 32); \
203 }
204
205 PIXEL_VAR_C( x264_pixel_var_16x16, 16, 16 )
206 PIXEL_VAR_C( x264_pixel_var_8x16, 8, 16 )
207 PIXEL_VAR_C( x264_pixel_var_8x8, 8, 8 )
208
209 /****************************************************************************
210 * pixel_var2_wxh
211 ****************************************************************************/
212 #define PIXEL_VAR2_C( name, w, h, shift ) \
213 static int name( pixel *pix1, intptr_t i_stride1, pixel *pix2, intptr_t i_stride2, int *ssd ) \
214 { \
215 int var = 0, sum = 0, sqr = 0; \
216 int y; \
217 int x; \
218 for( y = 0; y < h; y++ ) \
219 { \
220 for( x = 0; x < w; x++ ) \
221 { \
222 int diff = pix1[x] - pix2[x]; \
223 sum += diff; \
224 sqr += diff * diff; \
225 } \
226 pix1 += i_stride1; \
227 pix2 += i_stride2; \
228 } \
229 var = sqr - ((int64_t)sum * sum >> shift); \
230 *ssd = sqr; \
231 return var; \
232 }
233
234 PIXEL_VAR2_C( x264_pixel_var2_8x16, 8, 16, 7 )
235 PIXEL_VAR2_C( x264_pixel_var2_8x8, 8, 8, 6 )
236
237 #if BIT_DEPTH > 8
238 typedef uint32_t sum_t;
239 typedef uint64_t sum2_t;
240 #else
241 typedef uint16_t sum_t;
242 typedef uint32_t sum2_t;
243 #endif
244 #define BITS_PER_SUM (8 * sizeof(sum_t))
245
246 #define HADAMARD4(d0, d1, d2, d3, s0, s1, s2, s3) {\
247 sum2_t t0 = s0 + s1;\
248 sum2_t t1 = s0 - s1;\
249 sum2_t t2 = s2 + s3;\
250 sum2_t t3 = s2 - s3;\
251 d0 = t0 + t2;\
252 d2 = t0 - t2;\
253 d1 = t1 + t3;\
254 d3 = t1 - t3;\
255 }
256
257 // in: a pseudo-simd number of the form x+(y<<16)
258 // return: abs(x)+(abs(y)<<16)
abs2(sum2_t a)259 static ALWAYS_INLINE sum2_t abs2( sum2_t a )
260 {
261 sum2_t s = ((a>>(BITS_PER_SUM-1))&(((sum2_t)1<<BITS_PER_SUM)+1))*((sum_t)-1);
262 return (a+s)^s;
263 }
264
265 /****************************************************************************
266 * pixel_satd_WxH: sum of 4x4 Hadamard transformed differences
267 ****************************************************************************/
268
x264_pixel_satd_4x4(pixel * pix1,intptr_t i_pix1,pixel * pix2,intptr_t i_pix2)269 static NOINLINE int x264_pixel_satd_4x4( pixel *pix1, intptr_t i_pix1, pixel *pix2, intptr_t i_pix2 )
270 {
271 sum2_t tmp[4][2];
272 sum2_t a0, a1, a2, a3, b0, b1;
273 sum2_t sum = 0;
274 int i;
275
276 for( i = 0; i < 4; i++, pix1 += i_pix1, pix2 += i_pix2 )
277 {
278 a0 = pix1[0] - pix2[0];
279 a1 = pix1[1] - pix2[1];
280 b0 = (a0+a1) + ((a0-a1)<<BITS_PER_SUM);
281 a2 = pix1[2] - pix2[2];
282 a3 = pix1[3] - pix2[3];
283 b1 = (a2+a3) + ((a2-a3)<<BITS_PER_SUM);
284 tmp[i][0] = b0 + b1;
285 tmp[i][1] = b0 - b1;
286 }
287 for( i = 0; i < 2; i++ )
288 {
289 HADAMARD4( a0, a1, a2, a3, tmp[0][i], tmp[1][i], tmp[2][i], tmp[3][i] );
290 a0 = abs2(a0) + abs2(a1) + abs2(a2) + abs2(a3);
291 sum += ((sum_t)a0) + (a0>>BITS_PER_SUM);
292 }
293 return sum >> 1;
294 }
295
x264_pixel_satd_8x4(pixel * pix1,intptr_t i_pix1,pixel * pix2,intptr_t i_pix2)296 static NOINLINE int x264_pixel_satd_8x4( pixel *pix1, intptr_t i_pix1, pixel *pix2, intptr_t i_pix2 )
297 {
298 sum2_t tmp[4][4];
299 sum2_t a0, a1, a2, a3;
300 sum2_t sum = 0;
301 int i;
302
303 for( i = 0; i < 4; i++, pix1 += i_pix1, pix2 += i_pix2 )
304 {
305 a0 = (pix1[0] - pix2[0]) + ((sum2_t)(pix1[4] - pix2[4]) << BITS_PER_SUM);
306 a1 = (pix1[1] - pix2[1]) + ((sum2_t)(pix1[5] - pix2[5]) << BITS_PER_SUM);
307 a2 = (pix1[2] - pix2[2]) + ((sum2_t)(pix1[6] - pix2[6]) << BITS_PER_SUM);
308 a3 = (pix1[3] - pix2[3]) + ((sum2_t)(pix1[7] - pix2[7]) << BITS_PER_SUM);
309 HADAMARD4( tmp[i][0], tmp[i][1], tmp[i][2], tmp[i][3], a0,a1,a2,a3 );
310 }
311 for( i = 0; i < 4; i++ )
312 {
313 HADAMARD4( a0, a1, a2, a3, tmp[0][i], tmp[1][i], tmp[2][i], tmp[3][i] );
314 sum += abs2(a0) + abs2(a1) + abs2(a2) + abs2(a3);
315 }
316 return (((sum_t)sum) + (sum>>BITS_PER_SUM)) >> 1;
317 }
318
319 #define PIXEL_SATD_C( w, h, sub )\
320 static int x264_pixel_satd_##w##x##h( pixel *pix1, intptr_t i_pix1, pixel *pix2, intptr_t i_pix2 )\
321 {\
322 int sum = sub( pix1, i_pix1, pix2, i_pix2 )\
323 + sub( pix1+4*i_pix1, i_pix1, pix2+4*i_pix2, i_pix2 );\
324 if( w==16 )\
325 sum+= sub( pix1+8, i_pix1, pix2+8, i_pix2 )\
326 + sub( pix1+8+4*i_pix1, i_pix1, pix2+8+4*i_pix2, i_pix2 );\
327 if( h==16 )\
328 sum+= sub( pix1+8*i_pix1, i_pix1, pix2+8*i_pix2, i_pix2 )\
329 + sub( pix1+12*i_pix1, i_pix1, pix2+12*i_pix2, i_pix2 );\
330 if( w==16 && h==16 )\
331 sum+= sub( pix1+8+8*i_pix1, i_pix1, pix2+8+8*i_pix2, i_pix2 )\
332 + sub( pix1+8+12*i_pix1, i_pix1, pix2+8+12*i_pix2, i_pix2 );\
333 return sum;\
334 }
335 PIXEL_SATD_C( 16, 16, x264_pixel_satd_8x4 )
336 PIXEL_SATD_C( 16, 8, x264_pixel_satd_8x4 )
337 PIXEL_SATD_C( 8, 16, x264_pixel_satd_8x4 )
338 PIXEL_SATD_C( 8, 8, x264_pixel_satd_8x4 )
339 PIXEL_SATD_C( 4, 16, x264_pixel_satd_4x4 )
340 PIXEL_SATD_C( 4, 8, x264_pixel_satd_4x4 )
341
sa8d_8x8(pixel * pix1,intptr_t i_pix1,pixel * pix2,intptr_t i_pix2)342 static NOINLINE int sa8d_8x8( pixel *pix1, intptr_t i_pix1, pixel *pix2, intptr_t i_pix2 )
343 {
344 sum2_t tmp[8][4];
345 sum2_t a0, a1, a2, a3, a4, a5, a6, a7, b0, b1, b2, b3;
346 sum2_t sum = 0;
347 int i;
348
349 for( i = 0; i < 8; i++, pix1 += i_pix1, pix2 += i_pix2 )
350 {
351 a0 = pix1[0] - pix2[0];
352 a1 = pix1[1] - pix2[1];
353 b0 = (a0+a1) + ((a0-a1)<<BITS_PER_SUM);
354 a2 = pix1[2] - pix2[2];
355 a3 = pix1[3] - pix2[3];
356 b1 = (a2+a3) + ((a2-a3)<<BITS_PER_SUM);
357 a4 = pix1[4] - pix2[4];
358 a5 = pix1[5] - pix2[5];
359 b2 = (a4+a5) + ((a4-a5)<<BITS_PER_SUM);
360 a6 = pix1[6] - pix2[6];
361 a7 = pix1[7] - pix2[7];
362 b3 = (a6+a7) + ((a6-a7)<<BITS_PER_SUM);
363 HADAMARD4( tmp[i][0], tmp[i][1], tmp[i][2], tmp[i][3], b0,b1,b2,b3 );
364 }
365 for( i = 0; i < 4; i++ )
366 {
367 HADAMARD4( a0, a1, a2, a3, tmp[0][i], tmp[1][i], tmp[2][i], tmp[3][i] );
368 HADAMARD4( a4, a5, a6, a7, tmp[4][i], tmp[5][i], tmp[6][i], tmp[7][i] );
369 b0 = abs2(a0+a4) + abs2(a0-a4);
370 b0 += abs2(a1+a5) + abs2(a1-a5);
371 b0 += abs2(a2+a6) + abs2(a2-a6);
372 b0 += abs2(a3+a7) + abs2(a3-a7);
373 sum += (sum_t)b0 + (b0>>BITS_PER_SUM);
374 }
375 return sum;
376 }
377
x264_pixel_sa8d_8x8(pixel * pix1,intptr_t i_pix1,pixel * pix2,intptr_t i_pix2)378 static int x264_pixel_sa8d_8x8( pixel *pix1, intptr_t i_pix1, pixel *pix2, intptr_t i_pix2 )
379 {
380 int sum = sa8d_8x8( pix1, i_pix1, pix2, i_pix2 );
381 return (sum+2)>>2;
382 }
383
x264_pixel_sa8d_16x16(pixel * pix1,intptr_t i_pix1,pixel * pix2,intptr_t i_pix2)384 static int x264_pixel_sa8d_16x16( pixel *pix1, intptr_t i_pix1, pixel *pix2, intptr_t i_pix2 )
385 {
386 int sum = sa8d_8x8( pix1, i_pix1, pix2, i_pix2 )
387 + sa8d_8x8( pix1+8, i_pix1, pix2+8, i_pix2 )
388 + sa8d_8x8( pix1+8*i_pix1, i_pix1, pix2+8*i_pix2, i_pix2 )
389 + sa8d_8x8( pix1+8+8*i_pix1, i_pix1, pix2+8+8*i_pix2, i_pix2 );
390 return (sum+2)>>2;
391 }
392
pixel_hadamard_ac(pixel * pix,intptr_t stride)393 static NOINLINE uint64_t pixel_hadamard_ac( pixel *pix, intptr_t stride )
394 {
395 sum2_t tmp[32];
396 sum2_t a0, a1, a2, a3, dc;
397 sum2_t sum4 = 0, sum8 = 0;
398 int i;
399
400 for( i = 0; i < 8; i++, pix+=stride )
401 {
402 sum2_t *t = tmp + (i&3) + (i&4)*4;
403 a0 = (pix[0]+pix[1]) + ((sum2_t)(pix[0]-pix[1])<<BITS_PER_SUM);
404 a1 = (pix[2]+pix[3]) + ((sum2_t)(pix[2]-pix[3])<<BITS_PER_SUM);
405 t[0] = a0 + a1;
406 t[4] = a0 - a1;
407 a2 = (pix[4]+pix[5]) + ((sum2_t)(pix[4]-pix[5])<<BITS_PER_SUM);
408 a3 = (pix[6]+pix[7]) + ((sum2_t)(pix[6]-pix[7])<<BITS_PER_SUM);
409 t[8] = a2 + a3;
410 t[12] = a2 - a3;
411 }
412 for( i = 0; i < 8; i++ )
413 {
414 HADAMARD4( a0, a1, a2, a3, tmp[i*4+0], tmp[i*4+1], tmp[i*4+2], tmp[i*4+3] );
415 tmp[i*4+0] = a0;
416 tmp[i*4+1] = a1;
417 tmp[i*4+2] = a2;
418 tmp[i*4+3] = a3;
419 sum4 += abs2(a0) + abs2(a1) + abs2(a2) + abs2(a3);
420 }
421 for( i = 0; i < 8; i++ )
422 {
423 HADAMARD4( a0,a1,a2,a3, tmp[i], tmp[8+i], tmp[16+i], tmp[24+i] );
424 sum8 += abs2(a0) + abs2(a1) + abs2(a2) + abs2(a3);
425 }
426 dc = (sum_t)(tmp[0] + tmp[8] + tmp[16] + tmp[24]);
427 sum4 = (sum_t)sum4 + (sum4>>BITS_PER_SUM) - dc;
428 sum8 = (sum_t)sum8 + (sum8>>BITS_PER_SUM) - dc;
429 return ((uint64_t)sum8<<32) + sum4;
430 }
431
432 #define HADAMARD_AC(w,h) \
433 static uint64_t x264_pixel_hadamard_ac_##w##x##h( pixel *pix, intptr_t stride )\
434 {\
435 uint64_t sum = pixel_hadamard_ac( pix, stride );\
436 if( w==16 )\
437 sum += pixel_hadamard_ac( pix+8, stride );\
438 if( h==16 )\
439 sum += pixel_hadamard_ac( pix+8*stride, stride );\
440 if( w==16 && h==16 )\
441 sum += pixel_hadamard_ac( pix+8*stride+8, stride );\
442 return ((sum>>34)<<32) + ((uint32_t)sum>>1);\
443 }
444 HADAMARD_AC( 16, 16 )
445 HADAMARD_AC( 16, 8 )
446 HADAMARD_AC( 8, 16 )
447 HADAMARD_AC( 8, 8 )
448
449
450 /****************************************************************************
451 * pixel_sad_x4
452 ****************************************************************************/
453 #define SAD_X( size ) \
454 static void x264_pixel_sad_x3_##size( pixel *fenc, pixel *pix0, pixel *pix1, pixel *pix2,\
455 intptr_t i_stride, int scores[3] )\
456 {\
457 scores[0] = x264_pixel_sad_##size( fenc, FENC_STRIDE, pix0, i_stride );\
458 scores[1] = x264_pixel_sad_##size( fenc, FENC_STRIDE, pix1, i_stride );\
459 scores[2] = x264_pixel_sad_##size( fenc, FENC_STRIDE, pix2, i_stride );\
460 }\
461 static void x264_pixel_sad_x4_##size( pixel *fenc, pixel *pix0, pixel *pix1,pixel *pix2, pixel *pix3,\
462 intptr_t i_stride, int scores[4] )\
463 {\
464 scores[0] = x264_pixel_sad_##size( fenc, FENC_STRIDE, pix0, i_stride );\
465 scores[1] = x264_pixel_sad_##size( fenc, FENC_STRIDE, pix1, i_stride );\
466 scores[2] = x264_pixel_sad_##size( fenc, FENC_STRIDE, pix2, i_stride );\
467 scores[3] = x264_pixel_sad_##size( fenc, FENC_STRIDE, pix3, i_stride );\
468 }
469
470 SAD_X( 16x16 )
471 SAD_X( 16x8 )
472 SAD_X( 8x16 )
473 SAD_X( 8x8 )
474 SAD_X( 8x4 )
475 SAD_X( 4x8 )
476 SAD_X( 4x4 )
477
478 /****************************************************************************
479 * pixel_satd_x4
480 * no faster than single satd, but needed for satd to be a drop-in replacement for sad
481 ****************************************************************************/
482
483 #define SATD_X( size, cpu ) \
484 static void x264_pixel_satd_x3_##size##cpu( pixel *fenc, pixel *pix0, pixel *pix1, pixel *pix2,\
485 intptr_t i_stride, int scores[3] )\
486 {\
487 scores[0] = x264_pixel_satd_##size##cpu( fenc, FENC_STRIDE, pix0, i_stride );\
488 scores[1] = x264_pixel_satd_##size##cpu( fenc, FENC_STRIDE, pix1, i_stride );\
489 scores[2] = x264_pixel_satd_##size##cpu( fenc, FENC_STRIDE, pix2, i_stride );\
490 }\
491 static void x264_pixel_satd_x4_##size##cpu( pixel *fenc, pixel *pix0, pixel *pix1, pixel *pix2, pixel *pix3,\
492 intptr_t i_stride, int scores[4] )\
493 {\
494 scores[0] = x264_pixel_satd_##size##cpu( fenc, FENC_STRIDE, pix0, i_stride );\
495 scores[1] = x264_pixel_satd_##size##cpu( fenc, FENC_STRIDE, pix1, i_stride );\
496 scores[2] = x264_pixel_satd_##size##cpu( fenc, FENC_STRIDE, pix2, i_stride );\
497 scores[3] = x264_pixel_satd_##size##cpu( fenc, FENC_STRIDE, pix3, i_stride );\
498 }
499
500 #define SATD_X_NO_CPU( size ) \
501 static void x264_pixel_satd_x3_##size( pixel *fenc, pixel *pix0, pixel *pix1, pixel *pix2,\
502 intptr_t i_stride, int scores[3] )\
503 {\
504 scores[0] = x264_pixel_satd_##size( fenc, FENC_STRIDE, pix0, i_stride );\
505 scores[1] = x264_pixel_satd_##size( fenc, FENC_STRIDE, pix1, i_stride );\
506 scores[2] = x264_pixel_satd_##size( fenc, FENC_STRIDE, pix2, i_stride );\
507 }\
508 static void x264_pixel_satd_x4_##size( pixel *fenc, pixel *pix0, pixel *pix1, pixel *pix2, pixel *pix3,\
509 intptr_t i_stride, int scores[4] )\
510 {\
511 scores[0] = x264_pixel_satd_##size( fenc, FENC_STRIDE, pix0, i_stride );\
512 scores[1] = x264_pixel_satd_##size( fenc, FENC_STRIDE, pix1, i_stride );\
513 scores[2] = x264_pixel_satd_##size( fenc, FENC_STRIDE, pix2, i_stride );\
514 scores[3] = x264_pixel_satd_##size( fenc, FENC_STRIDE, pix3, i_stride );\
515 }
516
517 #define SATD_X_DECL6( cpu )\
518 SATD_X( 16x16, cpu )\
519 SATD_X( 16x8, cpu )\
520 SATD_X( 8x16, cpu )\
521 SATD_X( 8x8, cpu )\
522 SATD_X( 8x4, cpu )\
523 SATD_X( 4x8, cpu )
524
525 #define SATD_X_DECL6_NO_CPU()\
526 SATD_X_NO_CPU( 16x16 )\
527 SATD_X_NO_CPU( 16x8 )\
528 SATD_X_NO_CPU( 8x16 )\
529 SATD_X_NO_CPU( 8x8 )\
530 SATD_X_NO_CPU( 8x4 )\
531 SATD_X_NO_CPU( 4x8 )
532
533 #define SATD_X_DECL7( cpu )\
534 SATD_X_DECL6( cpu )\
535 SATD_X( 4x4, cpu )
536
537 #define SATD_X_DECL7_NO_CPU()\
538 SATD_X_DECL6_NO_CPU()\
539 SATD_X_NO_CPU( 4x4 )
540
SATD_X_DECL7_NO_CPU()541 SATD_X_DECL7_NO_CPU()
542 #if HAVE_MMX
543 SATD_X_DECL7( _mmx2 )
544 #if !HIGH_BIT_DEPTH
545 SATD_X_DECL6( _sse2 )
546 SATD_X_DECL7( _ssse3 )
547 SATD_X_DECL6( _ssse3_atom )
548 SATD_X_DECL7( _sse4 )
549 SATD_X_DECL7( _avx )
550 SATD_X_DECL7( _xop )
551 #endif // !HIGH_BIT_DEPTH
552 #endif
553
554 #if !HIGH_BIT_DEPTH
555 #if HAVE_ARMV6 || ARCH_AARCH64
556 SATD_X_DECL7( _neon )
557 #endif
558 #endif // !HIGH_BIT_DEPTH
559
560 #define INTRA_MBCMP_8x8( mbcmp, cpu, cpu2 )\
561 void x264_intra_##mbcmp##_x3_8x8##cpu( pixel *fenc, pixel edge[36], int res[3] )\
562 {\
563 ALIGNED_ARRAY_16( pixel, pix, [8*FDEC_STRIDE] );\
564 x264_predict_8x8_v##cpu2( pix, edge );\
565 res[0] = x264_pixel_##mbcmp##_8x8##cpu( pix, FDEC_STRIDE, fenc, FENC_STRIDE );\
566 x264_predict_8x8_h##cpu2( pix, edge );\
567 res[1] = x264_pixel_##mbcmp##_8x8##cpu( pix, FDEC_STRIDE, fenc, FENC_STRIDE );\
568 x264_predict_8x8_dc##cpu2( pix, edge );\
569 res[2] = x264_pixel_##mbcmp##_8x8##cpu( pix, FDEC_STRIDE, fenc, FENC_STRIDE );\
570 }
571
572 INTRA_MBCMP_8x8( sad,, _c )
573 INTRA_MBCMP_8x8(sa8d,, _c )
574
575 #if HIGH_BIT_DEPTH && HAVE_MMX
576 #define x264_predict_8x8_v_sse2 x264_predict_8x8_v_sse
577 INTRA_MBCMP_8x8( sad, _mmx2, _c )
578 INTRA_MBCMP_8x8(sa8d, _sse2, _sse2 )
579 #endif
580 #if !HIGH_BIT_DEPTH && (HAVE_ARMV6 || ARCH_AARCH64)
581 INTRA_MBCMP_8x8( sad, _neon, _neon )
582 INTRA_MBCMP_8x8(sa8d, _neon, _neon )
583 #endif
584
585 #define INTRA_MBCMP( mbcmp, size, pred1, pred2, pred3, chroma, cpu, cpu2 )\
586 void x264_intra_##mbcmp##_x3_##size##chroma##cpu( pixel *fenc, pixel *fdec, int res[3] )\
587 {\
588 x264_predict_##size##chroma##_##pred1##cpu2( fdec );\
589 res[0] = x264_pixel_##mbcmp##_##size##cpu( fdec, FDEC_STRIDE, fenc, FENC_STRIDE );\
590 x264_predict_##size##chroma##_##pred2##cpu2( fdec );\
591 res[1] = x264_pixel_##mbcmp##_##size##cpu( fdec, FDEC_STRIDE, fenc, FENC_STRIDE );\
592 x264_predict_##size##chroma##_##pred3##cpu2( fdec );\
593 res[2] = x264_pixel_##mbcmp##_##size##cpu( fdec, FDEC_STRIDE, fenc, FENC_STRIDE );\
594 }
595
596 #define INTRA_MBCMP_NO_CPU( mbcmp, size, pred1, pred2, pred3, chroma, cpu2 )\
597 void x264_intra_##mbcmp##_x3_##size##chroma( pixel *fenc, pixel *fdec, int res[3] )\
598 {\
599 x264_predict_##size##chroma##_##pred1##cpu2( fdec );\
600 res[0] = x264_pixel_##mbcmp##_##size( fdec, FDEC_STRIDE, fenc, FENC_STRIDE );\
601 x264_predict_##size##chroma##_##pred2##cpu2( fdec );\
602 res[1] = x264_pixel_##mbcmp##_##size( fdec, FDEC_STRIDE, fenc, FENC_STRIDE );\
603 x264_predict_##size##chroma##_##pred3##cpu2( fdec );\
604 res[2] = x264_pixel_##mbcmp##_##size( fdec, FDEC_STRIDE, fenc, FENC_STRIDE );\
605 }
606
607 #define INTRA_MBCMP_NO_CHROMA_NO_CPU( mbcmp, size, pred1, pred2, pred3, cpu2 )\
608 void x264_intra_##mbcmp##_x3_##size( pixel *fenc, pixel *fdec, int res[3] )\
609 {\
610 x264_predict_##size##_##pred1##cpu2( fdec );\
611 res[0] = x264_pixel_##mbcmp##_##size( fdec, FDEC_STRIDE, fenc, FENC_STRIDE );\
612 x264_predict_##size##_##pred2##cpu2( fdec );\
613 res[1] = x264_pixel_##mbcmp##_##size( fdec, FDEC_STRIDE, fenc, FENC_STRIDE );\
614 x264_predict_##size##_##pred3##cpu2( fdec );\
615 res[2] = x264_pixel_##mbcmp##_##size( fdec, FDEC_STRIDE, fenc, FENC_STRIDE );\
616 }
617
618 INTRA_MBCMP_NO_CHROMA_NO_CPU( sad, 4x4, v, h, dc, _c )
619 INTRA_MBCMP_NO_CHROMA_NO_CPU(satd, 4x4, v, h, dc, _c )
620 INTRA_MBCMP_NO_CPU( sad, 8x8, dc, h, v, c, _c )
621 INTRA_MBCMP_NO_CPU(satd, 8x8, dc, h, v, c, _c )
622 INTRA_MBCMP_NO_CPU( sad, 8x16, dc, h, v, c, _c )
623 INTRA_MBCMP_NO_CPU(satd, 8x16, dc, h, v, c, _c )
624 INTRA_MBCMP_NO_CHROMA_NO_CPU( sad, 16x16, v, h, dc, _c )
625 INTRA_MBCMP_NO_CHROMA_NO_CPU(satd, 16x16, v, h, dc, _c )
626
627 #if HAVE_MMX
628 #if HIGH_BIT_DEPTH
629 #define x264_predict_8x8c_v_mmx2 x264_predict_8x8c_v_mmx
630 #define x264_predict_8x16c_v_mmx2 x264_predict_8x16c_v_c
631 #define x264_predict_8x8c_v_sse2 x264_predict_8x8c_v_sse
632 #define x264_predict_8x16c_v_sse2 x264_predict_8x16c_v_sse
633 #define x264_predict_16x16_v_sse2 x264_predict_16x16_v_sse
634 INTRA_MBCMP( sad, 4x4, v, h, dc, , _mmx2, _c )
635 INTRA_MBCMP( sad, 8x8, dc, h, v, c, _mmx2, _mmx2 )
636 INTRA_MBCMP( sad, 8x16, dc, h, v, c, _mmx2, _mmx2 )
637 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _mmx2, _mmx2 )
638 INTRA_MBCMP( sad, 16x16, v, h, dc, , _mmx2, _mmx2 )
639 INTRA_MBCMP( sad, 8x8, dc, h, v, c, _sse2, _sse2 )
640 INTRA_MBCMP( sad, 8x16, dc, h, v, c, _sse2, _sse2 )
641 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _sse2, _sse2 )
642 INTRA_MBCMP( sad, 16x16, v, h, dc, , _sse2, _sse2 )
643 INTRA_MBCMP( sad, 8x8, dc, h, v, c, _ssse3, _sse2 )
644 INTRA_MBCMP( sad, 8x16, dc, h, v, c, _ssse3, _sse2 )
645 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _ssse3, _sse2 )
646 INTRA_MBCMP( sad, 16x16, v, h, dc, , _ssse3, _sse2 )
647 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _sse4, _sse2 )
648 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _avx, _sse2 )
649 #else
650 #define x264_predict_8x16c_v_mmx2 x264_predict_8x16c_v_mmx
651 INTRA_MBCMP( sad, 8x16, dc, h, v, c, _mmx2, _mmx2 )
652 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _mmx2, _mmx2 )
653 INTRA_MBCMP( sad, 8x16, dc, h, v, c, _sse2, _mmx2 )
654 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _sse2, _mmx2 )
655 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _ssse3, _mmx2 )
656 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _sse4, _mmx2 )
657 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _avx, _mmx2 )
658 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _xop, _mmx2 )
659 #endif
660 #endif
661 #if !HIGH_BIT_DEPTH && HAVE_ARMV6
662 INTRA_MBCMP( sad, 4x4, v, h, dc, , _neon, _armv6 )
663 INTRA_MBCMP(satd, 4x4, v, h, dc, , _neon, _armv6 )
664 INTRA_MBCMP( sad, 8x8, dc, h, v, c, _neon, _neon )
665 INTRA_MBCMP(satd, 8x8, dc, h, v, c, _neon, _neon )
666 INTRA_MBCMP( sad, 8x16, dc, h, v, c, _neon, _c )
667 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _neon, _c )
668 INTRA_MBCMP( sad, 16x16, v, h, dc, , _neon, _neon )
669 INTRA_MBCMP(satd, 16x16, v, h, dc, , _neon, _neon )
670 #endif
671 #if !HIGH_BIT_DEPTH && ARCH_AARCH64
672 INTRA_MBCMP( sad, 4x4, v, h, dc, , _neon, _neon )
673 INTRA_MBCMP(satd, 4x4, v, h, dc, , _neon, _neon )
674 INTRA_MBCMP( sad, 8x8, dc, h, v, c, _neon, _neon )
675 INTRA_MBCMP(satd, 8x8, dc, h, v, c, _neon, _neon )
676 INTRA_MBCMP( sad, 8x16, dc, h, v, c, _neon, _c )
677 INTRA_MBCMP(satd, 8x16, dc, h, v, c, _neon, _c )
678 INTRA_MBCMP( sad, 16x16, v, h, dc, , _neon, _neon )
679 INTRA_MBCMP(satd, 16x16, v, h, dc, , _neon, _neon )
680 #endif
681
682 // No C implementation of intra_satd_x9. See checkasm for its behavior,
683 // or see x264_mb_analyse_intra for the entirely different algorithm we
684 // use when lacking an asm implementation of it.
685
686
687
688 /****************************************************************************
689 * structural similarity metric
690 ****************************************************************************/
691 static void ssim_4x4x2_core( const pixel *pix1, intptr_t stride1,
692 const pixel *pix2, intptr_t stride2,
693 int sums[2][4] )
694 {
695 int z;
696
697 for( z = 0; z < 2; z++ )
698 {
699 uint32_t s1 = 0, s2 = 0, ss = 0, s12 = 0;
700 int y;
701 int x;
702 for( y = 0; y < 4; y++ )
703 for( x = 0; x < 4; x++ )
704 {
705 int a = pix1[x+y*stride1];
706 int b = pix2[x+y*stride2];
707 s1 += a;
708 s2 += b;
709 ss += a*a;
710 ss += b*b;
711 s12 += a*b;
712 }
713 sums[z][0] = s1;
714 sums[z][1] = s2;
715 sums[z][2] = ss;
716 sums[z][3] = s12;
717 pix1 += 4;
718 pix2 += 4;
719 }
720 }
721
ssim_end1(int s1,int s2,int ss,int s12)722 static float ssim_end1( int s1, int s2, int ss, int s12 )
723 {
724 /* Maximum value for 10-bit is: ss*64 = (2^10-1)^2*16*4*64 = 4286582784, which will overflow in some cases.
725 * s1*s1, s2*s2, and s1*s2 also obtain this value for edge cases: ((2^10-1)*16*4)^2 = 4286582784.
726 * Maximum value for 9-bit is: ss*64 = (2^9-1)^2*16*4*64 = 1069551616, which will not overflow. */
727 #if BIT_DEPTH > 9
728 #define type float
729 static const float ssim_c1 = .01*.01*PIXEL_MAX*PIXEL_MAX*64;
730 static const float ssim_c2 = .03*.03*PIXEL_MAX*PIXEL_MAX*64*63;
731 #else
732 #define type int
733 static const int ssim_c1 = (int)(.01*.01*PIXEL_MAX*PIXEL_MAX*64 + .5);
734 static const int ssim_c2 = (int)(.03*.03*PIXEL_MAX*PIXEL_MAX*64*63 + .5);
735 #endif
736 type fs1 = s1;
737 type fs2 = s2;
738 type fss = ss;
739 type fs12 = s12;
740 type vars = fss*64 - fs1*fs1 - fs2*fs2;
741 type covar = fs12*64 - fs1*fs2;
742 return (float)(2*fs1*fs2 + ssim_c1) * (float)(2*covar + ssim_c2)
743 / ((float)(fs1*fs1 + fs2*fs2 + ssim_c1) * (float)(vars + ssim_c2));
744 #undef type
745 }
746
ssim_end4(int sum0[5][4],int sum1[5][4],int width)747 static float ssim_end4( int sum0[5][4], int sum1[5][4], int width )
748 {
749 float ssim = 0.0;
750 int i;
751
752 for( i = 0; i < width; i++ )
753 ssim += ssim_end1( sum0[i][0] + sum0[i+1][0] + sum1[i][0] + sum1[i+1][0],
754 sum0[i][1] + sum0[i+1][1] + sum1[i][1] + sum1[i+1][1],
755 sum0[i][2] + sum0[i+1][2] + sum1[i][2] + sum1[i+1][2],
756 sum0[i][3] + sum0[i+1][3] + sum1[i][3] + sum1[i+1][3] );
757 return ssim;
758 }
759
x264_pixel_ssim_wxh(x264_pixel_function_t * pf,pixel * pix1,intptr_t stride1,pixel * pix2,intptr_t stride2,int width,int height,void * buf,int * cnt)760 float x264_pixel_ssim_wxh( x264_pixel_function_t *pf,
761 pixel *pix1, intptr_t stride1,
762 pixel *pix2, intptr_t stride2,
763 int width, int height, void *buf, int *cnt )
764 {
765 int z = 0;
766 float ssim = 0.0;
767 int (*sum0)[4] = buf;
768 int (*sum1)[4] = sum0 + (width >> 2) + 3;
769 int y;
770
771 width >>= 2;
772 height >>= 2;
773 for( y = 1; y < height; y++ )
774 {
775 int x;
776 for( ; z <= y; z++ )
777 {
778 XCHG( void*, sum0, sum1 );
779 for( x = 0; x < width; x+=2 )
780 pf->ssim_4x4x2_core( &pix1[4*(x+z*stride1)], stride1, &pix2[4*(x+z*stride2)], stride2, &sum0[x] );
781 }
782 for( x = 0; x < width-1; x += 4 )
783 ssim += pf->ssim_end4( sum0+x, sum1+x, X264_MIN(4,width-x-1) );
784 }
785 *cnt = (height-1) * (width-1);
786 return ssim;
787 }
788
pixel_vsad(pixel * src,intptr_t stride,int height)789 static int pixel_vsad( pixel *src, intptr_t stride, int height )
790 {
791 int score = 0;
792 int i;
793 int j;
794
795 for( i = 1; i < height; i++, src += stride )
796 for( j = 0; j < 16; j++ )
797 score += abs(src[j] - src[j+stride]);
798 return score;
799 }
800
x264_field_vsad(x264_t * h,int mb_x,int mb_y)801 int x264_field_vsad( x264_t *h, int mb_x, int mb_y )
802 {
803 int score_field, score_frame;
804 int stride = h->fenc->i_stride[0];
805 int mb_stride = h->mb.i_mb_stride;
806 pixel *fenc = h->fenc->plane[0] + 16 * (mb_x + mb_y * stride);
807 int mb_xy = mb_x + mb_y*mb_stride;
808
809 /* We don't want to analyze pixels outside the frame, as it gives inaccurate results. */
810 int mbpair_height = X264_MIN( h->param.i_height - mb_y * 16, 32 );
811 score_frame = h->pixf.vsad( fenc, stride, mbpair_height );
812 score_field = h->pixf.vsad( fenc, stride*2, mbpair_height >> 1 );
813 score_field += h->pixf.vsad( fenc+stride, stride*2, mbpair_height >> 1 );
814
815 if( mb_x > 0 )
816 score_field += 512 - h->mb.field[mb_xy -1]*1024;
817 if( mb_y > 0 )
818 score_field += 512 - h->mb.field[mb_xy-mb_stride]*1024;
819
820 return (score_field < score_frame);
821 }
822
pixel_asd8(pixel * pix1,intptr_t stride1,pixel * pix2,intptr_t stride2,int height)823 static int pixel_asd8( pixel *pix1, intptr_t stride1, pixel *pix2, intptr_t stride2, int height )
824 {
825 int sum = 0;
826 int y;
827 int x;
828
829 for( y = 0; y < height; y++, pix1 += stride1, pix2 += stride2 )
830 for( x = 0; x < 8; x++ )
831 sum += pix1[x] - pix2[x];
832 return abs( sum );
833 }
834
835 /****************************************************************************
836 * successive elimination
837 ****************************************************************************/
x264_pixel_ads4(int enc_dc[4],uint16_t * sums,int delta,uint16_t * cost_mvx,int16_t * mvs,int width,int thresh)838 static int x264_pixel_ads4( int enc_dc[4], uint16_t *sums, int delta,
839 uint16_t *cost_mvx, int16_t *mvs, int width, int thresh )
840 {
841 int nmv = 0;
842 int i;
843
844 for( i = 0; i < width; i++, sums++ )
845 {
846 int ads = abs( enc_dc[0] - sums[0] )
847 + abs( enc_dc[1] - sums[8] )
848 + abs( enc_dc[2] - sums[delta] )
849 + abs( enc_dc[3] - sums[delta+8] )
850 + cost_mvx[i];
851 if( ads < thresh )
852 mvs[nmv++] = i;
853 }
854 return nmv;
855 }
856
x264_pixel_ads2(int enc_dc[2],uint16_t * sums,int delta,uint16_t * cost_mvx,int16_t * mvs,int width,int thresh)857 static int x264_pixel_ads2( int enc_dc[2], uint16_t *sums, int delta,
858 uint16_t *cost_mvx, int16_t *mvs, int width, int thresh )
859 {
860 int nmv = 0;
861 int i;
862
863 for( i = 0; i < width; i++, sums++ )
864 {
865 int ads = abs( enc_dc[0] - sums[0] )
866 + abs( enc_dc[1] - sums[delta] )
867 + cost_mvx[i];
868 if( ads < thresh )
869 mvs[nmv++] = i;
870 }
871 return nmv;
872 }
873
x264_pixel_ads1(int enc_dc[1],uint16_t * sums,int delta,uint16_t * cost_mvx,int16_t * mvs,int width,int thresh)874 static int x264_pixel_ads1( int enc_dc[1], uint16_t *sums, int delta,
875 uint16_t *cost_mvx, int16_t *mvs, int width, int thresh )
876 {
877 int nmv = 0;
878 int i;
879
880 for( i = 0; i<width; i++, sums++ )
881 {
882 int ads = abs( enc_dc[0] - sums[0] )
883 + cost_mvx[i];
884 if( ads < thresh )
885 mvs[nmv++] = i;
886 }
887 return nmv;
888 }
889
890
891 /****************************************************************************
892 * x264_pixel_init:
893 ****************************************************************************/
x264_pixel_init(int cpu,x264_pixel_function_t * pixf)894 void x264_pixel_init( int cpu, x264_pixel_function_t *pixf )
895 {
896 memset( pixf, 0, sizeof(*pixf) );
897
898 #define INIT2_NAME( name1, name2, cpu ) \
899 pixf->name1[PIXEL_16x16] = x264_pixel_##name2##_16x16##cpu;\
900 pixf->name1[PIXEL_16x8] = x264_pixel_##name2##_16x8##cpu;
901 #define INIT4_NAME( name1, name2, cpu ) \
902 INIT2_NAME( name1, name2, cpu ) \
903 pixf->name1[PIXEL_8x16] = x264_pixel_##name2##_8x16##cpu;\
904 pixf->name1[PIXEL_8x8] = x264_pixel_##name2##_8x8##cpu;
905 #define INIT5_NAME( name1, name2, cpu ) \
906 INIT4_NAME( name1, name2, cpu ) \
907 pixf->name1[PIXEL_8x4] = x264_pixel_##name2##_8x4##cpu;
908 #define INIT6_NAME( name1, name2, cpu ) \
909 INIT5_NAME( name1, name2, cpu ) \
910 pixf->name1[PIXEL_4x8] = x264_pixel_##name2##_4x8##cpu;
911 #define INIT7_NAME( name1, name2, cpu ) \
912 INIT6_NAME( name1, name2, cpu ) \
913 pixf->name1[PIXEL_4x4] = x264_pixel_##name2##_4x4##cpu;
914 #define INIT8_NAME( name1, name2, cpu ) \
915 INIT7_NAME( name1, name2, cpu ) \
916 pixf->name1[PIXEL_4x16] = x264_pixel_##name2##_4x16##cpu;
917 #define INIT2( name, cpu ) INIT2_NAME( name, name, cpu )
918 #define INIT4( name, cpu ) INIT4_NAME( name, name, cpu )
919 #define INIT5( name, cpu ) INIT5_NAME( name, name, cpu )
920 #define INIT6( name, cpu ) INIT6_NAME( name, name, cpu )
921 #define INIT7( name, cpu ) INIT7_NAME( name, name, cpu )
922 #define INIT8( name, cpu ) INIT8_NAME( name, name, cpu )
923
924 #define INIT_ADS( cpu ) \
925 pixf->ads[PIXEL_16x16] = x264_pixel_ads4##cpu;\
926 pixf->ads[PIXEL_16x8] = x264_pixel_ads2##cpu;\
927 pixf->ads[PIXEL_8x8] = x264_pixel_ads1##cpu;
928
929 INIT8( sad, );
930 INIT8_NAME( sad_aligned, sad, );
931 INIT7( sad_x3, );
932 INIT7( sad_x4, );
933 INIT8( ssd, );
934 INIT8( satd, );
935 INIT7( satd_x3, );
936 INIT7( satd_x4, );
937 INIT4( hadamard_ac, );
938 INIT_ADS( );
939
940 pixf->sa8d[PIXEL_16x16] = x264_pixel_sa8d_16x16;
941 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8;
942 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16;
943 pixf->var[PIXEL_8x16] = x264_pixel_var_8x16;
944 pixf->var[PIXEL_8x8] = x264_pixel_var_8x8;
945 pixf->var2[PIXEL_8x16] = x264_pixel_var2_8x16;
946 pixf->var2[PIXEL_8x8] = x264_pixel_var2_8x8;
947
948 pixf->ssd_nv12_core = pixel_ssd_nv12_core;
949 pixf->ssim_4x4x2_core = ssim_4x4x2_core;
950 pixf->ssim_end4 = ssim_end4;
951 pixf->vsad = pixel_vsad;
952 pixf->asd8 = pixel_asd8;
953
954 pixf->intra_sad_x3_4x4 = x264_intra_sad_x3_4x4;
955 pixf->intra_satd_x3_4x4 = x264_intra_satd_x3_4x4;
956 pixf->intra_sad_x3_8x8 = x264_intra_sad_x3_8x8;
957 pixf->intra_sa8d_x3_8x8 = x264_intra_sa8d_x3_8x8;
958 pixf->intra_sad_x3_8x8c = x264_intra_sad_x3_8x8c;
959 pixf->intra_satd_x3_8x8c = x264_intra_satd_x3_8x8c;
960 pixf->intra_sad_x3_8x16c = x264_intra_sad_x3_8x16c;
961 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c;
962 pixf->intra_sad_x3_16x16 = x264_intra_sad_x3_16x16;
963 pixf->intra_satd_x3_16x16 = x264_intra_satd_x3_16x16;
964
965 #if HIGH_BIT_DEPTH
966 #if HAVE_MMX
967 if( cpu&X264_CPU_MMX2 )
968 {
969 INIT7( sad, _mmx2 );
970 INIT7_NAME( sad_aligned, sad, _mmx2 );
971 INIT7( sad_x3, _mmx2 );
972 INIT7( sad_x4, _mmx2 );
973 INIT8( satd, _mmx2 );
974 INIT7( satd_x3, _mmx2 );
975 INIT7( satd_x4, _mmx2 );
976 INIT4( hadamard_ac, _mmx2 );
977 INIT8( ssd, _mmx2 );
978 INIT_ADS( _mmx2 );
979
980 pixf->ssd_nv12_core = x264_pixel_ssd_nv12_core_mmx2;
981 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16_mmx2;
982 pixf->var[PIXEL_8x8] = x264_pixel_var_8x8_mmx2;
983 #if ARCH_X86
984 pixf->var2[PIXEL_8x8] = x264_pixel_var2_8x8_mmx2;
985 pixf->var2[PIXEL_8x16] = x264_pixel_var2_8x16_mmx2;
986 #endif
987
988 pixf->intra_sad_x3_4x4 = x264_intra_sad_x3_4x4_mmx2;
989 pixf->intra_satd_x3_4x4 = x264_intra_satd_x3_4x4_mmx2;
990 pixf->intra_sad_x3_8x8 = x264_intra_sad_x3_8x8_mmx2;
991 pixf->intra_sad_x3_8x8c = x264_intra_sad_x3_8x8c_mmx2;
992 pixf->intra_satd_x3_8x8c = x264_intra_satd_x3_8x8c_mmx2;
993 pixf->intra_sad_x3_8x16c = x264_intra_sad_x3_8x16c_mmx2;
994 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_mmx2;
995 pixf->intra_sad_x3_16x16 = x264_intra_sad_x3_16x16_mmx2;
996 pixf->intra_satd_x3_16x16 = x264_intra_satd_x3_16x16_mmx2;
997 }
998 if( cpu&X264_CPU_SSE2 )
999 {
1000 INIT4_NAME( sad_aligned, sad, _sse2_aligned );
1001 INIT5( ssd, _sse2 );
1002 INIT6( satd, _sse2 );
1003 pixf->satd[PIXEL_4x16] = x264_pixel_satd_4x16_sse2;
1004
1005 pixf->sa8d[PIXEL_16x16] = x264_pixel_sa8d_16x16_sse2;
1006 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_sse2;
1007 #if ARCH_X86_64
1008 pixf->intra_sa8d_x3_8x8 = x264_intra_sa8d_x3_8x8_sse2;
1009 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_sse2;
1010 #endif
1011 pixf->intra_sad_x3_4x4 = x264_intra_sad_x3_4x4_sse2;
1012 pixf->ssd_nv12_core = x264_pixel_ssd_nv12_core_sse2;
1013 pixf->ssim_4x4x2_core = x264_pixel_ssim_4x4x2_core_sse2;
1014 pixf->ssim_end4 = x264_pixel_ssim_end4_sse2;
1015 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16_sse2;
1016 pixf->var[PIXEL_8x8] = x264_pixel_var_8x8_sse2;
1017 pixf->var2[PIXEL_8x8] = x264_pixel_var2_8x8_sse2;
1018 pixf->var2[PIXEL_8x16] = x264_pixel_var2_8x16_sse2;
1019 pixf->intra_sad_x3_8x8 = x264_intra_sad_x3_8x8_sse2;
1020 }
1021 if( (cpu&X264_CPU_SSE2) && !(cpu&X264_CPU_SSE2_IS_SLOW) )
1022 {
1023 INIT5( sad, _sse2 );
1024 INIT2( sad_x3, _sse2 );
1025 INIT2( sad_x4, _sse2 );
1026 INIT_ADS( _sse2 );
1027
1028 if( !(cpu&X264_CPU_STACK_MOD4) )
1029 {
1030 INIT4( hadamard_ac, _sse2 );
1031 }
1032 pixf->vsad = x264_pixel_vsad_sse2;
1033 pixf->asd8 = x264_pixel_asd8_sse2;
1034 pixf->intra_sad_x3_8x8 = x264_intra_sad_x3_8x8_sse2;
1035 pixf->intra_sad_x3_8x8c = x264_intra_sad_x3_8x8c_sse2;
1036 pixf->intra_sad_x3_8x16c = x264_intra_sad_x3_8x16c_sse2;
1037 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_sse2;
1038 pixf->intra_sad_x3_16x16 = x264_intra_sad_x3_16x16_sse2;
1039 }
1040 if( cpu&X264_CPU_SSE2_IS_FAST )
1041 {
1042 pixf->sad[PIXEL_8x16] = x264_pixel_sad_8x16_sse2;
1043 pixf->sad_x3[PIXEL_8x16] = x264_pixel_sad_x3_8x16_sse2;
1044 pixf->sad_x3[PIXEL_8x8] = x264_pixel_sad_x3_8x8_sse2;
1045 pixf->sad_x3[PIXEL_8x4] = x264_pixel_sad_x3_8x4_sse2;
1046 pixf->sad_x4[PIXEL_8x16] = x264_pixel_sad_x4_8x16_sse2;
1047 pixf->sad_x4[PIXEL_8x8] = x264_pixel_sad_x4_8x8_sse2;
1048 pixf->sad_x4[PIXEL_8x4] = x264_pixel_sad_x4_8x4_sse2;
1049 }
1050 if( cpu&X264_CPU_SSSE3 )
1051 {
1052 INIT4_NAME( sad_aligned, sad, _ssse3_aligned );
1053 pixf->sad_aligned[PIXEL_4x4] = x264_pixel_sad_4x4_ssse3;
1054 pixf->sad_aligned[PIXEL_4x8] = x264_pixel_sad_4x8_ssse3;
1055 INIT7( sad, _ssse3 );
1056 INIT7( sad_x3, _ssse3 );
1057 INIT7( sad_x4, _ssse3 );
1058 INIT_ADS( _ssse3 );
1059 INIT6( satd, _ssse3 );
1060 pixf->satd[PIXEL_4x16] = x264_pixel_satd_4x16_ssse3;
1061
1062 if( !(cpu&X264_CPU_STACK_MOD4) )
1063 {
1064 INIT4( hadamard_ac, _ssse3 );
1065 }
1066 pixf->vsad = x264_pixel_vsad_ssse3;
1067 pixf->asd8 = x264_pixel_asd8_ssse3;
1068 pixf->intra_sad_x3_4x4 = x264_intra_sad_x3_4x4_ssse3;
1069 pixf->sa8d[PIXEL_16x16]= x264_pixel_sa8d_16x16_ssse3;
1070 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_ssse3;
1071 #if ARCH_X86_64
1072 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_ssse3;
1073 #endif
1074 pixf->intra_sad_x3_4x4 = x264_intra_sad_x3_4x4_ssse3;
1075 pixf->intra_sad_x3_8x8 = x264_intra_sad_x3_8x8_ssse3;
1076 pixf->intra_sad_x3_8x8c = x264_intra_sad_x3_8x8c_ssse3;
1077 pixf->intra_sad_x3_8x16c = x264_intra_sad_x3_8x16c_ssse3;
1078 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_ssse3;
1079 pixf->intra_sad_x3_16x16 = x264_intra_sad_x3_16x16_ssse3;
1080 }
1081 if( cpu&X264_CPU_SSE4 )
1082 {
1083 INIT6( satd, _sse4 );
1084 pixf->satd[PIXEL_4x16] = x264_pixel_satd_4x16_sse4;
1085 if( !(cpu&X264_CPU_STACK_MOD4) )
1086 {
1087 INIT4( hadamard_ac, _sse4 );
1088 }
1089 pixf->sa8d[PIXEL_16x16]= x264_pixel_sa8d_16x16_sse4;
1090 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_sse4;
1091 #if ARCH_X86_64
1092 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_sse4;
1093 #endif
1094 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_sse4;
1095 }
1096 if( cpu&X264_CPU_AVX )
1097 {
1098 INIT5_NAME( sad_aligned, sad, _ssse3 ); /* AVX-capable CPUs doesn't benefit from an aligned version */
1099 INIT_ADS( _avx );
1100 INIT6( satd, _avx );
1101 pixf->satd[PIXEL_4x16] = x264_pixel_satd_4x16_avx;
1102 if( !(cpu&X264_CPU_STACK_MOD4) )
1103 {
1104 INIT4( hadamard_ac, _avx );
1105 }
1106 pixf->intra_sad_x3_4x4 = x264_intra_sad_x3_4x4_avx;
1107 pixf->sa8d[PIXEL_16x16]= x264_pixel_sa8d_16x16_avx;
1108 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_avx;
1109 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16_avx;
1110 pixf->var[PIXEL_8x8] = x264_pixel_var_8x8_avx;
1111 pixf->ssd_nv12_core = x264_pixel_ssd_nv12_core_avx;
1112 pixf->ssim_4x4x2_core = x264_pixel_ssim_4x4x2_core_avx;
1113 pixf->ssim_end4 = x264_pixel_ssim_end4_avx;
1114 #if ARCH_X86_64
1115 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_avx;
1116 #endif
1117 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_avx;
1118 }
1119 if( cpu&X264_CPU_XOP )
1120 {
1121 INIT5( sad_x3, _xop );
1122 INIT5( sad_x4, _xop );
1123 pixf->ssd_nv12_core = x264_pixel_ssd_nv12_core_xop;
1124 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16_xop;
1125 pixf->var[PIXEL_8x8] = x264_pixel_var_8x8_xop;
1126 pixf->vsad = x264_pixel_vsad_xop;
1127 pixf->asd8 = x264_pixel_asd8_xop;
1128 #if ARCH_X86_64
1129 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_xop;
1130 #endif
1131 }
1132 if( cpu&X264_CPU_AVX2 )
1133 {
1134 INIT2( ssd, _avx2 );
1135 INIT2( sad, _avx2 );
1136 INIT2_NAME( sad_aligned, sad, _avx2 );
1137 INIT2( sad_x3, _avx2 );
1138 INIT2( sad_x4, _avx2 );
1139 pixf->vsad = x264_pixel_vsad_avx2;
1140 pixf->ssd_nv12_core = x264_pixel_ssd_nv12_core_avx2;
1141 pixf->intra_sad_x3_8x8 = x264_intra_sad_x3_8x8_avx2;
1142 }
1143 #endif // HAVE_MMX
1144 #else // !HIGH_BIT_DEPTH
1145 #if HAVE_MMX
1146 if( cpu&X264_CPU_MMX )
1147 {
1148 INIT8( ssd, _mmx );
1149 }
1150
1151 if( cpu&X264_CPU_MMX2 )
1152 {
1153 INIT8( sad, _mmx2 );
1154 INIT8_NAME( sad_aligned, sad, _mmx2 );
1155 INIT7( sad_x3, _mmx2 );
1156 INIT7( sad_x4, _mmx2 );
1157 INIT8( satd, _mmx2 );
1158 INIT7( satd_x3, _mmx2 );
1159 INIT7( satd_x4, _mmx2 );
1160 INIT4( hadamard_ac, _mmx2 );
1161 INIT_ADS( _mmx2 );
1162 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16_mmx2;
1163 pixf->var[PIXEL_8x16] = x264_pixel_var_8x16_mmx2;
1164 pixf->var[PIXEL_8x8] = x264_pixel_var_8x8_mmx2;
1165 pixf->ssd_nv12_core = x264_pixel_ssd_nv12_core_mmx2;
1166 #if ARCH_X86
1167 pixf->sa8d[PIXEL_16x16] = x264_pixel_sa8d_16x16_mmx2;
1168 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_mmx2;
1169 pixf->intra_sa8d_x3_8x8 = x264_intra_sa8d_x3_8x8_mmx2;
1170 pixf->ssim_4x4x2_core = x264_pixel_ssim_4x4x2_core_mmx2;
1171 pixf->var2[PIXEL_8x8] = x264_pixel_var2_8x8_mmx2;
1172 pixf->var2[PIXEL_8x16] = x264_pixel_var2_8x16_mmx2;
1173 pixf->vsad = x264_pixel_vsad_mmx2;
1174
1175 if( cpu&X264_CPU_CACHELINE_32 )
1176 {
1177 INIT5( sad, _cache32_mmx2 );
1178 INIT4( sad_x3, _cache32_mmx2 );
1179 INIT4( sad_x4, _cache32_mmx2 );
1180 }
1181 else if( cpu&X264_CPU_CACHELINE_64 && !(cpu&X264_CPU_SLOW_ATOM) )
1182 {
1183 INIT5( sad, _cache64_mmx2 );
1184 INIT4( sad_x3, _cache64_mmx2 );
1185 INIT4( sad_x4, _cache64_mmx2 );
1186 }
1187 #else
1188 if( cpu&X264_CPU_CACHELINE_64 && !(cpu&X264_CPU_SLOW_ATOM) )
1189 {
1190 pixf->sad[PIXEL_8x16] = x264_pixel_sad_8x16_cache64_mmx2;
1191 pixf->sad[PIXEL_8x8] = x264_pixel_sad_8x8_cache64_mmx2;
1192 pixf->sad[PIXEL_8x4] = x264_pixel_sad_8x4_cache64_mmx2;
1193 pixf->sad_x3[PIXEL_8x16] = x264_pixel_sad_x3_8x16_cache64_mmx2;
1194 pixf->sad_x3[PIXEL_8x8] = x264_pixel_sad_x3_8x8_cache64_mmx2;
1195 pixf->sad_x4[PIXEL_8x16] = x264_pixel_sad_x4_8x16_cache64_mmx2;
1196 pixf->sad_x4[PIXEL_8x8] = x264_pixel_sad_x4_8x8_cache64_mmx2;
1197 }
1198 #endif
1199 pixf->intra_satd_x3_16x16 = x264_intra_satd_x3_16x16_mmx2;
1200 pixf->intra_sad_x3_16x16 = x264_intra_sad_x3_16x16_mmx2;
1201 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_mmx2;
1202 pixf->intra_sad_x3_8x16c = x264_intra_sad_x3_8x16c_mmx2;
1203 pixf->intra_satd_x3_8x8c = x264_intra_satd_x3_8x8c_mmx2;
1204 pixf->intra_sad_x3_8x8c = x264_intra_sad_x3_8x8c_mmx2;
1205 pixf->intra_sad_x3_8x8 = x264_intra_sad_x3_8x8_mmx2;
1206 pixf->intra_satd_x3_4x4 = x264_intra_satd_x3_4x4_mmx2;
1207 pixf->intra_sad_x3_4x4 = x264_intra_sad_x3_4x4_mmx2;
1208 }
1209
1210 if( cpu&X264_CPU_SSE2 )
1211 {
1212 INIT5( ssd, _sse2slow );
1213 INIT2_NAME( sad_aligned, sad, _sse2_aligned );
1214 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16_sse2;
1215 pixf->ssd_nv12_core = x264_pixel_ssd_nv12_core_sse2;
1216 pixf->ssim_4x4x2_core = x264_pixel_ssim_4x4x2_core_sse2;
1217 pixf->ssim_end4 = x264_pixel_ssim_end4_sse2;
1218 pixf->sa8d[PIXEL_16x16] = x264_pixel_sa8d_16x16_sse2;
1219 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_sse2;
1220 #if ARCH_X86_64
1221 pixf->intra_sa8d_x3_8x8 = x264_intra_sa8d_x3_8x8_sse2;
1222 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_sse2;
1223 #endif
1224 pixf->var2[PIXEL_8x8] = x264_pixel_var2_8x8_sse2;
1225 pixf->var2[PIXEL_8x16] = x264_pixel_var2_8x16_sse2;
1226 pixf->vsad = x264_pixel_vsad_sse2;
1227 pixf->asd8 = x264_pixel_asd8_sse2;
1228 }
1229
1230 if( (cpu&X264_CPU_SSE2) && !(cpu&X264_CPU_SSE2_IS_SLOW) )
1231 {
1232 INIT2( sad, _sse2 );
1233 INIT2( sad_x3, _sse2 );
1234 INIT2( sad_x4, _sse2 );
1235 INIT6( satd, _sse2 );
1236 pixf->satd[PIXEL_4x16] = x264_pixel_satd_4x16_sse2;
1237 INIT6( satd_x3, _sse2 );
1238 INIT6( satd_x4, _sse2 );
1239 INIT4( hadamard_ac, _sse2 );
1240 INIT_ADS( _sse2 );
1241 pixf->var[PIXEL_8x8] = x264_pixel_var_8x8_sse2;
1242 pixf->var[PIXEL_8x16] = x264_pixel_var_8x16_sse2;
1243 pixf->intra_sad_x3_16x16 = x264_intra_sad_x3_16x16_sse2;
1244 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_sse2;
1245 pixf->intra_sad_x3_8x16c = x264_intra_sad_x3_8x16c_sse2;
1246 if( cpu&X264_CPU_CACHELINE_64 )
1247 {
1248 INIT2( ssd, _sse2); /* faster for width 16 on p4 */
1249 #if ARCH_X86
1250 INIT2( sad, _cache64_sse2 );
1251 INIT2( sad_x3, _cache64_sse2 );
1252 INIT2( sad_x4, _cache64_sse2 );
1253 #endif
1254 if( cpu&X264_CPU_SSE2_IS_FAST )
1255 {
1256 pixf->sad_x3[PIXEL_8x16] = x264_pixel_sad_x3_8x16_cache64_sse2;
1257 pixf->sad_x4[PIXEL_8x16] = x264_pixel_sad_x4_8x16_cache64_sse2;
1258 }
1259 }
1260 }
1261
1262 if( cpu&X264_CPU_SSE2_IS_FAST && !(cpu&X264_CPU_CACHELINE_64) )
1263 {
1264 pixf->sad_aligned[PIXEL_8x16] = x264_pixel_sad_8x16_sse2;
1265 pixf->sad[PIXEL_8x16] = x264_pixel_sad_8x16_sse2;
1266 pixf->sad_x3[PIXEL_8x16] = x264_pixel_sad_x3_8x16_sse2;
1267 pixf->sad_x3[PIXEL_8x8] = x264_pixel_sad_x3_8x8_sse2;
1268 pixf->sad_x3[PIXEL_8x4] = x264_pixel_sad_x3_8x4_sse2;
1269 pixf->sad_x4[PIXEL_8x16] = x264_pixel_sad_x4_8x16_sse2;
1270 pixf->sad_x4[PIXEL_8x8] = x264_pixel_sad_x4_8x8_sse2;
1271 pixf->sad_x4[PIXEL_8x4] = x264_pixel_sad_x4_8x4_sse2;
1272 }
1273
1274 if( (cpu&X264_CPU_SSE3) && (cpu&X264_CPU_CACHELINE_64) )
1275 {
1276 INIT2( sad, _sse3 );
1277 INIT2( sad_x3, _sse3 );
1278 INIT2( sad_x4, _sse3 );
1279 }
1280
1281 if( cpu&X264_CPU_SSSE3 )
1282 {
1283 INIT4( hadamard_ac, _ssse3 );
1284 if( !(cpu&X264_CPU_STACK_MOD4) )
1285 {
1286 pixf->intra_sad_x9_4x4 = x264_intra_sad_x9_4x4_ssse3;
1287 pixf->intra_satd_x9_4x4 = x264_intra_satd_x9_4x4_ssse3;
1288 pixf->intra_sad_x9_8x8 = x264_intra_sad_x9_8x8_ssse3;
1289 #if ARCH_X86_64
1290 pixf->intra_sa8d_x9_8x8 = x264_intra_sa8d_x9_8x8_ssse3;
1291 #endif
1292 }
1293 INIT_ADS( _ssse3 );
1294 if( cpu&X264_CPU_SLOW_ATOM )
1295 {
1296 pixf->sa8d[PIXEL_16x16]= x264_pixel_sa8d_16x16_ssse3_atom;
1297 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_ssse3_atom;
1298 INIT6( satd, _ssse3_atom );
1299 pixf->satd[PIXEL_4x16] = x264_pixel_satd_4x16_ssse3_atom;
1300 INIT6( satd_x3, _ssse3_atom );
1301 INIT6( satd_x4, _ssse3_atom );
1302 INIT4( hadamard_ac, _ssse3_atom );
1303 #if ARCH_X86_64
1304 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_ssse3_atom;
1305 #endif
1306 }
1307 else
1308 {
1309 INIT8( ssd, _ssse3 );
1310 pixf->sa8d[PIXEL_16x16]= x264_pixel_sa8d_16x16_ssse3;
1311 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_ssse3;
1312 INIT8( satd, _ssse3 );
1313 INIT7( satd_x3, _ssse3 );
1314 INIT7( satd_x4, _ssse3 );
1315 #if ARCH_X86_64
1316 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_ssse3;
1317 #endif
1318 }
1319 pixf->intra_satd_x3_16x16 = x264_intra_satd_x3_16x16_ssse3;
1320 if( !(cpu&X264_CPU_SLOW_PSHUFB) )
1321 pixf->intra_sad_x3_16x16 = x264_intra_sad_x3_16x16_ssse3;
1322 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_ssse3;
1323 pixf->intra_satd_x3_8x8c = x264_intra_satd_x3_8x8c_ssse3;
1324 pixf->intra_sad_x3_8x8c = x264_intra_sad_x3_8x8c_ssse3;
1325 pixf->var2[PIXEL_8x8] = x264_pixel_var2_8x8_ssse3;
1326 pixf->var2[PIXEL_8x16] = x264_pixel_var2_8x16_ssse3;
1327 pixf->asd8 = x264_pixel_asd8_ssse3;
1328 if( cpu&X264_CPU_CACHELINE_64 )
1329 {
1330 INIT2( sad, _cache64_ssse3 );
1331 INIT2( sad_x3, _cache64_ssse3 );
1332 INIT2( sad_x4, _cache64_ssse3 );
1333 }
1334 else
1335 {
1336 INIT2( sad_x3, _ssse3 );
1337 INIT5( sad_x4, _ssse3 );
1338 }
1339 if( (cpu&X264_CPU_SLOW_ATOM) || (cpu&X264_CPU_SLOW_SHUFFLE) )
1340 {
1341 INIT5( ssd, _sse2 ); /* on conroe, sse2 is faster for width8/16 */
1342 }
1343 }
1344
1345 if( cpu&X264_CPU_SSE4 )
1346 {
1347 INIT8( satd, _sse4 );
1348 INIT7( satd_x3, _sse4 );
1349 INIT7( satd_x4, _sse4 );
1350 INIT4( hadamard_ac, _sse4 );
1351 if( !(cpu&X264_CPU_STACK_MOD4) )
1352 {
1353 pixf->intra_sad_x9_4x4 = x264_intra_sad_x9_4x4_sse4;
1354 pixf->intra_satd_x9_4x4 = x264_intra_satd_x9_4x4_sse4;
1355 pixf->intra_sad_x9_8x8 = x264_intra_sad_x9_8x8_sse4;
1356 #if ARCH_X86_64
1357 pixf->intra_sa8d_x9_8x8 = x264_intra_sa8d_x9_8x8_sse4;
1358 #endif
1359 }
1360 pixf->sa8d[PIXEL_16x16]= x264_pixel_sa8d_16x16_sse4;
1361 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_sse4;
1362 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_sse4;
1363 #if ARCH_X86_64
1364 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_sse4;
1365 #endif
1366 }
1367
1368 if( cpu&X264_CPU_AVX )
1369 {
1370 INIT2_NAME( sad_aligned, sad, _sse2 ); /* AVX-capable CPUs doesn't benefit from an aligned version */
1371 INIT2( sad_x3, _avx );
1372 INIT2( sad_x4, _avx );
1373 INIT8( satd, _avx );
1374 INIT7( satd_x3, _avx );
1375 INIT7( satd_x4, _avx );
1376 INIT_ADS( _avx );
1377 INIT4( hadamard_ac, _avx );
1378 if( !(cpu&X264_CPU_STACK_MOD4) )
1379 {
1380 pixf->intra_sad_x9_4x4 = x264_intra_sad_x9_4x4_avx;
1381 pixf->intra_satd_x9_4x4 = x264_intra_satd_x9_4x4_avx;
1382 pixf->intra_sad_x9_8x8 = x264_intra_sad_x9_8x8_avx;
1383 #if ARCH_X86_64
1384 pixf->intra_sa8d_x9_8x8 = x264_intra_sa8d_x9_8x8_avx;
1385 #endif
1386 }
1387 INIT5( ssd, _avx );
1388 pixf->sa8d[PIXEL_16x16]= x264_pixel_sa8d_16x16_avx;
1389 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_avx;
1390 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_avx;
1391 pixf->ssd_nv12_core = x264_pixel_ssd_nv12_core_avx;
1392 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16_avx;
1393 pixf->var[PIXEL_8x16] = x264_pixel_var_8x16_avx;
1394 pixf->var[PIXEL_8x8] = x264_pixel_var_8x8_avx;
1395 pixf->ssim_4x4x2_core = x264_pixel_ssim_4x4x2_core_avx;
1396 pixf->ssim_end4 = x264_pixel_ssim_end4_avx;
1397 #if ARCH_X86_64
1398 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_avx;
1399 #endif
1400 }
1401
1402 if( cpu&X264_CPU_XOP )
1403 {
1404 INIT7( satd, _xop );
1405 INIT7( satd_x3, _xop );
1406 INIT7( satd_x4, _xop );
1407 INIT4( hadamard_ac, _xop );
1408 if( !(cpu&X264_CPU_STACK_MOD4) )
1409 {
1410 pixf->intra_satd_x9_4x4 = x264_intra_satd_x9_4x4_xop;
1411 }
1412 INIT5( ssd, _xop );
1413 pixf->sa8d[PIXEL_16x16]= x264_pixel_sa8d_16x16_xop;
1414 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_xop;
1415 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_xop;
1416 pixf->ssd_nv12_core = x264_pixel_ssd_nv12_core_xop;
1417 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16_xop;
1418 pixf->var[PIXEL_8x16] = x264_pixel_var_8x16_xop;
1419 pixf->var[PIXEL_8x8] = x264_pixel_var_8x8_xop;
1420 pixf->var2[PIXEL_8x8] = x264_pixel_var2_8x8_xop;
1421 pixf->var2[PIXEL_8x16] = x264_pixel_var2_8x16_xop;
1422 #if ARCH_X86_64
1423 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_xop;
1424 #endif
1425 }
1426
1427 if( cpu&X264_CPU_AVX2 )
1428 {
1429 INIT2( ssd, _avx2 );
1430 INIT2( sad_x3, _avx2 );
1431 INIT2( sad_x4, _avx2 );
1432 INIT4( satd, _avx2 );
1433 INIT2( hadamard_ac, _avx2 );
1434 INIT_ADS( _avx2 );
1435 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_avx2;
1436 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16_avx2;
1437 pixf->var2[PIXEL_8x16] = x264_pixel_var2_8x16_avx2;
1438 pixf->var2[PIXEL_8x8] = x264_pixel_var2_8x8_avx2;
1439 pixf->intra_sad_x3_16x16 = x264_intra_sad_x3_16x16_avx2;
1440 pixf->intra_sad_x9_8x8 = x264_intra_sad_x9_8x8_avx2;
1441 pixf->intra_sad_x3_8x8c = x264_intra_sad_x3_8x8c_avx2;
1442 pixf->ssd_nv12_core = x264_pixel_ssd_nv12_core_avx2;
1443 #if ARCH_X86_64
1444 pixf->sa8d_satd[PIXEL_16x16] = x264_pixel_sa8d_satd_16x16_avx2;
1445 #endif
1446 }
1447 #endif //HAVE_MMX
1448
1449 #if HAVE_ARMV6
1450 if( cpu&X264_CPU_ARMV6 )
1451 {
1452 pixf->sad[PIXEL_4x8] = x264_pixel_sad_4x8_armv6;
1453 pixf->sad[PIXEL_4x4] = x264_pixel_sad_4x4_armv6;
1454 pixf->sad_aligned[PIXEL_4x8] = x264_pixel_sad_4x8_armv6;
1455 pixf->sad_aligned[PIXEL_4x4] = x264_pixel_sad_4x4_armv6;
1456 }
1457 if( cpu&X264_CPU_NEON )
1458 {
1459 INIT5( sad, _neon );
1460 INIT5( sad_aligned, _neon );
1461 INIT7( sad_x3, _neon );
1462 INIT7( sad_x4, _neon );
1463 INIT7( ssd, _neon );
1464 INIT7( satd, _neon );
1465 INIT7( satd_x3, _neon );
1466 INIT7( satd_x4, _neon );
1467 INIT4( hadamard_ac, _neon );
1468 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_neon;
1469 pixf->sa8d[PIXEL_16x16] = x264_pixel_sa8d_16x16_neon;
1470 pixf->var[PIXEL_8x8] = x264_pixel_var_8x8_neon;
1471 pixf->var[PIXEL_8x16] = x264_pixel_var_8x16_neon;
1472 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16_neon;
1473 pixf->var2[PIXEL_8x8] = x264_pixel_var2_8x8_neon;
1474 pixf->var2[PIXEL_8x16] = x264_pixel_var2_8x16_neon;
1475
1476 pixf->intra_sad_x3_4x4 = x264_intra_sad_x3_4x4_neon;
1477 pixf->intra_satd_x3_4x4 = x264_intra_satd_x3_4x4_neon;
1478 pixf->intra_sad_x3_8x8 = x264_intra_sad_x3_8x8_neon;
1479 pixf->intra_sa8d_x3_8x8 = x264_intra_sa8d_x3_8x8_neon;
1480 pixf->intra_sad_x3_8x8c = x264_intra_sad_x3_8x8c_neon;
1481 pixf->intra_satd_x3_8x8c = x264_intra_satd_x3_8x8c_neon;
1482 pixf->intra_sad_x3_8x16c = x264_intra_sad_x3_8x16c_neon;
1483 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_neon;
1484 pixf->intra_sad_x3_16x16 = x264_intra_sad_x3_16x16_neon;
1485 pixf->intra_satd_x3_16x16 = x264_intra_satd_x3_16x16_neon;
1486
1487 pixf->ssim_4x4x2_core = x264_pixel_ssim_4x4x2_core_neon;
1488 pixf->ssim_end4 = x264_pixel_ssim_end4_neon;
1489
1490 if( cpu&X264_CPU_FAST_NEON_MRC )
1491 {
1492 pixf->sad[PIXEL_4x8] = x264_pixel_sad_4x8_neon;
1493 pixf->sad[PIXEL_4x4] = x264_pixel_sad_4x4_neon;
1494 pixf->sad_aligned[PIXEL_4x8] = x264_pixel_sad_aligned_4x8_neon;
1495 pixf->sad_aligned[PIXEL_4x4] = x264_pixel_sad_aligned_4x4_neon;
1496 }
1497 else // really just scheduled for dual issue / A8
1498 {
1499 INIT5( sad_aligned, _neon_dual );
1500 }
1501 }
1502 #endif
1503
1504 #if ARCH_AARCH64
1505 if( cpu&X264_CPU_NEON )
1506 {
1507 INIT7( sad, _neon );
1508 // AArch64 has no distinct instructions for aligned load/store
1509 INIT7_NAME( sad_aligned, sad, _neon );
1510 INIT7( sad_x3, _neon );
1511 INIT7( sad_x4, _neon );
1512 INIT7( ssd, _neon );
1513 INIT7( satd, _neon );
1514 INIT7( satd_x3, _neon );
1515 INIT7( satd_x4, _neon );
1516 INIT4( hadamard_ac, _neon );
1517
1518 pixf->sa8d[PIXEL_8x8] = x264_pixel_sa8d_8x8_neon;
1519 pixf->sa8d[PIXEL_16x16] = x264_pixel_sa8d_16x16_neon;
1520
1521 pixf->var[PIXEL_8x8] = x264_pixel_var_8x8_neon;
1522 pixf->var[PIXEL_8x16] = x264_pixel_var_8x16_neon;
1523 pixf->var[PIXEL_16x16] = x264_pixel_var_16x16_neon;
1524 pixf->var2[PIXEL_8x8] = x264_pixel_var2_8x8_neon;
1525 pixf->var2[PIXEL_8x16] = x264_pixel_var2_8x16_neon;
1526
1527 pixf->intra_sad_x3_4x4 = x264_intra_sad_x3_4x4_neon;
1528 pixf->intra_satd_x3_4x4 = x264_intra_satd_x3_4x4_neon;
1529 pixf->intra_sad_x3_8x8 = x264_intra_sad_x3_8x8_neon;
1530 pixf->intra_sa8d_x3_8x8 = x264_intra_sa8d_x3_8x8_neon;
1531 pixf->intra_sad_x3_8x8c = x264_intra_sad_x3_8x8c_neon;
1532 pixf->intra_satd_x3_8x8c = x264_intra_satd_x3_8x8c_neon;
1533 pixf->intra_sad_x3_8x16c = x264_intra_sad_x3_8x16c_neon;
1534 pixf->intra_satd_x3_8x16c = x264_intra_satd_x3_8x16c_neon;
1535 pixf->intra_sad_x3_16x16 = x264_intra_sad_x3_16x16_neon;
1536 pixf->intra_satd_x3_16x16 = x264_intra_satd_x3_16x16_neon;
1537
1538 pixf->ssim_4x4x2_core = x264_pixel_ssim_4x4x2_core_neon;
1539 pixf->ssim_end4 = x264_pixel_ssim_end4_neon;
1540 }
1541 #endif // ARCH_AARCH64
1542
1543 #endif // HIGH_BIT_DEPTH
1544 #if HAVE_ALTIVEC
1545 if( cpu&X264_CPU_ALTIVEC )
1546 {
1547 x264_pixel_altivec_init( pixf );
1548 }
1549 #endif
1550
1551 pixf->ads[PIXEL_8x16] =
1552 pixf->ads[PIXEL_8x4] =
1553 pixf->ads[PIXEL_4x8] = pixf->ads[PIXEL_16x8];
1554 pixf->ads[PIXEL_4x4] = pixf->ads[PIXEL_8x8];
1555 }
1556
1557