1;***************************************************************************** 2;* dct-32.asm: x86_32 transform and zigzag 3;***************************************************************************** 4;* Copyright (C) 2003-2014 x264 project 5;* 6;* Authors: Loren Merritt <lorenm@u.washington.edu> 7;* Holger Lubitz <holger@lubitz.org> 8;* Laurent Aimar <fenrir@via.ecp.fr> 9;* Min Chen <chenm001.163.com> 10;* Christian Heine <sennindemokrit@gmx.net> 11;* 12;* This program is free software; you can redistribute it and/or modify 13;* it under the terms of the GNU General Public License as published by 14;* the Free Software Foundation; either version 2 of the License, or 15;* (at your option) any later version. 16;* 17;* This program is distributed in the hope that it will be useful, 18;* but WITHOUT ANY WARRANTY; without even the implied warranty of 19;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20;* GNU General Public License for more details. 21;* 22;* You should have received a copy of the GNU General Public License 23;* along with this program; if not, write to the Free Software 24;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA. 25;* 26;* This program is also available under a commercial proprietary license. 27;* For more information, contact us at licensing@x264.com. 28;***************************************************************************** 29 30%include "x86inc.asm" 31%include "x86util.asm" 32 33SECTION .text 34 35cextern pd_32 36cextern pw_pixel_max 37cextern pw_2 38cextern pw_m2 39cextern pw_32 40cextern hsub_mul 41 42%macro SPILL_SHUFFLE 3-* ; ptr, list of regs, list of memory offsets 43 %xdefine %%base %1 44 %rep %0/2 45 %xdefine %%tmp m%2 46 %rotate %0/2 47 mova [%%base + %2*16], %%tmp 48 %rotate 1-%0/2 49 %endrep 50%endmacro 51 52%macro UNSPILL_SHUFFLE 3-* 53 %xdefine %%base %1 54 %rep %0/2 55 %xdefine %%tmp m%2 56 %rotate %0/2 57 mova %%tmp, [%%base + %2*16] 58 %rotate 1-%0/2 59 %endrep 60%endmacro 61 62%macro SPILL 2+ ; assume offsets are the same as reg numbers 63 SPILL_SHUFFLE %1, %2, %2 64%endmacro 65 66%macro UNSPILL 2+ 67 UNSPILL_SHUFFLE %1, %2, %2 68%endmacro 69 70; in: size, m0..m7 71; out: 0,4,6 in memory at %10,%11,%12, rest in regs 72%macro DCT8_1D 12 73 SUMSUB_BA %1, %9, %2 ; %9 = s07, %2 = d07 74 SUMSUB_BA %1, %8, %3 ; %8 = s16, %3 = d16 75 SUMSUB_BA %1, %7, %4 ; %7 = s25, %4 = d25 76 SUMSUB_BA %1, %6, %5 ; %6 = s34, %5 = d34 77 SUMSUB_BA %1, %6, %9 ; %6 = a0, %9 = a2 78 SUMSUB_BA %1, %7, %8 ; %7 = a1, %8 = a3 79 SUMSUB_BA %1, %7, %6 ; %7 = dst0, %6 = dst4 80 mova %10, m%7 81 mova %11, m%6 82 psra%1 m%7, m%8, 1 ; a3>>1 83 padd%1 m%7, m%9 ; a2 + (a3>>1) 84 psra%1 m%9, 1 ; a2>>1 85 psub%1 m%9, m%8 ; (a2>>1) - a3 86 mova %12, m%9 87 psra%1 m%6, m%4, 1 88 padd%1 m%6, m%4 ; d25+(d25>>1) 89 psub%1 m%8, m%2, m%5 ; a5 = d07-d34-(d25+(d25>>1)) 90 psub%1 m%8, m%6 91 psra%1 m%6, m%3, 1 92 padd%1 m%6, m%3 ; d16+(d16>>1) 93 padd%1 m%9, m%2, m%5 94 psub%1 m%9, m%6 ; a6 = d07+d34-(d16+(d16>>1)) 95 psra%1 m%6, m%2, 1 96 padd%1 m%6, m%2 ; d07+(d07>>1) 97 padd%1 m%6, m%3 98 padd%1 m%6, m%4 ; a4 = d16+d25+(d07+(d07>>1)) 99 psra%1 m%2, m%5, 1 100 padd%1 m%2, m%5 ; d34+(d34>>1) 101 padd%1 m%2, m%3 102 psub%1 m%2, m%4 ; a7 = d16-d25+(d34+(d34>>1)) 103 psra%1 m%5, m%2, 2 104 padd%1 m%5, m%6 ; a4 + (a7>>2) 105 psra%1 m%4, m%9, 2 106 padd%1 m%4, m%8 ; a5 + (a6>>2) 107 psra%1 m%6, 2 108 psra%1 m%8, 2 109 psub%1 m%6, m%2 ; (a4>>2) - a7 110 psub%1 m%9, m%8 ; a6 - (a5>>2) 111 SWAP %3, %5, %4, %7, %9, %6 112%endmacro 113 114; in: size, m[1,2,3,5,6,7], 0,4 in mem at %10,%11 115; out: m0..m7 116%macro IDCT8_1D 11 117 psra%1 m%2, m%4, 1 118 psra%1 m%6, m%8, 1 119 psub%1 m%2, m%8 120 padd%1 m%6, m%4 121 psra%1 m%8, m%3, 1 122 padd%1 m%8, m%3 123 padd%1 m%8, m%5 124 padd%1 m%8, m%7 125 psra%1 m%4, m%7, 1 126 padd%1 m%4, m%7 127 padd%1 m%4, m%9 128 psub%1 m%4, m%3 129 psub%1 m%3, m%5 130 psub%1 m%7, m%5 131 padd%1 m%3, m%9 132 psub%1 m%7, m%9 133 psra%1 m%5, 1 134 psra%1 m%9, 1 135 psub%1 m%3, m%5 136 psub%1 m%7, m%9 137 psra%1 m%5, m%8, 2 138 psra%1 m%9, m%4, 2 139 padd%1 m%5, m%7 140 padd%1 m%9, m%3 141 psra%1 m%7, 2 142 psra%1 m%3, 2 143 psub%1 m%8, m%7 144 psub%1 m%3, m%4 145 mova m%4, %10 146 mova m%7, %11 147 SUMSUB_BA %1, %7, %4 148 SUMSUB_BA %1, %6, %7 149 SUMSUB_BA %1, %2, %4 150 SUMSUB_BA %1, %8, %6 151 SUMSUB_BA %1, %3, %2 152 SUMSUB_BA %1, %9, %4 153 SUMSUB_BA %1, %5, %7 154 SWAP %2, %4 155 SWAP %6, %8 156 SWAP %2, %6, %7 157 SWAP %4, %9, %8 158%endmacro 159 160%if HIGH_BIT_DEPTH 161 162%macro SUB8x8_DCT8 0 163cglobal sub8x8_dct8, 3,3,8 164global current_function %+ .skip_prologue 165.skip_prologue: 166 LOAD_DIFF8x4 0,1,2,3, none,none, r1, r2 167 LOAD_DIFF8x4 4,5,6,7, none,none, r1, r2 168 169 DCT8_1D w, 0,1,2,3,4,5,6,7, [r0],[r0+0x10],[r0+0x50] 170 mova m0, [r0] 171 172 mova [r0+0x30], m5 173 mova [r0+0x70], m7 174 TRANSPOSE4x4W 0,1,2,3,4 175 WIDEN_SXWD 0,4 176 WIDEN_SXWD 1,5 177 WIDEN_SXWD 2,6 178 WIDEN_SXWD 3,7 179 DCT8_1D d, 0,4,1,5,2,6,3,7, [r0],[r0+0x80],[r0+0xC0] 180 mova [r0+0x20], m4 181 mova [r0+0x40], m1 182 mova [r0+0x60], m5 183 mova [r0+0xA0], m6 184 mova [r0+0xE0], m7 185 mova m4, [r0+0x10] 186 mova m5, [r0+0x30] 187 mova m6, [r0+0x50] 188 mova m7, [r0+0x70] 189 190 TRANSPOSE4x4W 4,5,6,7,0 191 WIDEN_SXWD 4,0 192 WIDEN_SXWD 5,1 193 WIDEN_SXWD 6,2 194 WIDEN_SXWD 7,3 195 DCT8_1D d,4,0,5,1,6,2,7,3, [r0+0x10],[r0+0x90],[r0+0xD0] 196 mova [r0+0x30], m0 197 mova [r0+0x50], m5 198 mova [r0+0x70], m1 199 mova [r0+0xB0], m2 200 mova [r0+0xF0], m3 201 ret 202%endmacro ; SUB8x8_DCT8 203 204INIT_XMM sse2 205SUB8x8_DCT8 206INIT_XMM sse4 207SUB8x8_DCT8 208INIT_XMM avx 209SUB8x8_DCT8 210 211%macro ADD8x8_IDCT8 0 212cglobal add8x8_idct8, 2,2 213 add r1, 128 214global current_function %+ .skip_prologue 215.skip_prologue: 216 UNSPILL_SHUFFLE r1, 1,2,3,5,6,7, -6,-4,-2,2,4,6 217 IDCT8_1D d,0,1,2,3,4,5,6,7,[r1-128],[r1+0] 218 mova [r1+0], m4 219 TRANSPOSE4x4D 0,1,2,3,4 220 paddd m0, [pd_32] 221 mova m4, [r1+0] 222 SPILL_SHUFFLE r1, 0,1,2,3, -8,-6,-4,-2 223 TRANSPOSE4x4D 4,5,6,7,3 224 paddd m4, [pd_32] 225 SPILL_SHUFFLE r1, 4,5,6,7, 0,2,4,6 226 UNSPILL_SHUFFLE r1, 1,2,3,5,6,7, -5,-3,-1,3,5,7 227 IDCT8_1D d,0,1,2,3,4,5,6,7,[r1-112],[r1+16] 228 mova [r1+16], m4 229 TRANSPOSE4x4D 0,1,2,3,4 230 mova m4, [r1+16] 231 mova [r1-112], m0 232 TRANSPOSE4x4D 4,5,6,7,0 233 SPILL_SHUFFLE r1, 4,5,6,7, 1,3,5,7 234 UNSPILL_SHUFFLE r1, 5,6,7, -6,-4,-2 235 IDCT8_1D d,4,5,6,7,0,1,2,3,[r1-128],[r1-112] 236 SPILL_SHUFFLE r1, 4,5,6,7,0,1,2,3, -8,-7,-6,-5,-4,-3,-2,-1 237 UNSPILL_SHUFFLE r1, 1,2,3,5,6,7, 2,4,6,3,5,7 238 IDCT8_1D d,0,1,2,3,4,5,6,7,[r1+0],[r1+16] 239 SPILL_SHUFFLE r1, 7,6,5, 7,6,5 240 mova m7, [pw_pixel_max] 241 pxor m6, m6 242 mova m5, [r1-128] 243 STORE_DIFF m5, m0, m6, m7, [r0+0*FDEC_STRIDEB] 244 mova m0, [r1-112] 245 STORE_DIFF m0, m1, m6, m7, [r0+1*FDEC_STRIDEB] 246 mova m0, [r1-96] 247 STORE_DIFF m0, m2, m6, m7, [r0+2*FDEC_STRIDEB] 248 mova m0, [r1-80] 249 STORE_DIFF m0, m3, m6, m7, [r0+3*FDEC_STRIDEB] 250 mova m0, [r1-64] 251 STORE_DIFF m0, m4, m6, m7, [r0+4*FDEC_STRIDEB] 252 mova m0, [r1-48] 253 mova m1, [r1+80] 254 STORE_DIFF m0, m1, m6, m7, [r0+5*FDEC_STRIDEB] 255 mova m0, [r1-32] 256 mova m1, [r1+96] 257 STORE_DIFF m0, m1, m6, m7, [r0+6*FDEC_STRIDEB] 258 mova m0, [r1-16] 259 mova m1, [r1+112] 260 STORE_DIFF m0, m1, m6, m7, [r0+7*FDEC_STRIDEB] 261 RET 262%endmacro ; ADD8x8_IDCT8 263 264INIT_XMM sse2 265ADD8x8_IDCT8 266INIT_XMM avx 267ADD8x8_IDCT8 268 269%else ; !HIGH_BIT_DEPTH 270 271INIT_MMX 272ALIGN 16 273load_diff_4x8_mmx: 274 LOAD_DIFF m0, m7, none, [r1+0*FENC_STRIDE], [r2+0*FDEC_STRIDE] 275 LOAD_DIFF m1, m7, none, [r1+1*FENC_STRIDE], [r2+1*FDEC_STRIDE] 276 LOAD_DIFF m2, m7, none, [r1+2*FENC_STRIDE], [r2+2*FDEC_STRIDE] 277 LOAD_DIFF m3, m7, none, [r1+3*FENC_STRIDE], [r2+3*FDEC_STRIDE] 278 LOAD_DIFF m4, m7, none, [r1+4*FENC_STRIDE], [r2+4*FDEC_STRIDE] 279 LOAD_DIFF m5, m7, none, [r1+5*FENC_STRIDE], [r2+5*FDEC_STRIDE] 280 movq [r0], m0 281 LOAD_DIFF m6, m7, none, [r1+6*FENC_STRIDE], [r2+6*FDEC_STRIDE] 282 LOAD_DIFF m7, m0, none, [r1+7*FENC_STRIDE], [r2+7*FDEC_STRIDE] 283 movq m0, [r0] 284 ret 285 286cglobal dct8_mmx 287 DCT8_1D w,0,1,2,3,4,5,6,7,[r0],[r0+0x40],[r0+0x60] 288 SAVE_MM_PERMUTATION 289 ret 290 291;----------------------------------------------------------------------------- 292; void sub8x8_dct8( int16_t dct[8][8], uint8_t *pix1, uint8_t *pix2 ) 293;----------------------------------------------------------------------------- 294cglobal sub8x8_dct8_mmx, 3,3 295global sub8x8_dct8_mmx.skip_prologue 296.skip_prologue: 297 RESET_MM_PERMUTATION 298 call load_diff_4x8_mmx 299 call dct8_mmx 300 UNSPILL r0, 0 301 TRANSPOSE4x4W 0,1,2,3,4 302 SPILL r0, 0,1,2,3 303 UNSPILL r0, 4,6 304 TRANSPOSE4x4W 4,5,6,7,0 305 SPILL r0, 4,5,6,7 306 RESET_MM_PERMUTATION 307 add r1, 4 308 add r2, 4 309 add r0, 8 310 call load_diff_4x8_mmx 311 sub r1, 4 312 sub r2, 4 313 call dct8_mmx 314 sub r0, 8 315 UNSPILL r0+8, 4,6 316 TRANSPOSE4x4W 4,5,6,7,0 317 SPILL r0+8, 4,5,6,7 318 UNSPILL r0+8, 0 319 TRANSPOSE4x4W 0,1,2,3,5 320 UNSPILL r0, 4,5,6,7 321 SPILL_SHUFFLE r0, 0,1,2,3, 4,5,6,7 322 movq mm4, m6 ; depends on the permutation to not produce conflicts 323 movq mm0, m4 324 movq mm1, m5 325 movq mm2, mm4 326 movq mm3, m7 327 RESET_MM_PERMUTATION 328 UNSPILL r0+8, 4,5,6,7 329 add r0, 8 330 call dct8_mmx 331 sub r0, 8 332 SPILL r0+8, 1,2,3,5,7 333 RESET_MM_PERMUTATION 334 UNSPILL r0, 0,1,2,3,4,5,6,7 335 call dct8_mmx 336 SPILL r0, 1,2,3,5,7 337 ret 338 339cglobal idct8_mmx 340 IDCT8_1D w,0,1,2,3,4,5,6,7,[r1+0],[r1+64] 341 SAVE_MM_PERMUTATION 342 ret 343 344%macro ADD_STORE_ROW 3 345 movq m1, [r0+%1*FDEC_STRIDE] 346 punpckhbw m2, m1, m0 347 punpcklbw m1, m0 348 paddw m1, %2 349 paddw m2, %3 350 packuswb m1, m2 351 movq [r0+%1*FDEC_STRIDE], m1 352%endmacro 353 354;----------------------------------------------------------------------------- 355; void add8x8_idct8( uint8_t *dst, int16_t dct[8][8] ) 356;----------------------------------------------------------------------------- 357cglobal add8x8_idct8_mmx, 2,2 358global add8x8_idct8_mmx.skip_prologue 359.skip_prologue: 360 INIT_MMX 361 add word [r1], 32 362 UNSPILL r1, 1,2,3,5,6,7 363 call idct8_mmx 364 SPILL r1, 7 365 TRANSPOSE4x4W 0,1,2,3,7 366 SPILL r1, 0,1,2,3 367 UNSPILL r1, 7 368 TRANSPOSE4x4W 4,5,6,7,0 369 SPILL r1, 4,5,6,7 370 INIT_MMX 371 UNSPILL r1+8, 1,2,3,5,6,7 372 add r1, 8 373 call idct8_mmx 374 sub r1, 8 375 SPILL r1+8, 7 376 TRANSPOSE4x4W 0,1,2,3,7 377 SPILL r1+8, 0,1,2,3 378 UNSPILL r1+8, 7 379 TRANSPOSE4x4W 4,5,6,7,0 380 SPILL r1+8, 4,5,6,7 381 INIT_MMX 382 movq m3, [r1+0x08] 383 movq m0, [r1+0x40] 384 movq [r1+0x40], m3 385 movq [r1+0x08], m0 386 ; memory layout at this time: 387 ; A0------ A1------ 388 ; B0------ F0------ 389 ; C0------ G0------ 390 ; D0------ H0------ 391 ; E0------ E1------ 392 ; B1------ F1------ 393 ; C1------ G1------ 394 ; D1------ H1------ 395 UNSPILL_SHUFFLE r1, 1,2,3, 5,6,7 396 UNSPILL r1+8, 5,6,7 397 add r1, 8 398 call idct8_mmx 399 sub r1, 8 400 psraw m0, 6 401 psraw m1, 6 402 psraw m2, 6 403 psraw m3, 6 404 psraw m4, 6 405 psraw m5, 6 406 psraw m6, 6 407 psraw m7, 6 408 movq [r1+0x08], m0 ; mm4 409 movq [r1+0x48], m4 ; mm5 410 movq [r1+0x58], m5 ; mm0 411 movq [r1+0x68], m6 ; mm2 412 movq [r1+0x78], m7 ; mm6 413 movq mm5, [r1+0x18] 414 movq mm6, [r1+0x28] 415 movq [r1+0x18], m1 ; mm1 416 movq [r1+0x28], m2 ; mm7 417 movq mm7, [r1+0x38] 418 movq [r1+0x38], m3 ; mm3 419 movq mm1, [r1+0x10] 420 movq mm2, [r1+0x20] 421 movq mm3, [r1+0x30] 422 call idct8_mmx 423 psraw m0, 6 424 psraw m1, 6 425 psraw m2, 6 426 psraw m3, 6 427 psraw m4, 6 428 psraw m5, 6 429 psraw m6, 6 430 psraw m7, 6 431 SPILL r1, 0,1,2 432 pxor m0, m0 433 ADD_STORE_ROW 0, [r1+0x00], [r1+0x08] 434 ADD_STORE_ROW 1, [r1+0x10], [r1+0x18] 435 ADD_STORE_ROW 2, [r1+0x20], [r1+0x28] 436 ADD_STORE_ROW 3, m3, [r1+0x38] 437 ADD_STORE_ROW 4, m4, [r1+0x48] 438 ADD_STORE_ROW 5, m5, [r1+0x58] 439 ADD_STORE_ROW 6, m6, [r1+0x68] 440 ADD_STORE_ROW 7, m7, [r1+0x78] 441 ret 442 443%macro DCT_SUB8 0 444cglobal sub8x8_dct, 3,3 445 add r2, 4*FDEC_STRIDE 446global current_function %+ .skip_prologue 447.skip_prologue: 448%if cpuflag(ssse3) 449 mova m7, [hsub_mul] 450%endif 451 LOAD_DIFF8x4 0, 1, 2, 3, 6, 7, r1, r2-4*FDEC_STRIDE 452 SPILL r0, 1,2 453 SWAP 2, 7 454 LOAD_DIFF8x4 4, 5, 6, 7, 1, 2, r1, r2-4*FDEC_STRIDE 455 UNSPILL r0, 1 456 SPILL r0, 7 457 SWAP 2, 7 458 UNSPILL r0, 2 459 DCT4_1D 0, 1, 2, 3, 7 460 TRANSPOSE2x4x4W 0, 1, 2, 3, 7 461 UNSPILL r0, 7 462 SPILL r0, 2 463 DCT4_1D 4, 5, 6, 7, 2 464 TRANSPOSE2x4x4W 4, 5, 6, 7, 2 465 UNSPILL r0, 2 466 SPILL r0, 6 467 DCT4_1D 0, 1, 2, 3, 6 468 UNSPILL r0, 6 469 STORE_DCT 0, 1, 2, 3, r0, 0 470 DCT4_1D 4, 5, 6, 7, 3 471 STORE_DCT 4, 5, 6, 7, r0, 64 472 ret 473 474;----------------------------------------------------------------------------- 475; void sub8x8_dct8( int16_t dct[8][8], uint8_t *pix1, uint8_t *pix2 ) 476;----------------------------------------------------------------------------- 477cglobal sub8x8_dct8, 3,3 478 add r2, 4*FDEC_STRIDE 479global current_function %+ .skip_prologue 480.skip_prologue: 481%if cpuflag(ssse3) 482 mova m7, [hsub_mul] 483 LOAD_DIFF8x4 0, 1, 2, 3, 4, 7, r1, r2-4*FDEC_STRIDE 484 SPILL r0, 0,1 485 SWAP 1, 7 486 LOAD_DIFF8x4 4, 5, 6, 7, 0, 1, r1, r2-4*FDEC_STRIDE 487 UNSPILL r0, 0,1 488%else 489 LOAD_DIFF m0, m7, none, [r1+0*FENC_STRIDE], [r2-4*FDEC_STRIDE] 490 LOAD_DIFF m1, m7, none, [r1+1*FENC_STRIDE], [r2-3*FDEC_STRIDE] 491 LOAD_DIFF m2, m7, none, [r1+2*FENC_STRIDE], [r2-2*FDEC_STRIDE] 492 LOAD_DIFF m3, m7, none, [r1+3*FENC_STRIDE], [r2-1*FDEC_STRIDE] 493 LOAD_DIFF m4, m7, none, [r1+4*FENC_STRIDE], [r2+0*FDEC_STRIDE] 494 LOAD_DIFF m5, m7, none, [r1+5*FENC_STRIDE], [r2+1*FDEC_STRIDE] 495 SPILL r0, 0 496 LOAD_DIFF m6, m7, none, [r1+6*FENC_STRIDE], [r2+2*FDEC_STRIDE] 497 LOAD_DIFF m7, m0, none, [r1+7*FENC_STRIDE], [r2+3*FDEC_STRIDE] 498 UNSPILL r0, 0 499%endif 500 DCT8_1D w,0,1,2,3,4,5,6,7,[r0],[r0+0x40],[r0+0x60] 501 UNSPILL r0, 0,4 502 TRANSPOSE8x8W 0,1,2,3,4,5,6,7,[r0+0x60],[r0+0x40],1 503 UNSPILL r0, 4 504 DCT8_1D w,0,1,2,3,4,5,6,7,[r0],[r0+0x40],[r0+0x60] 505 SPILL r0, 1,2,3,5,7 506 ret 507%endmacro 508 509INIT_XMM sse2 510%define movdqa movaps 511%define punpcklqdq movlhps 512DCT_SUB8 513%undef movdqa 514%undef punpcklqdq 515INIT_XMM ssse3 516DCT_SUB8 517INIT_XMM avx 518DCT_SUB8 519INIT_XMM xop 520DCT_SUB8 521 522;----------------------------------------------------------------------------- 523; void add8x8_idct( uint8_t *pix, int16_t dct[4][4][4] ) 524;----------------------------------------------------------------------------- 525%macro ADD8x8 0 526cglobal add8x8_idct, 2,2 527 add r0, 4*FDEC_STRIDE 528global current_function %+ .skip_prologue 529.skip_prologue: 530 UNSPILL_SHUFFLE r1, 0,2,1,3, 0,1,2,3 531 SBUTTERFLY qdq, 0, 1, 4 532 SBUTTERFLY qdq, 2, 3, 4 533 UNSPILL_SHUFFLE r1, 4,6,5,7, 4,5,6,7 534 SPILL r1, 0 535 SBUTTERFLY qdq, 4, 5, 0 536 SBUTTERFLY qdq, 6, 7, 0 537 UNSPILL r1,0 538 IDCT4_1D w,0,1,2,3,r1 539 SPILL r1, 4 540 TRANSPOSE2x4x4W 0,1,2,3,4 541 UNSPILL r1, 4 542 IDCT4_1D w,4,5,6,7,r1 543 SPILL r1, 0 544 TRANSPOSE2x4x4W 4,5,6,7,0 545 UNSPILL r1, 0 546 paddw m0, [pw_32] 547 IDCT4_1D w,0,1,2,3,r1 548 paddw m4, [pw_32] 549 IDCT4_1D w,4,5,6,7,r1 550 SPILL r1, 6,7 551 pxor m7, m7 552 DIFFx2 m0, m1, m6, m7, [r0-4*FDEC_STRIDE], [r0-3*FDEC_STRIDE]; m5 553 DIFFx2 m2, m3, m6, m7, [r0-2*FDEC_STRIDE], [r0-1*FDEC_STRIDE]; m5 554 UNSPILL_SHUFFLE r1, 0,2, 6,7 555 DIFFx2 m4, m5, m6, m7, [r0+0*FDEC_STRIDE], [r0+1*FDEC_STRIDE]; m5 556 DIFFx2 m0, m2, m6, m7, [r0+2*FDEC_STRIDE], [r0+3*FDEC_STRIDE]; m5 557 STORE_IDCT m1, m3, m5, m2 558 ret 559%endmacro ; ADD8x8 560 561INIT_XMM sse2 562ADD8x8 563INIT_XMM avx 564ADD8x8 565 566;----------------------------------------------------------------------------- 567; void add8x8_idct8( uint8_t *p_dst, int16_t dct[8][8] ) 568;----------------------------------------------------------------------------- 569%macro ADD8x8_IDCT8 0 570cglobal add8x8_idct8, 2,2 571 add r0, 4*FDEC_STRIDE 572global current_function %+ .skip_prologue 573.skip_prologue: 574 UNSPILL r1, 1,2,3,5,6,7 575 IDCT8_1D w,0,1,2,3,4,5,6,7,[r1+0],[r1+64] 576 SPILL r1, 6 577 TRANSPOSE8x8W 0,1,2,3,4,5,6,7,[r1+0x60],[r1+0x40],1 578 paddw m0, [pw_32] 579 SPILL r1, 0 580 IDCT8_1D w,0,1,2,3,4,5,6,7,[r1+0],[r1+64] 581 SPILL r1, 6,7 582 pxor m7, m7 583 DIFFx2 m0, m1, m6, m7, [r0-4*FDEC_STRIDE], [r0-3*FDEC_STRIDE]; m5 584 DIFFx2 m2, m3, m6, m7, [r0-2*FDEC_STRIDE], [r0-1*FDEC_STRIDE]; m5 585 UNSPILL_SHUFFLE r1, 0,2, 6,7 586 DIFFx2 m4, m5, m6, m7, [r0+0*FDEC_STRIDE], [r0+1*FDEC_STRIDE]; m5 587 DIFFx2 m0, m2, m6, m7, [r0+2*FDEC_STRIDE], [r0+3*FDEC_STRIDE]; m5 588 STORE_IDCT m1, m3, m5, m2 589 ret 590%endmacro ; ADD8x8_IDCT8 591 592INIT_XMM sse2 593ADD8x8_IDCT8 594INIT_XMM avx 595ADD8x8_IDCT8 596%endif ; !HIGH_BIT_DEPTH 597