1; 2; jchuff-sse2-64.asm - Huffman entropy encoding (64-bit SSE2) 3; 4; Copyright (C) 2009-2011, 2014-2016, D. R. Commander. 5; Copyright (C) 2015, Matthieu Darbois. 6; 7; Based on the x86 SIMD extension for IJG JPEG library 8; Copyright (C) 1999-2006, MIYASAKA Masaru. 9; For conditions of distribution and use, see copyright notice in jsimdext.inc 10; 11; This file should be assembled with NASM (Netwide Assembler), 12; can *not* be assembled with Microsoft's MASM or any compatible 13; assembler (including Borland's Turbo Assembler). 14; NASM is available from http://nasm.sourceforge.net/ or 15; http://sourceforge.net/project/showfiles.php?group_id=6208 16; 17; This file contains an SSE2 implementation for Huffman coding of one block. 18; The following code is based directly on jchuff.c; see jchuff.c for more 19; details. 20; 21; [TAB8] 22 23%include "jsimdext.inc" 24 25; -------------------------------------------------------------------------- 26 SECTION SEG_CONST 27 28 alignz 16 29 global EXTN(jconst_huff_encode_one_block) 30 31EXTN(jconst_huff_encode_one_block): 32 33%include "jpeg_nbits_table.inc" 34 35 alignz 16 36 37; -------------------------------------------------------------------------- 38 SECTION SEG_TEXT 39 BITS 64 40 41; These macros perform the same task as the emit_bits() function in the 42; original libjpeg code. In addition to reducing overhead by explicitly 43; inlining the code, additional performance is achieved by taking into 44; account the size of the bit buffer and waiting until it is almost full 45; before emptying it. This mostly benefits 64-bit platforms, since 6 46; bytes can be stored in a 64-bit bit buffer before it has to be emptied. 47 48%macro EMIT_BYTE 0 49 sub put_bits, 8 ; put_bits -= 8; 50 mov rdx, put_buffer 51 mov ecx, put_bits 52 shr rdx, cl ; c = (JOCTET)GETJOCTET(put_buffer >> put_bits); 53 mov byte [buffer], dl ; *buffer++ = c; 54 add buffer, 1 55 cmp dl, 0xFF ; need to stuff a zero byte? 56 jne %%.EMIT_BYTE_END 57 mov byte [buffer], 0 ; *buffer++ = 0; 58 add buffer, 1 59%%.EMIT_BYTE_END: 60%endmacro 61 62%macro PUT_BITS 1 63 add put_bits, ecx ; put_bits += size; 64 shl put_buffer, cl ; put_buffer = (put_buffer << size); 65 or put_buffer, %1 66%endmacro 67 68%macro CHECKBUF31 0 69 cmp put_bits, 32 ; if (put_bits > 31) { 70 jl %%.CHECKBUF31_END 71 EMIT_BYTE 72 EMIT_BYTE 73 EMIT_BYTE 74 EMIT_BYTE 75%%.CHECKBUF31_END: 76%endmacro 77 78%macro CHECKBUF47 0 79 cmp put_bits, 48 ; if (put_bits > 47) { 80 jl %%.CHECKBUF47_END 81 EMIT_BYTE 82 EMIT_BYTE 83 EMIT_BYTE 84 EMIT_BYTE 85 EMIT_BYTE 86 EMIT_BYTE 87%%.CHECKBUF47_END: 88%endmacro 89 90%macro EMIT_BITS 2 91 CHECKBUF47 92 mov ecx, %2 93 PUT_BITS %1 94%endmacro 95 96%macro kloop_prepare 37 ;(ko, jno0, ..., jno31, xmm0, xmm1, xmm2, xmm3) 97 pxor xmm8, xmm8 ; __m128i neg = _mm_setzero_si128(); 98 pxor xmm9, xmm9 ; __m128i neg = _mm_setzero_si128(); 99 pxor xmm10, xmm10 ; __m128i neg = _mm_setzero_si128(); 100 pxor xmm11, xmm11 ; __m128i neg = _mm_setzero_si128(); 101 pinsrw %34, word [r12 + %2 * SIZEOF_WORD], 0 ; xmm_shadow[0] = block[jno0]; 102 pinsrw %35, word [r12 + %10 * SIZEOF_WORD], 0 ; xmm_shadow[8] = block[jno8]; 103 pinsrw %36, word [r12 + %18 * SIZEOF_WORD], 0 ; xmm_shadow[16] = block[jno16]; 104 pinsrw %37, word [r12 + %26 * SIZEOF_WORD], 0 ; xmm_shadow[24] = block[jno24]; 105 pinsrw %34, word [r12 + %3 * SIZEOF_WORD], 1 ; xmm_shadow[1] = block[jno1]; 106 pinsrw %35, word [r12 + %11 * SIZEOF_WORD], 1 ; xmm_shadow[9] = block[jno9]; 107 pinsrw %36, word [r12 + %19 * SIZEOF_WORD], 1 ; xmm_shadow[17] = block[jno17]; 108 pinsrw %37, word [r12 + %27 * SIZEOF_WORD], 1 ; xmm_shadow[25] = block[jno25]; 109 pinsrw %34, word [r12 + %4 * SIZEOF_WORD], 2 ; xmm_shadow[2] = block[jno2]; 110 pinsrw %35, word [r12 + %12 * SIZEOF_WORD], 2 ; xmm_shadow[10] = block[jno10]; 111 pinsrw %36, word [r12 + %20 * SIZEOF_WORD], 2 ; xmm_shadow[18] = block[jno18]; 112 pinsrw %37, word [r12 + %28 * SIZEOF_WORD], 2 ; xmm_shadow[26] = block[jno26]; 113 pinsrw %34, word [r12 + %5 * SIZEOF_WORD], 3 ; xmm_shadow[3] = block[jno3]; 114 pinsrw %35, word [r12 + %13 * SIZEOF_WORD], 3 ; xmm_shadow[11] = block[jno11]; 115 pinsrw %36, word [r12 + %21 * SIZEOF_WORD], 3 ; xmm_shadow[19] = block[jno19]; 116 pinsrw %37, word [r12 + %29 * SIZEOF_WORD], 3 ; xmm_shadow[27] = block[jno27]; 117 pinsrw %34, word [r12 + %6 * SIZEOF_WORD], 4 ; xmm_shadow[4] = block[jno4]; 118 pinsrw %35, word [r12 + %14 * SIZEOF_WORD], 4 ; xmm_shadow[12] = block[jno12]; 119 pinsrw %36, word [r12 + %22 * SIZEOF_WORD], 4 ; xmm_shadow[20] = block[jno20]; 120 pinsrw %37, word [r12 + %30 * SIZEOF_WORD], 4 ; xmm_shadow[28] = block[jno28]; 121 pinsrw %34, word [r12 + %7 * SIZEOF_WORD], 5 ; xmm_shadow[5] = block[jno5]; 122 pinsrw %35, word [r12 + %15 * SIZEOF_WORD], 5 ; xmm_shadow[13] = block[jno13]; 123 pinsrw %36, word [r12 + %23 * SIZEOF_WORD], 5 ; xmm_shadow[21] = block[jno21]; 124 pinsrw %37, word [r12 + %31 * SIZEOF_WORD], 5 ; xmm_shadow[29] = block[jno29]; 125 pinsrw %34, word [r12 + %8 * SIZEOF_WORD], 6 ; xmm_shadow[6] = block[jno6]; 126 pinsrw %35, word [r12 + %16 * SIZEOF_WORD], 6 ; xmm_shadow[14] = block[jno14]; 127 pinsrw %36, word [r12 + %24 * SIZEOF_WORD], 6 ; xmm_shadow[22] = block[jno22]; 128 pinsrw %37, word [r12 + %32 * SIZEOF_WORD], 6 ; xmm_shadow[30] = block[jno30]; 129 pinsrw %34, word [r12 + %9 * SIZEOF_WORD], 7 ; xmm_shadow[7] = block[jno7]; 130 pinsrw %35, word [r12 + %17 * SIZEOF_WORD], 7 ; xmm_shadow[15] = block[jno15]; 131 pinsrw %36, word [r12 + %25 * SIZEOF_WORD], 7 ; xmm_shadow[23] = block[jno23]; 132%if %1 != 32 133 pinsrw %37, word [r12 + %33 * SIZEOF_WORD], 7 ; xmm_shadow[31] = block[jno31]; 134%else 135 pinsrw %37, ebx, 7 ; xmm_shadow[31] = block[jno31]; 136%endif 137 pcmpgtw xmm8, %34 ; neg = _mm_cmpgt_epi16(neg, x1); 138 pcmpgtw xmm9, %35 ; neg = _mm_cmpgt_epi16(neg, x1); 139 pcmpgtw xmm10, %36 ; neg = _mm_cmpgt_epi16(neg, x1); 140 pcmpgtw xmm11, %37 ; neg = _mm_cmpgt_epi16(neg, x1); 141 paddw %34, xmm8 ; x1 = _mm_add_epi16(x1, neg); 142 paddw %35, xmm9 ; x1 = _mm_add_epi16(x1, neg); 143 paddw %36, xmm10 ; x1 = _mm_add_epi16(x1, neg); 144 paddw %37, xmm11 ; x1 = _mm_add_epi16(x1, neg); 145 pxor %34, xmm8 ; x1 = _mm_xor_si128(x1, neg); 146 pxor %35, xmm9 ; x1 = _mm_xor_si128(x1, neg); 147 pxor %36, xmm10 ; x1 = _mm_xor_si128(x1, neg); 148 pxor %37, xmm11 ; x1 = _mm_xor_si128(x1, neg); 149 pxor xmm8, %34 ; neg = _mm_xor_si128(neg, x1); 150 pxor xmm9, %35 ; neg = _mm_xor_si128(neg, x1); 151 pxor xmm10, %36 ; neg = _mm_xor_si128(neg, x1); 152 pxor xmm11, %37 ; neg = _mm_xor_si128(neg, x1); 153 movdqa XMMWORD [t1 + %1 * SIZEOF_WORD], %34 ; _mm_storeu_si128((__m128i *)(t1 + ko), x1); 154 movdqa XMMWORD [t1 + (%1 + 8) * SIZEOF_WORD], %35 ; _mm_storeu_si128((__m128i *)(t1 + ko + 8), x1); 155 movdqa XMMWORD [t1 + (%1 + 16) * SIZEOF_WORD], %36 ; _mm_storeu_si128((__m128i *)(t1 + ko + 16), x1); 156 movdqa XMMWORD [t1 + (%1 + 24) * SIZEOF_WORD], %37 ; _mm_storeu_si128((__m128i *)(t1 + ko + 24), x1); 157 movdqa XMMWORD [t2 + %1 * SIZEOF_WORD], xmm8 ; _mm_storeu_si128((__m128i *)(t2 + ko), neg); 158 movdqa XMMWORD [t2 + (%1 + 8) * SIZEOF_WORD], xmm9 ; _mm_storeu_si128((__m128i *)(t2 + ko + 8), neg); 159 movdqa XMMWORD [t2 + (%1 + 16) * SIZEOF_WORD], xmm10 ; _mm_storeu_si128((__m128i *)(t2 + ko + 16), neg); 160 movdqa XMMWORD [t2 + (%1 + 24) * SIZEOF_WORD], xmm11 ; _mm_storeu_si128((__m128i *)(t2 + ko + 24), neg); 161%endmacro 162 163; 164; Encode a single block's worth of coefficients. 165; 166; GLOBAL(JOCTET*) 167; jsimd_huff_encode_one_block_sse2 (working_state *state, JOCTET *buffer, 168; JCOEFPTR block, int last_dc_val, 169; c_derived_tbl *dctbl, c_derived_tbl *actbl) 170; 171 172; r10 = working_state *state 173; r11 = JOCTET *buffer 174; r12 = JCOEFPTR block 175; r13 = int last_dc_val 176; r14 = c_derived_tbl *dctbl 177; r15 = c_derived_tbl *actbl 178 179%define t1 rbp-(DCTSIZE2*SIZEOF_WORD) 180%define t2 t1-(DCTSIZE2*SIZEOF_WORD) 181%define put_buffer r8 182%define put_bits r9d 183%define buffer rax 184 185 align 16 186 global EXTN(jsimd_huff_encode_one_block_sse2) 187 188EXTN(jsimd_huff_encode_one_block_sse2): 189 push rbp 190 mov rax,rsp ; rax = original rbp 191 sub rsp, byte 4 192 and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits 193 mov [rsp],rax 194 mov rbp,rsp ; rbp = aligned rbp 195 lea rsp, [t2] 196 collect_args 197%ifdef WIN64 198 movaps XMMWORD [rsp-1*SIZEOF_XMMWORD], xmm8 199 movaps XMMWORD [rsp-2*SIZEOF_XMMWORD], xmm9 200 movaps XMMWORD [rsp-3*SIZEOF_XMMWORD], xmm10 201 movaps XMMWORD [rsp-4*SIZEOF_XMMWORD], xmm11 202 sub rsp, 4*SIZEOF_XMMWORD 203%endif 204 push rbx 205 206 mov buffer, r11 ; r11 is now sratch 207 208 mov put_buffer, MMWORD [r10+16] ; put_buffer = state->cur.put_buffer; 209 mov put_bits, DWORD [r10+24] ; put_bits = state->cur.put_bits; 210 push r10 ; r10 is now scratch 211 212 ; Encode the DC coefficient difference per section F.1.2.1 213 movsx edi, word [r12] ; temp = temp2 = block[0] - last_dc_val; 214 sub edi, r13d ; r13 is not used anymore 215 mov ebx, edi 216 217 ; This is a well-known technique for obtaining the absolute value 218 ; without a branch. It is derived from an assembly language technique 219 ; presented in "How to Optimize for the Pentium Processors", 220 ; Copyright (c) 1996, 1997 by Agner Fog. 221 mov esi, edi 222 sar esi, 31 ; temp3 = temp >> (CHAR_BIT * sizeof(int) - 1); 223 xor edi, esi ; temp ^= temp3; 224 sub edi, esi ; temp -= temp3; 225 226 ; For a negative input, want temp2 = bitwise complement of abs(input) 227 ; This code assumes we are on a two's complement machine 228 add ebx, esi ; temp2 += temp3; 229 230 ; Find the number of bits needed for the magnitude of the coefficient 231 lea r11, [rel jpeg_nbits_table] 232 movzx rdi, byte [r11 + rdi] ; nbits = JPEG_NBITS(temp); 233 ; Emit the Huffman-coded symbol for the number of bits 234 mov r11d, INT [r14 + rdi * 4] ; code = dctbl->ehufco[nbits]; 235 movzx esi, byte [r14 + rdi + 1024] ; size = dctbl->ehufsi[nbits]; 236 EMIT_BITS r11, esi ; EMIT_BITS(code, size) 237 238 ; Mask off any extra bits in code 239 mov esi, 1 240 mov ecx, edi 241 shl esi, cl 242 dec esi 243 and ebx, esi ; temp2 &= (((JLONG) 1)<<nbits) - 1; 244 245 ; Emit that number of bits of the value, if positive, 246 ; or the complement of its magnitude, if negative. 247 EMIT_BITS rbx, edi ; EMIT_BITS(temp2, nbits) 248 249 ; Prepare data 250 xor ebx, ebx 251 kloop_prepare 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, \ 252 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, \ 253 27, 20, 13, 6, 7, 14, 21, 28, 35, \ 254 xmm0, xmm1, xmm2, xmm3 255 kloop_prepare 32, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, \ 256 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, \ 257 53, 60, 61, 54, 47, 55, 62, 63, 63, \ 258 xmm4, xmm5, xmm6, xmm7 259 260 pxor xmm8, xmm8 261 pcmpeqw xmm0, xmm8 ; tmp0 = _mm_cmpeq_epi16(tmp0, zero); 262 pcmpeqw xmm1, xmm8 ; tmp1 = _mm_cmpeq_epi16(tmp1, zero); 263 pcmpeqw xmm2, xmm8 ; tmp2 = _mm_cmpeq_epi16(tmp2, zero); 264 pcmpeqw xmm3, xmm8 ; tmp3 = _mm_cmpeq_epi16(tmp3, zero); 265 pcmpeqw xmm4, xmm8 ; tmp4 = _mm_cmpeq_epi16(tmp4, zero); 266 pcmpeqw xmm5, xmm8 ; tmp5 = _mm_cmpeq_epi16(tmp5, zero); 267 pcmpeqw xmm6, xmm8 ; tmp6 = _mm_cmpeq_epi16(tmp6, zero); 268 pcmpeqw xmm7, xmm8 ; tmp7 = _mm_cmpeq_epi16(tmp7, zero); 269 packsswb xmm0, xmm1 ; tmp0 = _mm_packs_epi16(tmp0, tmp1); 270 packsswb xmm2, xmm3 ; tmp2 = _mm_packs_epi16(tmp2, tmp3); 271 packsswb xmm4, xmm5 ; tmp4 = _mm_packs_epi16(tmp4, tmp5); 272 packsswb xmm6, xmm7 ; tmp6 = _mm_packs_epi16(tmp6, tmp7); 273 pmovmskb r11d, xmm0 ; index = ((uint64_t)_mm_movemask_epi8(tmp0)) << 0; 274 pmovmskb r12d, xmm2 ; index = ((uint64_t)_mm_movemask_epi8(tmp2)) << 16; 275 pmovmskb r13d, xmm4 ; index = ((uint64_t)_mm_movemask_epi8(tmp4)) << 32; 276 pmovmskb r14d, xmm6 ; index = ((uint64_t)_mm_movemask_epi8(tmp6)) << 48; 277 shl r12, 16 278 shl r14, 16 279 or r11, r12 280 or r13, r14 281 shl r13, 32 282 or r11, r13 283 not r11 ; index = ~index; 284 285 ;mov MMWORD [ t1 + DCTSIZE2 * SIZEOF_WORD ], r11 286 ;jmp .EFN 287 288 mov r13d, INT [r15 + 240 * 4] ; code_0xf0 = actbl->ehufco[0xf0]; 289 movzx r14d, byte [r15 + 1024 + 240] ; size_0xf0 = actbl->ehufsi[0xf0]; 290 lea rsi, [t1] 291.BLOOP: 292 bsf r12, r11 ; r = __builtin_ctzl(index); 293 jz .ELOOP 294 mov rcx, r12 295 lea rsi, [rsi+r12*2] ; k += r; 296 shr r11, cl ; index >>= r; 297 movzx rdi, word [rsi] ; temp = t1[k]; 298 lea rbx, [rel jpeg_nbits_table] 299 movzx rdi, byte [rbx + rdi] ; nbits = JPEG_NBITS(temp); 300.BRLOOP: 301 cmp r12, 16 ; while (r > 15) { 302 jl .ERLOOP 303 EMIT_BITS r13, r14d ; EMIT_BITS(code_0xf0, size_0xf0) 304 sub r12, 16 ; r -= 16; 305 jmp .BRLOOP 306.ERLOOP: 307 ; Emit Huffman symbol for run length / number of bits 308 CHECKBUF31 ; uses rcx, rdx 309 310 shl r12, 4 ; temp3 = (r << 4) + nbits; 311 add r12, rdi 312 mov ebx, INT [r15 + r12 * 4] ; code = actbl->ehufco[temp3]; 313 movzx ecx, byte [r15 + r12 + 1024] ; size = actbl->ehufsi[temp3]; 314 PUT_BITS rbx 315 316 ;EMIT_CODE(code, size) 317 318 movsx ebx, word [rsi-DCTSIZE2*2] ; temp2 = t2[k]; 319 ; Mask off any extra bits in code 320 mov rcx, rdi 321 mov rdx, 1 322 shl rdx, cl 323 dec rdx 324 and rbx, rdx ; temp2 &= (((JLONG) 1)<<nbits) - 1; 325 PUT_BITS rbx ; PUT_BITS(temp2, nbits) 326 327 shr r11, 1 ; index >>= 1; 328 add rsi, 2 ; ++k; 329 jmp .BLOOP 330.ELOOP: 331 ; If the last coef(s) were zero, emit an end-of-block code 332 lea rdi, [t1 + (DCTSIZE2-1) * 2] ; r = DCTSIZE2-1-k; 333 cmp rdi, rsi ; if (r > 0) { 334 je .EFN 335 mov ebx, INT [r15] ; code = actbl->ehufco[0]; 336 movzx r12d, byte [r15 + 1024] ; size = actbl->ehufsi[0]; 337 EMIT_BITS rbx, r12d 338.EFN: 339 pop r10 340 ; Save put_buffer & put_bits 341 mov MMWORD [r10+16], put_buffer ; state->cur.put_buffer = put_buffer; 342 mov DWORD [r10+24], put_bits ; state->cur.put_bits = put_bits; 343 344 pop rbx 345%ifdef WIN64 346 movaps xmm11, XMMWORD [rsp+0*SIZEOF_XMMWORD] 347 movaps xmm10, XMMWORD [rsp+1*SIZEOF_XMMWORD] 348 movaps xmm9, XMMWORD [rsp+2*SIZEOF_XMMWORD] 349 movaps xmm8, XMMWORD [rsp+3*SIZEOF_XMMWORD] 350 add rsp, 4*SIZEOF_XMMWORD 351%endif 352 uncollect_args 353 mov rsp,rbp ; rsp <- aligned rbp 354 pop rsp ; rsp <- original rbp 355 pop rbp 356 ret 357 358; For some reason, the OS X linker does not honor the request to align the 359; segment unless we do this. 360 align 16 361