1;*****************************************************************************
2;* x86inc.asm: x264asm abstraction layer
3;*****************************************************************************
4;* Copyright (C) 2005-2018 x264 project
5;*
6;* Authors: Loren Merritt <lorenm@u.washington.edu>
7;*          Henrik Gramner <henrik@gramner.com>
8;*          Anton Mitrofanov <BugMaster@narod.ru>
9;*          Fiona Glaser <fiona@x264.com>
10;*
11;* Permission to use, copy, modify, and/or distribute this software for any
12;* purpose with or without fee is hereby granted, provided that the above
13;* copyright notice and this permission notice appear in all copies.
14;*
15;* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
16;* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
17;* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
18;* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19;* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
20;* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
21;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
22;*****************************************************************************
23
24; This is a header file for the x264ASM assembly language, which uses
25; NASM/YASM syntax combined with a large number of macros to provide easy
26; abstraction between different calling conventions (x86_32, win64, linux64).
27; It also has various other useful features to simplify writing the kind of
28; DSP functions that are most often used in x264.
29
30; Unlike the rest of x264, this file is available under an ISC license, as it
31; has significant usefulness outside of x264 and we want it to be available
32; to the largest audience possible.  Of course, if you modify it for your own
33; purposes to add a new feature, we strongly encourage contributing a patch
34; as this feature might be useful for others as well.  Send patches or ideas
35; to x264-devel@videolan.org .
36
37%ifndef private_prefix
38    %define private_prefix x264
39%endif
40
41%ifndef public_prefix
42    %define public_prefix private_prefix
43%endif
44
45%if HAVE_ALIGNED_STACK
46    %define STACK_ALIGNMENT 16
47%endif
48%ifndef STACK_ALIGNMENT
49    %if ARCH_X86_64
50        %define STACK_ALIGNMENT 16
51    %else
52        %define STACK_ALIGNMENT 4
53    %endif
54%endif
55
56%define WIN64  0
57%define UNIX64 0
58%if ARCH_X86_64
59    %ifidn __OUTPUT_FORMAT__,win32
60        %define WIN64  1
61    %elifidn __OUTPUT_FORMAT__,win64
62        %define WIN64  1
63    %elifidn __OUTPUT_FORMAT__,x64
64        %define WIN64  1
65    %else
66        %define UNIX64 1
67    %endif
68%endif
69
70%define FORMAT_ELF 0
71%ifidn __OUTPUT_FORMAT__,elf
72    %define FORMAT_ELF 1
73%elifidn __OUTPUT_FORMAT__,elf32
74    %define FORMAT_ELF 1
75%elifidn __OUTPUT_FORMAT__,elf64
76    %define FORMAT_ELF 1
77%endif
78
79%ifdef PREFIX
80    %define mangle(x) _ %+ x
81%else
82    %define mangle(x) x
83%endif
84
85; aout does not support align=
86; NOTE: This section is out of sync with x264, in order to
87; keep supporting OS/2.
88%macro SECTION_RODATA 0-1 16
89    %ifidn __OUTPUT_FORMAT__,aout
90        SECTION .text
91    %elifidn __OUTPUT_FORMAT__,coff
92        SECTION .text
93    %elifidn __OUTPUT_FORMAT__,win32
94        SECTION .rdata align=%1
95    %elif WIN64
96        SECTION .rdata align=%1
97    %else
98        SECTION .rodata align=%1
99    %endif
100%endmacro
101
102%if WIN64
103    %define PIC
104%elif ARCH_X86_64 == 0
105; x86_32 doesn't require PIC.
106; Some distros prefer shared objects to be PIC, but nothing breaks if
107; the code contains a few textrels, so we'll skip that complexity.
108    %undef PIC
109%endif
110%ifdef PIC
111    default rel
112%endif
113
114%macro CPUNOP 1
115    %if HAVE_CPUNOP
116        CPU %1
117    %endif
118%endmacro
119
120; Macros to eliminate most code duplication between x86_32 and x86_64:
121; Currently this works only for leaf functions which load all their arguments
122; into registers at the start, and make no other use of the stack. Luckily that
123; covers most of x264's asm.
124
125; PROLOGUE:
126; %1 = number of arguments. loads them from stack if needed.
127; %2 = number of registers used. pushes callee-saved regs if needed.
128; %3 = number of xmm registers used. pushes callee-saved xmm regs if needed.
129; %4 = (optional) stack size to be allocated. The stack will be aligned before
130;      allocating the specified stack size. If the required stack alignment is
131;      larger than the known stack alignment the stack will be manually aligned
132;      and an extra register will be allocated to hold the original stack
133;      pointer (to not invalidate r0m etc.). To prevent the use of an extra
134;      register as stack pointer, request a negative stack size.
135; %4+/%5+ = list of names to define to registers
136; PROLOGUE can also be invoked by adding the same options to cglobal
137
138; e.g.
139; cglobal foo, 2,3,7,0x40, dst, src, tmp
140; declares a function (foo) that automatically loads two arguments (dst and
141; src) into registers, uses one additional register (tmp) plus 7 vector
142; registers (m0-m6) and allocates 0x40 bytes of stack space.
143
144; TODO Some functions can use some args directly from the stack. If they're the
145; last args then you can just not declare them, but if they're in the middle
146; we need more flexible macro.
147
148; RET:
149; Pops anything that was pushed by PROLOGUE, and returns.
150
151; REP_RET:
152; Use this instead of RET if it's a branch target.
153
154; registers:
155; rN and rNq are the native-size register holding function argument N
156; rNd, rNw, rNb are dword, word, and byte size
157; rNh is the high 8 bits of the word size
158; rNm is the original location of arg N (a register or on the stack), dword
159; rNmp is native size
160
161%macro DECLARE_REG 2-3
162    %define r%1q %2
163    %define r%1d %2d
164    %define r%1w %2w
165    %define r%1b %2b
166    %define r%1h %2h
167    %define %2q %2
168    %if %0 == 2
169        %define r%1m  %2d
170        %define r%1mp %2
171    %elif ARCH_X86_64 ; memory
172        %define r%1m [rstk + stack_offset + %3]
173        %define r%1mp qword r %+ %1 %+ m
174    %else
175        %define r%1m [rstk + stack_offset + %3]
176        %define r%1mp dword r %+ %1 %+ m
177    %endif
178    %define r%1  %2
179%endmacro
180
181%macro DECLARE_REG_SIZE 3
182    %define r%1q r%1
183    %define e%1q r%1
184    %define r%1d e%1
185    %define e%1d e%1
186    %define r%1w %1
187    %define e%1w %1
188    %define r%1h %3
189    %define e%1h %3
190    %define r%1b %2
191    %define e%1b %2
192    %if ARCH_X86_64 == 0
193        %define r%1 e%1
194    %endif
195%endmacro
196
197DECLARE_REG_SIZE ax, al, ah
198DECLARE_REG_SIZE bx, bl, bh
199DECLARE_REG_SIZE cx, cl, ch
200DECLARE_REG_SIZE dx, dl, dh
201DECLARE_REG_SIZE si, sil, null
202DECLARE_REG_SIZE di, dil, null
203DECLARE_REG_SIZE bp, bpl, null
204
205; t# defines for when per-arch register allocation is more complex than just function arguments
206
207%macro DECLARE_REG_TMP 1-*
208    %assign %%i 0
209    %rep %0
210        CAT_XDEFINE t, %%i, r%1
211        %assign %%i %%i+1
212        %rotate 1
213    %endrep
214%endmacro
215
216%macro DECLARE_REG_TMP_SIZE 0-*
217    %rep %0
218        %define t%1q t%1 %+ q
219        %define t%1d t%1 %+ d
220        %define t%1w t%1 %+ w
221        %define t%1h t%1 %+ h
222        %define t%1b t%1 %+ b
223        %rotate 1
224    %endrep
225%endmacro
226
227DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14
228
229%if ARCH_X86_64
230    %define gprsize 8
231%else
232    %define gprsize 4
233%endif
234
235%macro PUSH 1
236    push %1
237    %ifidn rstk, rsp
238        %assign stack_offset stack_offset+gprsize
239    %endif
240%endmacro
241
242%macro POP 1
243    pop %1
244    %ifidn rstk, rsp
245        %assign stack_offset stack_offset-gprsize
246    %endif
247%endmacro
248
249%macro PUSH_IF_USED 1-*
250    %rep %0
251        %if %1 < regs_used
252            PUSH r%1
253        %endif
254        %rotate 1
255    %endrep
256%endmacro
257
258%macro POP_IF_USED 1-*
259    %rep %0
260        %if %1 < regs_used
261            pop r%1
262        %endif
263        %rotate 1
264    %endrep
265%endmacro
266
267%macro LOAD_IF_USED 1-*
268    %rep %0
269        %if %1 < num_args
270            mov r%1, r %+ %1 %+ mp
271        %endif
272        %rotate 1
273    %endrep
274%endmacro
275
276%macro SUB 2
277    sub %1, %2
278    %ifidn %1, rstk
279        %assign stack_offset stack_offset+(%2)
280    %endif
281%endmacro
282
283%macro ADD 2
284    add %1, %2
285    %ifidn %1, rstk
286        %assign stack_offset stack_offset-(%2)
287    %endif
288%endmacro
289
290%macro movifnidn 2
291    %ifnidn %1, %2
292        mov %1, %2
293    %endif
294%endmacro
295
296%macro movsxdifnidn 2
297    %ifnidn %1, %2
298        movsxd %1, %2
299    %endif
300%endmacro
301
302%macro ASSERT 1
303    %if (%1) == 0
304        %error assertion ``%1'' failed
305    %endif
306%endmacro
307
308%macro DEFINE_ARGS 0-*
309    %ifdef n_arg_names
310        %assign %%i 0
311        %rep n_arg_names
312            CAT_UNDEF arg_name %+ %%i, q
313            CAT_UNDEF arg_name %+ %%i, d
314            CAT_UNDEF arg_name %+ %%i, w
315            CAT_UNDEF arg_name %+ %%i, h
316            CAT_UNDEF arg_name %+ %%i, b
317            CAT_UNDEF arg_name %+ %%i, m
318            CAT_UNDEF arg_name %+ %%i, mp
319            CAT_UNDEF arg_name, %%i
320            %assign %%i %%i+1
321        %endrep
322    %endif
323
324    %xdefine %%stack_offset stack_offset
325    %undef stack_offset ; so that the current value of stack_offset doesn't get baked in by xdefine
326    %assign %%i 0
327    %rep %0
328        %xdefine %1q r %+ %%i %+ q
329        %xdefine %1d r %+ %%i %+ d
330        %xdefine %1w r %+ %%i %+ w
331        %xdefine %1h r %+ %%i %+ h
332        %xdefine %1b r %+ %%i %+ b
333        %xdefine %1m r %+ %%i %+ m
334        %xdefine %1mp r %+ %%i %+ mp
335        CAT_XDEFINE arg_name, %%i, %1
336        %assign %%i %%i+1
337        %rotate 1
338    %endrep
339    %xdefine stack_offset %%stack_offset
340    %assign n_arg_names %0
341%endmacro
342
343%define required_stack_alignment ((mmsize + 15) & ~15)
344%define vzeroupper_required (mmsize > 16 && (ARCH_X86_64 == 0 || xmm_regs_used > 16 || notcpuflag(avx512)))
345%define high_mm_regs (16*cpuflag(avx512))
346
347%macro ALLOC_STACK 1-2 0 ; stack_size, n_xmm_regs (for win64 only)
348    %ifnum %1
349        %if %1 != 0
350            %assign %%pad 0
351            %assign stack_size %1
352            %if stack_size < 0
353                %assign stack_size -stack_size
354            %endif
355            %if WIN64
356                %assign %%pad %%pad + 32 ; shadow space
357                %if mmsize != 8
358                    %assign xmm_regs_used %2
359                    %if xmm_regs_used > 8
360                        %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers
361                    %endif
362                %endif
363            %endif
364            %if required_stack_alignment <= STACK_ALIGNMENT
365                ; maintain the current stack alignment
366                %assign stack_size_padded stack_size + %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1))
367                SUB rsp, stack_size_padded
368            %else
369                %assign %%reg_num (regs_used - 1)
370                %xdefine rstk r %+ %%reg_num
371                ; align stack, and save original stack location directly above
372                ; it, i.e. in [rsp+stack_size_padded], so we can restore the
373                ; stack in a single instruction (i.e. mov rsp, rstk or mov
374                ; rsp, [rsp+stack_size_padded])
375                %if %1 < 0 ; need to store rsp on stack
376                    %xdefine rstkm [rsp + stack_size + %%pad]
377                    %assign %%pad %%pad + gprsize
378                %else ; can keep rsp in rstk during whole function
379                    %xdefine rstkm rstk
380                %endif
381                %assign stack_size_padded stack_size + ((%%pad + required_stack_alignment-1) & ~(required_stack_alignment-1))
382                mov rstk, rsp
383                and rsp, ~(required_stack_alignment-1)
384                sub rsp, stack_size_padded
385                movifnidn rstkm, rstk
386            %endif
387            WIN64_PUSH_XMM
388        %endif
389    %endif
390%endmacro
391
392%macro SETUP_STACK_POINTER 1
393    %ifnum %1
394        %if %1 != 0 && required_stack_alignment > STACK_ALIGNMENT
395            %if %1 > 0
396                ; Reserve an additional register for storing the original stack pointer, but avoid using
397                ; eax/rax for this purpose since it can potentially get overwritten as a return value.
398                %assign regs_used (regs_used + 1)
399                %if ARCH_X86_64 && regs_used == 7
400                    %assign regs_used 8
401                %elif ARCH_X86_64 == 0 && regs_used == 1
402                    %assign regs_used 2
403                %endif
404            %endif
405            %if ARCH_X86_64 && regs_used < 5 + UNIX64 * 3
406                ; Ensure that we don't clobber any registers containing arguments. For UNIX64 we also preserve r6 (rax)
407                ; since it's used as a hidden argument in vararg functions to specify the number of vector registers used.
408                %assign regs_used 5 + UNIX64 * 3
409            %endif
410        %endif
411    %endif
412%endmacro
413
414%if WIN64 ; Windows x64 ;=================================================
415
416DECLARE_REG 0,  rcx
417DECLARE_REG 1,  rdx
418DECLARE_REG 2,  R8
419DECLARE_REG 3,  R9
420DECLARE_REG 4,  R10, 40
421DECLARE_REG 5,  R11, 48
422DECLARE_REG 6,  rax, 56
423DECLARE_REG 7,  rdi, 64
424DECLARE_REG 8,  rsi, 72
425DECLARE_REG 9,  rbx, 80
426DECLARE_REG 10, rbp, 88
427DECLARE_REG 11, R14, 96
428DECLARE_REG 12, R15, 104
429DECLARE_REG 13, R12, 112
430DECLARE_REG 14, R13, 120
431
432%macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names...
433    %assign num_args %1
434    %assign regs_used %2
435    ASSERT regs_used >= num_args
436    SETUP_STACK_POINTER %4
437    ASSERT regs_used <= 15
438    PUSH_IF_USED 7, 8, 9, 10, 11, 12, 13, 14
439    ALLOC_STACK %4, %3
440    %if mmsize != 8 && stack_size == 0
441        WIN64_SPILL_XMM %3
442    %endif
443    LOAD_IF_USED 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14
444    %if %0 > 4
445        %ifnum %4
446            DEFINE_ARGS %5
447        %else
448            DEFINE_ARGS %4, %5
449        %endif
450    %elifnnum %4
451        DEFINE_ARGS %4
452    %endif
453%endmacro
454
455%macro WIN64_PUSH_XMM 0
456    ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated.
457    %if xmm_regs_used > 6 + high_mm_regs
458        movaps [rstk + stack_offset +  8], xmm6
459    %endif
460    %if xmm_regs_used > 7 + high_mm_regs
461        movaps [rstk + stack_offset + 24], xmm7
462    %endif
463    %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8
464    %if %%xmm_regs_on_stack > 0
465        %assign %%i 8
466        %rep %%xmm_regs_on_stack
467            movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i
468            %assign %%i %%i+1
469        %endrep
470    %endif
471%endmacro
472
473%macro WIN64_SPILL_XMM 1
474    %assign xmm_regs_used %1
475    ASSERT xmm_regs_used <= 16 + high_mm_regs
476    %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8
477    %if %%xmm_regs_on_stack > 0
478        ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack.
479        %assign %%pad %%xmm_regs_on_stack*16 + 32
480        %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1))
481        SUB rsp, stack_size_padded
482    %endif
483    WIN64_PUSH_XMM
484%endmacro
485
486%macro WIN64_RESTORE_XMM_INTERNAL 0
487    %assign %%pad_size 0
488    %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8
489    %if %%xmm_regs_on_stack > 0
490        %assign %%i xmm_regs_used - high_mm_regs
491        %rep %%xmm_regs_on_stack
492            %assign %%i %%i-1
493            movaps xmm %+ %%i, [rsp + (%%i-8)*16 + stack_size + 32]
494        %endrep
495    %endif
496    %if stack_size_padded > 0
497        %if stack_size > 0 && required_stack_alignment > STACK_ALIGNMENT
498            mov rsp, rstkm
499        %else
500            add rsp, stack_size_padded
501            %assign %%pad_size stack_size_padded
502        %endif
503    %endif
504    %if xmm_regs_used > 7 + high_mm_regs
505        movaps xmm7, [rsp + stack_offset - %%pad_size + 24]
506    %endif
507    %if xmm_regs_used > 6 + high_mm_regs
508        movaps xmm6, [rsp + stack_offset - %%pad_size +  8]
509    %endif
510%endmacro
511
512%macro WIN64_RESTORE_XMM 0
513    WIN64_RESTORE_XMM_INTERNAL
514    %assign stack_offset (stack_offset-stack_size_padded)
515    %assign stack_size_padded 0
516    %assign xmm_regs_used 0
517%endmacro
518
519%define has_epilogue regs_used > 7 || stack_size > 0 || vzeroupper_required || xmm_regs_used > 6+high_mm_regs
520
521%macro RET 0
522    WIN64_RESTORE_XMM_INTERNAL
523    POP_IF_USED 14, 13, 12, 11, 10, 9, 8, 7
524    %if vzeroupper_required
525        vzeroupper
526    %endif
527    AUTO_REP_RET
528%endmacro
529
530%elif ARCH_X86_64 ; *nix x64 ;=============================================
531
532DECLARE_REG 0,  rdi
533DECLARE_REG 1,  rsi
534DECLARE_REG 2,  rdx
535DECLARE_REG 3,  rcx
536DECLARE_REG 4,  R8
537DECLARE_REG 5,  R9
538DECLARE_REG 6,  rax, 8
539DECLARE_REG 7,  R10, 16
540DECLARE_REG 8,  R11, 24
541DECLARE_REG 9,  rbx, 32
542DECLARE_REG 10, rbp, 40
543DECLARE_REG 11, R14, 48
544DECLARE_REG 12, R15, 56
545DECLARE_REG 13, R12, 64
546DECLARE_REG 14, R13, 72
547
548%macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names...
549    %assign num_args %1
550    %assign regs_used %2
551    %assign xmm_regs_used %3
552    ASSERT regs_used >= num_args
553    SETUP_STACK_POINTER %4
554    ASSERT regs_used <= 15
555    PUSH_IF_USED 9, 10, 11, 12, 13, 14
556    ALLOC_STACK %4
557    LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14
558    %if %0 > 4
559        %ifnum %4
560            DEFINE_ARGS %5
561        %else
562            DEFINE_ARGS %4, %5
563        %endif
564    %elifnnum %4
565        DEFINE_ARGS %4
566    %endif
567%endmacro
568
569%define has_epilogue regs_used > 9 || stack_size > 0 || vzeroupper_required
570
571%macro RET 0
572    %if stack_size_padded > 0
573        %if required_stack_alignment > STACK_ALIGNMENT
574            mov rsp, rstkm
575        %else
576            add rsp, stack_size_padded
577        %endif
578    %endif
579    POP_IF_USED 14, 13, 12, 11, 10, 9
580    %if vzeroupper_required
581        vzeroupper
582    %endif
583    AUTO_REP_RET
584%endmacro
585
586%else ; X86_32 ;==============================================================
587
588DECLARE_REG 0, eax, 4
589DECLARE_REG 1, ecx, 8
590DECLARE_REG 2, edx, 12
591DECLARE_REG 3, ebx, 16
592DECLARE_REG 4, esi, 20
593DECLARE_REG 5, edi, 24
594DECLARE_REG 6, ebp, 28
595%define rsp esp
596
597%macro DECLARE_ARG 1-*
598    %rep %0
599        %define r%1m [rstk + stack_offset + 4*%1 + 4]
600        %define r%1mp dword r%1m
601        %rotate 1
602    %endrep
603%endmacro
604
605DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14
606
607%macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names...
608    %assign num_args %1
609    %assign regs_used %2
610    ASSERT regs_used >= num_args
611    %if num_args > 7
612        %assign num_args 7
613    %endif
614    %if regs_used > 7
615        %assign regs_used 7
616    %endif
617    SETUP_STACK_POINTER %4
618    ASSERT regs_used <= 7
619    PUSH_IF_USED 3, 4, 5, 6
620    ALLOC_STACK %4
621    LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6
622    %if %0 > 4
623        %ifnum %4
624            DEFINE_ARGS %5
625        %else
626            DEFINE_ARGS %4, %5
627        %endif
628    %elifnnum %4
629        DEFINE_ARGS %4
630    %endif
631%endmacro
632
633%define has_epilogue regs_used > 3 || stack_size > 0 || vzeroupper_required
634
635%macro RET 0
636    %if stack_size_padded > 0
637        %if required_stack_alignment > STACK_ALIGNMENT
638            mov rsp, rstkm
639        %else
640            add rsp, stack_size_padded
641        %endif
642    %endif
643    POP_IF_USED 6, 5, 4, 3
644    %if vzeroupper_required
645        vzeroupper
646    %endif
647    AUTO_REP_RET
648%endmacro
649
650%endif ;======================================================================
651
652%if WIN64 == 0
653    %macro WIN64_SPILL_XMM 1
654    %endmacro
655    %macro WIN64_RESTORE_XMM 0
656    %endmacro
657    %macro WIN64_PUSH_XMM 0
658    %endmacro
659%endif
660
661; On AMD cpus <=K10, an ordinary ret is slow if it immediately follows either
662; a branch or a branch target. So switch to a 2-byte form of ret in that case.
663; We can automatically detect "follows a branch", but not a branch target.
664; (SSSE3 is a sufficient condition to know that your cpu doesn't have this problem.)
665%macro REP_RET 0
666    %if has_epilogue || cpuflag(ssse3)
667        RET
668    %else
669        rep ret
670    %endif
671    annotate_function_size
672%endmacro
673
674%define last_branch_adr $$
675%macro AUTO_REP_RET 0
676    %if notcpuflag(ssse3)
677        times ((last_branch_adr-$)>>31)+1 rep ; times 1 iff $ == last_branch_adr.
678    %endif
679    ret
680    annotate_function_size
681%endmacro
682
683%macro BRANCH_INSTR 0-*
684    %rep %0
685        %macro %1 1-2 %1
686            %2 %1
687            %if notcpuflag(ssse3)
688                %%branch_instr equ $
689                %xdefine last_branch_adr %%branch_instr
690            %endif
691        %endmacro
692        %rotate 1
693    %endrep
694%endmacro
695
696BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp
697
698%macro TAIL_CALL 2 ; callee, is_nonadjacent
699    %if has_epilogue
700        call %1
701        RET
702    %elif %2
703        jmp %1
704    %endif
705    annotate_function_size
706%endmacro
707
708;=============================================================================
709; arch-independent part
710;=============================================================================
711
712%assign function_align 16
713
714; Begin a function.
715; Applies any symbol mangling needed for C linkage, and sets up a define such that
716; subsequent uses of the function name automatically refer to the mangled version.
717; Appends cpuflags to the function name if cpuflags has been specified.
718; The "" empty default parameter is a workaround for nasm, which fails if SUFFIX
719; is empty and we call cglobal_internal with just %1 %+ SUFFIX (without %2).
720%macro cglobal 1-2+ "" ; name, [PROLOGUE args]
721    cglobal_internal 1, %1 %+ SUFFIX, %2
722%endmacro
723%macro cvisible 1-2+ "" ; name, [PROLOGUE args]
724    cglobal_internal 0, %1 %+ SUFFIX, %2
725%endmacro
726%macro cglobal_internal 2-3+
727    annotate_function_size
728    %if %1
729        %xdefine %%FUNCTION_PREFIX private_prefix
730        ; Chromium patch to ensure symbols are correctly hidden for macho files,
731        ; normal yasm does not have a private_extern flag.
732        %ifidn __OUTPUT_FORMAT__,macho
733            %xdefine %%VISIBILITY private_extern
734        %elifidn __OUTPUT_FORMAT__,macho32
735            %xdefine %%VISIBILITY private_extern
736        %elifidn __OUTPUT_FORMAT__,macho64
737            %xdefine %%VISIBILITY private_extern
738        %else
739            %xdefine %%VISIBILITY hidden
740        %endif
741    %else
742        %xdefine %%FUNCTION_PREFIX public_prefix
743        %xdefine %%VISIBILITY
744    %endif
745    %ifndef cglobaled_%2
746        %xdefine %2 mangle(%%FUNCTION_PREFIX %+ _ %+ %2)
747        %xdefine %2.skip_prologue %2 %+ .skip_prologue
748        CAT_XDEFINE cglobaled_, %2, 1
749    %endif
750    %xdefine current_function %2
751    ; Chromium patch to ensure symbols are correctly hidden for macho files,
752    ; normal yasm does not have a private_extern flag.
753    %xdefine current_function_section __SECT__
754    %if FORMAT_ELF
755        global %2:function %%VISIBILITY
756    %elifidn __OUTPUT_FORMAT__,elf32
757        global %2:function %%VISIBILITY
758    %elifidn __OUTPUT_FORMAT__,elf64
759        global %2:function %%VISIBILITY
760    %elifidn __OUTPUT_FORMAT__,macho
761        global %2:%%VISIBILITY
762    %elifidn __OUTPUT_FORMAT__,macho32
763        global %2:%%VISIBILITY
764    %elifidn __OUTPUT_FORMAT__,macho64
765        global %2:%%VISIBILITY
766    %else
767        global %2
768    %endif
769    align function_align
770    %2:
771    RESET_MM_PERMUTATION        ; needed for x86-64, also makes disassembly somewhat nicer
772    %xdefine rstk rsp           ; copy of the original stack pointer, used when greater alignment than the known stack alignment is required
773    %assign stack_offset 0      ; stack pointer offset relative to the return address
774    %assign stack_size 0        ; amount of stack space that can be freely used inside a function
775    %assign stack_size_padded 0 ; total amount of allocated stack space, including space for callee-saved xmm registers on WIN64 and alignment padding
776    %assign xmm_regs_used 0     ; number of XMM registers requested, used for dealing with callee-saved registers on WIN64 and vzeroupper
777    %ifnidn %3, ""
778        PROLOGUE %3
779    %endif
780%endmacro
781
782; Create a global symbol from a local label with the correct name mangling and type
783%macro cglobal_label 1
784    %if FORMAT_ELF
785        global current_function %+ %1:function hidden
786    %else
787        global current_function %+ %1
788    %endif
789    %1:
790%endmacro
791
792%macro cextern 1
793    %xdefine %1 mangle(private_prefix %+ _ %+ %1)
794    CAT_XDEFINE cglobaled_, %1, 1
795    extern %1
796%endmacro
797
798; like cextern, but without the prefix
799%macro cextern_naked 1
800    %ifdef PREFIX
801        %xdefine %1 mangle(%1)
802    %endif
803    CAT_XDEFINE cglobaled_, %1, 1
804    extern %1
805%endmacro
806
807%macro const 1-2+
808    %xdefine %1 mangle(private_prefix %+ _ %+ %1)
809    %if FORMAT_ELF
810        global %1:data hidden
811    %else
812        global %1
813    %endif
814    %1: %2
815%endmacro
816
817; This is needed for ELF, otherwise the GNU linker assumes the stack is executable by default.
818%if FORMAT_ELF
819    [SECTION .note.GNU-stack noalloc noexec nowrite progbits]
820%endif
821
822; Tell debuggers how large the function was.
823; This may be invoked multiple times per function; we rely on later instances overriding earlier ones.
824; This is invoked by RET and similar macros, and also cglobal does it for the previous function,
825; but if the last function in a source file doesn't use any of the standard macros for its epilogue,
826; then its size might be unspecified.
827%macro annotate_function_size 0
828    %ifdef __YASM_VER__
829        %ifdef current_function
830            %if FORMAT_ELF
831                current_function_section
832                %%ecf equ $
833                size current_function %%ecf - current_function
834                __SECT__
835            %endif
836        %endif
837    %endif
838%endmacro
839
840; cpuflags
841
842%assign cpuflags_mmx      (1<<0)
843%assign cpuflags_mmx2     (1<<1) | cpuflags_mmx
844%assign cpuflags_3dnow    (1<<2) | cpuflags_mmx
845%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow
846%assign cpuflags_sse      (1<<4) | cpuflags_mmx2
847%assign cpuflags_sse2     (1<<5) | cpuflags_sse
848%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2
849%assign cpuflags_lzcnt    (1<<7) | cpuflags_sse2
850%assign cpuflags_sse3     (1<<8) | cpuflags_sse2
851%assign cpuflags_ssse3    (1<<9) | cpuflags_sse3
852%assign cpuflags_sse4     (1<<10)| cpuflags_ssse3
853%assign cpuflags_sse42    (1<<11)| cpuflags_sse4
854%assign cpuflags_aesni    (1<<12)| cpuflags_sse42
855%assign cpuflags_avx      (1<<13)| cpuflags_sse42
856%assign cpuflags_xop      (1<<14)| cpuflags_avx
857%assign cpuflags_fma4     (1<<15)| cpuflags_avx
858%assign cpuflags_fma3     (1<<16)| cpuflags_avx
859%assign cpuflags_bmi1     (1<<17)| cpuflags_avx|cpuflags_lzcnt
860%assign cpuflags_bmi2     (1<<18)| cpuflags_bmi1
861%assign cpuflags_avx2     (1<<19)| cpuflags_fma3|cpuflags_bmi2
862%assign cpuflags_avx512   (1<<20)| cpuflags_avx2 ; F, CD, BW, DQ, VL
863
864%assign cpuflags_cache32  (1<<21)
865%assign cpuflags_cache64  (1<<22)
866%assign cpuflags_aligned  (1<<23) ; not a cpu feature, but a function variant
867%assign cpuflags_atom     (1<<24)
868
869; Returns a boolean value expressing whether or not the specified cpuflag is enabled.
870%define    cpuflag(x) (((((cpuflags & (cpuflags_ %+ x)) ^ (cpuflags_ %+ x)) - 1) >> 31) & 1)
871%define notcpuflag(x) (cpuflag(x) ^ 1)
872
873; Takes an arbitrary number of cpuflags from the above list.
874; All subsequent functions (up to the next INIT_CPUFLAGS) is built for the specified cpu.
875; You shouldn't need to invoke this macro directly, it's a subroutine for INIT_MMX &co.
876%macro INIT_CPUFLAGS 0-*
877    %xdefine SUFFIX
878    %undef cpuname
879    %assign cpuflags 0
880
881    %if %0 >= 1
882        %rep %0
883            %ifdef cpuname
884                %xdefine cpuname cpuname %+ _%1
885            %else
886                %xdefine cpuname %1
887            %endif
888            %assign cpuflags cpuflags | cpuflags_%1
889            %rotate 1
890        %endrep
891        %xdefine SUFFIX _ %+ cpuname
892
893        %if cpuflag(avx)
894            %assign avx_enabled 1
895        %endif
896        %if (mmsize == 16 && notcpuflag(sse2)) || (mmsize == 32 && notcpuflag(avx2))
897            %define mova movaps
898            %define movu movups
899            %define movnta movntps
900        %endif
901        %if cpuflag(aligned)
902            %define movu mova
903        %elif cpuflag(sse3) && notcpuflag(ssse3)
904            %define movu lddqu
905        %endif
906    %endif
907
908    %if ARCH_X86_64 || cpuflag(sse2)
909        CPUNOP amdnop
910    %else
911        CPUNOP basicnop
912    %endif
913%endmacro
914
915; Merge mmx, sse*, and avx*
916; m# is a simd register of the currently selected size
917; xm# is the corresponding xmm register if mmsize >= 16, otherwise the same as m#
918; ym# is the corresponding ymm register if mmsize >= 32, otherwise the same as m#
919; zm# is the corresponding zmm register if mmsize >= 64, otherwise the same as m#
920; (All 4 remain in sync through SWAP.)
921
922%macro CAT_XDEFINE 3
923    %xdefine %1%2 %3
924%endmacro
925
926%macro CAT_UNDEF 2
927    %undef %1%2
928%endmacro
929
930%macro DEFINE_MMREGS 1 ; mmtype
931    %assign %%prev_mmregs 0
932    %ifdef num_mmregs
933        %assign %%prev_mmregs num_mmregs
934    %endif
935
936    %assign num_mmregs 8
937    %if ARCH_X86_64 && mmsize >= 16
938        %assign num_mmregs 16
939        %if cpuflag(avx512) || mmsize == 64
940            %assign num_mmregs 32
941        %endif
942    %endif
943
944    %assign %%i 0
945    %rep num_mmregs
946        CAT_XDEFINE m, %%i, %1 %+ %%i
947        CAT_XDEFINE nn%1, %%i, %%i
948        %assign %%i %%i+1
949    %endrep
950    %if %%prev_mmregs > num_mmregs
951        %rep %%prev_mmregs - num_mmregs
952            CAT_UNDEF m, %%i
953            CAT_UNDEF nn %+ mmtype, %%i
954            %assign %%i %%i+1
955        %endrep
956    %endif
957    %xdefine mmtype %1
958%endmacro
959
960; Prefer registers 16-31 over 0-15 to avoid having to use vzeroupper
961%macro AVX512_MM_PERMUTATION 0-1 0 ; start_reg
962    %if ARCH_X86_64 && cpuflag(avx512)
963        %assign %%i %1
964        %rep 16-%1
965            %assign %%i_high %%i+16
966            SWAP %%i, %%i_high
967            %assign %%i %%i+1
968        %endrep
969    %endif
970%endmacro
971
972%macro INIT_MMX 0-1+
973    %assign avx_enabled 0
974    %define RESET_MM_PERMUTATION INIT_MMX %1
975    %define mmsize 8
976    %define mova movq
977    %define movu movq
978    %define movh movd
979    %define movnta movntq
980    INIT_CPUFLAGS %1
981    DEFINE_MMREGS mm
982%endmacro
983
984%macro INIT_XMM 0-1+
985    %assign avx_enabled 0
986    %define RESET_MM_PERMUTATION INIT_XMM %1
987    %define mmsize 16
988    %define mova movdqa
989    %define movu movdqu
990    %define movh movq
991    %define movnta movntdq
992    INIT_CPUFLAGS %1
993    DEFINE_MMREGS xmm
994    %if WIN64
995        AVX512_MM_PERMUTATION 6 ; Swap callee-saved registers with volatile registers
996    %endif
997%endmacro
998
999%macro INIT_YMM 0-1+
1000    %assign avx_enabled 1
1001    %define RESET_MM_PERMUTATION INIT_YMM %1
1002    %define mmsize 32
1003    %define mova movdqa
1004    %define movu movdqu
1005    %undef movh
1006    %define movnta movntdq
1007    INIT_CPUFLAGS %1
1008    DEFINE_MMREGS ymm
1009    AVX512_MM_PERMUTATION
1010%endmacro
1011
1012%macro INIT_ZMM 0-1+
1013    %assign avx_enabled 1
1014    %define RESET_MM_PERMUTATION INIT_ZMM %1
1015    %define mmsize 64
1016    %define mova movdqa
1017    %define movu movdqu
1018    %undef movh
1019    %define movnta movntdq
1020    INIT_CPUFLAGS %1
1021    DEFINE_MMREGS zmm
1022    AVX512_MM_PERMUTATION
1023%endmacro
1024
1025INIT_XMM
1026
1027%macro DECLARE_MMCAST 1
1028    %define  mmmm%1   mm%1
1029    %define  mmxmm%1  mm%1
1030    %define  mmymm%1  mm%1
1031    %define  mmzmm%1  mm%1
1032    %define xmmmm%1   mm%1
1033    %define xmmxmm%1 xmm%1
1034    %define xmmymm%1 xmm%1
1035    %define xmmzmm%1 xmm%1
1036    %define ymmmm%1   mm%1
1037    %define ymmxmm%1 xmm%1
1038    %define ymmymm%1 ymm%1
1039    %define ymmzmm%1 ymm%1
1040    %define zmmmm%1   mm%1
1041    %define zmmxmm%1 xmm%1
1042    %define zmmymm%1 ymm%1
1043    %define zmmzmm%1 zmm%1
1044    %define xm%1 xmm %+ m%1
1045    %define ym%1 ymm %+ m%1
1046    %define zm%1 zmm %+ m%1
1047%endmacro
1048
1049%assign i 0
1050%rep 32
1051    DECLARE_MMCAST i
1052    %assign i i+1
1053%endrep
1054
1055; I often want to use macros that permute their arguments. e.g. there's no
1056; efficient way to implement butterfly or transpose or dct without swapping some
1057; arguments.
1058;
1059; I would like to not have to manually keep track of the permutations:
1060; If I insert a permutation in the middle of a function, it should automatically
1061; change everything that follows. For more complex macros I may also have multiple
1062; implementations, e.g. the SSE2 and SSSE3 versions may have different permutations.
1063;
1064; Hence these macros. Insert a PERMUTE or some SWAPs at the end of a macro that
1065; permutes its arguments. It's equivalent to exchanging the contents of the
1066; registers, except that this way you exchange the register names instead, so it
1067; doesn't cost any cycles.
1068
1069%macro PERMUTE 2-* ; takes a list of pairs to swap
1070    %rep %0/2
1071        %xdefine %%tmp%2 m%2
1072        %rotate 2
1073    %endrep
1074    %rep %0/2
1075        %xdefine m%1 %%tmp%2
1076        CAT_XDEFINE nn, m%1, %1
1077        %rotate 2
1078    %endrep
1079%endmacro
1080
1081%macro SWAP 2+ ; swaps a single chain (sometimes more concise than pairs)
1082    %ifnum %1 ; SWAP 0, 1, ...
1083        SWAP_INTERNAL_NUM %1, %2
1084    %else ; SWAP m0, m1, ...
1085        SWAP_INTERNAL_NAME %1, %2
1086    %endif
1087%endmacro
1088
1089%macro SWAP_INTERNAL_NUM 2-*
1090    %rep %0-1
1091        %xdefine %%tmp m%1
1092        %xdefine m%1 m%2
1093        %xdefine m%2 %%tmp
1094        CAT_XDEFINE nn, m%1, %1
1095        CAT_XDEFINE nn, m%2, %2
1096        %rotate 1
1097    %endrep
1098%endmacro
1099
1100%macro SWAP_INTERNAL_NAME 2-*
1101    %xdefine %%args nn %+ %1
1102    %rep %0-1
1103        %xdefine %%args %%args, nn %+ %2
1104        %rotate 1
1105    %endrep
1106    SWAP_INTERNAL_NUM %%args
1107%endmacro
1108
1109; If SAVE_MM_PERMUTATION is placed at the end of a function, then any later
1110; calls to that function will automatically load the permutation, so values can
1111; be returned in mmregs.
1112%macro SAVE_MM_PERMUTATION 0-1
1113    %if %0
1114        %xdefine %%f %1_m
1115    %else
1116        %xdefine %%f current_function %+ _m
1117    %endif
1118    %assign %%i 0
1119    %rep num_mmregs
1120        CAT_XDEFINE %%f, %%i, m %+ %%i
1121        %assign %%i %%i+1
1122    %endrep
1123%endmacro
1124
1125%macro LOAD_MM_PERMUTATION 1 ; name to load from
1126    %ifdef %1_m0
1127        %assign %%i 0
1128        %rep num_mmregs
1129            CAT_XDEFINE m, %%i, %1_m %+ %%i
1130            CAT_XDEFINE nn, m %+ %%i, %%i
1131            %assign %%i %%i+1
1132        %endrep
1133    %endif
1134%endmacro
1135
1136; Append cpuflags to the callee's name iff the appended name is known and the plain name isn't
1137%macro call 1
1138    %ifid %1
1139        call_internal %1 %+ SUFFIX, %1
1140    %else
1141        call %1
1142    %endif
1143%endmacro
1144%macro call_internal 2
1145    %xdefine %%i %2
1146    %ifndef cglobaled_%2
1147        %ifdef cglobaled_%1
1148            %xdefine %%i %1
1149        %endif
1150    %endif
1151    call %%i
1152    LOAD_MM_PERMUTATION %%i
1153%endmacro
1154
1155; Substitutions that reduce instruction size but are functionally equivalent
1156%macro add 2
1157    %ifnum %2
1158        %if %2==128
1159            sub %1, -128
1160        %else
1161            add %1, %2
1162        %endif
1163    %else
1164        add %1, %2
1165    %endif
1166%endmacro
1167
1168%macro sub 2
1169    %ifnum %2
1170        %if %2==128
1171            add %1, -128
1172        %else
1173            sub %1, %2
1174        %endif
1175    %else
1176        sub %1, %2
1177    %endif
1178%endmacro
1179
1180;=============================================================================
1181; AVX abstraction layer
1182;=============================================================================
1183
1184%assign i 0
1185%rep 32
1186    %if i < 8
1187        CAT_XDEFINE sizeofmm, i, 8
1188        CAT_XDEFINE regnumofmm, i, i
1189    %endif
1190    CAT_XDEFINE sizeofxmm, i, 16
1191    CAT_XDEFINE sizeofymm, i, 32
1192    CAT_XDEFINE sizeofzmm, i, 64
1193    CAT_XDEFINE regnumofxmm, i, i
1194    CAT_XDEFINE regnumofymm, i, i
1195    CAT_XDEFINE regnumofzmm, i, i
1196    %assign i i+1
1197%endrep
1198%undef i
1199
1200%macro CHECK_AVX_INSTR_EMU 3-*
1201    %xdefine %%opcode %1
1202    %xdefine %%dst %2
1203    %rep %0-2
1204        %ifidn %%dst, %3
1205            %error non-avx emulation of ``%%opcode'' is not supported
1206        %endif
1207        %rotate 1
1208    %endrep
1209%endmacro
1210
1211;%1 == instruction
1212;%2 == minimal instruction set
1213;%3 == 1 if float, 0 if int
1214;%4 == 1 if 4-operand emulation, 0 if 3-operand emulation, 255 otherwise (no emulation)
1215;%5 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not
1216;%6+: operands
1217%macro RUN_AVX_INSTR 6-9+
1218    %ifnum sizeof%7
1219        %assign __sizeofreg sizeof%7
1220    %elifnum sizeof%6
1221        %assign __sizeofreg sizeof%6
1222    %else
1223        %assign __sizeofreg mmsize
1224    %endif
1225    %assign __emulate_avx 0
1226    %if avx_enabled && __sizeofreg >= 16
1227        %xdefine __instr v%1
1228    %else
1229        %xdefine __instr %1
1230        %if %0 >= 8+%4
1231            %assign __emulate_avx 1
1232        %endif
1233    %endif
1234    %ifnidn %2, fnord
1235        %ifdef cpuname
1236            %if notcpuflag(%2)
1237                %error use of ``%1'' %2 instruction in cpuname function: current_function
1238            %elif cpuflags_%2 < cpuflags_sse && notcpuflag(sse2) && __sizeofreg > 8
1239                %error use of ``%1'' sse2 instruction in cpuname function: current_function
1240            %endif
1241        %endif
1242    %endif
1243
1244    %if __emulate_avx
1245        %xdefine __src1 %7
1246        %xdefine __src2 %8
1247        %if %5 && %4 == 0
1248            %ifnidn %6, %7
1249                %ifidn %6, %8
1250                    %xdefine __src1 %8
1251                    %xdefine __src2 %7
1252                %elifnnum sizeof%8
1253                    ; 3-operand AVX instructions with a memory arg can only have it in src2,
1254                    ; whereas SSE emulation prefers to have it in src1 (i.e. the mov).
1255                    ; So, if the instruction is commutative with a memory arg, swap them.
1256                    %xdefine __src1 %8
1257                    %xdefine __src2 %7
1258                %endif
1259            %endif
1260        %endif
1261        %ifnidn %6, __src1
1262            %if %0 >= 9
1263                CHECK_AVX_INSTR_EMU {%1 %6, %7, %8, %9}, %6, __src2, %9
1264            %else
1265                CHECK_AVX_INSTR_EMU {%1 %6, %7, %8}, %6, __src2
1266            %endif
1267            %if __sizeofreg == 8
1268                MOVQ %6, __src1
1269            %elif %3
1270                MOVAPS %6, __src1
1271            %else
1272                MOVDQA %6, __src1
1273            %endif
1274        %endif
1275        %if %0 >= 9
1276            %1 %6, __src2, %9
1277        %else
1278            %1 %6, __src2
1279        %endif
1280    %elif %0 >= 9
1281        __instr %6, %7, %8, %9
1282    %elif %0 == 8
1283        __instr %6, %7, %8
1284    %elif %0 == 7
1285        __instr %6, %7
1286    %else
1287        __instr %6
1288    %endif
1289%endmacro
1290
1291;%1 == instruction
1292;%2 == minimal instruction set
1293;%3 == 1 if float, 0 if int
1294;%4 == 1 if 4-operand emulation, 0 if 3-operand emulation, 255 otherwise (no emulation)
1295;%5 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not
1296%macro AVX_INSTR 1-5 fnord, 0, 255, 0
1297    %macro %1 1-10 fnord, fnord, fnord, fnord, %1, %2, %3, %4, %5
1298        %ifidn %2, fnord
1299            RUN_AVX_INSTR %6, %7, %8, %9, %10, %1
1300        %elifidn %3, fnord
1301            RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2
1302        %elifidn %4, fnord
1303            RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3
1304        %elifidn %5, fnord
1305            RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3, %4
1306        %else
1307            RUN_AVX_INSTR %6, %7, %8, %9, %10, %1, %2, %3, %4, %5
1308        %endif
1309    %endmacro
1310%endmacro
1311
1312; Instructions with both VEX/EVEX and legacy encodings
1313; Non-destructive instructions are written without parameters
1314AVX_INSTR addpd, sse2, 1, 0, 1
1315AVX_INSTR addps, sse, 1, 0, 1
1316AVX_INSTR addsd, sse2, 1, 0, 0
1317AVX_INSTR addss, sse, 1, 0, 0
1318AVX_INSTR addsubpd, sse3, 1, 0, 0
1319AVX_INSTR addsubps, sse3, 1, 0, 0
1320AVX_INSTR aesdec, aesni, 0, 0, 0
1321AVX_INSTR aesdeclast, aesni, 0, 0, 0
1322AVX_INSTR aesenc, aesni, 0, 0, 0
1323AVX_INSTR aesenclast, aesni, 0, 0, 0
1324AVX_INSTR aesimc, aesni
1325AVX_INSTR aeskeygenassist, aesni
1326AVX_INSTR andnpd, sse2, 1, 0, 0
1327AVX_INSTR andnps, sse, 1, 0, 0
1328AVX_INSTR andpd, sse2, 1, 0, 1
1329AVX_INSTR andps, sse, 1, 0, 1
1330AVX_INSTR blendpd, sse4, 1, 1, 0
1331AVX_INSTR blendps, sse4, 1, 1, 0
1332AVX_INSTR blendvpd, sse4 ; can't be emulated
1333AVX_INSTR blendvps, sse4 ; can't be emulated
1334AVX_INSTR cmpeqpd, sse2, 1, 0, 1
1335AVX_INSTR cmpeqps, sse, 1, 0, 1
1336AVX_INSTR cmpeqsd, sse2, 1, 0, 0
1337AVX_INSTR cmpeqss, sse, 1, 0, 0
1338AVX_INSTR cmplepd, sse2, 1, 0, 0
1339AVX_INSTR cmpleps, sse, 1, 0, 0
1340AVX_INSTR cmplesd, sse2, 1, 0, 0
1341AVX_INSTR cmpless, sse, 1, 0, 0
1342AVX_INSTR cmpltpd, sse2, 1, 0, 0
1343AVX_INSTR cmpltps, sse, 1, 0, 0
1344AVX_INSTR cmpltsd, sse2, 1, 0, 0
1345AVX_INSTR cmpltss, sse, 1, 0, 0
1346AVX_INSTR cmpneqpd, sse2, 1, 0, 1
1347AVX_INSTR cmpneqps, sse, 1, 0, 1
1348AVX_INSTR cmpneqsd, sse2, 1, 0, 0
1349AVX_INSTR cmpneqss, sse, 1, 0, 0
1350AVX_INSTR cmpnlepd, sse2, 1, 0, 0
1351AVX_INSTR cmpnleps, sse, 1, 0, 0
1352AVX_INSTR cmpnlesd, sse2, 1, 0, 0
1353AVX_INSTR cmpnless, sse, 1, 0, 0
1354AVX_INSTR cmpnltpd, sse2, 1, 0, 0
1355AVX_INSTR cmpnltps, sse, 1, 0, 0
1356AVX_INSTR cmpnltsd, sse2, 1, 0, 0
1357AVX_INSTR cmpnltss, sse, 1, 0, 0
1358AVX_INSTR cmpordpd, sse2 1, 0, 1
1359AVX_INSTR cmpordps, sse 1, 0, 1
1360AVX_INSTR cmpordsd, sse2 1, 0, 0
1361AVX_INSTR cmpordss, sse 1, 0, 0
1362AVX_INSTR cmppd, sse2, 1, 1, 0
1363AVX_INSTR cmpps, sse, 1, 1, 0
1364AVX_INSTR cmpsd, sse2, 1, 1, 0
1365AVX_INSTR cmpss, sse, 1, 1, 0
1366AVX_INSTR cmpunordpd, sse2, 1, 0, 1
1367AVX_INSTR cmpunordps, sse, 1, 0, 1
1368AVX_INSTR cmpunordsd, sse2, 1, 0, 0
1369AVX_INSTR cmpunordss, sse, 1, 0, 0
1370AVX_INSTR comisd, sse2
1371AVX_INSTR comiss, sse
1372AVX_INSTR cvtdq2pd, sse2
1373AVX_INSTR cvtdq2ps, sse2
1374AVX_INSTR cvtpd2dq, sse2
1375AVX_INSTR cvtpd2ps, sse2
1376AVX_INSTR cvtps2dq, sse2
1377AVX_INSTR cvtps2pd, sse2
1378AVX_INSTR cvtsd2si, sse2
1379AVX_INSTR cvtsd2ss, sse2, 1, 0, 0
1380AVX_INSTR cvtsi2sd, sse2, 1, 0, 0
1381AVX_INSTR cvtsi2ss, sse, 1, 0, 0
1382AVX_INSTR cvtss2sd, sse2, 1, 0, 0
1383AVX_INSTR cvtss2si, sse
1384AVX_INSTR cvttpd2dq, sse2
1385AVX_INSTR cvttps2dq, sse2
1386AVX_INSTR cvttsd2si, sse2
1387AVX_INSTR cvttss2si, sse
1388AVX_INSTR divpd, sse2, 1, 0, 0
1389AVX_INSTR divps, sse, 1, 0, 0
1390AVX_INSTR divsd, sse2, 1, 0, 0
1391AVX_INSTR divss, sse, 1, 0, 0
1392AVX_INSTR dppd, sse4, 1, 1, 0
1393AVX_INSTR dpps, sse4, 1, 1, 0
1394AVX_INSTR extractps, sse4
1395AVX_INSTR haddpd, sse3, 1, 0, 0
1396AVX_INSTR haddps, sse3, 1, 0, 0
1397AVX_INSTR hsubpd, sse3, 1, 0, 0
1398AVX_INSTR hsubps, sse3, 1, 0, 0
1399AVX_INSTR insertps, sse4, 1, 1, 0
1400AVX_INSTR lddqu, sse3
1401AVX_INSTR ldmxcsr, sse
1402AVX_INSTR maskmovdqu, sse2
1403AVX_INSTR maxpd, sse2, 1, 0, 1
1404AVX_INSTR maxps, sse, 1, 0, 1
1405AVX_INSTR maxsd, sse2, 1, 0, 0
1406AVX_INSTR maxss, sse, 1, 0, 0
1407AVX_INSTR minpd, sse2, 1, 0, 1
1408AVX_INSTR minps, sse, 1, 0, 1
1409AVX_INSTR minsd, sse2, 1, 0, 0
1410AVX_INSTR minss, sse, 1, 0, 0
1411AVX_INSTR movapd, sse2
1412AVX_INSTR movaps, sse
1413AVX_INSTR movd, mmx
1414AVX_INSTR movddup, sse3
1415AVX_INSTR movdqa, sse2
1416AVX_INSTR movdqu, sse2
1417AVX_INSTR movhlps, sse, 1, 0, 0
1418AVX_INSTR movhpd, sse2, 1, 0, 0
1419AVX_INSTR movhps, sse, 1, 0, 0
1420AVX_INSTR movlhps, sse, 1, 0, 0
1421AVX_INSTR movlpd, sse2, 1, 0, 0
1422AVX_INSTR movlps, sse, 1, 0, 0
1423AVX_INSTR movmskpd, sse2
1424AVX_INSTR movmskps, sse
1425AVX_INSTR movntdq, sse2
1426AVX_INSTR movntdqa, sse4
1427AVX_INSTR movntpd, sse2
1428AVX_INSTR movntps, sse
1429AVX_INSTR movq, mmx
1430AVX_INSTR movsd, sse2, 1, 0, 0
1431AVX_INSTR movshdup, sse3
1432AVX_INSTR movsldup, sse3
1433AVX_INSTR movss, sse, 1, 0, 0
1434AVX_INSTR movupd, sse2
1435AVX_INSTR movups, sse
1436AVX_INSTR mpsadbw, sse4, 0, 1, 0
1437AVX_INSTR mulpd, sse2, 1, 0, 1
1438AVX_INSTR mulps, sse, 1, 0, 1
1439AVX_INSTR mulsd, sse2, 1, 0, 0
1440AVX_INSTR mulss, sse, 1, 0, 0
1441AVX_INSTR orpd, sse2, 1, 0, 1
1442AVX_INSTR orps, sse, 1, 0, 1
1443AVX_INSTR pabsb, ssse3
1444AVX_INSTR pabsd, ssse3
1445AVX_INSTR pabsw, ssse3
1446AVX_INSTR packsswb, mmx, 0, 0, 0
1447AVX_INSTR packssdw, mmx, 0, 0, 0
1448AVX_INSTR packuswb, mmx, 0, 0, 0
1449AVX_INSTR packusdw, sse4, 0, 0, 0
1450AVX_INSTR paddb, mmx, 0, 0, 1
1451AVX_INSTR paddw, mmx, 0, 0, 1
1452AVX_INSTR paddd, mmx, 0, 0, 1
1453AVX_INSTR paddq, sse2, 0, 0, 1
1454AVX_INSTR paddsb, mmx, 0, 0, 1
1455AVX_INSTR paddsw, mmx, 0, 0, 1
1456AVX_INSTR paddusb, mmx, 0, 0, 1
1457AVX_INSTR paddusw, mmx, 0, 0, 1
1458AVX_INSTR palignr, ssse3, 0, 1, 0
1459AVX_INSTR pand, mmx, 0, 0, 1
1460AVX_INSTR pandn, mmx, 0, 0, 0
1461AVX_INSTR pavgb, mmx2, 0, 0, 1
1462AVX_INSTR pavgw, mmx2, 0, 0, 1
1463AVX_INSTR pblendvb, sse4 ; can't be emulated
1464AVX_INSTR pblendw, sse4, 0, 1, 0
1465AVX_INSTR pclmulqdq, fnord, 0, 1, 0
1466AVX_INSTR pclmulhqhqdq, fnord, 0, 0, 0
1467AVX_INSTR pclmulhqlqdq, fnord, 0, 0, 0
1468AVX_INSTR pclmullqhqdq, fnord, 0, 0, 0
1469AVX_INSTR pclmullqlqdq, fnord, 0, 0, 0
1470AVX_INSTR pcmpestri, sse42
1471AVX_INSTR pcmpestrm, sse42
1472AVX_INSTR pcmpistri, sse42
1473AVX_INSTR pcmpistrm, sse42
1474AVX_INSTR pcmpeqb, mmx, 0, 0, 1
1475AVX_INSTR pcmpeqw, mmx, 0, 0, 1
1476AVX_INSTR pcmpeqd, mmx, 0, 0, 1
1477AVX_INSTR pcmpeqq, sse4, 0, 0, 1
1478AVX_INSTR pcmpgtb, mmx, 0, 0, 0
1479AVX_INSTR pcmpgtw, mmx, 0, 0, 0
1480AVX_INSTR pcmpgtd, mmx, 0, 0, 0
1481AVX_INSTR pcmpgtq, sse42, 0, 0, 0
1482AVX_INSTR pextrb, sse4
1483AVX_INSTR pextrd, sse4
1484AVX_INSTR pextrq, sse4
1485AVX_INSTR pextrw, mmx2
1486AVX_INSTR phaddw, ssse3, 0, 0, 0
1487AVX_INSTR phaddd, ssse3, 0, 0, 0
1488AVX_INSTR phaddsw, ssse3, 0, 0, 0
1489AVX_INSTR phminposuw, sse4
1490AVX_INSTR phsubw, ssse3, 0, 0, 0
1491AVX_INSTR phsubd, ssse3, 0, 0, 0
1492AVX_INSTR phsubsw, ssse3, 0, 0, 0
1493AVX_INSTR pinsrb, sse4, 0, 1, 0
1494AVX_INSTR pinsrd, sse4, 0, 1, 0
1495AVX_INSTR pinsrq, sse4, 0, 1, 0
1496AVX_INSTR pinsrw, mmx2, 0, 1, 0
1497AVX_INSTR pmaddwd, mmx, 0, 0, 1
1498AVX_INSTR pmaddubsw, ssse3, 0, 0, 0
1499AVX_INSTR pmaxsb, sse4, 0, 0, 1
1500AVX_INSTR pmaxsw, mmx2, 0, 0, 1
1501AVX_INSTR pmaxsd, sse4, 0, 0, 1
1502AVX_INSTR pmaxub, mmx2, 0, 0, 1
1503AVX_INSTR pmaxuw, sse4, 0, 0, 1
1504AVX_INSTR pmaxud, sse4, 0, 0, 1
1505AVX_INSTR pminsb, sse4, 0, 0, 1
1506AVX_INSTR pminsw, mmx2, 0, 0, 1
1507AVX_INSTR pminsd, sse4, 0, 0, 1
1508AVX_INSTR pminub, mmx2, 0, 0, 1
1509AVX_INSTR pminuw, sse4, 0, 0, 1
1510AVX_INSTR pminud, sse4, 0, 0, 1
1511AVX_INSTR pmovmskb, mmx2
1512AVX_INSTR pmovsxbw, sse4
1513AVX_INSTR pmovsxbd, sse4
1514AVX_INSTR pmovsxbq, sse4
1515AVX_INSTR pmovsxwd, sse4
1516AVX_INSTR pmovsxwq, sse4
1517AVX_INSTR pmovsxdq, sse4
1518AVX_INSTR pmovzxbw, sse4
1519AVX_INSTR pmovzxbd, sse4
1520AVX_INSTR pmovzxbq, sse4
1521AVX_INSTR pmovzxwd, sse4
1522AVX_INSTR pmovzxwq, sse4
1523AVX_INSTR pmovzxdq, sse4
1524AVX_INSTR pmuldq, sse4, 0, 0, 1
1525AVX_INSTR pmulhrsw, ssse3, 0, 0, 1
1526AVX_INSTR pmulhuw, mmx2, 0, 0, 1
1527AVX_INSTR pmulhw, mmx, 0, 0, 1
1528AVX_INSTR pmullw, mmx, 0, 0, 1
1529AVX_INSTR pmulld, sse4, 0, 0, 1
1530AVX_INSTR pmuludq, sse2, 0, 0, 1
1531AVX_INSTR por, mmx, 0, 0, 1
1532AVX_INSTR psadbw, mmx2, 0, 0, 1
1533AVX_INSTR pshufb, ssse3, 0, 0, 0
1534AVX_INSTR pshufd, sse2
1535AVX_INSTR pshufhw, sse2
1536AVX_INSTR pshuflw, sse2
1537AVX_INSTR psignb, ssse3, 0, 0, 0
1538AVX_INSTR psignw, ssse3, 0, 0, 0
1539AVX_INSTR psignd, ssse3, 0, 0, 0
1540AVX_INSTR psllw, mmx, 0, 0, 0
1541AVX_INSTR pslld, mmx, 0, 0, 0
1542AVX_INSTR psllq, mmx, 0, 0, 0
1543AVX_INSTR pslldq, sse2, 0, 0, 0
1544AVX_INSTR psraw, mmx, 0, 0, 0
1545AVX_INSTR psrad, mmx, 0, 0, 0
1546AVX_INSTR psrlw, mmx, 0, 0, 0
1547AVX_INSTR psrld, mmx, 0, 0, 0
1548AVX_INSTR psrlq, mmx, 0, 0, 0
1549AVX_INSTR psrldq, sse2, 0, 0, 0
1550AVX_INSTR psubb, mmx, 0, 0, 0
1551AVX_INSTR psubw, mmx, 0, 0, 0
1552AVX_INSTR psubd, mmx, 0, 0, 0
1553AVX_INSTR psubq, sse2, 0, 0, 0
1554AVX_INSTR psubsb, mmx, 0, 0, 0
1555AVX_INSTR psubsw, mmx, 0, 0, 0
1556AVX_INSTR psubusb, mmx, 0, 0, 0
1557AVX_INSTR psubusw, mmx, 0, 0, 0
1558AVX_INSTR ptest, sse4
1559AVX_INSTR punpckhbw, mmx, 0, 0, 0
1560AVX_INSTR punpckhwd, mmx, 0, 0, 0
1561AVX_INSTR punpckhdq, mmx, 0, 0, 0
1562AVX_INSTR punpckhqdq, sse2, 0, 0, 0
1563AVX_INSTR punpcklbw, mmx, 0, 0, 0
1564AVX_INSTR punpcklwd, mmx, 0, 0, 0
1565AVX_INSTR punpckldq, mmx, 0, 0, 0
1566AVX_INSTR punpcklqdq, sse2, 0, 0, 0
1567AVX_INSTR pxor, mmx, 0, 0, 1
1568AVX_INSTR rcpps, sse
1569AVX_INSTR rcpss, sse, 1, 0, 0
1570AVX_INSTR roundpd, sse4
1571AVX_INSTR roundps, sse4
1572AVX_INSTR roundsd, sse4, 1, 1, 0
1573AVX_INSTR roundss, sse4, 1, 1, 0
1574AVX_INSTR rsqrtps, sse
1575AVX_INSTR rsqrtss, sse, 1, 0, 0
1576AVX_INSTR shufpd, sse2, 1, 1, 0
1577AVX_INSTR shufps, sse, 1, 1, 0
1578AVX_INSTR sqrtpd, sse2
1579AVX_INSTR sqrtps, sse
1580AVX_INSTR sqrtsd, sse2, 1, 0, 0
1581AVX_INSTR sqrtss, sse, 1, 0, 0
1582AVX_INSTR stmxcsr, sse
1583AVX_INSTR subpd, sse2, 1, 0, 0
1584AVX_INSTR subps, sse, 1, 0, 0
1585AVX_INSTR subsd, sse2, 1, 0, 0
1586AVX_INSTR subss, sse, 1, 0, 0
1587AVX_INSTR ucomisd, sse2
1588AVX_INSTR ucomiss, sse
1589AVX_INSTR unpckhpd, sse2, 1, 0, 0
1590AVX_INSTR unpckhps, sse, 1, 0, 0
1591AVX_INSTR unpcklpd, sse2, 1, 0, 0
1592AVX_INSTR unpcklps, sse, 1, 0, 0
1593AVX_INSTR xorpd, sse2, 1, 0, 1
1594AVX_INSTR xorps, sse, 1, 0, 1
1595
1596; 3DNow instructions, for sharing code between AVX, SSE and 3DN
1597AVX_INSTR pfadd, 3dnow, 1, 0, 1
1598AVX_INSTR pfsub, 3dnow, 1, 0, 0
1599AVX_INSTR pfmul, 3dnow, 1, 0, 1
1600
1601; base-4 constants for shuffles
1602%assign i 0
1603%rep 256
1604    %assign j ((i>>6)&3)*1000 + ((i>>4)&3)*100 + ((i>>2)&3)*10 + (i&3)
1605    %if j < 10
1606        CAT_XDEFINE q000, j, i
1607    %elif j < 100
1608        CAT_XDEFINE q00, j, i
1609    %elif j < 1000
1610        CAT_XDEFINE q0, j, i
1611    %else
1612        CAT_XDEFINE q, j, i
1613    %endif
1614    %assign i i+1
1615%endrep
1616%undef i
1617%undef j
1618
1619%macro FMA_INSTR 3
1620    %macro %1 4-7 %1, %2, %3
1621        %if cpuflag(xop)
1622            v%5 %1, %2, %3, %4
1623        %elifnidn %1, %4
1624            %6 %1, %2, %3
1625            %7 %1, %4
1626        %else
1627            %error non-xop emulation of ``%5 %1, %2, %3, %4'' is not supported
1628        %endif
1629    %endmacro
1630%endmacro
1631
1632FMA_INSTR  pmacsww,  pmullw, paddw
1633FMA_INSTR  pmacsdd,  pmulld, paddd ; sse4 emulation
1634FMA_INSTR pmacsdql,  pmuldq, paddq ; sse4 emulation
1635FMA_INSTR pmadcswd, pmaddwd, paddd
1636
1637; tzcnt is equivalent to "rep bsf" and is backwards-compatible with bsf.
1638; This lets us use tzcnt without bumping the yasm version requirement yet.
1639%define tzcnt rep bsf
1640
1641; Macros for consolidating FMA3 and FMA4 using 4-operand (dst, src1, src2, src3) syntax.
1642; FMA3 is only possible if dst is the same as one of the src registers.
1643; Either src2 or src3 can be a memory operand.
1644%macro FMA4_INSTR 2-*
1645    %push fma4_instr
1646    %xdefine %$prefix %1
1647    %rep %0 - 1
1648        %macro %$prefix%2 4-6 %$prefix, %2
1649            %if notcpuflag(fma3) && notcpuflag(fma4)
1650                %error use of ``%5%6'' fma instruction in cpuname function: current_function
1651            %elif cpuflag(fma4)
1652                v%5%6 %1, %2, %3, %4
1653            %elifidn %1, %2
1654                ; If %3 or %4 is a memory operand it needs to be encoded as the last operand.
1655                %ifnum sizeof%3
1656                    v%{5}213%6 %2, %3, %4
1657                %else
1658                    v%{5}132%6 %2, %4, %3
1659                %endif
1660            %elifidn %1, %3
1661                v%{5}213%6 %3, %2, %4
1662            %elifidn %1, %4
1663                v%{5}231%6 %4, %2, %3
1664            %else
1665                %error fma3 emulation of ``%5%6 %1, %2, %3, %4'' is not supported
1666            %endif
1667        %endmacro
1668        %rotate 1
1669    %endrep
1670    %pop
1671%endmacro
1672
1673FMA4_INSTR fmadd,    pd, ps, sd, ss
1674FMA4_INSTR fmaddsub, pd, ps
1675FMA4_INSTR fmsub,    pd, ps, sd, ss
1676FMA4_INSTR fmsubadd, pd, ps
1677FMA4_INSTR fnmadd,   pd, ps, sd, ss
1678FMA4_INSTR fnmsub,   pd, ps, sd, ss
1679
1680; Macros for converting VEX instructions to equivalent EVEX ones.
1681%macro EVEX_INSTR 2-3 0 ; vex, evex, prefer_evex
1682    %macro %1 2-7 fnord, fnord, %1, %2, %3
1683        %ifidn %3, fnord
1684            %define %%args %1, %2
1685        %elifidn %4, fnord
1686            %define %%args %1, %2, %3
1687        %else
1688            %define %%args %1, %2, %3, %4
1689        %endif
1690        %assign %%evex_required cpuflag(avx512) & %7
1691        %ifnum regnumof%1
1692            %if regnumof%1 >= 16 || sizeof%1 > 32
1693                %assign %%evex_required 1
1694            %endif
1695        %endif
1696        %ifnum regnumof%2
1697            %if regnumof%2 >= 16 || sizeof%2 > 32
1698                %assign %%evex_required 1
1699            %endif
1700        %endif
1701        %if %%evex_required
1702            %6 %%args
1703        %else
1704            %5 %%args ; Prefer VEX over EVEX due to shorter instruction length
1705        %endif
1706    %endmacro
1707%endmacro
1708
1709EVEX_INSTR vbroadcastf128, vbroadcastf32x4
1710EVEX_INSTR vbroadcasti128, vbroadcasti32x4
1711EVEX_INSTR vextractf128,   vextractf32x4
1712EVEX_INSTR vextracti128,   vextracti32x4
1713EVEX_INSTR vinsertf128,    vinsertf32x4
1714EVEX_INSTR vinserti128,    vinserti32x4
1715EVEX_INSTR vmovdqa,        vmovdqa32
1716EVEX_INSTR vmovdqu,        vmovdqu32
1717EVEX_INSTR vpand,          vpandd
1718EVEX_INSTR vpandn,         vpandnd
1719EVEX_INSTR vpor,           vpord
1720EVEX_INSTR vpxor,          vpxord
1721EVEX_INSTR vrcpps,         vrcp14ps,   1 ; EVEX versions have higher precision
1722EVEX_INSTR vrcpss,         vrcp14ss,   1
1723EVEX_INSTR vrsqrtps,       vrsqrt14ps, 1
1724EVEX_INSTR vrsqrtss,       vrsqrt14ss, 1
1725
1726; workaround: vpbroadcastq is broken in x86_32 due to a yasm bug (fixed in 1.3.0)
1727%ifdef __YASM_VER__
1728    %if __YASM_VERSION_ID__ < 0x01030000 && ARCH_X86_64 == 0
1729        %macro vpbroadcastq 2
1730            %if sizeof%1 == 16
1731                movddup %1, %2
1732            %else
1733                vbroadcastsd %1, %2
1734            %endif
1735        %endmacro
1736    %endif
1737%endif
1738