109467b48Spatrick//===- X86InstrCompiler.td - Compiler Pseudos and Patterns -*- tablegen -*-===// 209467b48Spatrick// 309467b48Spatrick// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 409467b48Spatrick// See https://llvm.org/LICENSE.txt for license information. 509467b48Spatrick// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 609467b48Spatrick// 709467b48Spatrick//===----------------------------------------------------------------------===// 809467b48Spatrick// 909467b48Spatrick// This file describes the various pseudo instructions used by the compiler, 1009467b48Spatrick// as well as Pat patterns used during instruction selection. 1109467b48Spatrick// 1209467b48Spatrick//===----------------------------------------------------------------------===// 1309467b48Spatrick 1409467b48Spatrick//===----------------------------------------------------------------------===// 1509467b48Spatrick// Pattern Matching Support 1609467b48Spatrick 1709467b48Spatrickdef GetLo32XForm : SDNodeXForm<imm, [{ 1809467b48Spatrick // Transformation function: get the low 32 bits. 1909467b48Spatrick return getI32Imm((uint32_t)N->getZExtValue(), SDLoc(N)); 2009467b48Spatrick}]>; 2109467b48Spatrick 2209467b48Spatrick 2309467b48Spatrick//===----------------------------------------------------------------------===// 2409467b48Spatrick// Random Pseudo Instructions. 2509467b48Spatrick 2609467b48Spatrick// PIC base construction. This expands to code that looks like this: 2709467b48Spatrick// call $next_inst 2809467b48Spatrick// popl %destreg" 2909467b48Spatricklet hasSideEffects = 0, isNotDuplicable = 1, Uses = [ESP, SSP], 3009467b48Spatrick SchedRW = [WriteJump] in 3109467b48Spatrick def MOVPC32r : Ii32<0xE8, Pseudo, (outs GR32:$reg), (ins i32imm:$label), 3209467b48Spatrick "", []>; 3309467b48Spatrick 3409467b48Spatrick// ADJCALLSTACKDOWN/UP implicitly use/def ESP because they may be expanded into 3509467b48Spatrick// a stack adjustment and the codegen must know that they may modify the stack 3609467b48Spatrick// pointer before prolog-epilog rewriting occurs. 3709467b48Spatrick// Pessimistically assume ADJCALLSTACKDOWN / ADJCALLSTACKUP will become 3809467b48Spatrick// sub / add which can clobber EFLAGS. 3909467b48Spatricklet Defs = [ESP, EFLAGS, SSP], Uses = [ESP, SSP], SchedRW = [WriteALU] in { 4009467b48Spatrickdef ADJCALLSTACKDOWN32 : I<0, Pseudo, (outs), 4109467b48Spatrick (ins i32imm:$amt1, i32imm:$amt2, i32imm:$amt3), 4209467b48Spatrick "#ADJCALLSTACKDOWN", []>, Requires<[NotLP64]>; 4309467b48Spatrickdef ADJCALLSTACKUP32 : I<0, Pseudo, (outs), (ins i32imm:$amt1, i32imm:$amt2), 4409467b48Spatrick "#ADJCALLSTACKUP", 4509467b48Spatrick [(X86callseq_end timm:$amt1, timm:$amt2)]>, 4609467b48Spatrick Requires<[NotLP64]>; 4709467b48Spatrick} 4809467b48Spatrickdef : Pat<(X86callseq_start timm:$amt1, timm:$amt2), 4909467b48Spatrick (ADJCALLSTACKDOWN32 i32imm:$amt1, i32imm:$amt2, 0)>, Requires<[NotLP64]>; 5009467b48Spatrick 5109467b48Spatrick 5209467b48Spatrick// ADJCALLSTACKDOWN/UP implicitly use/def RSP because they may be expanded into 5309467b48Spatrick// a stack adjustment and the codegen must know that they may modify the stack 5409467b48Spatrick// pointer before prolog-epilog rewriting occurs. 5509467b48Spatrick// Pessimistically assume ADJCALLSTACKDOWN / ADJCALLSTACKUP will become 5609467b48Spatrick// sub / add which can clobber EFLAGS. 5709467b48Spatricklet Defs = [RSP, EFLAGS, SSP], Uses = [RSP, SSP], SchedRW = [WriteALU] in { 5809467b48Spatrickdef ADJCALLSTACKDOWN64 : I<0, Pseudo, (outs), 5909467b48Spatrick (ins i32imm:$amt1, i32imm:$amt2, i32imm:$amt3), 6009467b48Spatrick "#ADJCALLSTACKDOWN", []>, Requires<[IsLP64]>; 6109467b48Spatrickdef ADJCALLSTACKUP64 : I<0, Pseudo, (outs), (ins i32imm:$amt1, i32imm:$amt2), 6209467b48Spatrick "#ADJCALLSTACKUP", 6309467b48Spatrick [(X86callseq_end timm:$amt1, timm:$amt2)]>, 6409467b48Spatrick Requires<[IsLP64]>; 6509467b48Spatrick} 6609467b48Spatrickdef : Pat<(X86callseq_start timm:$amt1, timm:$amt2), 6709467b48Spatrick (ADJCALLSTACKDOWN64 i32imm:$amt1, i32imm:$amt2, 0)>, Requires<[IsLP64]>; 6809467b48Spatrick 6909467b48Spatricklet SchedRW = [WriteSystem] in { 7009467b48Spatrick 7109467b48Spatrick// x86-64 va_start lowering magic. 7209467b48Spatricklet usesCustomInserter = 1, Defs = [EFLAGS] in { 7309467b48Spatrickdef VASTART_SAVE_XMM_REGS : I<0, Pseudo, 7409467b48Spatrick (outs), 7509467b48Spatrick (ins GR8:$al, 7609467b48Spatrick i64imm:$regsavefi, i64imm:$offset, 7709467b48Spatrick variable_ops), 7809467b48Spatrick "#VASTART_SAVE_XMM_REGS $al, $regsavefi, $offset", 7909467b48Spatrick [(X86vastart_save_xmm_regs GR8:$al, 8009467b48Spatrick imm:$regsavefi, 8109467b48Spatrick imm:$offset), 8209467b48Spatrick (implicit EFLAGS)]>; 8309467b48Spatrick 8409467b48Spatrick// The VAARG_64 pseudo-instruction takes the address of the va_list, 8509467b48Spatrick// and places the address of the next argument into a register. 8609467b48Spatricklet Defs = [EFLAGS] in 8709467b48Spatrickdef VAARG_64 : I<0, Pseudo, 8809467b48Spatrick (outs GR64:$dst), 8909467b48Spatrick (ins i8mem:$ap, i32imm:$size, i8imm:$mode, i32imm:$align), 9009467b48Spatrick "#VAARG_64 $dst, $ap, $size, $mode, $align", 9109467b48Spatrick [(set GR64:$dst, 9209467b48Spatrick (X86vaarg64 addr:$ap, imm:$size, imm:$mode, imm:$align)), 9309467b48Spatrick (implicit EFLAGS)]>; 9409467b48Spatrick 9509467b48Spatrick 9609467b48Spatrick// When using segmented stacks these are lowered into instructions which first 9709467b48Spatrick// check if the current stacklet has enough free memory. If it does, memory is 9809467b48Spatrick// allocated by bumping the stack pointer. Otherwise memory is allocated from 9909467b48Spatrick// the heap. 10009467b48Spatrick 10109467b48Spatricklet Defs = [EAX, ESP, EFLAGS], Uses = [ESP] in 10209467b48Spatrickdef SEG_ALLOCA_32 : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$size), 10309467b48Spatrick "# variable sized alloca for segmented stacks", 10409467b48Spatrick [(set GR32:$dst, 10509467b48Spatrick (X86SegAlloca GR32:$size))]>, 10609467b48Spatrick Requires<[NotLP64]>; 10709467b48Spatrick 10809467b48Spatricklet Defs = [RAX, RSP, EFLAGS], Uses = [RSP] in 10909467b48Spatrickdef SEG_ALLOCA_64 : I<0, Pseudo, (outs GR64:$dst), (ins GR64:$size), 11009467b48Spatrick "# variable sized alloca for segmented stacks", 11109467b48Spatrick [(set GR64:$dst, 11209467b48Spatrick (X86SegAlloca GR64:$size))]>, 11309467b48Spatrick Requires<[In64BitMode]>; 114097a140dSpatrick 115097a140dSpatrick// To protect against stack clash, dynamic allocation should perform a memory 116097a140dSpatrick// probe at each page. 117097a140dSpatrick 118097a140dSpatricklet Defs = [EAX, ESP, EFLAGS], Uses = [ESP] in 119097a140dSpatrickdef PROBED_ALLOCA_32 : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$size), 120097a140dSpatrick "# variable sized alloca with probing", 121097a140dSpatrick [(set GR32:$dst, 122097a140dSpatrick (X86ProbedAlloca GR32:$size))]>, 123097a140dSpatrick Requires<[NotLP64]>; 124097a140dSpatrick 125097a140dSpatricklet Defs = [RAX, RSP, EFLAGS], Uses = [RSP] in 126097a140dSpatrickdef PROBED_ALLOCA_64 : I<0, Pseudo, (outs GR64:$dst), (ins GR64:$size), 127097a140dSpatrick "# variable sized alloca with probing", 128097a140dSpatrick [(set GR64:$dst, 129097a140dSpatrick (X86ProbedAlloca GR64:$size))]>, 130097a140dSpatrick Requires<[In64BitMode]>; 13109467b48Spatrick} 13209467b48Spatrick 133097a140dSpatricklet hasNoSchedulingInfo = 1 in 134097a140dSpatrickdef STACKALLOC_W_PROBING : I<0, Pseudo, (outs), (ins i64imm:$stacksize), 135097a140dSpatrick "# fixed size alloca with probing", 136097a140dSpatrick []>; 137097a140dSpatrick 13809467b48Spatrick// Dynamic stack allocation yields a _chkstk or _alloca call for all Windows 13909467b48Spatrick// targets. These calls are needed to probe the stack when allocating more than 14009467b48Spatrick// 4k bytes in one go. Touching the stack at 4K increments is necessary to 14109467b48Spatrick// ensure that the guard pages used by the OS virtual memory manager are 14209467b48Spatrick// allocated in correct sequence. 14309467b48Spatrick// The main point of having separate instruction are extra unmodelled effects 14409467b48Spatrick// (compared to ordinary calls) like stack pointer change. 14509467b48Spatrick 14609467b48Spatricklet Defs = [EAX, ESP, EFLAGS], Uses = [ESP] in 14709467b48Spatrickdef WIN_ALLOCA_32 : I<0, Pseudo, (outs), (ins GR32:$size), 14809467b48Spatrick "# dynamic stack allocation", 14909467b48Spatrick [(X86WinAlloca GR32:$size)]>, 15009467b48Spatrick Requires<[NotLP64]>; 15109467b48Spatrick 15209467b48Spatricklet Defs = [RAX, RSP, EFLAGS], Uses = [RSP] in 15309467b48Spatrickdef WIN_ALLOCA_64 : I<0, Pseudo, (outs), (ins GR64:$size), 15409467b48Spatrick "# dynamic stack allocation", 15509467b48Spatrick [(X86WinAlloca GR64:$size)]>, 15609467b48Spatrick Requires<[In64BitMode]>; 15709467b48Spatrick} // SchedRW 15809467b48Spatrick 15909467b48Spatrick// These instructions XOR the frame pointer into a GPR. They are used in some 16009467b48Spatrick// stack protection schemes. These are post-RA pseudos because we only know the 16109467b48Spatrick// frame register after register allocation. 16209467b48Spatricklet Constraints = "$src = $dst", isMoveImm = 1, isPseudo = 1, Defs = [EFLAGS] in { 16309467b48Spatrick def XOR32_FP : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$src), 16409467b48Spatrick "xorl\t$$FP, $src", []>, 16509467b48Spatrick Requires<[NotLP64]>, Sched<[WriteALU]>; 16609467b48Spatrick def XOR64_FP : I<0, Pseudo, (outs GR64:$dst), (ins GR64:$src), 16709467b48Spatrick "xorq\t$$FP $src", []>, 16809467b48Spatrick Requires<[In64BitMode]>, Sched<[WriteALU]>; 16909467b48Spatrick} 17009467b48Spatrick 17109467b48Spatrick//===----------------------------------------------------------------------===// 17209467b48Spatrick// EH Pseudo Instructions 17309467b48Spatrick// 17409467b48Spatricklet SchedRW = [WriteSystem] in { 17509467b48Spatricklet isTerminator = 1, isReturn = 1, isBarrier = 1, 17609467b48Spatrick hasCtrlDep = 1, isCodeGenOnly = 1 in { 17709467b48Spatrickdef EH_RETURN : I<0xC3, RawFrm, (outs), (ins GR32:$addr), 17809467b48Spatrick "ret\t#eh_return, addr: $addr", 17909467b48Spatrick [(X86ehret GR32:$addr)]>, Sched<[WriteJumpLd]>; 18009467b48Spatrick 18109467b48Spatrick} 18209467b48Spatrick 18309467b48Spatricklet isTerminator = 1, isReturn = 1, isBarrier = 1, 18409467b48Spatrick hasCtrlDep = 1, isCodeGenOnly = 1 in { 18509467b48Spatrickdef EH_RETURN64 : I<0xC3, RawFrm, (outs), (ins GR64:$addr), 18609467b48Spatrick "ret\t#eh_return, addr: $addr", 18709467b48Spatrick [(X86ehret GR64:$addr)]>, Sched<[WriteJumpLd]>; 18809467b48Spatrick 18909467b48Spatrick} 19009467b48Spatrick 19109467b48Spatricklet isTerminator = 1, hasSideEffects = 1, isBarrier = 1, hasCtrlDep = 1, 19209467b48Spatrick isCodeGenOnly = 1, isReturn = 1, isEHScopeReturn = 1 in { 19309467b48Spatrick def CLEANUPRET : I<0, Pseudo, (outs), (ins), "# CLEANUPRET", [(cleanupret)]>; 19409467b48Spatrick 19509467b48Spatrick // CATCHRET needs a custom inserter for SEH. 19609467b48Spatrick let usesCustomInserter = 1 in 19709467b48Spatrick def CATCHRET : I<0, Pseudo, (outs), (ins brtarget32:$dst, brtarget32:$from), 19809467b48Spatrick "# CATCHRET", 19909467b48Spatrick [(catchret bb:$dst, bb:$from)]>; 20009467b48Spatrick} 20109467b48Spatrick 20209467b48Spatricklet hasSideEffects = 1, isBarrier = 1, isCodeGenOnly = 1, 20309467b48Spatrick usesCustomInserter = 1 in { 20409467b48Spatrick def EH_SjLj_SetJmp32 : I<0, Pseudo, (outs GR32:$dst), (ins i32mem:$buf), 20509467b48Spatrick "#EH_SJLJ_SETJMP32", 20609467b48Spatrick [(set GR32:$dst, (X86eh_sjlj_setjmp addr:$buf))]>, 20709467b48Spatrick Requires<[Not64BitMode]>; 20809467b48Spatrick def EH_SjLj_SetJmp64 : I<0, Pseudo, (outs GR32:$dst), (ins i64mem:$buf), 20909467b48Spatrick "#EH_SJLJ_SETJMP64", 21009467b48Spatrick [(set GR32:$dst, (X86eh_sjlj_setjmp addr:$buf))]>, 21109467b48Spatrick Requires<[In64BitMode]>; 21209467b48Spatrick let isTerminator = 1 in { 21309467b48Spatrick def EH_SjLj_LongJmp32 : I<0, Pseudo, (outs), (ins i32mem:$buf), 21409467b48Spatrick "#EH_SJLJ_LONGJMP32", 21509467b48Spatrick [(X86eh_sjlj_longjmp addr:$buf)]>, 21609467b48Spatrick Requires<[Not64BitMode]>; 21709467b48Spatrick def EH_SjLj_LongJmp64 : I<0, Pseudo, (outs), (ins i64mem:$buf), 21809467b48Spatrick "#EH_SJLJ_LONGJMP64", 21909467b48Spatrick [(X86eh_sjlj_longjmp addr:$buf)]>, 22009467b48Spatrick Requires<[In64BitMode]>; 22109467b48Spatrick } 22209467b48Spatrick} 22309467b48Spatrick 22409467b48Spatricklet isBranch = 1, isTerminator = 1, isCodeGenOnly = 1 in { 22509467b48Spatrick def EH_SjLj_Setup : I<0, Pseudo, (outs), (ins brtarget:$dst), 22609467b48Spatrick "#EH_SjLj_Setup\t$dst", []>; 22709467b48Spatrick} 22809467b48Spatrick} // SchedRW 22909467b48Spatrick 23009467b48Spatrick//===----------------------------------------------------------------------===// 23109467b48Spatrick// Pseudo instructions used by unwind info. 23209467b48Spatrick// 23309467b48Spatricklet isPseudo = 1, SchedRW = [WriteSystem] in { 23409467b48Spatrick def SEH_PushReg : I<0, Pseudo, (outs), (ins i32imm:$reg), 23509467b48Spatrick "#SEH_PushReg $reg", []>; 23609467b48Spatrick def SEH_SaveReg : I<0, Pseudo, (outs), (ins i32imm:$reg, i32imm:$dst), 23709467b48Spatrick "#SEH_SaveReg $reg, $dst", []>; 23809467b48Spatrick def SEH_SaveXMM : I<0, Pseudo, (outs), (ins i32imm:$reg, i32imm:$dst), 23909467b48Spatrick "#SEH_SaveXMM $reg, $dst", []>; 24009467b48Spatrick def SEH_StackAlloc : I<0, Pseudo, (outs), (ins i32imm:$size), 24109467b48Spatrick "#SEH_StackAlloc $size", []>; 24209467b48Spatrick def SEH_StackAlign : I<0, Pseudo, (outs), (ins i32imm:$align), 24309467b48Spatrick "#SEH_StackAlign $align", []>; 24409467b48Spatrick def SEH_SetFrame : I<0, Pseudo, (outs), (ins i32imm:$reg, i32imm:$offset), 24509467b48Spatrick "#SEH_SetFrame $reg, $offset", []>; 24609467b48Spatrick def SEH_PushFrame : I<0, Pseudo, (outs), (ins i1imm:$mode), 24709467b48Spatrick "#SEH_PushFrame $mode", []>; 24809467b48Spatrick def SEH_EndPrologue : I<0, Pseudo, (outs), (ins), 24909467b48Spatrick "#SEH_EndPrologue", []>; 25009467b48Spatrick def SEH_Epilogue : I<0, Pseudo, (outs), (ins), 25109467b48Spatrick "#SEH_Epilogue", []>; 25209467b48Spatrick} 25309467b48Spatrick 25409467b48Spatrick//===----------------------------------------------------------------------===// 25509467b48Spatrick// Pseudo instructions used by segmented stacks. 25609467b48Spatrick// 25709467b48Spatrick 25809467b48Spatrick// This is lowered into a RET instruction by MCInstLower. We need 25909467b48Spatrick// this so that we don't have to have a MachineBasicBlock which ends 26009467b48Spatrick// with a RET and also has successors. 26109467b48Spatricklet isPseudo = 1, SchedRW = [WriteJumpLd] in { 26209467b48Spatrickdef MORESTACK_RET: I<0, Pseudo, (outs), (ins), "", []>; 26309467b48Spatrick 26409467b48Spatrick// This instruction is lowered to a RET followed by a MOV. The two 26509467b48Spatrick// instructions are not generated on a higher level since then the 26609467b48Spatrick// verifier sees a MachineBasicBlock ending with a non-terminator. 26709467b48Spatrickdef MORESTACK_RET_RESTORE_R10 : I<0, Pseudo, (outs), (ins), "", []>; 26809467b48Spatrick} 26909467b48Spatrick 27009467b48Spatrick//===----------------------------------------------------------------------===// 271*adae0cfdSpatrick// Pseudo instruction used by retguard 272*adae0cfdSpatrick 273*adae0cfdSpatrick// This is lowered to a JE 2; INT3; INT3. Prior to this pseudo should be a 274*adae0cfdSpatrick// compare instruction to ensure the retguard cookie is correct. 275*adae0cfdSpatrick// We use a pseudo here in order to avoid splitting the BB just before the return. 276*adae0cfdSpatrick// Splitting the BB and inserting a JE_1 over a new INT3 BB occasionally 277*adae0cfdSpatrick// resulted in incorrect code when a value from a byte register (CL) was 278*adae0cfdSpatrick// used as a return value. When emitted as a split BB, the single byte 279*adae0cfdSpatrick// register would sometimes be widened to 4 bytes, which would corrupt 280*adae0cfdSpatrick// the return value (ie mov %ecx, %eax instead of mov %cl, %al). 281*adae0cfdSpatricklet isCodeGenOnly = 1, Uses = [EFLAGS] in { 282*adae0cfdSpatrickdef RETGUARD_JMP_TRAP: I<0, Pseudo, (outs), (ins), "", []>; 283*adae0cfdSpatrick} 284*adae0cfdSpatrick 285*adae0cfdSpatricklet isCodeGenOnly = 1 in { 286*adae0cfdSpatrickdef JMP_TRAP: I<0, Pseudo, (outs), (ins), "", []>; 287*adae0cfdSpatrick} 288*adae0cfdSpatrick 289*adae0cfdSpatrick//===----------------------------------------------------------------------===// 29009467b48Spatrick// Alias Instructions 29109467b48Spatrick//===----------------------------------------------------------------------===// 29209467b48Spatrick 29309467b48Spatrick// Alias instruction mapping movr0 to xor. 29409467b48Spatrick// FIXME: remove when we can teach regalloc that xor reg, reg is ok. 29509467b48Spatricklet Defs = [EFLAGS], isReMaterializable = 1, isAsCheapAsAMove = 1, 29609467b48Spatrick isPseudo = 1, isMoveImm = 1, AddedComplexity = 10 in 29709467b48Spatrickdef MOV32r0 : I<0, Pseudo, (outs GR32:$dst), (ins), "", 29809467b48Spatrick [(set GR32:$dst, 0)]>, Sched<[WriteZero]>; 29909467b48Spatrick 30009467b48Spatrick// Other widths can also make use of the 32-bit xor, which may have a smaller 30109467b48Spatrick// encoding and avoid partial register updates. 30209467b48Spatricklet AddedComplexity = 10 in { 30309467b48Spatrickdef : Pat<(i8 0), (EXTRACT_SUBREG (MOV32r0), sub_8bit)>; 30409467b48Spatrickdef : Pat<(i16 0), (EXTRACT_SUBREG (MOV32r0), sub_16bit)>; 30509467b48Spatrickdef : Pat<(i64 0), (SUBREG_TO_REG (i64 0), (MOV32r0), sub_32bit)>; 30609467b48Spatrick} 30709467b48Spatrick 30809467b48Spatricklet Predicates = [OptForSize, Not64BitMode], 30909467b48Spatrick AddedComplexity = 10 in { 31009467b48Spatrick let SchedRW = [WriteALU] in { 31109467b48Spatrick // Pseudo instructions for materializing 1 and -1 using XOR+INC/DEC, 31209467b48Spatrick // which only require 3 bytes compared to MOV32ri which requires 5. 31309467b48Spatrick let Defs = [EFLAGS], isReMaterializable = 1, isPseudo = 1 in { 31409467b48Spatrick def MOV32r1 : I<0, Pseudo, (outs GR32:$dst), (ins), "", 31509467b48Spatrick [(set GR32:$dst, 1)]>; 31609467b48Spatrick def MOV32r_1 : I<0, Pseudo, (outs GR32:$dst), (ins), "", 31709467b48Spatrick [(set GR32:$dst, -1)]>; 31809467b48Spatrick } 31909467b48Spatrick } // SchedRW 32009467b48Spatrick 32109467b48Spatrick // MOV16ri is 4 bytes, so the instructions above are smaller. 32209467b48Spatrick def : Pat<(i16 1), (EXTRACT_SUBREG (MOV32r1), sub_16bit)>; 32309467b48Spatrick def : Pat<(i16 -1), (EXTRACT_SUBREG (MOV32r_1), sub_16bit)>; 32409467b48Spatrick} 32509467b48Spatrick 32609467b48Spatricklet isReMaterializable = 1, isPseudo = 1, AddedComplexity = 5, 32709467b48Spatrick SchedRW = [WriteALU] in { 32809467b48Spatrick// AddedComplexity higher than MOV64ri but lower than MOV32r0 and MOV32r1. 32909467b48Spatrickdef MOV32ImmSExti8 : I<0, Pseudo, (outs GR32:$dst), (ins i32i8imm:$src), "", 33009467b48Spatrick [(set GR32:$dst, i32immSExt8:$src)]>, 33109467b48Spatrick Requires<[OptForMinSize, NotWin64WithoutFP]>; 33209467b48Spatrickdef MOV64ImmSExti8 : I<0, Pseudo, (outs GR64:$dst), (ins i64i8imm:$src), "", 33309467b48Spatrick [(set GR64:$dst, i64immSExt8:$src)]>, 33409467b48Spatrick Requires<[OptForMinSize, NotWin64WithoutFP]>; 33509467b48Spatrick} 33609467b48Spatrick 33709467b48Spatrick// Materialize i64 constant where top 32-bits are zero. This could theoretically 33809467b48Spatrick// use MOV32ri with a SUBREG_TO_REG to represent the zero-extension, however 33909467b48Spatrick// that would make it more difficult to rematerialize. 340097a140dSpatricklet AddedComplexity = 1, isReMaterializable = 1, isAsCheapAsAMove = 1, 341097a140dSpatrick isPseudo = 1, SchedRW = [WriteMove] in 342097a140dSpatrickdef MOV32ri64 : I<0, Pseudo, (outs GR64:$dst), (ins i64i32imm:$src), "", 343097a140dSpatrick [(set GR64:$dst, i64immZExt32:$src)]>; 34409467b48Spatrick 345097a140dSpatrick// This 64-bit pseudo-move can also be used for labels in the x86-64 small code 346097a140dSpatrick// model. 347097a140dSpatrickdef mov64imm32 : ComplexPattern<i64, 1, "selectMOV64Imm32", [X86Wrapper]>; 34809467b48Spatrickdef : Pat<(i64 mov64imm32:$src), (MOV32ri64 mov64imm32:$src)>; 34909467b48Spatrick 35009467b48Spatrick// Use sbb to materialize carry bit. 351097a140dSpatricklet Uses = [EFLAGS], Defs = [EFLAGS], isPseudo = 1, SchedRW = [WriteADC], 352097a140dSpatrick hasSideEffects = 0 in { 35309467b48Spatrick// FIXME: These are pseudo ops that should be replaced with Pat<> patterns. 35409467b48Spatrick// However, Pat<> can't replicate the destination reg into the inputs of the 35509467b48Spatrick// result. 356097a140dSpatrickdef SETB_C32r : I<0, Pseudo, (outs GR32:$dst), (ins), "", []>; 357097a140dSpatrickdef SETB_C64r : I<0, Pseudo, (outs GR64:$dst), (ins), "", []>; 35809467b48Spatrick} // isCodeGenOnly 35909467b48Spatrick 36009467b48Spatrick//===----------------------------------------------------------------------===// 36109467b48Spatrick// String Pseudo Instructions 36209467b48Spatrick// 36309467b48Spatricklet SchedRW = [WriteMicrocoded] in { 36409467b48Spatricklet Defs = [ECX,EDI,ESI], Uses = [ECX,EDI,ESI], isCodeGenOnly = 1 in { 36509467b48Spatrickdef REP_MOVSB_32 : I<0xA4, RawFrm, (outs), (ins), 36609467b48Spatrick "{rep;movsb (%esi), %es:(%edi)|rep movsb es:[edi], [esi]}", 36709467b48Spatrick [(X86rep_movs i8)]>, REP, AdSize32, 36809467b48Spatrick Requires<[NotLP64]>; 36909467b48Spatrickdef REP_MOVSW_32 : I<0xA5, RawFrm, (outs), (ins), 37009467b48Spatrick "{rep;movsw (%esi), %es:(%edi)|rep movsw es:[edi], [esi]}", 37109467b48Spatrick [(X86rep_movs i16)]>, REP, AdSize32, OpSize16, 37209467b48Spatrick Requires<[NotLP64]>; 37309467b48Spatrickdef REP_MOVSD_32 : I<0xA5, RawFrm, (outs), (ins), 37409467b48Spatrick "{rep;movsl (%esi), %es:(%edi)|rep movsd es:[edi], [esi]}", 37509467b48Spatrick [(X86rep_movs i32)]>, REP, AdSize32, OpSize32, 37609467b48Spatrick Requires<[NotLP64]>; 37709467b48Spatrickdef REP_MOVSQ_32 : RI<0xA5, RawFrm, (outs), (ins), 37809467b48Spatrick "{rep;movsq (%esi), %es:(%edi)|rep movsq es:[edi], [esi]}", 37909467b48Spatrick [(X86rep_movs i64)]>, REP, AdSize32, 38009467b48Spatrick Requires<[NotLP64, In64BitMode]>; 38109467b48Spatrick} 38209467b48Spatrick 38309467b48Spatricklet Defs = [RCX,RDI,RSI], Uses = [RCX,RDI,RSI], isCodeGenOnly = 1 in { 38409467b48Spatrickdef REP_MOVSB_64 : I<0xA4, RawFrm, (outs), (ins), 38509467b48Spatrick "{rep;movsb (%rsi), %es:(%rdi)|rep movsb es:[rdi], [rsi]}", 38609467b48Spatrick [(X86rep_movs i8)]>, REP, AdSize64, 38709467b48Spatrick Requires<[IsLP64]>; 38809467b48Spatrickdef REP_MOVSW_64 : I<0xA5, RawFrm, (outs), (ins), 38909467b48Spatrick "{rep;movsw (%rsi), %es:(%rdi)|rep movsw es:[rdi], [rsi]}", 39009467b48Spatrick [(X86rep_movs i16)]>, REP, AdSize64, OpSize16, 39109467b48Spatrick Requires<[IsLP64]>; 39209467b48Spatrickdef REP_MOVSD_64 : I<0xA5, RawFrm, (outs), (ins), 39309467b48Spatrick "{rep;movsl (%rsi), %es:(%rdi)|rep movsdi es:[rdi], [rsi]}", 39409467b48Spatrick [(X86rep_movs i32)]>, REP, AdSize64, OpSize32, 39509467b48Spatrick Requires<[IsLP64]>; 39609467b48Spatrickdef REP_MOVSQ_64 : RI<0xA5, RawFrm, (outs), (ins), 39709467b48Spatrick "{rep;movsq (%rsi), %es:(%rdi)|rep movsq es:[rdi], [rsi]}", 39809467b48Spatrick [(X86rep_movs i64)]>, REP, AdSize64, 39909467b48Spatrick Requires<[IsLP64]>; 40009467b48Spatrick} 40109467b48Spatrick 40209467b48Spatrick// FIXME: Should use "(X86rep_stos AL)" as the pattern. 40309467b48Spatricklet Defs = [ECX,EDI], isCodeGenOnly = 1 in { 40409467b48Spatrick let Uses = [AL,ECX,EDI] in 40509467b48Spatrick def REP_STOSB_32 : I<0xAA, RawFrm, (outs), (ins), 40609467b48Spatrick "{rep;stosb %al, %es:(%edi)|rep stosb es:[edi], al}", 40709467b48Spatrick [(X86rep_stos i8)]>, REP, AdSize32, 40809467b48Spatrick Requires<[NotLP64]>; 40909467b48Spatrick let Uses = [AX,ECX,EDI] in 41009467b48Spatrick def REP_STOSW_32 : I<0xAB, RawFrm, (outs), (ins), 41109467b48Spatrick "{rep;stosw %ax, %es:(%edi)|rep stosw es:[edi], ax}", 41209467b48Spatrick [(X86rep_stos i16)]>, REP, AdSize32, OpSize16, 41309467b48Spatrick Requires<[NotLP64]>; 41409467b48Spatrick let Uses = [EAX,ECX,EDI] in 41509467b48Spatrick def REP_STOSD_32 : I<0xAB, RawFrm, (outs), (ins), 41609467b48Spatrick "{rep;stosl %eax, %es:(%edi)|rep stosd es:[edi], eax}", 41709467b48Spatrick [(X86rep_stos i32)]>, REP, AdSize32, OpSize32, 41809467b48Spatrick Requires<[NotLP64]>; 41909467b48Spatrick let Uses = [RAX,RCX,RDI] in 42009467b48Spatrick def REP_STOSQ_32 : RI<0xAB, RawFrm, (outs), (ins), 42109467b48Spatrick "{rep;stosq %rax, %es:(%edi)|rep stosq es:[edi], rax}", 42209467b48Spatrick [(X86rep_stos i64)]>, REP, AdSize32, 42309467b48Spatrick Requires<[NotLP64, In64BitMode]>; 42409467b48Spatrick} 42509467b48Spatrick 42609467b48Spatricklet Defs = [RCX,RDI], isCodeGenOnly = 1 in { 42709467b48Spatrick let Uses = [AL,RCX,RDI] in 42809467b48Spatrick def REP_STOSB_64 : I<0xAA, RawFrm, (outs), (ins), 42909467b48Spatrick "{rep;stosb %al, %es:(%rdi)|rep stosb es:[rdi], al}", 43009467b48Spatrick [(X86rep_stos i8)]>, REP, AdSize64, 43109467b48Spatrick Requires<[IsLP64]>; 43209467b48Spatrick let Uses = [AX,RCX,RDI] in 43309467b48Spatrick def REP_STOSW_64 : I<0xAB, RawFrm, (outs), (ins), 43409467b48Spatrick "{rep;stosw %ax, %es:(%rdi)|rep stosw es:[rdi], ax}", 43509467b48Spatrick [(X86rep_stos i16)]>, REP, AdSize64, OpSize16, 43609467b48Spatrick Requires<[IsLP64]>; 43709467b48Spatrick let Uses = [RAX,RCX,RDI] in 43809467b48Spatrick def REP_STOSD_64 : I<0xAB, RawFrm, (outs), (ins), 43909467b48Spatrick "{rep;stosl %eax, %es:(%rdi)|rep stosd es:[rdi], eax}", 44009467b48Spatrick [(X86rep_stos i32)]>, REP, AdSize64, OpSize32, 44109467b48Spatrick Requires<[IsLP64]>; 44209467b48Spatrick 44309467b48Spatrick let Uses = [RAX,RCX,RDI] in 44409467b48Spatrick def REP_STOSQ_64 : RI<0xAB, RawFrm, (outs), (ins), 44509467b48Spatrick "{rep;stosq %rax, %es:(%rdi)|rep stosq es:[rdi], rax}", 44609467b48Spatrick [(X86rep_stos i64)]>, REP, AdSize64, 44709467b48Spatrick Requires<[IsLP64]>; 44809467b48Spatrick} 44909467b48Spatrick} // SchedRW 45009467b48Spatrick 45109467b48Spatrick//===----------------------------------------------------------------------===// 45209467b48Spatrick// Thread Local Storage Instructions 45309467b48Spatrick// 45409467b48Spatricklet SchedRW = [WriteSystem] in { 45509467b48Spatrick 45609467b48Spatrick// ELF TLS Support 45709467b48Spatrick// All calls clobber the non-callee saved registers. ESP is marked as 45809467b48Spatrick// a use to prevent stack-pointer assignments that appear immediately 45909467b48Spatrick// before calls from potentially appearing dead. 46009467b48Spatricklet Defs = [EAX, ECX, EDX, FP0, FP1, FP2, FP3, FP4, FP5, FP6, FP7, 46109467b48Spatrick ST0, ST1, ST2, ST3, ST4, ST5, ST6, ST7, 46209467b48Spatrick MM0, MM1, MM2, MM3, MM4, MM5, MM6, MM7, 46309467b48Spatrick XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7, 46409467b48Spatrick XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15, EFLAGS, DF], 46509467b48Spatrick usesCustomInserter = 1, Uses = [ESP, SSP] in { 46609467b48Spatrickdef TLS_addr32 : I<0, Pseudo, (outs), (ins i32mem:$sym), 46709467b48Spatrick "# TLS_addr32", 46809467b48Spatrick [(X86tlsaddr tls32addr:$sym)]>, 46909467b48Spatrick Requires<[Not64BitMode]>; 47009467b48Spatrickdef TLS_base_addr32 : I<0, Pseudo, (outs), (ins i32mem:$sym), 47109467b48Spatrick "# TLS_base_addr32", 47209467b48Spatrick [(X86tlsbaseaddr tls32baseaddr:$sym)]>, 47309467b48Spatrick Requires<[Not64BitMode]>; 47409467b48Spatrick} 47509467b48Spatrick 47609467b48Spatrick// All calls clobber the non-callee saved registers. RSP is marked as 47709467b48Spatrick// a use to prevent stack-pointer assignments that appear immediately 47809467b48Spatrick// before calls from potentially appearing dead. 47909467b48Spatricklet Defs = [RAX, RCX, RDX, RSI, RDI, R8, R9, R10, R11, 48009467b48Spatrick FP0, FP1, FP2, FP3, FP4, FP5, FP6, FP7, 48109467b48Spatrick ST0, ST1, ST2, ST3, ST4, ST5, ST6, ST7, 48209467b48Spatrick MM0, MM1, MM2, MM3, MM4, MM5, MM6, MM7, 48309467b48Spatrick XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7, 48409467b48Spatrick XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15, EFLAGS, DF], 48509467b48Spatrick usesCustomInserter = 1, Uses = [RSP, SSP] in { 48609467b48Spatrickdef TLS_addr64 : I<0, Pseudo, (outs), (ins i64mem:$sym), 48709467b48Spatrick "# TLS_addr64", 48809467b48Spatrick [(X86tlsaddr tls64addr:$sym)]>, 48909467b48Spatrick Requires<[In64BitMode]>; 49009467b48Spatrickdef TLS_base_addr64 : I<0, Pseudo, (outs), (ins i64mem:$sym), 49109467b48Spatrick "# TLS_base_addr64", 49209467b48Spatrick [(X86tlsbaseaddr tls64baseaddr:$sym)]>, 49309467b48Spatrick Requires<[In64BitMode]>; 49409467b48Spatrick} 49509467b48Spatrick 49609467b48Spatrick// Darwin TLS Support 49709467b48Spatrick// For i386, the address of the thunk is passed on the stack, on return the 49809467b48Spatrick// address of the variable is in %eax. %ecx is trashed during the function 49909467b48Spatrick// call. All other registers are preserved. 50009467b48Spatricklet Defs = [EAX, ECX, EFLAGS, DF], 50109467b48Spatrick Uses = [ESP, SSP], 50209467b48Spatrick usesCustomInserter = 1 in 50309467b48Spatrickdef TLSCall_32 : I<0, Pseudo, (outs), (ins i32mem:$sym), 50409467b48Spatrick "# TLSCall_32", 50509467b48Spatrick [(X86TLSCall addr:$sym)]>, 50609467b48Spatrick Requires<[Not64BitMode]>; 50709467b48Spatrick 50809467b48Spatrick// For x86_64, the address of the thunk is passed in %rdi, but the 50909467b48Spatrick// pseudo directly use the symbol, so do not add an implicit use of 51009467b48Spatrick// %rdi. The lowering will do the right thing with RDI. 51109467b48Spatrick// On return the address of the variable is in %rax. All other 51209467b48Spatrick// registers are preserved. 51309467b48Spatricklet Defs = [RAX, EFLAGS, DF], 51409467b48Spatrick Uses = [RSP, SSP], 51509467b48Spatrick usesCustomInserter = 1 in 51609467b48Spatrickdef TLSCall_64 : I<0, Pseudo, (outs), (ins i64mem:$sym), 51709467b48Spatrick "# TLSCall_64", 51809467b48Spatrick [(X86TLSCall addr:$sym)]>, 51909467b48Spatrick Requires<[In64BitMode]>; 52009467b48Spatrick} // SchedRW 52109467b48Spatrick 52209467b48Spatrick//===----------------------------------------------------------------------===// 52309467b48Spatrick// Conditional Move Pseudo Instructions 52409467b48Spatrick 52509467b48Spatrick// CMOV* - Used to implement the SELECT DAG operation. Expanded after 52609467b48Spatrick// instruction selection into a branch sequence. 52709467b48Spatrickmulticlass CMOVrr_PSEUDO<RegisterClass RC, ValueType VT> { 52809467b48Spatrick def CMOV#NAME : I<0, Pseudo, 52909467b48Spatrick (outs RC:$dst), (ins RC:$t, RC:$f, i8imm:$cond), 53009467b48Spatrick "#CMOV_"#NAME#" PSEUDO!", 53109467b48Spatrick [(set RC:$dst, (VT (X86cmov RC:$t, RC:$f, timm:$cond, 53209467b48Spatrick EFLAGS)))]>; 53309467b48Spatrick} 53409467b48Spatrick 53509467b48Spatricklet usesCustomInserter = 1, hasNoSchedulingInfo = 1, Uses = [EFLAGS] in { 53609467b48Spatrick // X86 doesn't have 8-bit conditional moves. Use a customInserter to 53709467b48Spatrick // emit control flow. An alternative to this is to mark i8 SELECT as Promote, 53809467b48Spatrick // however that requires promoting the operands, and can induce additional 53909467b48Spatrick // i8 register pressure. 54009467b48Spatrick defm _GR8 : CMOVrr_PSEUDO<GR8, i8>; 54109467b48Spatrick 54209467b48Spatrick let Predicates = [NoCMov] in { 54309467b48Spatrick defm _GR32 : CMOVrr_PSEUDO<GR32, i32>; 54409467b48Spatrick defm _GR16 : CMOVrr_PSEUDO<GR16, i16>; 54509467b48Spatrick } // Predicates = [NoCMov] 54609467b48Spatrick 54709467b48Spatrick // fcmov doesn't handle all possible EFLAGS, provide a fallback if there is no 54809467b48Spatrick // SSE1/SSE2. 54909467b48Spatrick let Predicates = [FPStackf32] in 55009467b48Spatrick defm _RFP32 : CMOVrr_PSEUDO<RFP32, f32>; 55109467b48Spatrick 55209467b48Spatrick let Predicates = [FPStackf64] in 55309467b48Spatrick defm _RFP64 : CMOVrr_PSEUDO<RFP64, f64>; 55409467b48Spatrick 55509467b48Spatrick defm _RFP80 : CMOVrr_PSEUDO<RFP80, f80>; 55609467b48Spatrick 557097a140dSpatrick let Predicates = [HasMMX] in 558097a140dSpatrick defm _VR64 : CMOVrr_PSEUDO<VR64, x86mmx>; 559097a140dSpatrick 560097a140dSpatrick let Predicates = [HasSSE1,NoAVX512] in 56109467b48Spatrick defm _FR32 : CMOVrr_PSEUDO<FR32, f32>; 562097a140dSpatrick let Predicates = [HasSSE2,NoAVX512] in 56309467b48Spatrick defm _FR64 : CMOVrr_PSEUDO<FR64, f64>; 56409467b48Spatrick let Predicates = [HasAVX512] in { 56509467b48Spatrick defm _FR32X : CMOVrr_PSEUDO<FR32X, f32>; 56609467b48Spatrick defm _FR64X : CMOVrr_PSEUDO<FR64X, f64>; 56709467b48Spatrick } 56809467b48Spatrick let Predicates = [NoVLX] in { 56909467b48Spatrick defm _VR128 : CMOVrr_PSEUDO<VR128, v2i64>; 57009467b48Spatrick defm _VR256 : CMOVrr_PSEUDO<VR256, v4i64>; 57109467b48Spatrick } 57209467b48Spatrick let Predicates = [HasVLX] in { 57309467b48Spatrick defm _VR128X : CMOVrr_PSEUDO<VR128X, v2i64>; 57409467b48Spatrick defm _VR256X : CMOVrr_PSEUDO<VR256X, v4i64>; 57509467b48Spatrick } 57609467b48Spatrick defm _VR512 : CMOVrr_PSEUDO<VR512, v8i64>; 577097a140dSpatrick defm _VK1 : CMOVrr_PSEUDO<VK1, v1i1>; 57809467b48Spatrick defm _VK2 : CMOVrr_PSEUDO<VK2, v2i1>; 57909467b48Spatrick defm _VK4 : CMOVrr_PSEUDO<VK4, v4i1>; 58009467b48Spatrick defm _VK8 : CMOVrr_PSEUDO<VK8, v8i1>; 58109467b48Spatrick defm _VK16 : CMOVrr_PSEUDO<VK16, v16i1>; 58209467b48Spatrick defm _VK32 : CMOVrr_PSEUDO<VK32, v32i1>; 58309467b48Spatrick defm _VK64 : CMOVrr_PSEUDO<VK64, v64i1>; 58409467b48Spatrick} // usesCustomInserter = 1, hasNoSchedulingInfo = 1, Uses = [EFLAGS] 58509467b48Spatrick 58609467b48Spatrickdef : Pat<(f128 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 58709467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 58809467b48Spatrick 58909467b48Spatricklet Predicates = [NoVLX] in { 59009467b48Spatrick def : Pat<(v16i8 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 59109467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 59209467b48Spatrick def : Pat<(v8i16 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 59309467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 59409467b48Spatrick def : Pat<(v4i32 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 59509467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 59609467b48Spatrick def : Pat<(v4f32 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 59709467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 59809467b48Spatrick def : Pat<(v2f64 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 59909467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 60009467b48Spatrick 60109467b48Spatrick def : Pat<(v32i8 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)), 60209467b48Spatrick (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>; 60309467b48Spatrick def : Pat<(v16i16 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)), 60409467b48Spatrick (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>; 60509467b48Spatrick def : Pat<(v8i32 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)), 60609467b48Spatrick (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>; 60709467b48Spatrick def : Pat<(v8f32 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)), 60809467b48Spatrick (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>; 60909467b48Spatrick def : Pat<(v4f64 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)), 61009467b48Spatrick (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>; 61109467b48Spatrick} 61209467b48Spatricklet Predicates = [HasVLX] in { 61309467b48Spatrick def : Pat<(v16i8 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)), 61409467b48Spatrick (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>; 61509467b48Spatrick def : Pat<(v8i16 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)), 61609467b48Spatrick (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>; 61709467b48Spatrick def : Pat<(v4i32 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)), 61809467b48Spatrick (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>; 61909467b48Spatrick def : Pat<(v4f32 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)), 62009467b48Spatrick (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>; 62109467b48Spatrick def : Pat<(v2f64 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)), 62209467b48Spatrick (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>; 62309467b48Spatrick 62409467b48Spatrick def : Pat<(v32i8 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)), 62509467b48Spatrick (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>; 62609467b48Spatrick def : Pat<(v16i16 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)), 62709467b48Spatrick (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>; 62809467b48Spatrick def : Pat<(v8i32 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)), 62909467b48Spatrick (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>; 63009467b48Spatrick def : Pat<(v8f32 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)), 63109467b48Spatrick (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>; 63209467b48Spatrick def : Pat<(v4f64 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)), 63309467b48Spatrick (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>; 63409467b48Spatrick} 63509467b48Spatrick 63609467b48Spatrickdef : Pat<(v64i8 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)), 63709467b48Spatrick (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>; 63809467b48Spatrickdef : Pat<(v32i16 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)), 63909467b48Spatrick (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>; 64009467b48Spatrickdef : Pat<(v16i32 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)), 64109467b48Spatrick (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>; 64209467b48Spatrickdef : Pat<(v16f32 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)), 64309467b48Spatrick (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>; 64409467b48Spatrickdef : Pat<(v8f64 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)), 64509467b48Spatrick (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>; 64609467b48Spatrick 64709467b48Spatrick//===----------------------------------------------------------------------===// 64809467b48Spatrick// Normal-Instructions-With-Lock-Prefix Pseudo Instructions 64909467b48Spatrick//===----------------------------------------------------------------------===// 65009467b48Spatrick 65109467b48Spatrick// FIXME: Use normal instructions and add lock prefix dynamically. 65209467b48Spatrick 65309467b48Spatrick// Memory barriers 65409467b48Spatrick 65509467b48Spatricklet isCodeGenOnly = 1, Defs = [EFLAGS] in 65609467b48Spatrickdef OR32mi8Locked : Ii8<0x83, MRM1m, (outs), (ins i32mem:$dst, i32i8imm:$zero), 65709467b48Spatrick "or{l}\t{$zero, $dst|$dst, $zero}", []>, 65809467b48Spatrick Requires<[Not64BitMode]>, OpSize32, LOCK, 65909467b48Spatrick Sched<[WriteALURMW]>; 66009467b48Spatrick 66109467b48Spatricklet hasSideEffects = 1 in 66209467b48Spatrickdef Int_MemBarrier : I<0, Pseudo, (outs), (ins), 66309467b48Spatrick "#MEMBARRIER", 66409467b48Spatrick [(X86MemBarrier)]>, Sched<[WriteLoad]>; 66509467b48Spatrick 66609467b48Spatrick// RegOpc corresponds to the mr version of the instruction 66709467b48Spatrick// ImmOpc corresponds to the mi version of the instruction 66809467b48Spatrick// ImmOpc8 corresponds to the mi8 version of the instruction 66909467b48Spatrick// ImmMod corresponds to the instruction format of the mi and mi8 versions 67009467b48Spatrickmulticlass LOCK_ArithBinOp<bits<8> RegOpc, bits<8> ImmOpc, bits<8> ImmOpc8, 67109467b48Spatrick Format ImmMod, SDNode Op, string mnemonic> { 67209467b48Spatricklet Defs = [EFLAGS], mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, 67309467b48Spatrick SchedRW = [WriteALURMW] in { 67409467b48Spatrick 67509467b48Spatrickdef NAME#8mr : I<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4}, 67609467b48Spatrick RegOpc{3}, RegOpc{2}, RegOpc{1}, 0 }, 67709467b48Spatrick MRMDestMem, (outs), (ins i8mem:$dst, GR8:$src2), 67809467b48Spatrick !strconcat(mnemonic, "{b}\t", 67909467b48Spatrick "{$src2, $dst|$dst, $src2}"), 68009467b48Spatrick [(set EFLAGS, (Op addr:$dst, GR8:$src2))]>, LOCK; 68109467b48Spatrick 68209467b48Spatrickdef NAME#16mr : I<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4}, 68309467b48Spatrick RegOpc{3}, RegOpc{2}, RegOpc{1}, 1 }, 68409467b48Spatrick MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src2), 68509467b48Spatrick !strconcat(mnemonic, "{w}\t", 68609467b48Spatrick "{$src2, $dst|$dst, $src2}"), 68709467b48Spatrick [(set EFLAGS, (Op addr:$dst, GR16:$src2))]>, 68809467b48Spatrick OpSize16, LOCK; 68909467b48Spatrick 69009467b48Spatrickdef NAME#32mr : I<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4}, 69109467b48Spatrick RegOpc{3}, RegOpc{2}, RegOpc{1}, 1 }, 69209467b48Spatrick MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src2), 69309467b48Spatrick !strconcat(mnemonic, "{l}\t", 69409467b48Spatrick "{$src2, $dst|$dst, $src2}"), 69509467b48Spatrick [(set EFLAGS, (Op addr:$dst, GR32:$src2))]>, 69609467b48Spatrick OpSize32, LOCK; 69709467b48Spatrick 69809467b48Spatrickdef NAME#64mr : RI<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4}, 69909467b48Spatrick RegOpc{3}, RegOpc{2}, RegOpc{1}, 1 }, 70009467b48Spatrick MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src2), 70109467b48Spatrick !strconcat(mnemonic, "{q}\t", 70209467b48Spatrick "{$src2, $dst|$dst, $src2}"), 70309467b48Spatrick [(set EFLAGS, (Op addr:$dst, GR64:$src2))]>, LOCK; 70409467b48Spatrick 70509467b48Spatrick// NOTE: These are order specific, we want the mi8 forms to be listed 70609467b48Spatrick// first so that they are slightly preferred to the mi forms. 70709467b48Spatrickdef NAME#16mi8 : Ii8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4}, 70809467b48Spatrick ImmOpc8{3}, ImmOpc8{2}, ImmOpc8{1}, 1 }, 70909467b48Spatrick ImmMod, (outs), (ins i16mem :$dst, i16i8imm :$src2), 71009467b48Spatrick !strconcat(mnemonic, "{w}\t", 71109467b48Spatrick "{$src2, $dst|$dst, $src2}"), 71209467b48Spatrick [(set EFLAGS, (Op addr:$dst, i16immSExt8:$src2))]>, 71309467b48Spatrick OpSize16, LOCK; 71409467b48Spatrick 71509467b48Spatrickdef NAME#32mi8 : Ii8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4}, 71609467b48Spatrick ImmOpc8{3}, ImmOpc8{2}, ImmOpc8{1}, 1 }, 71709467b48Spatrick ImmMod, (outs), (ins i32mem :$dst, i32i8imm :$src2), 71809467b48Spatrick !strconcat(mnemonic, "{l}\t", 71909467b48Spatrick "{$src2, $dst|$dst, $src2}"), 72009467b48Spatrick [(set EFLAGS, (Op addr:$dst, i32immSExt8:$src2))]>, 72109467b48Spatrick OpSize32, LOCK; 72209467b48Spatrick 72309467b48Spatrickdef NAME#64mi8 : RIi8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4}, 72409467b48Spatrick ImmOpc8{3}, ImmOpc8{2}, ImmOpc8{1}, 1 }, 72509467b48Spatrick ImmMod, (outs), (ins i64mem :$dst, i64i8imm :$src2), 72609467b48Spatrick !strconcat(mnemonic, "{q}\t", 72709467b48Spatrick "{$src2, $dst|$dst, $src2}"), 72809467b48Spatrick [(set EFLAGS, (Op addr:$dst, i64immSExt8:$src2))]>, 72909467b48Spatrick LOCK; 73009467b48Spatrick 73109467b48Spatrickdef NAME#8mi : Ii8<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4}, 73209467b48Spatrick ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 0 }, 73309467b48Spatrick ImmMod, (outs), (ins i8mem :$dst, i8imm :$src2), 73409467b48Spatrick !strconcat(mnemonic, "{b}\t", 73509467b48Spatrick "{$src2, $dst|$dst, $src2}"), 73609467b48Spatrick [(set EFLAGS, (Op addr:$dst, (i8 imm:$src2)))]>, LOCK; 73709467b48Spatrick 73809467b48Spatrickdef NAME#16mi : Ii16<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4}, 73909467b48Spatrick ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 1 }, 74009467b48Spatrick ImmMod, (outs), (ins i16mem :$dst, i16imm :$src2), 74109467b48Spatrick !strconcat(mnemonic, "{w}\t", 74209467b48Spatrick "{$src2, $dst|$dst, $src2}"), 74309467b48Spatrick [(set EFLAGS, (Op addr:$dst, (i16 imm:$src2)))]>, 74409467b48Spatrick OpSize16, LOCK; 74509467b48Spatrick 74609467b48Spatrickdef NAME#32mi : Ii32<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4}, 74709467b48Spatrick ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 1 }, 74809467b48Spatrick ImmMod, (outs), (ins i32mem :$dst, i32imm :$src2), 74909467b48Spatrick !strconcat(mnemonic, "{l}\t", 75009467b48Spatrick "{$src2, $dst|$dst, $src2}"), 75109467b48Spatrick [(set EFLAGS, (Op addr:$dst, (i32 imm:$src2)))]>, 75209467b48Spatrick OpSize32, LOCK; 75309467b48Spatrick 75409467b48Spatrickdef NAME#64mi32 : RIi32S<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4}, 75509467b48Spatrick ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 1 }, 75609467b48Spatrick ImmMod, (outs), (ins i64mem :$dst, i64i32imm :$src2), 75709467b48Spatrick !strconcat(mnemonic, "{q}\t", 75809467b48Spatrick "{$src2, $dst|$dst, $src2}"), 75909467b48Spatrick [(set EFLAGS, (Op addr:$dst, i64immSExt32:$src2))]>, 76009467b48Spatrick LOCK; 76109467b48Spatrick} 76209467b48Spatrick 76309467b48Spatrick} 76409467b48Spatrick 76509467b48Spatrickdefm LOCK_ADD : LOCK_ArithBinOp<0x00, 0x80, 0x83, MRM0m, X86lock_add, "add">; 76609467b48Spatrickdefm LOCK_SUB : LOCK_ArithBinOp<0x28, 0x80, 0x83, MRM5m, X86lock_sub, "sub">; 76709467b48Spatrickdefm LOCK_OR : LOCK_ArithBinOp<0x08, 0x80, 0x83, MRM1m, X86lock_or , "or">; 76809467b48Spatrickdefm LOCK_AND : LOCK_ArithBinOp<0x20, 0x80, 0x83, MRM4m, X86lock_and, "and">; 76909467b48Spatrickdefm LOCK_XOR : LOCK_ArithBinOp<0x30, 0x80, 0x83, MRM6m, X86lock_xor, "xor">; 77009467b48Spatrick 77109467b48Spatrickdef X86lock_add_nocf : PatFrag<(ops node:$lhs, node:$rhs), 77209467b48Spatrick (X86lock_add node:$lhs, node:$rhs), [{ 77309467b48Spatrick return hasNoCarryFlagUses(SDValue(N, 0)); 77409467b48Spatrick}]>; 77509467b48Spatrick 77609467b48Spatrickdef X86lock_sub_nocf : PatFrag<(ops node:$lhs, node:$rhs), 77709467b48Spatrick (X86lock_sub node:$lhs, node:$rhs), [{ 77809467b48Spatrick return hasNoCarryFlagUses(SDValue(N, 0)); 77909467b48Spatrick}]>; 78009467b48Spatrick 78109467b48Spatricklet Predicates = [UseIncDec] in { 78209467b48Spatrick let Defs = [EFLAGS], mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, 78309467b48Spatrick SchedRW = [WriteALURMW] in { 78409467b48Spatrick def LOCK_INC8m : I<0xFE, MRM0m, (outs), (ins i8mem :$dst), 78509467b48Spatrick "inc{b}\t$dst", 78609467b48Spatrick [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i8 1)))]>, 78709467b48Spatrick LOCK; 78809467b48Spatrick def LOCK_INC16m : I<0xFF, MRM0m, (outs), (ins i16mem:$dst), 78909467b48Spatrick "inc{w}\t$dst", 79009467b48Spatrick [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i16 1)))]>, 79109467b48Spatrick OpSize16, LOCK; 79209467b48Spatrick def LOCK_INC32m : I<0xFF, MRM0m, (outs), (ins i32mem:$dst), 79309467b48Spatrick "inc{l}\t$dst", 79409467b48Spatrick [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i32 1)))]>, 79509467b48Spatrick OpSize32, LOCK; 79609467b48Spatrick def LOCK_INC64m : RI<0xFF, MRM0m, (outs), (ins i64mem:$dst), 79709467b48Spatrick "inc{q}\t$dst", 79809467b48Spatrick [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i64 1)))]>, 79909467b48Spatrick LOCK; 80009467b48Spatrick 80109467b48Spatrick def LOCK_DEC8m : I<0xFE, MRM1m, (outs), (ins i8mem :$dst), 80209467b48Spatrick "dec{b}\t$dst", 80309467b48Spatrick [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i8 1)))]>, 80409467b48Spatrick LOCK; 80509467b48Spatrick def LOCK_DEC16m : I<0xFF, MRM1m, (outs), (ins i16mem:$dst), 80609467b48Spatrick "dec{w}\t$dst", 80709467b48Spatrick [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i16 1)))]>, 80809467b48Spatrick OpSize16, LOCK; 80909467b48Spatrick def LOCK_DEC32m : I<0xFF, MRM1m, (outs), (ins i32mem:$dst), 81009467b48Spatrick "dec{l}\t$dst", 81109467b48Spatrick [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i32 1)))]>, 81209467b48Spatrick OpSize32, LOCK; 81309467b48Spatrick def LOCK_DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), 81409467b48Spatrick "dec{q}\t$dst", 81509467b48Spatrick [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i64 1)))]>, 81609467b48Spatrick LOCK; 81709467b48Spatrick } 81809467b48Spatrick 81909467b48Spatrick // Additional patterns for -1 constant. 82009467b48Spatrick def : Pat<(X86lock_add addr:$dst, (i8 -1)), (LOCK_DEC8m addr:$dst)>; 82109467b48Spatrick def : Pat<(X86lock_add addr:$dst, (i16 -1)), (LOCK_DEC16m addr:$dst)>; 82209467b48Spatrick def : Pat<(X86lock_add addr:$dst, (i32 -1)), (LOCK_DEC32m addr:$dst)>; 82309467b48Spatrick def : Pat<(X86lock_add addr:$dst, (i64 -1)), (LOCK_DEC64m addr:$dst)>; 82409467b48Spatrick def : Pat<(X86lock_sub addr:$dst, (i8 -1)), (LOCK_INC8m addr:$dst)>; 82509467b48Spatrick def : Pat<(X86lock_sub addr:$dst, (i16 -1)), (LOCK_INC16m addr:$dst)>; 82609467b48Spatrick def : Pat<(X86lock_sub addr:$dst, (i32 -1)), (LOCK_INC32m addr:$dst)>; 82709467b48Spatrick def : Pat<(X86lock_sub addr:$dst, (i64 -1)), (LOCK_INC64m addr:$dst)>; 82809467b48Spatrick} 82909467b48Spatrick 83009467b48Spatrick// Atomic compare and swap. 83109467b48Spatrickmulticlass LCMPXCHG_UnOp<bits<8> Opc, Format Form, string mnemonic, 83209467b48Spatrick SDPatternOperator frag, X86MemOperand x86memop> { 83309467b48Spatricklet isCodeGenOnly = 1, usesCustomInserter = 1 in { 83409467b48Spatrick def NAME : I<Opc, Form, (outs), (ins x86memop:$ptr), 83509467b48Spatrick !strconcat(mnemonic, "\t$ptr"), 83609467b48Spatrick [(frag addr:$ptr)]>, TB, LOCK; 83709467b48Spatrick} 83809467b48Spatrick} 83909467b48Spatrick 84009467b48Spatrickmulticlass LCMPXCHG_BinOp<bits<8> Opc8, bits<8> Opc, Format Form, 84109467b48Spatrick string mnemonic, SDPatternOperator frag> { 84209467b48Spatricklet isCodeGenOnly = 1, SchedRW = [WriteCMPXCHGRMW] in { 84309467b48Spatrick let Defs = [AL, EFLAGS], Uses = [AL] in 84409467b48Spatrick def NAME#8 : I<Opc8, Form, (outs), (ins i8mem:$ptr, GR8:$swap), 84509467b48Spatrick !strconcat(mnemonic, "{b}\t{$swap, $ptr|$ptr, $swap}"), 84609467b48Spatrick [(frag addr:$ptr, GR8:$swap, 1)]>, TB, LOCK; 84709467b48Spatrick let Defs = [AX, EFLAGS], Uses = [AX] in 84809467b48Spatrick def NAME#16 : I<Opc, Form, (outs), (ins i16mem:$ptr, GR16:$swap), 84909467b48Spatrick !strconcat(mnemonic, "{w}\t{$swap, $ptr|$ptr, $swap}"), 85009467b48Spatrick [(frag addr:$ptr, GR16:$swap, 2)]>, TB, OpSize16, LOCK; 85109467b48Spatrick let Defs = [EAX, EFLAGS], Uses = [EAX] in 85209467b48Spatrick def NAME#32 : I<Opc, Form, (outs), (ins i32mem:$ptr, GR32:$swap), 85309467b48Spatrick !strconcat(mnemonic, "{l}\t{$swap, $ptr|$ptr, $swap}"), 85409467b48Spatrick [(frag addr:$ptr, GR32:$swap, 4)]>, TB, OpSize32, LOCK; 85509467b48Spatrick let Defs = [RAX, EFLAGS], Uses = [RAX] in 85609467b48Spatrick def NAME#64 : RI<Opc, Form, (outs), (ins i64mem:$ptr, GR64:$swap), 85709467b48Spatrick !strconcat(mnemonic, "{q}\t{$swap, $ptr|$ptr, $swap}"), 85809467b48Spatrick [(frag addr:$ptr, GR64:$swap, 8)]>, TB, LOCK; 85909467b48Spatrick} 86009467b48Spatrick} 86109467b48Spatrick 86209467b48Spatricklet Defs = [EAX, EDX, EFLAGS], Uses = [EAX, EBX, ECX, EDX], 86309467b48Spatrick Predicates = [HasCmpxchg8b], SchedRW = [WriteCMPXCHGRMW] in { 86409467b48Spatrickdefm LCMPXCHG8B : LCMPXCHG_UnOp<0xC7, MRM1m, "cmpxchg8b", X86cas8, i64mem>; 86509467b48Spatrick} 86609467b48Spatrick 86709467b48Spatrick// This pseudo must be used when the frame uses RBX as 86809467b48Spatrick// the base pointer. Indeed, in such situation RBX is a reserved 86909467b48Spatrick// register and the register allocator will ignore any use/def of 87009467b48Spatrick// it. In other words, the register will not fix the clobbering of 87109467b48Spatrick// RBX that will happen when setting the arguments for the instrucion. 87209467b48Spatrick// 873097a140dSpatrick// Unlike the actual related instruction, we mark that this one 87409467b48Spatrick// defines EBX (instead of using EBX). 87509467b48Spatrick// The rationale is that we will define RBX during the expansion of 87609467b48Spatrick// the pseudo. The argument feeding EBX is ebx_input. 87709467b48Spatrick// 87809467b48Spatrick// The additional argument, $ebx_save, is a temporary register used to 87909467b48Spatrick// save the value of RBX across the actual instruction. 88009467b48Spatrick// 88109467b48Spatrick// To make sure the register assigned to $ebx_save does not interfere with 88209467b48Spatrick// the definition of the actual instruction, we use a definition $dst which 88309467b48Spatrick// is tied to $rbx_save. That way, the live-range of $rbx_save spans across 88409467b48Spatrick// the instruction and we are sure we will have a valid register to restore 88509467b48Spatrick// the value of RBX. 88609467b48Spatricklet Defs = [EAX, EDX, EBX, EFLAGS], Uses = [EAX, ECX, EDX], 88709467b48Spatrick Predicates = [HasCmpxchg8b], SchedRW = [WriteCMPXCHGRMW], 88809467b48Spatrick isCodeGenOnly = 1, isPseudo = 1, Constraints = "$ebx_save = $dst", 88909467b48Spatrick usesCustomInserter = 1 in { 89009467b48Spatrickdef LCMPXCHG8B_SAVE_EBX : 89109467b48Spatrick I<0, Pseudo, (outs GR32:$dst), 89209467b48Spatrick (ins i64mem:$ptr, GR32:$ebx_input, GR32:$ebx_save), 89309467b48Spatrick !strconcat("cmpxchg8b", "\t$ptr"), 89409467b48Spatrick [(set GR32:$dst, (X86cas8save_ebx addr:$ptr, GR32:$ebx_input, 89509467b48Spatrick GR32:$ebx_save))]>; 89609467b48Spatrick} 89709467b48Spatrick 89809467b48Spatrick 89909467b48Spatricklet Defs = [RAX, RDX, EFLAGS], Uses = [RAX, RBX, RCX, RDX], 90009467b48Spatrick Predicates = [HasCmpxchg16b,In64BitMode], SchedRW = [WriteCMPXCHGRMW] in { 90109467b48Spatrickdefm LCMPXCHG16B : LCMPXCHG_UnOp<0xC7, MRM1m, "cmpxchg16b", 90209467b48Spatrick X86cas16, i128mem>, REX_W; 90309467b48Spatrick} 90409467b48Spatrick 90509467b48Spatrick// Same as LCMPXCHG8B_SAVE_RBX but for the 16 Bytes variant. 90609467b48Spatricklet Defs = [RAX, RDX, RBX, EFLAGS], Uses = [RAX, RCX, RDX], 90709467b48Spatrick Predicates = [HasCmpxchg16b,In64BitMode], SchedRW = [WriteCMPXCHGRMW], 90809467b48Spatrick isCodeGenOnly = 1, isPseudo = 1, Constraints = "$rbx_save = $dst", 90909467b48Spatrick usesCustomInserter = 1 in { 91009467b48Spatrickdef LCMPXCHG16B_SAVE_RBX : 91109467b48Spatrick I<0, Pseudo, (outs GR64:$dst), 91209467b48Spatrick (ins i128mem:$ptr, GR64:$rbx_input, GR64:$rbx_save), 91309467b48Spatrick !strconcat("cmpxchg16b", "\t$ptr"), 91409467b48Spatrick [(set GR64:$dst, (X86cas16save_rbx addr:$ptr, GR64:$rbx_input, 91509467b48Spatrick GR64:$rbx_save))]>; 91609467b48Spatrick} 91709467b48Spatrick 91809467b48Spatrickdefm LCMPXCHG : LCMPXCHG_BinOp<0xB0, 0xB1, MRMDestMem, "cmpxchg", X86cas>; 91909467b48Spatrick 92009467b48Spatrick// Atomic exchange and add 92109467b48Spatrickmulticlass ATOMIC_LOAD_BINOP<bits<8> opc8, bits<8> opc, string mnemonic, 92209467b48Spatrick string frag> { 92309467b48Spatrick let Constraints = "$val = $dst", Defs = [EFLAGS], isCodeGenOnly = 1, 92409467b48Spatrick SchedRW = [WriteALURMW] in { 92509467b48Spatrick def NAME#8 : I<opc8, MRMSrcMem, (outs GR8:$dst), 92609467b48Spatrick (ins GR8:$val, i8mem:$ptr), 92709467b48Spatrick !strconcat(mnemonic, "{b}\t{$val, $ptr|$ptr, $val}"), 92809467b48Spatrick [(set GR8:$dst, 92909467b48Spatrick (!cast<PatFrag>(frag # "_8") addr:$ptr, GR8:$val))]>; 93009467b48Spatrick def NAME#16 : I<opc, MRMSrcMem, (outs GR16:$dst), 93109467b48Spatrick (ins GR16:$val, i16mem:$ptr), 93209467b48Spatrick !strconcat(mnemonic, "{w}\t{$val, $ptr|$ptr, $val}"), 93309467b48Spatrick [(set 93409467b48Spatrick GR16:$dst, 93509467b48Spatrick (!cast<PatFrag>(frag # "_16") addr:$ptr, GR16:$val))]>, 93609467b48Spatrick OpSize16; 93709467b48Spatrick def NAME#32 : I<opc, MRMSrcMem, (outs GR32:$dst), 93809467b48Spatrick (ins GR32:$val, i32mem:$ptr), 93909467b48Spatrick !strconcat(mnemonic, "{l}\t{$val, $ptr|$ptr, $val}"), 94009467b48Spatrick [(set 94109467b48Spatrick GR32:$dst, 94209467b48Spatrick (!cast<PatFrag>(frag # "_32") addr:$ptr, GR32:$val))]>, 94309467b48Spatrick OpSize32; 94409467b48Spatrick def NAME#64 : RI<opc, MRMSrcMem, (outs GR64:$dst), 94509467b48Spatrick (ins GR64:$val, i64mem:$ptr), 94609467b48Spatrick !strconcat(mnemonic, "{q}\t{$val, $ptr|$ptr, $val}"), 94709467b48Spatrick [(set 94809467b48Spatrick GR64:$dst, 94909467b48Spatrick (!cast<PatFrag>(frag # "_64") addr:$ptr, GR64:$val))]>; 95009467b48Spatrick } 95109467b48Spatrick} 95209467b48Spatrick 95309467b48Spatrickdefm LXADD : ATOMIC_LOAD_BINOP<0xc0, 0xc1, "xadd", "atomic_load_add">, TB, LOCK; 95409467b48Spatrick 95509467b48Spatrick/* The following multiclass tries to make sure that in code like 95609467b48Spatrick * x.store (immediate op x.load(acquire), release) 95709467b48Spatrick * and 95809467b48Spatrick * x.store (register op x.load(acquire), release) 95909467b48Spatrick * an operation directly on memory is generated instead of wasting a register. 96009467b48Spatrick * It is not automatic as atomic_store/load are only lowered to MOV instructions 96109467b48Spatrick * extremely late to prevent them from being accidentally reordered in the backend 96209467b48Spatrick * (see below the RELEASE_MOV* / ACQUIRE_MOV* pseudo-instructions) 96309467b48Spatrick */ 96409467b48Spatrickmulticlass RELEASE_BINOP_MI<string Name, SDNode op> { 96509467b48Spatrick def : Pat<(atomic_store_8 addr:$dst, 96609467b48Spatrick (op (atomic_load_8 addr:$dst), (i8 imm:$src))), 96709467b48Spatrick (!cast<Instruction>(Name#"8mi") addr:$dst, imm:$src)>; 96809467b48Spatrick def : Pat<(atomic_store_16 addr:$dst, 96909467b48Spatrick (op (atomic_load_16 addr:$dst), (i16 imm:$src))), 97009467b48Spatrick (!cast<Instruction>(Name#"16mi") addr:$dst, imm:$src)>; 97109467b48Spatrick def : Pat<(atomic_store_32 addr:$dst, 97209467b48Spatrick (op (atomic_load_32 addr:$dst), (i32 imm:$src))), 97309467b48Spatrick (!cast<Instruction>(Name#"32mi") addr:$dst, imm:$src)>; 97409467b48Spatrick def : Pat<(atomic_store_64 addr:$dst, 97509467b48Spatrick (op (atomic_load_64 addr:$dst), (i64immSExt32:$src))), 97609467b48Spatrick (!cast<Instruction>(Name#"64mi32") addr:$dst, (i64immSExt32:$src))>; 97709467b48Spatrick 97809467b48Spatrick def : Pat<(atomic_store_8 addr:$dst, 97909467b48Spatrick (op (atomic_load_8 addr:$dst), (i8 GR8:$src))), 98009467b48Spatrick (!cast<Instruction>(Name#"8mr") addr:$dst, GR8:$src)>; 98109467b48Spatrick def : Pat<(atomic_store_16 addr:$dst, 98209467b48Spatrick (op (atomic_load_16 addr:$dst), (i16 GR16:$src))), 98309467b48Spatrick (!cast<Instruction>(Name#"16mr") addr:$dst, GR16:$src)>; 98409467b48Spatrick def : Pat<(atomic_store_32 addr:$dst, 98509467b48Spatrick (op (atomic_load_32 addr:$dst), (i32 GR32:$src))), 98609467b48Spatrick (!cast<Instruction>(Name#"32mr") addr:$dst, GR32:$src)>; 98709467b48Spatrick def : Pat<(atomic_store_64 addr:$dst, 98809467b48Spatrick (op (atomic_load_64 addr:$dst), (i64 GR64:$src))), 98909467b48Spatrick (!cast<Instruction>(Name#"64mr") addr:$dst, GR64:$src)>; 99009467b48Spatrick} 99109467b48Spatrickdefm : RELEASE_BINOP_MI<"ADD", add>; 99209467b48Spatrickdefm : RELEASE_BINOP_MI<"AND", and>; 99309467b48Spatrickdefm : RELEASE_BINOP_MI<"OR", or>; 99409467b48Spatrickdefm : RELEASE_BINOP_MI<"XOR", xor>; 99509467b48Spatrickdefm : RELEASE_BINOP_MI<"SUB", sub>; 99609467b48Spatrick 99709467b48Spatrick// Atomic load + floating point patterns. 99809467b48Spatrick// FIXME: This could also handle SIMD operations with *ps and *pd instructions. 99909467b48Spatrickmulticlass ATOMIC_LOAD_FP_BINOP_MI<string Name, SDNode op> { 100009467b48Spatrick def : Pat<(op FR32:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))), 100109467b48Spatrick (!cast<Instruction>(Name#"SSrm") FR32:$src1, addr:$src2)>, 100209467b48Spatrick Requires<[UseSSE1]>; 100309467b48Spatrick def : Pat<(op FR32:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))), 100409467b48Spatrick (!cast<Instruction>("V"#Name#"SSrm") FR32:$src1, addr:$src2)>, 100509467b48Spatrick Requires<[UseAVX]>; 100609467b48Spatrick def : Pat<(op FR32X:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))), 100709467b48Spatrick (!cast<Instruction>("V"#Name#"SSZrm") FR32X:$src1, addr:$src2)>, 100809467b48Spatrick Requires<[HasAVX512]>; 100909467b48Spatrick 101009467b48Spatrick def : Pat<(op FR64:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))), 101109467b48Spatrick (!cast<Instruction>(Name#"SDrm") FR64:$src1, addr:$src2)>, 101209467b48Spatrick Requires<[UseSSE1]>; 101309467b48Spatrick def : Pat<(op FR64:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))), 101409467b48Spatrick (!cast<Instruction>("V"#Name#"SDrm") FR64:$src1, addr:$src2)>, 101509467b48Spatrick Requires<[UseAVX]>; 101609467b48Spatrick def : Pat<(op FR64X:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))), 101709467b48Spatrick (!cast<Instruction>("V"#Name#"SDZrm") FR64X:$src1, addr:$src2)>, 101809467b48Spatrick Requires<[HasAVX512]>; 101909467b48Spatrick} 102009467b48Spatrickdefm : ATOMIC_LOAD_FP_BINOP_MI<"ADD", fadd>; 102109467b48Spatrick// FIXME: Add fsub, fmul, fdiv, ... 102209467b48Spatrick 102309467b48Spatrickmulticlass RELEASE_UNOP<string Name, dag dag8, dag dag16, dag dag32, 102409467b48Spatrick dag dag64> { 102509467b48Spatrick def : Pat<(atomic_store_8 addr:$dst, dag8), 102609467b48Spatrick (!cast<Instruction>(Name#8m) addr:$dst)>; 102709467b48Spatrick def : Pat<(atomic_store_16 addr:$dst, dag16), 102809467b48Spatrick (!cast<Instruction>(Name#16m) addr:$dst)>; 102909467b48Spatrick def : Pat<(atomic_store_32 addr:$dst, dag32), 103009467b48Spatrick (!cast<Instruction>(Name#32m) addr:$dst)>; 103109467b48Spatrick def : Pat<(atomic_store_64 addr:$dst, dag64), 103209467b48Spatrick (!cast<Instruction>(Name#64m) addr:$dst)>; 103309467b48Spatrick} 103409467b48Spatrick 103509467b48Spatricklet Predicates = [UseIncDec] in { 103609467b48Spatrick defm : RELEASE_UNOP<"INC", 103709467b48Spatrick (add (atomic_load_8 addr:$dst), (i8 1)), 103809467b48Spatrick (add (atomic_load_16 addr:$dst), (i16 1)), 103909467b48Spatrick (add (atomic_load_32 addr:$dst), (i32 1)), 104009467b48Spatrick (add (atomic_load_64 addr:$dst), (i64 1))>; 104109467b48Spatrick defm : RELEASE_UNOP<"DEC", 104209467b48Spatrick (add (atomic_load_8 addr:$dst), (i8 -1)), 104309467b48Spatrick (add (atomic_load_16 addr:$dst), (i16 -1)), 104409467b48Spatrick (add (atomic_load_32 addr:$dst), (i32 -1)), 104509467b48Spatrick (add (atomic_load_64 addr:$dst), (i64 -1))>; 104609467b48Spatrick} 104709467b48Spatrick 104809467b48Spatrickdefm : RELEASE_UNOP<"NEG", 104909467b48Spatrick (ineg (i8 (atomic_load_8 addr:$dst))), 105009467b48Spatrick (ineg (i16 (atomic_load_16 addr:$dst))), 105109467b48Spatrick (ineg (i32 (atomic_load_32 addr:$dst))), 105209467b48Spatrick (ineg (i64 (atomic_load_64 addr:$dst)))>; 105309467b48Spatrickdefm : RELEASE_UNOP<"NOT", 105409467b48Spatrick (not (i8 (atomic_load_8 addr:$dst))), 105509467b48Spatrick (not (i16 (atomic_load_16 addr:$dst))), 105609467b48Spatrick (not (i32 (atomic_load_32 addr:$dst))), 105709467b48Spatrick (not (i64 (atomic_load_64 addr:$dst)))>; 105809467b48Spatrick 105909467b48Spatrickdef : Pat<(atomic_store_8 addr:$dst, (i8 imm:$src)), 106009467b48Spatrick (MOV8mi addr:$dst, imm:$src)>; 106109467b48Spatrickdef : Pat<(atomic_store_16 addr:$dst, (i16 imm:$src)), 106209467b48Spatrick (MOV16mi addr:$dst, imm:$src)>; 106309467b48Spatrickdef : Pat<(atomic_store_32 addr:$dst, (i32 imm:$src)), 106409467b48Spatrick (MOV32mi addr:$dst, imm:$src)>; 106509467b48Spatrickdef : Pat<(atomic_store_64 addr:$dst, (i64immSExt32:$src)), 106609467b48Spatrick (MOV64mi32 addr:$dst, i64immSExt32:$src)>; 106709467b48Spatrick 106809467b48Spatrickdef : Pat<(atomic_store_8 addr:$dst, GR8:$src), 106909467b48Spatrick (MOV8mr addr:$dst, GR8:$src)>; 107009467b48Spatrickdef : Pat<(atomic_store_16 addr:$dst, GR16:$src), 107109467b48Spatrick (MOV16mr addr:$dst, GR16:$src)>; 107209467b48Spatrickdef : Pat<(atomic_store_32 addr:$dst, GR32:$src), 107309467b48Spatrick (MOV32mr addr:$dst, GR32:$src)>; 107409467b48Spatrickdef : Pat<(atomic_store_64 addr:$dst, GR64:$src), 107509467b48Spatrick (MOV64mr addr:$dst, GR64:$src)>; 107609467b48Spatrick 107709467b48Spatrickdef : Pat<(i8 (atomic_load_8 addr:$src)), (MOV8rm addr:$src)>; 107809467b48Spatrickdef : Pat<(i16 (atomic_load_16 addr:$src)), (MOV16rm addr:$src)>; 107909467b48Spatrickdef : Pat<(i32 (atomic_load_32 addr:$src)), (MOV32rm addr:$src)>; 108009467b48Spatrickdef : Pat<(i64 (atomic_load_64 addr:$src)), (MOV64rm addr:$src)>; 108109467b48Spatrick 108209467b48Spatrick// Floating point loads/stores. 108309467b48Spatrickdef : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), 108409467b48Spatrick (MOVSSmr addr:$dst, FR32:$src)>, Requires<[UseSSE1]>; 108509467b48Spatrickdef : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), 108609467b48Spatrick (VMOVSSmr addr:$dst, FR32:$src)>, Requires<[UseAVX]>; 108709467b48Spatrickdef : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), 108809467b48Spatrick (VMOVSSZmr addr:$dst, FR32:$src)>, Requires<[HasAVX512]>; 108909467b48Spatrick 109009467b48Spatrickdef : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), 109109467b48Spatrick (MOVSDmr addr:$dst, FR64:$src)>, Requires<[UseSSE2]>; 109209467b48Spatrickdef : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), 109309467b48Spatrick (VMOVSDmr addr:$dst, FR64:$src)>, Requires<[UseAVX]>; 109409467b48Spatrickdef : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), 109509467b48Spatrick (VMOVSDmr addr:$dst, FR64:$src)>, Requires<[HasAVX512]>; 109609467b48Spatrick 109709467b48Spatrickdef : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))), 109809467b48Spatrick (MOVSSrm_alt addr:$src)>, Requires<[UseSSE1]>; 109909467b48Spatrickdef : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))), 110009467b48Spatrick (VMOVSSrm_alt addr:$src)>, Requires<[UseAVX]>; 110109467b48Spatrickdef : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))), 110209467b48Spatrick (VMOVSSZrm_alt addr:$src)>, Requires<[HasAVX512]>; 110309467b48Spatrick 110409467b48Spatrickdef : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))), 110509467b48Spatrick (MOVSDrm_alt addr:$src)>, Requires<[UseSSE2]>; 110609467b48Spatrickdef : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))), 110709467b48Spatrick (VMOVSDrm_alt addr:$src)>, Requires<[UseAVX]>; 110809467b48Spatrickdef : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))), 110909467b48Spatrick (VMOVSDZrm_alt addr:$src)>, Requires<[HasAVX512]>; 111009467b48Spatrick 111109467b48Spatrick//===----------------------------------------------------------------------===// 111209467b48Spatrick// DAG Pattern Matching Rules 111309467b48Spatrick//===----------------------------------------------------------------------===// 111409467b48Spatrick 111509467b48Spatrick// Use AND/OR to store 0/-1 in memory when optimizing for minsize. This saves 111609467b48Spatrick// binary size compared to a regular MOV, but it introduces an unnecessary 111709467b48Spatrick// load, so is not suitable for regular or optsize functions. 111809467b48Spatricklet Predicates = [OptForMinSize] in { 111909467b48Spatrickdef : Pat<(simple_store (i16 0), addr:$dst), (AND16mi8 addr:$dst, 0)>; 112009467b48Spatrickdef : Pat<(simple_store (i32 0), addr:$dst), (AND32mi8 addr:$dst, 0)>; 112109467b48Spatrickdef : Pat<(simple_store (i64 0), addr:$dst), (AND64mi8 addr:$dst, 0)>; 112209467b48Spatrickdef : Pat<(simple_store (i16 -1), addr:$dst), (OR16mi8 addr:$dst, -1)>; 112309467b48Spatrickdef : Pat<(simple_store (i32 -1), addr:$dst), (OR32mi8 addr:$dst, -1)>; 112409467b48Spatrickdef : Pat<(simple_store (i64 -1), addr:$dst), (OR64mi8 addr:$dst, -1)>; 112509467b48Spatrick} 112609467b48Spatrick 112709467b48Spatrick// In kernel code model, we can get the address of a label 112809467b48Spatrick// into a register with 'movq'. FIXME: This is a hack, the 'imm' predicate of 112909467b48Spatrick// the MOV64ri32 should accept these. 113009467b48Spatrickdef : Pat<(i64 (X86Wrapper tconstpool :$dst)), 113109467b48Spatrick (MOV64ri32 tconstpool :$dst)>, Requires<[KernelCode]>; 113209467b48Spatrickdef : Pat<(i64 (X86Wrapper tjumptable :$dst)), 113309467b48Spatrick (MOV64ri32 tjumptable :$dst)>, Requires<[KernelCode]>; 113409467b48Spatrickdef : Pat<(i64 (X86Wrapper tglobaladdr :$dst)), 113509467b48Spatrick (MOV64ri32 tglobaladdr :$dst)>, Requires<[KernelCode]>; 113609467b48Spatrickdef : Pat<(i64 (X86Wrapper texternalsym:$dst)), 113709467b48Spatrick (MOV64ri32 texternalsym:$dst)>, Requires<[KernelCode]>; 113809467b48Spatrickdef : Pat<(i64 (X86Wrapper mcsym:$dst)), 113909467b48Spatrick (MOV64ri32 mcsym:$dst)>, Requires<[KernelCode]>; 114009467b48Spatrickdef : Pat<(i64 (X86Wrapper tblockaddress:$dst)), 114109467b48Spatrick (MOV64ri32 tblockaddress:$dst)>, Requires<[KernelCode]>; 114209467b48Spatrick 114309467b48Spatrick// If we have small model and -static mode, it is safe to store global addresses 114409467b48Spatrick// directly as immediates. FIXME: This is really a hack, the 'imm' predicate 114509467b48Spatrick// for MOV64mi32 should handle this sort of thing. 114609467b48Spatrickdef : Pat<(store (i64 (X86Wrapper tconstpool:$src)), addr:$dst), 114709467b48Spatrick (MOV64mi32 addr:$dst, tconstpool:$src)>, 114809467b48Spatrick Requires<[NearData, IsNotPIC]>; 114909467b48Spatrickdef : Pat<(store (i64 (X86Wrapper tjumptable:$src)), addr:$dst), 115009467b48Spatrick (MOV64mi32 addr:$dst, tjumptable:$src)>, 115109467b48Spatrick Requires<[NearData, IsNotPIC]>; 115209467b48Spatrickdef : Pat<(store (i64 (X86Wrapper tglobaladdr:$src)), addr:$dst), 115309467b48Spatrick (MOV64mi32 addr:$dst, tglobaladdr:$src)>, 115409467b48Spatrick Requires<[NearData, IsNotPIC]>; 115509467b48Spatrickdef : Pat<(store (i64 (X86Wrapper texternalsym:$src)), addr:$dst), 115609467b48Spatrick (MOV64mi32 addr:$dst, texternalsym:$src)>, 115709467b48Spatrick Requires<[NearData, IsNotPIC]>; 115809467b48Spatrickdef : Pat<(store (i64 (X86Wrapper mcsym:$src)), addr:$dst), 115909467b48Spatrick (MOV64mi32 addr:$dst, mcsym:$src)>, 116009467b48Spatrick Requires<[NearData, IsNotPIC]>; 116109467b48Spatrickdef : Pat<(store (i64 (X86Wrapper tblockaddress:$src)), addr:$dst), 116209467b48Spatrick (MOV64mi32 addr:$dst, tblockaddress:$src)>, 116309467b48Spatrick Requires<[NearData, IsNotPIC]>; 116409467b48Spatrick 116509467b48Spatrickdef : Pat<(i32 (X86RecoverFrameAlloc mcsym:$dst)), (MOV32ri mcsym:$dst)>; 116609467b48Spatrickdef : Pat<(i64 (X86RecoverFrameAlloc mcsym:$dst)), (MOV64ri mcsym:$dst)>; 116709467b48Spatrick 116809467b48Spatrick// Calls 116909467b48Spatrick 117009467b48Spatrick// tls has some funny stuff here... 117109467b48Spatrick// This corresponds to movabs $foo@tpoff, %rax 117209467b48Spatrickdef : Pat<(i64 (X86Wrapper tglobaltlsaddr :$dst)), 117309467b48Spatrick (MOV64ri32 tglobaltlsaddr :$dst)>; 117409467b48Spatrick// This corresponds to add $foo@tpoff, %rax 117509467b48Spatrickdef : Pat<(add GR64:$src1, (X86Wrapper tglobaltlsaddr :$dst)), 117609467b48Spatrick (ADD64ri32 GR64:$src1, tglobaltlsaddr :$dst)>; 117709467b48Spatrick 117809467b48Spatrick 117909467b48Spatrick// Direct PC relative function call for small code model. 32-bit displacement 118009467b48Spatrick// sign extended to 64-bit. 118109467b48Spatrickdef : Pat<(X86call (i64 tglobaladdr:$dst)), 118209467b48Spatrick (CALL64pcrel32 tglobaladdr:$dst)>; 118309467b48Spatrickdef : Pat<(X86call (i64 texternalsym:$dst)), 118409467b48Spatrick (CALL64pcrel32 texternalsym:$dst)>; 118509467b48Spatrick 118609467b48Spatrick// Tailcall stuff. The TCRETURN instructions execute after the epilog, so they 118709467b48Spatrick// can never use callee-saved registers. That is the purpose of the GR64_TC 118809467b48Spatrick// register classes. 118909467b48Spatrick// 119009467b48Spatrick// The only volatile register that is never used by the calling convention is 119109467b48Spatrick// %r11. This happens when calling a vararg function with 6 arguments. 119209467b48Spatrick// 119309467b48Spatrick// Match an X86tcret that uses less than 7 volatile registers. 119409467b48Spatrickdef X86tcret_6regs : PatFrag<(ops node:$ptr, node:$off), 119509467b48Spatrick (X86tcret node:$ptr, node:$off), [{ 119609467b48Spatrick // X86tcret args: (*chain, ptr, imm, regs..., glue) 119709467b48Spatrick unsigned NumRegs = 0; 119809467b48Spatrick for (unsigned i = 3, e = N->getNumOperands(); i != e; ++i) 119909467b48Spatrick if (isa<RegisterSDNode>(N->getOperand(i)) && ++NumRegs > 6) 120009467b48Spatrick return false; 120109467b48Spatrick return true; 120209467b48Spatrick}]>; 120309467b48Spatrick 120409467b48Spatrickdef : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off), 120509467b48Spatrick (TCRETURNri ptr_rc_tailcall:$dst, imm:$off)>, 12067299aa8dSpatrick Requires<[Not64BitMode, NotUseIndirectThunkCalls]>; 120709467b48Spatrick 120809467b48Spatrick// FIXME: This is disabled for 32-bit PIC mode because the global base 120909467b48Spatrick// register which is part of the address mode may be assigned a 121009467b48Spatrick// callee-saved register. 121109467b48Spatrickdef : Pat<(X86tcret (load addr:$dst), imm:$off), 121209467b48Spatrick (TCRETURNmi addr:$dst, imm:$off)>, 12137299aa8dSpatrick Requires<[Not64BitMode, IsNotPIC, NotUseIndirectThunkCalls]>; 121409467b48Spatrick 121509467b48Spatrickdef : Pat<(X86tcret (i32 tglobaladdr:$dst), imm:$off), 121609467b48Spatrick (TCRETURNdi tglobaladdr:$dst, imm:$off)>, 121709467b48Spatrick Requires<[NotLP64]>; 121809467b48Spatrick 121909467b48Spatrickdef : Pat<(X86tcret (i32 texternalsym:$dst), imm:$off), 122009467b48Spatrick (TCRETURNdi texternalsym:$dst, imm:$off)>, 122109467b48Spatrick Requires<[NotLP64]>; 122209467b48Spatrick 122309467b48Spatrickdef : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off), 122409467b48Spatrick (TCRETURNri64 ptr_rc_tailcall:$dst, imm:$off)>, 12257299aa8dSpatrick Requires<[In64BitMode, NotUseIndirectThunkCalls]>; 122609467b48Spatrick 122709467b48Spatrick// Don't fold loads into X86tcret requiring more than 6 regs. 122809467b48Spatrick// There wouldn't be enough scratch registers for base+index. 122909467b48Spatrickdef : Pat<(X86tcret_6regs (load addr:$dst), imm:$off), 123009467b48Spatrick (TCRETURNmi64 addr:$dst, imm:$off)>, 12317299aa8dSpatrick Requires<[In64BitMode, NotUseIndirectThunkCalls]>; 123209467b48Spatrick 123309467b48Spatrickdef : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off), 12347299aa8dSpatrick (INDIRECT_THUNK_TCRETURN64 ptr_rc_tailcall:$dst, imm:$off)>, 12357299aa8dSpatrick Requires<[In64BitMode, UseIndirectThunkCalls]>; 123609467b48Spatrick 123709467b48Spatrickdef : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off), 12387299aa8dSpatrick (INDIRECT_THUNK_TCRETURN32 ptr_rc_tailcall:$dst, imm:$off)>, 12397299aa8dSpatrick Requires<[Not64BitMode, UseIndirectThunkCalls]>; 124009467b48Spatrick 124109467b48Spatrickdef : Pat<(X86tcret (i64 tglobaladdr:$dst), imm:$off), 124209467b48Spatrick (TCRETURNdi64 tglobaladdr:$dst, imm:$off)>, 124309467b48Spatrick Requires<[IsLP64]>; 124409467b48Spatrick 124509467b48Spatrickdef : Pat<(X86tcret (i64 texternalsym:$dst), imm:$off), 124609467b48Spatrick (TCRETURNdi64 texternalsym:$dst, imm:$off)>, 124709467b48Spatrick Requires<[IsLP64]>; 124809467b48Spatrick 124909467b48Spatrick// Normal calls, with various flavors of addresses. 125009467b48Spatrickdef : Pat<(X86call (i32 tglobaladdr:$dst)), 125109467b48Spatrick (CALLpcrel32 tglobaladdr:$dst)>; 125209467b48Spatrickdef : Pat<(X86call (i32 texternalsym:$dst)), 125309467b48Spatrick (CALLpcrel32 texternalsym:$dst)>; 125409467b48Spatrickdef : Pat<(X86call (i32 imm:$dst)), 125509467b48Spatrick (CALLpcrel32 imm:$dst)>, Requires<[CallImmAddr]>; 125609467b48Spatrick 125709467b48Spatrick// Comparisons. 125809467b48Spatrick 125909467b48Spatrick// TEST R,R is smaller than CMP R,0 126009467b48Spatrickdef : Pat<(X86cmp GR8:$src1, 0), 126109467b48Spatrick (TEST8rr GR8:$src1, GR8:$src1)>; 126209467b48Spatrickdef : Pat<(X86cmp GR16:$src1, 0), 126309467b48Spatrick (TEST16rr GR16:$src1, GR16:$src1)>; 126409467b48Spatrickdef : Pat<(X86cmp GR32:$src1, 0), 126509467b48Spatrick (TEST32rr GR32:$src1, GR32:$src1)>; 126609467b48Spatrickdef : Pat<(X86cmp GR64:$src1, 0), 126709467b48Spatrick (TEST64rr GR64:$src1, GR64:$src1)>; 126809467b48Spatrick 126909467b48Spatrick// zextload bool -> zextload byte 127009467b48Spatrick// i1 stored in one byte in zero-extended form. 127109467b48Spatrick// Upper bits cleanup should be executed before Store. 127209467b48Spatrickdef : Pat<(zextloadi8i1 addr:$src), (MOV8rm addr:$src)>; 127309467b48Spatrickdef : Pat<(zextloadi16i1 addr:$src), 127409467b48Spatrick (EXTRACT_SUBREG (MOVZX32rm8 addr:$src), sub_16bit)>; 127509467b48Spatrickdef : Pat<(zextloadi32i1 addr:$src), (MOVZX32rm8 addr:$src)>; 127609467b48Spatrickdef : Pat<(zextloadi64i1 addr:$src), 127709467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rm8 addr:$src), sub_32bit)>; 127809467b48Spatrick 127909467b48Spatrick// extload bool -> extload byte 128009467b48Spatrick// When extloading from 16-bit and smaller memory locations into 64-bit 128109467b48Spatrick// registers, use zero-extending loads so that the entire 64-bit register is 128209467b48Spatrick// defined, avoiding partial-register updates. 128309467b48Spatrick 128409467b48Spatrickdef : Pat<(extloadi8i1 addr:$src), (MOV8rm addr:$src)>; 128509467b48Spatrickdef : Pat<(extloadi16i1 addr:$src), 128609467b48Spatrick (EXTRACT_SUBREG (MOVZX32rm8 addr:$src), sub_16bit)>; 128709467b48Spatrickdef : Pat<(extloadi32i1 addr:$src), (MOVZX32rm8 addr:$src)>; 128809467b48Spatrickdef : Pat<(extloadi16i8 addr:$src), 128909467b48Spatrick (EXTRACT_SUBREG (MOVZX32rm8 addr:$src), sub_16bit)>; 129009467b48Spatrickdef : Pat<(extloadi32i8 addr:$src), (MOVZX32rm8 addr:$src)>; 129109467b48Spatrickdef : Pat<(extloadi32i16 addr:$src), (MOVZX32rm16 addr:$src)>; 129209467b48Spatrick 129309467b48Spatrick// For other extloads, use subregs, since the high contents of the register are 129409467b48Spatrick// defined after an extload. 129509467b48Spatrick// NOTE: The extloadi64i32 pattern needs to be first as it will try to form 129609467b48Spatrick// 32-bit loads for 4 byte aligned i8/i16 loads. 129709467b48Spatrickdef : Pat<(extloadi64i32 addr:$src), 129809467b48Spatrick (SUBREG_TO_REG (i64 0), (MOV32rm addr:$src), sub_32bit)>; 129909467b48Spatrickdef : Pat<(extloadi64i1 addr:$src), 130009467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rm8 addr:$src), sub_32bit)>; 130109467b48Spatrickdef : Pat<(extloadi64i8 addr:$src), 130209467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rm8 addr:$src), sub_32bit)>; 130309467b48Spatrickdef : Pat<(extloadi64i16 addr:$src), 130409467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rm16 addr:$src), sub_32bit)>; 130509467b48Spatrick 130609467b48Spatrick// anyext. Define these to do an explicit zero-extend to 130709467b48Spatrick// avoid partial-register updates. 130809467b48Spatrickdef : Pat<(i16 (anyext GR8 :$src)), (EXTRACT_SUBREG 130909467b48Spatrick (MOVZX32rr8 GR8 :$src), sub_16bit)>; 131009467b48Spatrickdef : Pat<(i32 (anyext GR8 :$src)), (MOVZX32rr8 GR8 :$src)>; 131109467b48Spatrick 131209467b48Spatrick// Except for i16 -> i32 since isel expect i16 ops to be promoted to i32. 131309467b48Spatrickdef : Pat<(i32 (anyext GR16:$src)), 131409467b48Spatrick (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR16:$src, sub_16bit)>; 131509467b48Spatrick 131609467b48Spatrickdef : Pat<(i64 (anyext GR8 :$src)), 131709467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rr8 GR8 :$src), sub_32bit)>; 131809467b48Spatrickdef : Pat<(i64 (anyext GR16:$src)), 131909467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rr16 GR16 :$src), sub_32bit)>; 132009467b48Spatrickdef : Pat<(i64 (anyext GR32:$src)), 132109467b48Spatrick (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR32:$src, sub_32bit)>; 132209467b48Spatrick 132309467b48Spatrick// If this is an anyext of the remainder of an 8-bit sdivrem, use a MOVSX 132409467b48Spatrick// instead of a MOVZX. The sdivrem lowering will emit emit a MOVSX to move 132509467b48Spatrick// %ah to the lower byte of a register. By using a MOVSX here we allow a 132609467b48Spatrick// post-isel peephole to merge the two MOVSX instructions into one. 132709467b48Spatrickdef anyext_sdiv : PatFrag<(ops node:$lhs), (anyext node:$lhs),[{ 132809467b48Spatrick return (N->getOperand(0).getOpcode() == ISD::SDIVREM && 132909467b48Spatrick N->getOperand(0).getResNo() == 1); 133009467b48Spatrick}]>; 133109467b48Spatrickdef : Pat<(i32 (anyext_sdiv GR8:$src)), (MOVSX32rr8 GR8:$src)>; 133209467b48Spatrick 133309467b48Spatrick// Any instruction that defines a 32-bit result leaves the high half of the 133409467b48Spatrick// register. Truncate can be lowered to EXTRACT_SUBREG. CopyFromReg may 133509467b48Spatrick// be copying from a truncate. Any other 32-bit operation will zero-extend 133609467b48Spatrick// up to 64 bits. AssertSext/AssertZext aren't saying anything about the upper 133709467b48Spatrick// 32 bits, they're probably just qualifying a CopyFromReg. 133809467b48Spatrickdef def32 : PatLeaf<(i32 GR32:$src), [{ 133909467b48Spatrick return N->getOpcode() != ISD::TRUNCATE && 134009467b48Spatrick N->getOpcode() != TargetOpcode::EXTRACT_SUBREG && 134109467b48Spatrick N->getOpcode() != ISD::CopyFromReg && 134209467b48Spatrick N->getOpcode() != ISD::AssertSext && 134309467b48Spatrick N->getOpcode() != ISD::AssertZext; 134409467b48Spatrick}]>; 134509467b48Spatrick 134609467b48Spatrick// In the case of a 32-bit def that is known to implicitly zero-extend, 134709467b48Spatrick// we can use a SUBREG_TO_REG. 134809467b48Spatrickdef : Pat<(i64 (zext def32:$src)), 134909467b48Spatrick (SUBREG_TO_REG (i64 0), GR32:$src, sub_32bit)>; 135009467b48Spatrickdef : Pat<(i64 (and (anyext def32:$src), 0x00000000FFFFFFFF)), 135109467b48Spatrick (SUBREG_TO_REG (i64 0), GR32:$src, sub_32bit)>; 135209467b48Spatrick 135309467b48Spatrick//===----------------------------------------------------------------------===// 135409467b48Spatrick// Pattern match OR as ADD 135509467b48Spatrick//===----------------------------------------------------------------------===// 135609467b48Spatrick 135709467b48Spatrick// If safe, we prefer to pattern match OR as ADD at isel time. ADD can be 135809467b48Spatrick// 3-addressified into an LEA instruction to avoid copies. However, we also 135909467b48Spatrick// want to finally emit these instructions as an or at the end of the code 136009467b48Spatrick// generator to make the generated code easier to read. To do this, we select 136109467b48Spatrick// into "disjoint bits" pseudo ops. 136209467b48Spatrick 136309467b48Spatrick// Treat an 'or' node is as an 'add' if the or'ed bits are known to be zero. 136409467b48Spatrickdef or_is_add : PatFrag<(ops node:$lhs, node:$rhs), (or node:$lhs, node:$rhs),[{ 136509467b48Spatrick if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 136609467b48Spatrick return CurDAG->MaskedValueIsZero(N->getOperand(0), CN->getAPIntValue()); 136709467b48Spatrick 136809467b48Spatrick KnownBits Known0 = CurDAG->computeKnownBits(N->getOperand(0), 0); 136909467b48Spatrick KnownBits Known1 = CurDAG->computeKnownBits(N->getOperand(1), 0); 137009467b48Spatrick return (~Known0.Zero & ~Known1.Zero) == 0; 137109467b48Spatrick}]>; 137209467b48Spatrick 137309467b48Spatrick 137409467b48Spatrick// (or x1, x2) -> (add x1, x2) if two operands are known not to share bits. 137509467b48Spatrick// Try this before the selecting to OR. 137609467b48Spatricklet SchedRW = [WriteALU] in { 137709467b48Spatrick 137809467b48Spatricklet isConvertibleToThreeAddress = 1, isPseudo = 1, 137909467b48Spatrick Constraints = "$src1 = $dst", Defs = [EFLAGS] in { 138009467b48Spatricklet isCommutable = 1 in { 138109467b48Spatrickdef ADD8rr_DB : I<0, Pseudo, (outs GR8:$dst), (ins GR8:$src1, GR8:$src2), 138209467b48Spatrick "", // orb/addb REG, REG 138309467b48Spatrick [(set GR8:$dst, (or_is_add GR8:$src1, GR8:$src2))]>; 138409467b48Spatrickdef ADD16rr_DB : I<0, Pseudo, (outs GR16:$dst), (ins GR16:$src1, GR16:$src2), 138509467b48Spatrick "", // orw/addw REG, REG 138609467b48Spatrick [(set GR16:$dst, (or_is_add GR16:$src1, GR16:$src2))]>; 138709467b48Spatrickdef ADD32rr_DB : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$src1, GR32:$src2), 138809467b48Spatrick "", // orl/addl REG, REG 138909467b48Spatrick [(set GR32:$dst, (or_is_add GR32:$src1, GR32:$src2))]>; 139009467b48Spatrickdef ADD64rr_DB : I<0, Pseudo, (outs GR64:$dst), (ins GR64:$src1, GR64:$src2), 139109467b48Spatrick "", // orq/addq REG, REG 139209467b48Spatrick [(set GR64:$dst, (or_is_add GR64:$src1, GR64:$src2))]>; 139309467b48Spatrick} // isCommutable 139409467b48Spatrick 139509467b48Spatrick// NOTE: These are order specific, we want the ri8 forms to be listed 139609467b48Spatrick// first so that they are slightly preferred to the ri forms. 139709467b48Spatrick 139809467b48Spatrickdef ADD8ri_DB : I<0, Pseudo, 139909467b48Spatrick (outs GR8:$dst), (ins GR8:$src1, i8imm:$src2), 140009467b48Spatrick "", // orb/addb REG, imm8 140109467b48Spatrick [(set GR8:$dst, (or_is_add GR8:$src1, imm:$src2))]>; 140209467b48Spatrickdef ADD16ri8_DB : I<0, Pseudo, 140309467b48Spatrick (outs GR16:$dst), (ins GR16:$src1, i16i8imm:$src2), 140409467b48Spatrick "", // orw/addw REG, imm8 140509467b48Spatrick [(set GR16:$dst,(or_is_add GR16:$src1,i16immSExt8:$src2))]>; 140609467b48Spatrickdef ADD16ri_DB : I<0, Pseudo, (outs GR16:$dst), (ins GR16:$src1, i16imm:$src2), 140709467b48Spatrick "", // orw/addw REG, imm 140809467b48Spatrick [(set GR16:$dst, (or_is_add GR16:$src1, imm:$src2))]>; 140909467b48Spatrick 141009467b48Spatrickdef ADD32ri8_DB : I<0, Pseudo, 141109467b48Spatrick (outs GR32:$dst), (ins GR32:$src1, i32i8imm:$src2), 141209467b48Spatrick "", // orl/addl REG, imm8 141309467b48Spatrick [(set GR32:$dst,(or_is_add GR32:$src1,i32immSExt8:$src2))]>; 141409467b48Spatrickdef ADD32ri_DB : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$src1, i32imm:$src2), 141509467b48Spatrick "", // orl/addl REG, imm 141609467b48Spatrick [(set GR32:$dst, (or_is_add GR32:$src1, imm:$src2))]>; 141709467b48Spatrick 141809467b48Spatrick 141909467b48Spatrickdef ADD64ri8_DB : I<0, Pseudo, 142009467b48Spatrick (outs GR64:$dst), (ins GR64:$src1, i64i8imm:$src2), 142109467b48Spatrick "", // orq/addq REG, imm8 142209467b48Spatrick [(set GR64:$dst, (or_is_add GR64:$src1, 142309467b48Spatrick i64immSExt8:$src2))]>; 142409467b48Spatrickdef ADD64ri32_DB : I<0, Pseudo, 142509467b48Spatrick (outs GR64:$dst), (ins GR64:$src1, i64i32imm:$src2), 142609467b48Spatrick "", // orq/addq REG, imm 142709467b48Spatrick [(set GR64:$dst, (or_is_add GR64:$src1, 142809467b48Spatrick i64immSExt32:$src2))]>; 142909467b48Spatrick} 143009467b48Spatrick} // AddedComplexity, SchedRW 143109467b48Spatrick 143209467b48Spatrick//===----------------------------------------------------------------------===// 143309467b48Spatrick// Pattern match SUB as XOR 143409467b48Spatrick//===----------------------------------------------------------------------===// 143509467b48Spatrick 143609467b48Spatrick// An immediate in the LHS of a subtract can't be encoded in the instruction. 143709467b48Spatrick// If there is no possibility of a borrow we can use an XOR instead of a SUB 143809467b48Spatrick// to enable the immediate to be folded. 143909467b48Spatrick// TODO: Move this to a DAG combine? 144009467b48Spatrick 144109467b48Spatrickdef sub_is_xor : PatFrag<(ops node:$lhs, node:$rhs), (sub node:$lhs, node:$rhs),[{ 144209467b48Spatrick if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) { 144309467b48Spatrick KnownBits Known = CurDAG->computeKnownBits(N->getOperand(1)); 144409467b48Spatrick 144509467b48Spatrick // If all possible ones in the RHS are set in the LHS then there can't be 144609467b48Spatrick // a borrow and we can use xor. 144709467b48Spatrick return (~Known.Zero).isSubsetOf(CN->getAPIntValue()); 144809467b48Spatrick } 144909467b48Spatrick 145009467b48Spatrick return false; 145109467b48Spatrick}]>; 145209467b48Spatrick 145309467b48Spatricklet AddedComplexity = 5 in { 145409467b48Spatrickdef : Pat<(sub_is_xor imm:$src2, GR8:$src1), 145509467b48Spatrick (XOR8ri GR8:$src1, imm:$src2)>; 145609467b48Spatrickdef : Pat<(sub_is_xor i16immSExt8:$src2, GR16:$src1), 145709467b48Spatrick (XOR16ri8 GR16:$src1, i16immSExt8:$src2)>; 145809467b48Spatrickdef : Pat<(sub_is_xor imm:$src2, GR16:$src1), 145909467b48Spatrick (XOR16ri GR16:$src1, imm:$src2)>; 146009467b48Spatrickdef : Pat<(sub_is_xor i32immSExt8:$src2, GR32:$src1), 146109467b48Spatrick (XOR32ri8 GR32:$src1, i32immSExt8:$src2)>; 146209467b48Spatrickdef : Pat<(sub_is_xor imm:$src2, GR32:$src1), 146309467b48Spatrick (XOR32ri GR32:$src1, imm:$src2)>; 146409467b48Spatrickdef : Pat<(sub_is_xor i64immSExt8:$src2, GR64:$src1), 146509467b48Spatrick (XOR64ri8 GR64:$src1, i64immSExt8:$src2)>; 146609467b48Spatrickdef : Pat<(sub_is_xor i64immSExt32:$src2, GR64:$src1), 146709467b48Spatrick (XOR64ri32 GR64:$src1, i64immSExt32:$src2)>; 146809467b48Spatrick} 146909467b48Spatrick 147009467b48Spatrick//===----------------------------------------------------------------------===// 147109467b48Spatrick// Some peepholes 147209467b48Spatrick//===----------------------------------------------------------------------===// 147309467b48Spatrick 147409467b48Spatrick// Odd encoding trick: -128 fits into an 8-bit immediate field while 147509467b48Spatrick// +128 doesn't, so in this special case use a sub instead of an add. 147609467b48Spatrickdef : Pat<(add GR16:$src1, 128), 147709467b48Spatrick (SUB16ri8 GR16:$src1, -128)>; 147809467b48Spatrickdef : Pat<(store (add (loadi16 addr:$dst), 128), addr:$dst), 147909467b48Spatrick (SUB16mi8 addr:$dst, -128)>; 148009467b48Spatrick 148109467b48Spatrickdef : Pat<(add GR32:$src1, 128), 148209467b48Spatrick (SUB32ri8 GR32:$src1, -128)>; 148309467b48Spatrickdef : Pat<(store (add (loadi32 addr:$dst), 128), addr:$dst), 148409467b48Spatrick (SUB32mi8 addr:$dst, -128)>; 148509467b48Spatrick 148609467b48Spatrickdef : Pat<(add GR64:$src1, 128), 148709467b48Spatrick (SUB64ri8 GR64:$src1, -128)>; 148809467b48Spatrickdef : Pat<(store (add (loadi64 addr:$dst), 128), addr:$dst), 148909467b48Spatrick (SUB64mi8 addr:$dst, -128)>; 149009467b48Spatrick 149109467b48Spatrickdef : Pat<(X86add_flag_nocf GR16:$src1, 128), 149209467b48Spatrick (SUB16ri8 GR16:$src1, -128)>; 149309467b48Spatrickdef : Pat<(X86add_flag_nocf GR32:$src1, 128), 149409467b48Spatrick (SUB32ri8 GR32:$src1, -128)>; 149509467b48Spatrickdef : Pat<(X86add_flag_nocf GR64:$src1, 128), 149609467b48Spatrick (SUB64ri8 GR64:$src1, -128)>; 149709467b48Spatrick 149809467b48Spatrick// The same trick applies for 32-bit immediate fields in 64-bit 149909467b48Spatrick// instructions. 150009467b48Spatrickdef : Pat<(add GR64:$src1, 0x0000000080000000), 150109467b48Spatrick (SUB64ri32 GR64:$src1, 0xffffffff80000000)>; 150209467b48Spatrickdef : Pat<(store (add (loadi64 addr:$dst), 0x0000000080000000), addr:$dst), 150309467b48Spatrick (SUB64mi32 addr:$dst, 0xffffffff80000000)>; 150409467b48Spatrick 150509467b48Spatrickdef : Pat<(X86add_flag_nocf GR64:$src1, 0x0000000080000000), 150609467b48Spatrick (SUB64ri32 GR64:$src1, 0xffffffff80000000)>; 150709467b48Spatrick 150809467b48Spatrick// To avoid needing to materialize an immediate in a register, use a 32-bit and 150909467b48Spatrick// with implicit zero-extension instead of a 64-bit and if the immediate has at 151009467b48Spatrick// least 32 bits of leading zeros. If in addition the last 32 bits can be 151109467b48Spatrick// represented with a sign extension of a 8 bit constant, use that. 151209467b48Spatrick// This can also reduce instruction size by eliminating the need for the REX 151309467b48Spatrick// prefix. 151409467b48Spatrick 151509467b48Spatrick// AddedComplexity is needed to give priority over i64immSExt8 and i64immSExt32. 151609467b48Spatricklet AddedComplexity = 1 in { 151709467b48Spatrickdef : Pat<(and GR64:$src, i64immZExt32SExt8:$imm), 151809467b48Spatrick (SUBREG_TO_REG 151909467b48Spatrick (i64 0), 152009467b48Spatrick (AND32ri8 152109467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_32bit), 152209467b48Spatrick (i32 (GetLo32XForm imm:$imm))), 152309467b48Spatrick sub_32bit)>; 152409467b48Spatrick 152509467b48Spatrickdef : Pat<(and GR64:$src, i64immZExt32:$imm), 152609467b48Spatrick (SUBREG_TO_REG 152709467b48Spatrick (i64 0), 152809467b48Spatrick (AND32ri 152909467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_32bit), 153009467b48Spatrick (i32 (GetLo32XForm imm:$imm))), 153109467b48Spatrick sub_32bit)>; 153209467b48Spatrick} // AddedComplexity = 1 153309467b48Spatrick 153409467b48Spatrick 153509467b48Spatrick// AddedComplexity is needed due to the increased complexity on the 153609467b48Spatrick// i64immZExt32SExt8 and i64immZExt32 patterns above. Applying this to all 153709467b48Spatrick// the MOVZX patterns keeps thems together in DAGIsel tables. 153809467b48Spatricklet AddedComplexity = 1 in { 153909467b48Spatrick// r & (2^16-1) ==> movz 154009467b48Spatrickdef : Pat<(and GR32:$src1, 0xffff), 154109467b48Spatrick (MOVZX32rr16 (EXTRACT_SUBREG GR32:$src1, sub_16bit))>; 154209467b48Spatrick// r & (2^8-1) ==> movz 154309467b48Spatrickdef : Pat<(and GR32:$src1, 0xff), 154409467b48Spatrick (MOVZX32rr8 (EXTRACT_SUBREG GR32:$src1, sub_8bit))>; 154509467b48Spatrick// r & (2^8-1) ==> movz 154609467b48Spatrickdef : Pat<(and GR16:$src1, 0xff), 154709467b48Spatrick (EXTRACT_SUBREG (MOVZX32rr8 (EXTRACT_SUBREG GR16:$src1, sub_8bit)), 154809467b48Spatrick sub_16bit)>; 154909467b48Spatrick 155009467b48Spatrick// r & (2^32-1) ==> movz 155109467b48Spatrickdef : Pat<(and GR64:$src, 0x00000000FFFFFFFF), 155209467b48Spatrick (SUBREG_TO_REG (i64 0), 155309467b48Spatrick (MOV32rr (EXTRACT_SUBREG GR64:$src, sub_32bit)), 155409467b48Spatrick sub_32bit)>; 155509467b48Spatrick// r & (2^16-1) ==> movz 155609467b48Spatrickdef : Pat<(and GR64:$src, 0xffff), 155709467b48Spatrick (SUBREG_TO_REG (i64 0), 155809467b48Spatrick (MOVZX32rr16 (i16 (EXTRACT_SUBREG GR64:$src, sub_16bit))), 155909467b48Spatrick sub_32bit)>; 156009467b48Spatrick// r & (2^8-1) ==> movz 156109467b48Spatrickdef : Pat<(and GR64:$src, 0xff), 156209467b48Spatrick (SUBREG_TO_REG (i64 0), 156309467b48Spatrick (MOVZX32rr8 (i8 (EXTRACT_SUBREG GR64:$src, sub_8bit))), 156409467b48Spatrick sub_32bit)>; 156509467b48Spatrick} // AddedComplexity = 1 156609467b48Spatrick 156709467b48Spatrick 156809467b48Spatrick// Try to use BTS/BTR/BTC for single bit operations on the upper 32-bits. 156909467b48Spatrick 157009467b48Spatrickdef BTRXForm : SDNodeXForm<imm, [{ 157109467b48Spatrick // Transformation function: Find the lowest 0. 157209467b48Spatrick return getI64Imm((uint8_t)N->getAPIntValue().countTrailingOnes(), SDLoc(N)); 157309467b48Spatrick}]>; 157409467b48Spatrick 157509467b48Spatrickdef BTCBTSXForm : SDNodeXForm<imm, [{ 157609467b48Spatrick // Transformation function: Find the lowest 1. 157709467b48Spatrick return getI64Imm((uint8_t)N->getAPIntValue().countTrailingZeros(), SDLoc(N)); 157809467b48Spatrick}]>; 157909467b48Spatrick 158009467b48Spatrickdef BTRMask64 : ImmLeaf<i64, [{ 158109467b48Spatrick return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm); 158209467b48Spatrick}]>; 158309467b48Spatrick 158409467b48Spatrickdef BTCBTSMask64 : ImmLeaf<i64, [{ 158509467b48Spatrick return !isInt<32>(Imm) && isPowerOf2_64(Imm); 158609467b48Spatrick}]>; 158709467b48Spatrick 158809467b48Spatrick// For now only do this for optsize. 158909467b48Spatricklet AddedComplexity = 1, Predicates=[OptForSize] in { 159009467b48Spatrick def : Pat<(and GR64:$src1, BTRMask64:$mask), 159109467b48Spatrick (BTR64ri8 GR64:$src1, (BTRXForm imm:$mask))>; 159209467b48Spatrick def : Pat<(or GR64:$src1, BTCBTSMask64:$mask), 159309467b48Spatrick (BTS64ri8 GR64:$src1, (BTCBTSXForm imm:$mask))>; 159409467b48Spatrick def : Pat<(xor GR64:$src1, BTCBTSMask64:$mask), 159509467b48Spatrick (BTC64ri8 GR64:$src1, (BTCBTSXForm imm:$mask))>; 159609467b48Spatrick} 159709467b48Spatrick 159809467b48Spatrick 159909467b48Spatrick// sext_inreg patterns 160009467b48Spatrickdef : Pat<(sext_inreg GR32:$src, i16), 160109467b48Spatrick (MOVSX32rr16 (EXTRACT_SUBREG GR32:$src, sub_16bit))>; 160209467b48Spatrickdef : Pat<(sext_inreg GR32:$src, i8), 160309467b48Spatrick (MOVSX32rr8 (EXTRACT_SUBREG GR32:$src, sub_8bit))>; 160409467b48Spatrick 160509467b48Spatrickdef : Pat<(sext_inreg GR16:$src, i8), 160609467b48Spatrick (EXTRACT_SUBREG (MOVSX32rr8 (EXTRACT_SUBREG GR16:$src, sub_8bit)), 160709467b48Spatrick sub_16bit)>; 160809467b48Spatrick 160909467b48Spatrickdef : Pat<(sext_inreg GR64:$src, i32), 161009467b48Spatrick (MOVSX64rr32 (EXTRACT_SUBREG GR64:$src, sub_32bit))>; 161109467b48Spatrickdef : Pat<(sext_inreg GR64:$src, i16), 161209467b48Spatrick (MOVSX64rr16 (EXTRACT_SUBREG GR64:$src, sub_16bit))>; 161309467b48Spatrickdef : Pat<(sext_inreg GR64:$src, i8), 161409467b48Spatrick (MOVSX64rr8 (EXTRACT_SUBREG GR64:$src, sub_8bit))>; 161509467b48Spatrick 161609467b48Spatrick// sext, sext_load, zext, zext_load 161709467b48Spatrickdef: Pat<(i16 (sext GR8:$src)), 161809467b48Spatrick (EXTRACT_SUBREG (MOVSX32rr8 GR8:$src), sub_16bit)>; 161909467b48Spatrickdef: Pat<(sextloadi16i8 addr:$src), 162009467b48Spatrick (EXTRACT_SUBREG (MOVSX32rm8 addr:$src), sub_16bit)>; 162109467b48Spatrickdef: Pat<(i16 (zext GR8:$src)), 162209467b48Spatrick (EXTRACT_SUBREG (MOVZX32rr8 GR8:$src), sub_16bit)>; 162309467b48Spatrickdef: Pat<(zextloadi16i8 addr:$src), 162409467b48Spatrick (EXTRACT_SUBREG (MOVZX32rm8 addr:$src), sub_16bit)>; 162509467b48Spatrick 162609467b48Spatrick// trunc patterns 162709467b48Spatrickdef : Pat<(i16 (trunc GR32:$src)), 162809467b48Spatrick (EXTRACT_SUBREG GR32:$src, sub_16bit)>; 162909467b48Spatrickdef : Pat<(i8 (trunc GR32:$src)), 163009467b48Spatrick (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src, GR32_ABCD)), 163109467b48Spatrick sub_8bit)>, 163209467b48Spatrick Requires<[Not64BitMode]>; 163309467b48Spatrickdef : Pat<(i8 (trunc GR16:$src)), 163409467b48Spatrick (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)), 163509467b48Spatrick sub_8bit)>, 163609467b48Spatrick Requires<[Not64BitMode]>; 163709467b48Spatrickdef : Pat<(i32 (trunc GR64:$src)), 163809467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_32bit)>; 163909467b48Spatrickdef : Pat<(i16 (trunc GR64:$src)), 164009467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_16bit)>; 164109467b48Spatrickdef : Pat<(i8 (trunc GR64:$src)), 164209467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_8bit)>; 164309467b48Spatrickdef : Pat<(i8 (trunc GR32:$src)), 164409467b48Spatrick (EXTRACT_SUBREG GR32:$src, sub_8bit)>, 164509467b48Spatrick Requires<[In64BitMode]>; 164609467b48Spatrickdef : Pat<(i8 (trunc GR16:$src)), 164709467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit)>, 164809467b48Spatrick Requires<[In64BitMode]>; 164909467b48Spatrick 165009467b48Spatrickdef immff00_ffff : ImmLeaf<i32, [{ 165109467b48Spatrick return Imm >= 0xff00 && Imm <= 0xffff; 165209467b48Spatrick}]>; 165309467b48Spatrick 165409467b48Spatrick// h-register tricks 165509467b48Spatrickdef : Pat<(i8 (trunc (srl_su GR16:$src, (i8 8)))), 165609467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit_hi)>, 165709467b48Spatrick Requires<[Not64BitMode]>; 165809467b48Spatrickdef : Pat<(i8 (trunc (srl_su (i32 (anyext GR16:$src)), (i8 8)))), 165909467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit_hi)>, 166009467b48Spatrick Requires<[Not64BitMode]>; 166109467b48Spatrickdef : Pat<(i8 (trunc (srl_su GR32:$src, (i8 8)))), 166209467b48Spatrick (EXTRACT_SUBREG GR32:$src, sub_8bit_hi)>, 166309467b48Spatrick Requires<[Not64BitMode]>; 166409467b48Spatrickdef : Pat<(srl GR16:$src, (i8 8)), 166509467b48Spatrick (EXTRACT_SUBREG 166609467b48Spatrick (MOVZX32rr8_NOREX (EXTRACT_SUBREG GR16:$src, sub_8bit_hi)), 166709467b48Spatrick sub_16bit)>; 166809467b48Spatrickdef : Pat<(i32 (zext (srl_su GR16:$src, (i8 8)))), 166909467b48Spatrick (MOVZX32rr8_NOREX (EXTRACT_SUBREG GR16:$src, sub_8bit_hi))>; 167009467b48Spatrickdef : Pat<(i32 (anyext (srl_su GR16:$src, (i8 8)))), 167109467b48Spatrick (MOVZX32rr8_NOREX (EXTRACT_SUBREG GR16:$src, sub_8bit_hi))>; 167209467b48Spatrickdef : Pat<(and (srl_su GR32:$src, (i8 8)), (i32 255)), 167309467b48Spatrick (MOVZX32rr8_NOREX (EXTRACT_SUBREG GR32:$src, sub_8bit_hi))>; 167409467b48Spatrickdef : Pat<(srl (and_su GR32:$src, immff00_ffff), (i8 8)), 167509467b48Spatrick (MOVZX32rr8_NOREX (EXTRACT_SUBREG GR32:$src, sub_8bit_hi))>; 167609467b48Spatrick 167709467b48Spatrick// h-register tricks. 167809467b48Spatrick// For now, be conservative on x86-64 and use an h-register extract only if the 167909467b48Spatrick// value is immediately zero-extended or stored, which are somewhat common 168009467b48Spatrick// cases. This uses a bunch of code to prevent a register requiring a REX prefix 168109467b48Spatrick// from being allocated in the same instruction as the h register, as there's 168209467b48Spatrick// currently no way to describe this requirement to the register allocator. 168309467b48Spatrick 168409467b48Spatrick// h-register extract and zero-extend. 168509467b48Spatrickdef : Pat<(and (srl_su GR64:$src, (i8 8)), (i64 255)), 168609467b48Spatrick (SUBREG_TO_REG 168709467b48Spatrick (i64 0), 168809467b48Spatrick (MOVZX32rr8_NOREX 168909467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_8bit_hi)), 169009467b48Spatrick sub_32bit)>; 169109467b48Spatrickdef : Pat<(i64 (zext (srl_su GR16:$src, (i8 8)))), 169209467b48Spatrick (SUBREG_TO_REG 169309467b48Spatrick (i64 0), 169409467b48Spatrick (MOVZX32rr8_NOREX 169509467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit_hi)), 169609467b48Spatrick sub_32bit)>; 169709467b48Spatrickdef : Pat<(i64 (anyext (srl_su GR16:$src, (i8 8)))), 169809467b48Spatrick (SUBREG_TO_REG 169909467b48Spatrick (i64 0), 170009467b48Spatrick (MOVZX32rr8_NOREX 170109467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit_hi)), 170209467b48Spatrick sub_32bit)>; 170309467b48Spatrick 170409467b48Spatrick// h-register extract and store. 170509467b48Spatrickdef : Pat<(store (i8 (trunc_su (srl_su GR64:$src, (i8 8)))), addr:$dst), 170609467b48Spatrick (MOV8mr_NOREX 170709467b48Spatrick addr:$dst, 170809467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_8bit_hi))>; 170909467b48Spatrickdef : Pat<(store (i8 (trunc_su (srl_su GR32:$src, (i8 8)))), addr:$dst), 171009467b48Spatrick (MOV8mr_NOREX 171109467b48Spatrick addr:$dst, 171209467b48Spatrick (EXTRACT_SUBREG GR32:$src, sub_8bit_hi))>, 171309467b48Spatrick Requires<[In64BitMode]>; 171409467b48Spatrickdef : Pat<(store (i8 (trunc_su (srl_su GR16:$src, (i8 8)))), addr:$dst), 171509467b48Spatrick (MOV8mr_NOREX 171609467b48Spatrick addr:$dst, 171709467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit_hi))>, 171809467b48Spatrick Requires<[In64BitMode]>; 171909467b48Spatrick 172009467b48Spatrick 172109467b48Spatrick// (shl x, 1) ==> (add x, x) 172209467b48Spatrick// Note that if x is undef (immediate or otherwise), we could theoretically 172309467b48Spatrick// end up with the two uses of x getting different values, producing a result 172409467b48Spatrick// where the least significant bit is not 0. However, the probability of this 172509467b48Spatrick// happening is considered low enough that this is officially not a 172609467b48Spatrick// "real problem". 172709467b48Spatrickdef : Pat<(shl GR8 :$src1, (i8 1)), (ADD8rr GR8 :$src1, GR8 :$src1)>; 172809467b48Spatrickdef : Pat<(shl GR16:$src1, (i8 1)), (ADD16rr GR16:$src1, GR16:$src1)>; 172909467b48Spatrickdef : Pat<(shl GR32:$src1, (i8 1)), (ADD32rr GR32:$src1, GR32:$src1)>; 173009467b48Spatrickdef : Pat<(shl GR64:$src1, (i8 1)), (ADD64rr GR64:$src1, GR64:$src1)>; 173109467b48Spatrick 173209467b48Spatrickdef shiftMask8 : PatFrag<(ops node:$lhs), (and node:$lhs, imm), [{ 173309467b48Spatrick return isUnneededShiftMask(N, 3); 173409467b48Spatrick}]>; 173509467b48Spatrick 173609467b48Spatrickdef shiftMask16 : PatFrag<(ops node:$lhs), (and node:$lhs, imm), [{ 173709467b48Spatrick return isUnneededShiftMask(N, 4); 173809467b48Spatrick}]>; 173909467b48Spatrick 174009467b48Spatrickdef shiftMask32 : PatFrag<(ops node:$lhs), (and node:$lhs, imm), [{ 174109467b48Spatrick return isUnneededShiftMask(N, 5); 174209467b48Spatrick}]>; 174309467b48Spatrick 174409467b48Spatrickdef shiftMask64 : PatFrag<(ops node:$lhs), (and node:$lhs, imm), [{ 174509467b48Spatrick return isUnneededShiftMask(N, 6); 174609467b48Spatrick}]>; 174709467b48Spatrick 174809467b48Spatrick 174909467b48Spatrick// Shift amount is implicitly masked. 175009467b48Spatrickmulticlass MaskedShiftAmountPats<SDNode frag, string name> { 175109467b48Spatrick // (shift x (and y, 31)) ==> (shift x, y) 175209467b48Spatrick def : Pat<(frag GR8:$src1, (shiftMask32 CL)), 175309467b48Spatrick (!cast<Instruction>(name # "8rCL") GR8:$src1)>; 175409467b48Spatrick def : Pat<(frag GR16:$src1, (shiftMask32 CL)), 175509467b48Spatrick (!cast<Instruction>(name # "16rCL") GR16:$src1)>; 175609467b48Spatrick def : Pat<(frag GR32:$src1, (shiftMask32 CL)), 175709467b48Spatrick (!cast<Instruction>(name # "32rCL") GR32:$src1)>; 175809467b48Spatrick def : Pat<(store (frag (loadi8 addr:$dst), (shiftMask32 CL)), addr:$dst), 175909467b48Spatrick (!cast<Instruction>(name # "8mCL") addr:$dst)>; 176009467b48Spatrick def : Pat<(store (frag (loadi16 addr:$dst), (shiftMask32 CL)), addr:$dst), 176109467b48Spatrick (!cast<Instruction>(name # "16mCL") addr:$dst)>; 176209467b48Spatrick def : Pat<(store (frag (loadi32 addr:$dst), (shiftMask32 CL)), addr:$dst), 176309467b48Spatrick (!cast<Instruction>(name # "32mCL") addr:$dst)>; 176409467b48Spatrick 176509467b48Spatrick // (shift x (and y, 63)) ==> (shift x, y) 176609467b48Spatrick def : Pat<(frag GR64:$src1, (shiftMask64 CL)), 176709467b48Spatrick (!cast<Instruction>(name # "64rCL") GR64:$src1)>; 176809467b48Spatrick def : Pat<(store (frag (loadi64 addr:$dst), (shiftMask64 CL)), addr:$dst), 176909467b48Spatrick (!cast<Instruction>(name # "64mCL") addr:$dst)>; 177009467b48Spatrick} 177109467b48Spatrick 177209467b48Spatrickdefm : MaskedShiftAmountPats<shl, "SHL">; 177309467b48Spatrickdefm : MaskedShiftAmountPats<srl, "SHR">; 177409467b48Spatrickdefm : MaskedShiftAmountPats<sra, "SAR">; 177509467b48Spatrick 177609467b48Spatrick// ROL/ROR instructions allow a stronger mask optimization than shift for 8- and 177709467b48Spatrick// 16-bit. We can remove a mask of any (bitwidth - 1) on the rotation amount 177809467b48Spatrick// because over-rotating produces the same result. This is noted in the Intel 177909467b48Spatrick// docs with: "tempCOUNT <- (COUNT & COUNTMASK) MOD SIZE". Masking the rotation 178009467b48Spatrick// amount could affect EFLAGS results, but that does not matter because we are 178109467b48Spatrick// not tracking flags for these nodes. 178209467b48Spatrickmulticlass MaskedRotateAmountPats<SDNode frag, string name> { 178309467b48Spatrick // (rot x (and y, BitWidth - 1)) ==> (rot x, y) 178409467b48Spatrick def : Pat<(frag GR8:$src1, (shiftMask8 CL)), 178509467b48Spatrick (!cast<Instruction>(name # "8rCL") GR8:$src1)>; 178609467b48Spatrick def : Pat<(frag GR16:$src1, (shiftMask16 CL)), 178709467b48Spatrick (!cast<Instruction>(name # "16rCL") GR16:$src1)>; 178809467b48Spatrick def : Pat<(frag GR32:$src1, (shiftMask32 CL)), 178909467b48Spatrick (!cast<Instruction>(name # "32rCL") GR32:$src1)>; 179009467b48Spatrick def : Pat<(store (frag (loadi8 addr:$dst), (shiftMask8 CL)), addr:$dst), 179109467b48Spatrick (!cast<Instruction>(name # "8mCL") addr:$dst)>; 179209467b48Spatrick def : Pat<(store (frag (loadi16 addr:$dst), (shiftMask16 CL)), addr:$dst), 179309467b48Spatrick (!cast<Instruction>(name # "16mCL") addr:$dst)>; 179409467b48Spatrick def : Pat<(store (frag (loadi32 addr:$dst), (shiftMask32 CL)), addr:$dst), 179509467b48Spatrick (!cast<Instruction>(name # "32mCL") addr:$dst)>; 179609467b48Spatrick 179709467b48Spatrick // (rot x (and y, 63)) ==> (rot x, y) 179809467b48Spatrick def : Pat<(frag GR64:$src1, (shiftMask64 CL)), 179909467b48Spatrick (!cast<Instruction>(name # "64rCL") GR64:$src1)>; 180009467b48Spatrick def : Pat<(store (frag (loadi64 addr:$dst), (shiftMask64 CL)), addr:$dst), 180109467b48Spatrick (!cast<Instruction>(name # "64mCL") addr:$dst)>; 180209467b48Spatrick} 180309467b48Spatrick 180409467b48Spatrick 180509467b48Spatrickdefm : MaskedRotateAmountPats<rotl, "ROL">; 180609467b48Spatrickdefm : MaskedRotateAmountPats<rotr, "ROR">; 180709467b48Spatrick 1808097a140dSpatrick// Double "funnel" shift amount is implicitly masked. 1809097a140dSpatrick// (fshl/fshr x (and y, 31)) ==> (fshl/fshr x, y) (NOTE: modulo32) 1810097a140dSpatrickdef : Pat<(X86fshl GR16:$src1, GR16:$src2, (shiftMask32 CL)), 1811097a140dSpatrick (SHLD16rrCL GR16:$src1, GR16:$src2)>; 1812097a140dSpatrickdef : Pat<(X86fshr GR16:$src2, GR16:$src1, (shiftMask32 CL)), 1813097a140dSpatrick (SHRD16rrCL GR16:$src1, GR16:$src2)>; 181409467b48Spatrick 1815097a140dSpatrick// (fshl/fshr x (and y, 31)) ==> (fshl/fshr x, y) 1816097a140dSpatrickdef : Pat<(fshl GR32:$src1, GR32:$src2, (shiftMask32 CL)), 1817097a140dSpatrick (SHLD32rrCL GR32:$src1, GR32:$src2)>; 1818097a140dSpatrickdef : Pat<(fshr GR32:$src2, GR32:$src1, (shiftMask32 CL)), 1819097a140dSpatrick (SHRD32rrCL GR32:$src1, GR32:$src2)>; 182009467b48Spatrick 1821097a140dSpatrick// (fshl/fshr x (and y, 63)) ==> (fshl/fshr x, y) 1822097a140dSpatrickdef : Pat<(fshl GR64:$src1, GR64:$src2, (shiftMask64 CL)), 1823097a140dSpatrick (SHLD64rrCL GR64:$src1, GR64:$src2)>; 1824097a140dSpatrickdef : Pat<(fshr GR64:$src2, GR64:$src1, (shiftMask64 CL)), 1825097a140dSpatrick (SHRD64rrCL GR64:$src1, GR64:$src2)>; 182609467b48Spatrick 182709467b48Spatricklet Predicates = [HasBMI2] in { 182809467b48Spatrick let AddedComplexity = 1 in { 182909467b48Spatrick def : Pat<(sra GR32:$src1, (shiftMask32 GR8:$src2)), 183009467b48Spatrick (SARX32rr GR32:$src1, 183109467b48Spatrick (INSERT_SUBREG 183209467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 183309467b48Spatrick def : Pat<(sra GR64:$src1, (shiftMask64 GR8:$src2)), 183409467b48Spatrick (SARX64rr GR64:$src1, 183509467b48Spatrick (INSERT_SUBREG 183609467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 183709467b48Spatrick 183809467b48Spatrick def : Pat<(srl GR32:$src1, (shiftMask32 GR8:$src2)), 183909467b48Spatrick (SHRX32rr GR32:$src1, 184009467b48Spatrick (INSERT_SUBREG 184109467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 184209467b48Spatrick def : Pat<(srl GR64:$src1, (shiftMask64 GR8:$src2)), 184309467b48Spatrick (SHRX64rr GR64:$src1, 184409467b48Spatrick (INSERT_SUBREG 184509467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 184609467b48Spatrick 184709467b48Spatrick def : Pat<(shl GR32:$src1, (shiftMask32 GR8:$src2)), 184809467b48Spatrick (SHLX32rr GR32:$src1, 184909467b48Spatrick (INSERT_SUBREG 185009467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 185109467b48Spatrick def : Pat<(shl GR64:$src1, (shiftMask64 GR8:$src2)), 185209467b48Spatrick (SHLX64rr GR64:$src1, 185309467b48Spatrick (INSERT_SUBREG 185409467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 185509467b48Spatrick } 185609467b48Spatrick 185709467b48Spatrick def : Pat<(sra (loadi32 addr:$src1), (shiftMask32 GR8:$src2)), 185809467b48Spatrick (SARX32rm addr:$src1, 185909467b48Spatrick (INSERT_SUBREG 186009467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 186109467b48Spatrick def : Pat<(sra (loadi64 addr:$src1), (shiftMask64 GR8:$src2)), 186209467b48Spatrick (SARX64rm addr:$src1, 186309467b48Spatrick (INSERT_SUBREG 186409467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 186509467b48Spatrick 186609467b48Spatrick def : Pat<(srl (loadi32 addr:$src1), (shiftMask32 GR8:$src2)), 186709467b48Spatrick (SHRX32rm addr:$src1, 186809467b48Spatrick (INSERT_SUBREG 186909467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 187009467b48Spatrick def : Pat<(srl (loadi64 addr:$src1), (shiftMask64 GR8:$src2)), 187109467b48Spatrick (SHRX64rm addr:$src1, 187209467b48Spatrick (INSERT_SUBREG 187309467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 187409467b48Spatrick 187509467b48Spatrick def : Pat<(shl (loadi32 addr:$src1), (shiftMask32 GR8:$src2)), 187609467b48Spatrick (SHLX32rm addr:$src1, 187709467b48Spatrick (INSERT_SUBREG 187809467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 187909467b48Spatrick def : Pat<(shl (loadi64 addr:$src1), (shiftMask64 GR8:$src2)), 188009467b48Spatrick (SHLX64rm addr:$src1, 188109467b48Spatrick (INSERT_SUBREG 188209467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 188309467b48Spatrick} 188409467b48Spatrick 188509467b48Spatrick// Use BTR/BTS/BTC for clearing/setting/toggling a bit in a variable location. 188609467b48Spatrickmulticlass one_bit_patterns<RegisterClass RC, ValueType VT, Instruction BTR, 188709467b48Spatrick Instruction BTS, Instruction BTC, 188809467b48Spatrick PatFrag ShiftMask> { 188909467b48Spatrick def : Pat<(and RC:$src1, (rotl -2, GR8:$src2)), 189009467b48Spatrick (BTR RC:$src1, 189109467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 189209467b48Spatrick def : Pat<(or RC:$src1, (shl 1, GR8:$src2)), 189309467b48Spatrick (BTS RC:$src1, 189409467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 189509467b48Spatrick def : Pat<(xor RC:$src1, (shl 1, GR8:$src2)), 189609467b48Spatrick (BTC RC:$src1, 189709467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 189809467b48Spatrick 189909467b48Spatrick // Similar to above, but removing unneeded masking of the shift amount. 190009467b48Spatrick def : Pat<(and RC:$src1, (rotl -2, (ShiftMask GR8:$src2))), 190109467b48Spatrick (BTR RC:$src1, 190209467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 190309467b48Spatrick def : Pat<(or RC:$src1, (shl 1, (ShiftMask GR8:$src2))), 190409467b48Spatrick (BTS RC:$src1, 190509467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 190609467b48Spatrick def : Pat<(xor RC:$src1, (shl 1, (ShiftMask GR8:$src2))), 190709467b48Spatrick (BTC RC:$src1, 190809467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 190909467b48Spatrick} 191009467b48Spatrick 191109467b48Spatrickdefm : one_bit_patterns<GR16, i16, BTR16rr, BTS16rr, BTC16rr, shiftMask16>; 191209467b48Spatrickdefm : one_bit_patterns<GR32, i32, BTR32rr, BTS32rr, BTC32rr, shiftMask32>; 191309467b48Spatrickdefm : one_bit_patterns<GR64, i64, BTR64rr, BTS64rr, BTC64rr, shiftMask64>; 191409467b48Spatrick 191509467b48Spatrick//===----------------------------------------------------------------------===// 191609467b48Spatrick// EFLAGS-defining Patterns 191709467b48Spatrick//===----------------------------------------------------------------------===// 191809467b48Spatrick 191909467b48Spatrick// add reg, reg 192009467b48Spatrickdef : Pat<(add GR8 :$src1, GR8 :$src2), (ADD8rr GR8 :$src1, GR8 :$src2)>; 192109467b48Spatrickdef : Pat<(add GR16:$src1, GR16:$src2), (ADD16rr GR16:$src1, GR16:$src2)>; 192209467b48Spatrickdef : Pat<(add GR32:$src1, GR32:$src2), (ADD32rr GR32:$src1, GR32:$src2)>; 192309467b48Spatrickdef : Pat<(add GR64:$src1, GR64:$src2), (ADD64rr GR64:$src1, GR64:$src2)>; 192409467b48Spatrick 192509467b48Spatrick// add reg, mem 192609467b48Spatrickdef : Pat<(add GR8:$src1, (loadi8 addr:$src2)), 192709467b48Spatrick (ADD8rm GR8:$src1, addr:$src2)>; 192809467b48Spatrickdef : Pat<(add GR16:$src1, (loadi16 addr:$src2)), 192909467b48Spatrick (ADD16rm GR16:$src1, addr:$src2)>; 193009467b48Spatrickdef : Pat<(add GR32:$src1, (loadi32 addr:$src2)), 193109467b48Spatrick (ADD32rm GR32:$src1, addr:$src2)>; 193209467b48Spatrickdef : Pat<(add GR64:$src1, (loadi64 addr:$src2)), 193309467b48Spatrick (ADD64rm GR64:$src1, addr:$src2)>; 193409467b48Spatrick 193509467b48Spatrick// add reg, imm 193609467b48Spatrickdef : Pat<(add GR8 :$src1, imm:$src2), (ADD8ri GR8:$src1 , imm:$src2)>; 193709467b48Spatrickdef : Pat<(add GR16:$src1, imm:$src2), (ADD16ri GR16:$src1, imm:$src2)>; 193809467b48Spatrickdef : Pat<(add GR32:$src1, imm:$src2), (ADD32ri GR32:$src1, imm:$src2)>; 193909467b48Spatrickdef : Pat<(add GR16:$src1, i16immSExt8:$src2), 194009467b48Spatrick (ADD16ri8 GR16:$src1, i16immSExt8:$src2)>; 194109467b48Spatrickdef : Pat<(add GR32:$src1, i32immSExt8:$src2), 194209467b48Spatrick (ADD32ri8 GR32:$src1, i32immSExt8:$src2)>; 194309467b48Spatrickdef : Pat<(add GR64:$src1, i64immSExt8:$src2), 194409467b48Spatrick (ADD64ri8 GR64:$src1, i64immSExt8:$src2)>; 194509467b48Spatrickdef : Pat<(add GR64:$src1, i64immSExt32:$src2), 194609467b48Spatrick (ADD64ri32 GR64:$src1, i64immSExt32:$src2)>; 194709467b48Spatrick 194809467b48Spatrick// sub reg, reg 194909467b48Spatrickdef : Pat<(sub GR8 :$src1, GR8 :$src2), (SUB8rr GR8 :$src1, GR8 :$src2)>; 195009467b48Spatrickdef : Pat<(sub GR16:$src1, GR16:$src2), (SUB16rr GR16:$src1, GR16:$src2)>; 195109467b48Spatrickdef : Pat<(sub GR32:$src1, GR32:$src2), (SUB32rr GR32:$src1, GR32:$src2)>; 195209467b48Spatrickdef : Pat<(sub GR64:$src1, GR64:$src2), (SUB64rr GR64:$src1, GR64:$src2)>; 195309467b48Spatrick 195409467b48Spatrick// sub reg, mem 195509467b48Spatrickdef : Pat<(sub GR8:$src1, (loadi8 addr:$src2)), 195609467b48Spatrick (SUB8rm GR8:$src1, addr:$src2)>; 195709467b48Spatrickdef : Pat<(sub GR16:$src1, (loadi16 addr:$src2)), 195809467b48Spatrick (SUB16rm GR16:$src1, addr:$src2)>; 195909467b48Spatrickdef : Pat<(sub GR32:$src1, (loadi32 addr:$src2)), 196009467b48Spatrick (SUB32rm GR32:$src1, addr:$src2)>; 196109467b48Spatrickdef : Pat<(sub GR64:$src1, (loadi64 addr:$src2)), 196209467b48Spatrick (SUB64rm GR64:$src1, addr:$src2)>; 196309467b48Spatrick 196409467b48Spatrick// sub reg, imm 196509467b48Spatrickdef : Pat<(sub GR8:$src1, imm:$src2), 196609467b48Spatrick (SUB8ri GR8:$src1, imm:$src2)>; 196709467b48Spatrickdef : Pat<(sub GR16:$src1, imm:$src2), 196809467b48Spatrick (SUB16ri GR16:$src1, imm:$src2)>; 196909467b48Spatrickdef : Pat<(sub GR32:$src1, imm:$src2), 197009467b48Spatrick (SUB32ri GR32:$src1, imm:$src2)>; 197109467b48Spatrickdef : Pat<(sub GR16:$src1, i16immSExt8:$src2), 197209467b48Spatrick (SUB16ri8 GR16:$src1, i16immSExt8:$src2)>; 197309467b48Spatrickdef : Pat<(sub GR32:$src1, i32immSExt8:$src2), 197409467b48Spatrick (SUB32ri8 GR32:$src1, i32immSExt8:$src2)>; 197509467b48Spatrickdef : Pat<(sub GR64:$src1, i64immSExt8:$src2), 197609467b48Spatrick (SUB64ri8 GR64:$src1, i64immSExt8:$src2)>; 197709467b48Spatrickdef : Pat<(sub GR64:$src1, i64immSExt32:$src2), 197809467b48Spatrick (SUB64ri32 GR64:$src1, i64immSExt32:$src2)>; 197909467b48Spatrick 198009467b48Spatrick// sub 0, reg 198109467b48Spatrickdef : Pat<(X86sub_flag 0, GR8 :$src), (NEG8r GR8 :$src)>; 198209467b48Spatrickdef : Pat<(X86sub_flag 0, GR16:$src), (NEG16r GR16:$src)>; 198309467b48Spatrickdef : Pat<(X86sub_flag 0, GR32:$src), (NEG32r GR32:$src)>; 198409467b48Spatrickdef : Pat<(X86sub_flag 0, GR64:$src), (NEG64r GR64:$src)>; 198509467b48Spatrick 198609467b48Spatrick// mul reg, reg 198709467b48Spatrickdef : Pat<(mul GR16:$src1, GR16:$src2), 198809467b48Spatrick (IMUL16rr GR16:$src1, GR16:$src2)>; 198909467b48Spatrickdef : Pat<(mul GR32:$src1, GR32:$src2), 199009467b48Spatrick (IMUL32rr GR32:$src1, GR32:$src2)>; 199109467b48Spatrickdef : Pat<(mul GR64:$src1, GR64:$src2), 199209467b48Spatrick (IMUL64rr GR64:$src1, GR64:$src2)>; 199309467b48Spatrick 199409467b48Spatrick// mul reg, mem 199509467b48Spatrickdef : Pat<(mul GR16:$src1, (loadi16 addr:$src2)), 199609467b48Spatrick (IMUL16rm GR16:$src1, addr:$src2)>; 199709467b48Spatrickdef : Pat<(mul GR32:$src1, (loadi32 addr:$src2)), 199809467b48Spatrick (IMUL32rm GR32:$src1, addr:$src2)>; 199909467b48Spatrickdef : Pat<(mul GR64:$src1, (loadi64 addr:$src2)), 200009467b48Spatrick (IMUL64rm GR64:$src1, addr:$src2)>; 200109467b48Spatrick 200209467b48Spatrick// mul reg, imm 200309467b48Spatrickdef : Pat<(mul GR16:$src1, imm:$src2), 200409467b48Spatrick (IMUL16rri GR16:$src1, imm:$src2)>; 200509467b48Spatrickdef : Pat<(mul GR32:$src1, imm:$src2), 200609467b48Spatrick (IMUL32rri GR32:$src1, imm:$src2)>; 200709467b48Spatrickdef : Pat<(mul GR16:$src1, i16immSExt8:$src2), 200809467b48Spatrick (IMUL16rri8 GR16:$src1, i16immSExt8:$src2)>; 200909467b48Spatrickdef : Pat<(mul GR32:$src1, i32immSExt8:$src2), 201009467b48Spatrick (IMUL32rri8 GR32:$src1, i32immSExt8:$src2)>; 201109467b48Spatrickdef : Pat<(mul GR64:$src1, i64immSExt8:$src2), 201209467b48Spatrick (IMUL64rri8 GR64:$src1, i64immSExt8:$src2)>; 201309467b48Spatrickdef : Pat<(mul GR64:$src1, i64immSExt32:$src2), 201409467b48Spatrick (IMUL64rri32 GR64:$src1, i64immSExt32:$src2)>; 201509467b48Spatrick 201609467b48Spatrick// reg = mul mem, imm 201709467b48Spatrickdef : Pat<(mul (loadi16 addr:$src1), imm:$src2), 201809467b48Spatrick (IMUL16rmi addr:$src1, imm:$src2)>; 201909467b48Spatrickdef : Pat<(mul (loadi32 addr:$src1), imm:$src2), 202009467b48Spatrick (IMUL32rmi addr:$src1, imm:$src2)>; 202109467b48Spatrickdef : Pat<(mul (loadi16 addr:$src1), i16immSExt8:$src2), 202209467b48Spatrick (IMUL16rmi8 addr:$src1, i16immSExt8:$src2)>; 202309467b48Spatrickdef : Pat<(mul (loadi32 addr:$src1), i32immSExt8:$src2), 202409467b48Spatrick (IMUL32rmi8 addr:$src1, i32immSExt8:$src2)>; 202509467b48Spatrickdef : Pat<(mul (loadi64 addr:$src1), i64immSExt8:$src2), 202609467b48Spatrick (IMUL64rmi8 addr:$src1, i64immSExt8:$src2)>; 202709467b48Spatrickdef : Pat<(mul (loadi64 addr:$src1), i64immSExt32:$src2), 202809467b48Spatrick (IMUL64rmi32 addr:$src1, i64immSExt32:$src2)>; 202909467b48Spatrick 203009467b48Spatrick// Increment/Decrement reg. 203109467b48Spatrick// Do not make INC/DEC if it is slow 203209467b48Spatricklet Predicates = [UseIncDec] in { 203309467b48Spatrick def : Pat<(add GR8:$src, 1), (INC8r GR8:$src)>; 203409467b48Spatrick def : Pat<(add GR16:$src, 1), (INC16r GR16:$src)>; 203509467b48Spatrick def : Pat<(add GR32:$src, 1), (INC32r GR32:$src)>; 203609467b48Spatrick def : Pat<(add GR64:$src, 1), (INC64r GR64:$src)>; 203709467b48Spatrick def : Pat<(add GR8:$src, -1), (DEC8r GR8:$src)>; 203809467b48Spatrick def : Pat<(add GR16:$src, -1), (DEC16r GR16:$src)>; 203909467b48Spatrick def : Pat<(add GR32:$src, -1), (DEC32r GR32:$src)>; 204009467b48Spatrick def : Pat<(add GR64:$src, -1), (DEC64r GR64:$src)>; 204109467b48Spatrick 204209467b48Spatrick def : Pat<(X86add_flag_nocf GR8:$src, -1), (DEC8r GR8:$src)>; 204309467b48Spatrick def : Pat<(X86add_flag_nocf GR16:$src, -1), (DEC16r GR16:$src)>; 204409467b48Spatrick def : Pat<(X86add_flag_nocf GR32:$src, -1), (DEC32r GR32:$src)>; 204509467b48Spatrick def : Pat<(X86add_flag_nocf GR64:$src, -1), (DEC64r GR64:$src)>; 204609467b48Spatrick def : Pat<(X86sub_flag_nocf GR8:$src, -1), (INC8r GR8:$src)>; 204709467b48Spatrick def : Pat<(X86sub_flag_nocf GR16:$src, -1), (INC16r GR16:$src)>; 204809467b48Spatrick def : Pat<(X86sub_flag_nocf GR32:$src, -1), (INC32r GR32:$src)>; 204909467b48Spatrick def : Pat<(X86sub_flag_nocf GR64:$src, -1), (INC64r GR64:$src)>; 205009467b48Spatrick} 205109467b48Spatrick 205209467b48Spatrick// or reg/reg. 205309467b48Spatrickdef : Pat<(or GR8 :$src1, GR8 :$src2), (OR8rr GR8 :$src1, GR8 :$src2)>; 205409467b48Spatrickdef : Pat<(or GR16:$src1, GR16:$src2), (OR16rr GR16:$src1, GR16:$src2)>; 205509467b48Spatrickdef : Pat<(or GR32:$src1, GR32:$src2), (OR32rr GR32:$src1, GR32:$src2)>; 205609467b48Spatrickdef : Pat<(or GR64:$src1, GR64:$src2), (OR64rr GR64:$src1, GR64:$src2)>; 205709467b48Spatrick 205809467b48Spatrick// or reg/mem 205909467b48Spatrickdef : Pat<(or GR8:$src1, (loadi8 addr:$src2)), 206009467b48Spatrick (OR8rm GR8:$src1, addr:$src2)>; 206109467b48Spatrickdef : Pat<(or GR16:$src1, (loadi16 addr:$src2)), 206209467b48Spatrick (OR16rm GR16:$src1, addr:$src2)>; 206309467b48Spatrickdef : Pat<(or GR32:$src1, (loadi32 addr:$src2)), 206409467b48Spatrick (OR32rm GR32:$src1, addr:$src2)>; 206509467b48Spatrickdef : Pat<(or GR64:$src1, (loadi64 addr:$src2)), 206609467b48Spatrick (OR64rm GR64:$src1, addr:$src2)>; 206709467b48Spatrick 206809467b48Spatrick// or reg/imm 206909467b48Spatrickdef : Pat<(or GR8:$src1 , imm:$src2), (OR8ri GR8 :$src1, imm:$src2)>; 207009467b48Spatrickdef : Pat<(or GR16:$src1, imm:$src2), (OR16ri GR16:$src1, imm:$src2)>; 207109467b48Spatrickdef : Pat<(or GR32:$src1, imm:$src2), (OR32ri GR32:$src1, imm:$src2)>; 207209467b48Spatrickdef : Pat<(or GR16:$src1, i16immSExt8:$src2), 207309467b48Spatrick (OR16ri8 GR16:$src1, i16immSExt8:$src2)>; 207409467b48Spatrickdef : Pat<(or GR32:$src1, i32immSExt8:$src2), 207509467b48Spatrick (OR32ri8 GR32:$src1, i32immSExt8:$src2)>; 207609467b48Spatrickdef : Pat<(or GR64:$src1, i64immSExt8:$src2), 207709467b48Spatrick (OR64ri8 GR64:$src1, i64immSExt8:$src2)>; 207809467b48Spatrickdef : Pat<(or GR64:$src1, i64immSExt32:$src2), 207909467b48Spatrick (OR64ri32 GR64:$src1, i64immSExt32:$src2)>; 208009467b48Spatrick 208109467b48Spatrick// xor reg/reg 208209467b48Spatrickdef : Pat<(xor GR8 :$src1, GR8 :$src2), (XOR8rr GR8 :$src1, GR8 :$src2)>; 208309467b48Spatrickdef : Pat<(xor GR16:$src1, GR16:$src2), (XOR16rr GR16:$src1, GR16:$src2)>; 208409467b48Spatrickdef : Pat<(xor GR32:$src1, GR32:$src2), (XOR32rr GR32:$src1, GR32:$src2)>; 208509467b48Spatrickdef : Pat<(xor GR64:$src1, GR64:$src2), (XOR64rr GR64:$src1, GR64:$src2)>; 208609467b48Spatrick 208709467b48Spatrick// xor reg/mem 208809467b48Spatrickdef : Pat<(xor GR8:$src1, (loadi8 addr:$src2)), 208909467b48Spatrick (XOR8rm GR8:$src1, addr:$src2)>; 209009467b48Spatrickdef : Pat<(xor GR16:$src1, (loadi16 addr:$src2)), 209109467b48Spatrick (XOR16rm GR16:$src1, addr:$src2)>; 209209467b48Spatrickdef : Pat<(xor GR32:$src1, (loadi32 addr:$src2)), 209309467b48Spatrick (XOR32rm GR32:$src1, addr:$src2)>; 209409467b48Spatrickdef : Pat<(xor GR64:$src1, (loadi64 addr:$src2)), 209509467b48Spatrick (XOR64rm GR64:$src1, addr:$src2)>; 209609467b48Spatrick 209709467b48Spatrick// xor reg/imm 209809467b48Spatrickdef : Pat<(xor GR8:$src1, imm:$src2), 209909467b48Spatrick (XOR8ri GR8:$src1, imm:$src2)>; 210009467b48Spatrickdef : Pat<(xor GR16:$src1, imm:$src2), 210109467b48Spatrick (XOR16ri GR16:$src1, imm:$src2)>; 210209467b48Spatrickdef : Pat<(xor GR32:$src1, imm:$src2), 210309467b48Spatrick (XOR32ri GR32:$src1, imm:$src2)>; 210409467b48Spatrickdef : Pat<(xor GR16:$src1, i16immSExt8:$src2), 210509467b48Spatrick (XOR16ri8 GR16:$src1, i16immSExt8:$src2)>; 210609467b48Spatrickdef : Pat<(xor GR32:$src1, i32immSExt8:$src2), 210709467b48Spatrick (XOR32ri8 GR32:$src1, i32immSExt8:$src2)>; 210809467b48Spatrickdef : Pat<(xor GR64:$src1, i64immSExt8:$src2), 210909467b48Spatrick (XOR64ri8 GR64:$src1, i64immSExt8:$src2)>; 211009467b48Spatrickdef : Pat<(xor GR64:$src1, i64immSExt32:$src2), 211109467b48Spatrick (XOR64ri32 GR64:$src1, i64immSExt32:$src2)>; 211209467b48Spatrick 211309467b48Spatrick// and reg/reg 211409467b48Spatrickdef : Pat<(and GR8 :$src1, GR8 :$src2), (AND8rr GR8 :$src1, GR8 :$src2)>; 211509467b48Spatrickdef : Pat<(and GR16:$src1, GR16:$src2), (AND16rr GR16:$src1, GR16:$src2)>; 211609467b48Spatrickdef : Pat<(and GR32:$src1, GR32:$src2), (AND32rr GR32:$src1, GR32:$src2)>; 211709467b48Spatrickdef : Pat<(and GR64:$src1, GR64:$src2), (AND64rr GR64:$src1, GR64:$src2)>; 211809467b48Spatrick 211909467b48Spatrick// and reg/mem 212009467b48Spatrickdef : Pat<(and GR8:$src1, (loadi8 addr:$src2)), 212109467b48Spatrick (AND8rm GR8:$src1, addr:$src2)>; 212209467b48Spatrickdef : Pat<(and GR16:$src1, (loadi16 addr:$src2)), 212309467b48Spatrick (AND16rm GR16:$src1, addr:$src2)>; 212409467b48Spatrickdef : Pat<(and GR32:$src1, (loadi32 addr:$src2)), 212509467b48Spatrick (AND32rm GR32:$src1, addr:$src2)>; 212609467b48Spatrickdef : Pat<(and GR64:$src1, (loadi64 addr:$src2)), 212709467b48Spatrick (AND64rm GR64:$src1, addr:$src2)>; 212809467b48Spatrick 212909467b48Spatrick// and reg/imm 213009467b48Spatrickdef : Pat<(and GR8:$src1, imm:$src2), 213109467b48Spatrick (AND8ri GR8:$src1, imm:$src2)>; 213209467b48Spatrickdef : Pat<(and GR16:$src1, imm:$src2), 213309467b48Spatrick (AND16ri GR16:$src1, imm:$src2)>; 213409467b48Spatrickdef : Pat<(and GR32:$src1, imm:$src2), 213509467b48Spatrick (AND32ri GR32:$src1, imm:$src2)>; 213609467b48Spatrickdef : Pat<(and GR16:$src1, i16immSExt8:$src2), 213709467b48Spatrick (AND16ri8 GR16:$src1, i16immSExt8:$src2)>; 213809467b48Spatrickdef : Pat<(and GR32:$src1, i32immSExt8:$src2), 213909467b48Spatrick (AND32ri8 GR32:$src1, i32immSExt8:$src2)>; 214009467b48Spatrickdef : Pat<(and GR64:$src1, i64immSExt8:$src2), 214109467b48Spatrick (AND64ri8 GR64:$src1, i64immSExt8:$src2)>; 214209467b48Spatrickdef : Pat<(and GR64:$src1, i64immSExt32:$src2), 214309467b48Spatrick (AND64ri32 GR64:$src1, i64immSExt32:$src2)>; 214409467b48Spatrick 214509467b48Spatrick// Bit scan instruction patterns to match explicit zero-undef behavior. 214609467b48Spatrickdef : Pat<(cttz_zero_undef GR16:$src), (BSF16rr GR16:$src)>; 214709467b48Spatrickdef : Pat<(cttz_zero_undef GR32:$src), (BSF32rr GR32:$src)>; 214809467b48Spatrickdef : Pat<(cttz_zero_undef GR64:$src), (BSF64rr GR64:$src)>; 214909467b48Spatrickdef : Pat<(cttz_zero_undef (loadi16 addr:$src)), (BSF16rm addr:$src)>; 215009467b48Spatrickdef : Pat<(cttz_zero_undef (loadi32 addr:$src)), (BSF32rm addr:$src)>; 215109467b48Spatrickdef : Pat<(cttz_zero_undef (loadi64 addr:$src)), (BSF64rm addr:$src)>; 215209467b48Spatrick 215309467b48Spatrick// When HasMOVBE is enabled it is possible to get a non-legalized 215409467b48Spatrick// register-register 16 bit bswap. This maps it to a ROL instruction. 215509467b48Spatricklet Predicates = [HasMOVBE] in { 215609467b48Spatrick def : Pat<(bswap GR16:$src), (ROL16ri GR16:$src, (i8 8))>; 215709467b48Spatrick} 2158