109467b48Spatrick//===- X86InstrCompiler.td - Compiler Pseudos and Patterns -*- tablegen -*-===// 209467b48Spatrick// 309467b48Spatrick// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 409467b48Spatrick// See https://llvm.org/LICENSE.txt for license information. 509467b48Spatrick// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 609467b48Spatrick// 709467b48Spatrick//===----------------------------------------------------------------------===// 809467b48Spatrick// 909467b48Spatrick// This file describes the various pseudo instructions used by the compiler, 1009467b48Spatrick// as well as Pat patterns used during instruction selection. 1109467b48Spatrick// 1209467b48Spatrick//===----------------------------------------------------------------------===// 1309467b48Spatrick 1409467b48Spatrick//===----------------------------------------------------------------------===// 1509467b48Spatrick// Pattern Matching Support 1609467b48Spatrick 1709467b48Spatrickdef GetLo32XForm : SDNodeXForm<imm, [{ 1809467b48Spatrick // Transformation function: get the low 32 bits. 1909467b48Spatrick return getI32Imm((uint32_t)N->getZExtValue(), SDLoc(N)); 2009467b48Spatrick}]>; 2109467b48Spatrick 2209467b48Spatrick 2309467b48Spatrick//===----------------------------------------------------------------------===// 2409467b48Spatrick// Random Pseudo Instructions. 2509467b48Spatrick 2609467b48Spatrick// PIC base construction. This expands to code that looks like this: 2709467b48Spatrick// call $next_inst 2809467b48Spatrick// popl %destreg" 2909467b48Spatricklet hasSideEffects = 0, isNotDuplicable = 1, Uses = [ESP, SSP], 3009467b48Spatrick SchedRW = [WriteJump] in 3109467b48Spatrick def MOVPC32r : Ii32<0xE8, Pseudo, (outs GR32:$reg), (ins i32imm:$label), 3209467b48Spatrick "", []>; 3309467b48Spatrick 3409467b48Spatrick// ADJCALLSTACKDOWN/UP implicitly use/def ESP because they may be expanded into 3509467b48Spatrick// a stack adjustment and the codegen must know that they may modify the stack 3609467b48Spatrick// pointer before prolog-epilog rewriting occurs. 3709467b48Spatrick// Pessimistically assume ADJCALLSTACKDOWN / ADJCALLSTACKUP will become 3809467b48Spatrick// sub / add which can clobber EFLAGS. 3909467b48Spatricklet Defs = [ESP, EFLAGS, SSP], Uses = [ESP, SSP], SchedRW = [WriteALU] in { 4009467b48Spatrickdef ADJCALLSTACKDOWN32 : I<0, Pseudo, (outs), 4109467b48Spatrick (ins i32imm:$amt1, i32imm:$amt2, i32imm:$amt3), 4209467b48Spatrick "#ADJCALLSTACKDOWN", []>, Requires<[NotLP64]>; 4309467b48Spatrickdef ADJCALLSTACKUP32 : I<0, Pseudo, (outs), (ins i32imm:$amt1, i32imm:$amt2), 4409467b48Spatrick "#ADJCALLSTACKUP", 4509467b48Spatrick [(X86callseq_end timm:$amt1, timm:$amt2)]>, 4609467b48Spatrick Requires<[NotLP64]>; 4709467b48Spatrick} 4809467b48Spatrickdef : Pat<(X86callseq_start timm:$amt1, timm:$amt2), 4909467b48Spatrick (ADJCALLSTACKDOWN32 i32imm:$amt1, i32imm:$amt2, 0)>, Requires<[NotLP64]>; 5009467b48Spatrick 5109467b48Spatrick 5209467b48Spatrick// ADJCALLSTACKDOWN/UP implicitly use/def RSP because they may be expanded into 5309467b48Spatrick// a stack adjustment and the codegen must know that they may modify the stack 5409467b48Spatrick// pointer before prolog-epilog rewriting occurs. 5509467b48Spatrick// Pessimistically assume ADJCALLSTACKDOWN / ADJCALLSTACKUP will become 5609467b48Spatrick// sub / add which can clobber EFLAGS. 5709467b48Spatricklet Defs = [RSP, EFLAGS, SSP], Uses = [RSP, SSP], SchedRW = [WriteALU] in { 5809467b48Spatrickdef ADJCALLSTACKDOWN64 : I<0, Pseudo, (outs), 5909467b48Spatrick (ins i32imm:$amt1, i32imm:$amt2, i32imm:$amt3), 6009467b48Spatrick "#ADJCALLSTACKDOWN", []>, Requires<[IsLP64]>; 6109467b48Spatrickdef ADJCALLSTACKUP64 : I<0, Pseudo, (outs), (ins i32imm:$amt1, i32imm:$amt2), 6209467b48Spatrick "#ADJCALLSTACKUP", 6309467b48Spatrick [(X86callseq_end timm:$amt1, timm:$amt2)]>, 6409467b48Spatrick Requires<[IsLP64]>; 6509467b48Spatrick} 6609467b48Spatrickdef : Pat<(X86callseq_start timm:$amt1, timm:$amt2), 6709467b48Spatrick (ADJCALLSTACKDOWN64 i32imm:$amt1, i32imm:$amt2, 0)>, Requires<[IsLP64]>; 6809467b48Spatrick 6909467b48Spatricklet SchedRW = [WriteSystem] in { 7009467b48Spatrick 7109467b48Spatrick// x86-64 va_start lowering magic. 72*a96b3639Srobertlet hasSideEffects = 1, mayStore = 1, Defs = [EFLAGS] in { 7309467b48Spatrickdef VASTART_SAVE_XMM_REGS : I<0, Pseudo, 7409467b48Spatrick (outs), 75*a96b3639Srobert (ins GR8:$al, i8mem:$regsavefi, variable_ops), 76*a96b3639Srobert "#VASTART_SAVE_XMM_REGS $al, $regsavefi", 77*a96b3639Srobert [(X86vastart_save_xmm_regs GR8:$al, addr:$regsavefi), 7809467b48Spatrick (implicit EFLAGS)]>; 79a0747c9fSpatrick} 8009467b48Spatrick 81a0747c9fSpatricklet usesCustomInserter = 1, Defs = [EFLAGS] in { 82a0747c9fSpatrick// The VAARG_64 and VAARG_X32 pseudo-instructions take the address of the 83a0747c9fSpatrick// va_list, and place the address of the next argument into a register. 84a0747c9fSpatricklet Defs = [EFLAGS] in { 8509467b48Spatrickdef VAARG_64 : I<0, Pseudo, 8609467b48Spatrick (outs GR64:$dst), 8709467b48Spatrick (ins i8mem:$ap, i32imm:$size, i8imm:$mode, i32imm:$align), 8809467b48Spatrick "#VAARG_64 $dst, $ap, $size, $mode, $align", 8909467b48Spatrick [(set GR64:$dst, 90a0747c9fSpatrick (X86vaarg64 addr:$ap, timm:$size, timm:$mode, timm:$align)), 91a0747c9fSpatrick (implicit EFLAGS)]>, Requires<[In64BitMode, IsLP64]>; 92a0747c9fSpatrickdef VAARG_X32 : I<0, Pseudo, 93a0747c9fSpatrick (outs GR32:$dst), 94a0747c9fSpatrick (ins i8mem:$ap, i32imm:$size, i8imm:$mode, i32imm:$align), 95a0747c9fSpatrick "#VAARG_X32 $dst, $ap, $size, $mode, $align", 96a0747c9fSpatrick [(set GR32:$dst, 97a0747c9fSpatrick (X86vaargx32 addr:$ap, timm:$size, timm:$mode, timm:$align)), 98a0747c9fSpatrick (implicit EFLAGS)]>, Requires<[In64BitMode, NotLP64]>; 99a0747c9fSpatrick} 10009467b48Spatrick 10109467b48Spatrick// When using segmented stacks these are lowered into instructions which first 10209467b48Spatrick// check if the current stacklet has enough free memory. If it does, memory is 10309467b48Spatrick// allocated by bumping the stack pointer. Otherwise memory is allocated from 10409467b48Spatrick// the heap. 10509467b48Spatrick 10609467b48Spatricklet Defs = [EAX, ESP, EFLAGS], Uses = [ESP] in 10709467b48Spatrickdef SEG_ALLOCA_32 : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$size), 10809467b48Spatrick "# variable sized alloca for segmented stacks", 10909467b48Spatrick [(set GR32:$dst, 11009467b48Spatrick (X86SegAlloca GR32:$size))]>, 11109467b48Spatrick Requires<[NotLP64]>; 11209467b48Spatrick 11309467b48Spatricklet Defs = [RAX, RSP, EFLAGS], Uses = [RSP] in 11409467b48Spatrickdef SEG_ALLOCA_64 : I<0, Pseudo, (outs GR64:$dst), (ins GR64:$size), 11509467b48Spatrick "# variable sized alloca for segmented stacks", 11609467b48Spatrick [(set GR64:$dst, 11709467b48Spatrick (X86SegAlloca GR64:$size))]>, 11809467b48Spatrick Requires<[In64BitMode]>; 119097a140dSpatrick 120097a140dSpatrick// To protect against stack clash, dynamic allocation should perform a memory 121097a140dSpatrick// probe at each page. 122097a140dSpatrick 123097a140dSpatricklet Defs = [EAX, ESP, EFLAGS], Uses = [ESP] in 124097a140dSpatrickdef PROBED_ALLOCA_32 : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$size), 125097a140dSpatrick "# variable sized alloca with probing", 126097a140dSpatrick [(set GR32:$dst, 127097a140dSpatrick (X86ProbedAlloca GR32:$size))]>, 128097a140dSpatrick Requires<[NotLP64]>; 129097a140dSpatrick 130097a140dSpatricklet Defs = [RAX, RSP, EFLAGS], Uses = [RSP] in 131097a140dSpatrickdef PROBED_ALLOCA_64 : I<0, Pseudo, (outs GR64:$dst), (ins GR64:$size), 132097a140dSpatrick "# variable sized alloca with probing", 133097a140dSpatrick [(set GR64:$dst, 134097a140dSpatrick (X86ProbedAlloca GR64:$size))]>, 135097a140dSpatrick Requires<[In64BitMode]>; 13609467b48Spatrick} 13709467b48Spatrick 138097a140dSpatricklet hasNoSchedulingInfo = 1 in 139097a140dSpatrickdef STACKALLOC_W_PROBING : I<0, Pseudo, (outs), (ins i64imm:$stacksize), 140097a140dSpatrick "# fixed size alloca with probing", 141097a140dSpatrick []>; 142097a140dSpatrick 14309467b48Spatrick// Dynamic stack allocation yields a _chkstk or _alloca call for all Windows 14409467b48Spatrick// targets. These calls are needed to probe the stack when allocating more than 14509467b48Spatrick// 4k bytes in one go. Touching the stack at 4K increments is necessary to 14609467b48Spatrick// ensure that the guard pages used by the OS virtual memory manager are 14709467b48Spatrick// allocated in correct sequence. 14809467b48Spatrick// The main point of having separate instruction are extra unmodelled effects 14909467b48Spatrick// (compared to ordinary calls) like stack pointer change. 15009467b48Spatrick 15109467b48Spatricklet Defs = [EAX, ESP, EFLAGS], Uses = [ESP] in 152*a96b3639Srobertdef DYN_ALLOCA_32 : I<0, Pseudo, (outs), (ins GR32:$size), 15309467b48Spatrick "# dynamic stack allocation", 154*a96b3639Srobert [(X86DynAlloca GR32:$size)]>, 15509467b48Spatrick Requires<[NotLP64]>; 15609467b48Spatrick 15709467b48Spatricklet Defs = [RAX, RSP, EFLAGS], Uses = [RSP] in 158*a96b3639Srobertdef DYN_ALLOCA_64 : I<0, Pseudo, (outs), (ins GR64:$size), 15909467b48Spatrick "# dynamic stack allocation", 160*a96b3639Srobert [(X86DynAlloca GR64:$size)]>, 16109467b48Spatrick Requires<[In64BitMode]>; 16209467b48Spatrick} // SchedRW 16309467b48Spatrick 16409467b48Spatrick// These instructions XOR the frame pointer into a GPR. They are used in some 16509467b48Spatrick// stack protection schemes. These are post-RA pseudos because we only know the 16609467b48Spatrick// frame register after register allocation. 16709467b48Spatricklet Constraints = "$src = $dst", isMoveImm = 1, isPseudo = 1, Defs = [EFLAGS] in { 16809467b48Spatrick def XOR32_FP : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$src), 16909467b48Spatrick "xorl\t$$FP, $src", []>, 17009467b48Spatrick Requires<[NotLP64]>, Sched<[WriteALU]>; 17109467b48Spatrick def XOR64_FP : I<0, Pseudo, (outs GR64:$dst), (ins GR64:$src), 17209467b48Spatrick "xorq\t$$FP $src", []>, 17309467b48Spatrick Requires<[In64BitMode]>, Sched<[WriteALU]>; 17409467b48Spatrick} 17509467b48Spatrick 17609467b48Spatrick//===----------------------------------------------------------------------===// 17709467b48Spatrick// EH Pseudo Instructions 17809467b48Spatrick// 17909467b48Spatricklet SchedRW = [WriteSystem] in { 18009467b48Spatricklet isTerminator = 1, isReturn = 1, isBarrier = 1, 18109467b48Spatrick hasCtrlDep = 1, isCodeGenOnly = 1 in { 18209467b48Spatrickdef EH_RETURN : I<0xC3, RawFrm, (outs), (ins GR32:$addr), 18309467b48Spatrick "ret\t#eh_return, addr: $addr", 18409467b48Spatrick [(X86ehret GR32:$addr)]>, Sched<[WriteJumpLd]>; 18509467b48Spatrick 18609467b48Spatrick} 18709467b48Spatrick 18809467b48Spatricklet isTerminator = 1, isReturn = 1, isBarrier = 1, 18909467b48Spatrick hasCtrlDep = 1, isCodeGenOnly = 1 in { 19009467b48Spatrickdef EH_RETURN64 : I<0xC3, RawFrm, (outs), (ins GR64:$addr), 19109467b48Spatrick "ret\t#eh_return, addr: $addr", 19209467b48Spatrick [(X86ehret GR64:$addr)]>, Sched<[WriteJumpLd]>; 19309467b48Spatrick 19409467b48Spatrick} 19509467b48Spatrick 19609467b48Spatricklet isTerminator = 1, hasSideEffects = 1, isBarrier = 1, hasCtrlDep = 1, 19709467b48Spatrick isCodeGenOnly = 1, isReturn = 1, isEHScopeReturn = 1 in { 19809467b48Spatrick def CLEANUPRET : I<0, Pseudo, (outs), (ins), "# CLEANUPRET", [(cleanupret)]>; 19909467b48Spatrick 20009467b48Spatrick // CATCHRET needs a custom inserter for SEH. 20109467b48Spatrick let usesCustomInserter = 1 in 20209467b48Spatrick def CATCHRET : I<0, Pseudo, (outs), (ins brtarget32:$dst, brtarget32:$from), 20309467b48Spatrick "# CATCHRET", 20409467b48Spatrick [(catchret bb:$dst, bb:$from)]>; 20509467b48Spatrick} 20609467b48Spatrick 20709467b48Spatricklet hasSideEffects = 1, isBarrier = 1, isCodeGenOnly = 1, 20809467b48Spatrick usesCustomInserter = 1 in { 20909467b48Spatrick def EH_SjLj_SetJmp32 : I<0, Pseudo, (outs GR32:$dst), (ins i32mem:$buf), 21009467b48Spatrick "#EH_SJLJ_SETJMP32", 21109467b48Spatrick [(set GR32:$dst, (X86eh_sjlj_setjmp addr:$buf))]>, 21209467b48Spatrick Requires<[Not64BitMode]>; 21309467b48Spatrick def EH_SjLj_SetJmp64 : I<0, Pseudo, (outs GR32:$dst), (ins i64mem:$buf), 21409467b48Spatrick "#EH_SJLJ_SETJMP64", 21509467b48Spatrick [(set GR32:$dst, (X86eh_sjlj_setjmp addr:$buf))]>, 21609467b48Spatrick Requires<[In64BitMode]>; 21709467b48Spatrick let isTerminator = 1 in { 21809467b48Spatrick def EH_SjLj_LongJmp32 : I<0, Pseudo, (outs), (ins i32mem:$buf), 21909467b48Spatrick "#EH_SJLJ_LONGJMP32", 22009467b48Spatrick [(X86eh_sjlj_longjmp addr:$buf)]>, 22109467b48Spatrick Requires<[Not64BitMode]>; 22209467b48Spatrick def EH_SjLj_LongJmp64 : I<0, Pseudo, (outs), (ins i64mem:$buf), 22309467b48Spatrick "#EH_SJLJ_LONGJMP64", 22409467b48Spatrick [(X86eh_sjlj_longjmp addr:$buf)]>, 22509467b48Spatrick Requires<[In64BitMode]>; 22609467b48Spatrick } 22709467b48Spatrick} 22809467b48Spatrick 22909467b48Spatricklet isBranch = 1, isTerminator = 1, isCodeGenOnly = 1 in { 23009467b48Spatrick def EH_SjLj_Setup : I<0, Pseudo, (outs), (ins brtarget:$dst), 23109467b48Spatrick "#EH_SjLj_Setup\t$dst", []>; 23209467b48Spatrick} 23309467b48Spatrick} // SchedRW 23409467b48Spatrick 23509467b48Spatrick//===----------------------------------------------------------------------===// 23609467b48Spatrick// Pseudo instructions used by unwind info. 23709467b48Spatrick// 23809467b48Spatricklet isPseudo = 1, SchedRW = [WriteSystem] in { 23909467b48Spatrick def SEH_PushReg : I<0, Pseudo, (outs), (ins i32imm:$reg), 24009467b48Spatrick "#SEH_PushReg $reg", []>; 24109467b48Spatrick def SEH_SaveReg : I<0, Pseudo, (outs), (ins i32imm:$reg, i32imm:$dst), 24209467b48Spatrick "#SEH_SaveReg $reg, $dst", []>; 24309467b48Spatrick def SEH_SaveXMM : I<0, Pseudo, (outs), (ins i32imm:$reg, i32imm:$dst), 24409467b48Spatrick "#SEH_SaveXMM $reg, $dst", []>; 24509467b48Spatrick def SEH_StackAlloc : I<0, Pseudo, (outs), (ins i32imm:$size), 24609467b48Spatrick "#SEH_StackAlloc $size", []>; 24709467b48Spatrick def SEH_StackAlign : I<0, Pseudo, (outs), (ins i32imm:$align), 24809467b48Spatrick "#SEH_StackAlign $align", []>; 24909467b48Spatrick def SEH_SetFrame : I<0, Pseudo, (outs), (ins i32imm:$reg, i32imm:$offset), 25009467b48Spatrick "#SEH_SetFrame $reg, $offset", []>; 25109467b48Spatrick def SEH_PushFrame : I<0, Pseudo, (outs), (ins i1imm:$mode), 25209467b48Spatrick "#SEH_PushFrame $mode", []>; 25309467b48Spatrick def SEH_EndPrologue : I<0, Pseudo, (outs), (ins), 25409467b48Spatrick "#SEH_EndPrologue", []>; 25509467b48Spatrick def SEH_Epilogue : I<0, Pseudo, (outs), (ins), 25609467b48Spatrick "#SEH_Epilogue", []>; 25709467b48Spatrick} 25809467b48Spatrick 25909467b48Spatrick//===----------------------------------------------------------------------===// 260*a96b3639Srobert// Pseudo instructions used by KCFI. 261*a96b3639Srobert//===----------------------------------------------------------------------===// 262*a96b3639Srobertlet 263*a96b3639Srobert Defs = [R10, R11, EFLAGS] in { 264*a96b3639Srobertdef KCFI_CHECK : PseudoI< 265*a96b3639Srobert (outs), (ins GR64:$ptr, i32imm:$type), []>, Sched<[]>; 266*a96b3639Srobert} 267*a96b3639Srobert 268*a96b3639Srobert//===----------------------------------------------------------------------===// 269*a96b3639Srobert// Pseudo instructions used by address sanitizer. 270*a96b3639Srobert//===----------------------------------------------------------------------===// 271*a96b3639Srobertlet 272*a96b3639Srobert Defs = [R10, R11, EFLAGS] in { 273*a96b3639Srobertdef ASAN_CHECK_MEMACCESS : PseudoI< 274*a96b3639Srobert (outs), (ins GR64PLTSafe:$addr, i32imm:$accessinfo), 275*a96b3639Srobert [(int_asan_check_memaccess GR64PLTSafe:$addr, (i32 timm:$accessinfo))]>, 276*a96b3639Srobert Sched<[]>; 277*a96b3639Srobert} 278*a96b3639Srobert 279*a96b3639Srobert//===----------------------------------------------------------------------===// 28009467b48Spatrick// Pseudo instructions used by segmented stacks. 28109467b48Spatrick// 28209467b48Spatrick 28309467b48Spatrick// This is lowered into a RET instruction by MCInstLower. We need 28409467b48Spatrick// this so that we don't have to have a MachineBasicBlock which ends 28509467b48Spatrick// with a RET and also has successors. 28609467b48Spatricklet isPseudo = 1, SchedRW = [WriteJumpLd] in { 28709467b48Spatrickdef MORESTACK_RET: I<0, Pseudo, (outs), (ins), "", []>; 28809467b48Spatrick 28909467b48Spatrick// This instruction is lowered to a RET followed by a MOV. The two 29009467b48Spatrick// instructions are not generated on a higher level since then the 29109467b48Spatrick// verifier sees a MachineBasicBlock ending with a non-terminator. 29209467b48Spatrickdef MORESTACK_RET_RESTORE_R10 : I<0, Pseudo, (outs), (ins), "", []>; 29309467b48Spatrick} 29409467b48Spatrick 29509467b48Spatrick//===----------------------------------------------------------------------===// 296adae0cfdSpatrick// Pseudo instruction used by retguard 297adae0cfdSpatrick 298adae0cfdSpatrick// This is lowered to a JE 2; INT3; INT3. Prior to this pseudo should be a 299adae0cfdSpatrick// compare instruction to ensure the retguard cookie is correct. 300adae0cfdSpatrick// We use a pseudo here in order to avoid splitting the BB just before the return. 301adae0cfdSpatrick// Splitting the BB and inserting a JE_1 over a new INT3 BB occasionally 302adae0cfdSpatrick// resulted in incorrect code when a value from a byte register (CL) was 303adae0cfdSpatrick// used as a return value. When emitted as a split BB, the single byte 304adae0cfdSpatrick// register would sometimes be widened to 4 bytes, which would corrupt 305adae0cfdSpatrick// the return value (ie mov %ecx, %eax instead of mov %cl, %al). 306a0747c9fSpatricklet isCodeGenOnly = 1, hasNoSchedulingInfo = 1, Uses = [EFLAGS] in { 307adae0cfdSpatrickdef RETGUARD_JMP_TRAP: I<0, Pseudo, (outs), (ins), "", []>; 308adae0cfdSpatrick} 309adae0cfdSpatrick 310a0747c9fSpatricklet isCodeGenOnly = 1, hasNoSchedulingInfo = 1 in { 311adae0cfdSpatrickdef JMP_TRAP: I<0, Pseudo, (outs), (ins), "", []>; 312adae0cfdSpatrick} 313adae0cfdSpatrick 314adae0cfdSpatrick//===----------------------------------------------------------------------===// 31509467b48Spatrick// Alias Instructions 31609467b48Spatrick//===----------------------------------------------------------------------===// 31709467b48Spatrick 31809467b48Spatrick// Alias instruction mapping movr0 to xor. 31909467b48Spatrick// FIXME: remove when we can teach regalloc that xor reg, reg is ok. 32009467b48Spatricklet Defs = [EFLAGS], isReMaterializable = 1, isAsCheapAsAMove = 1, 32109467b48Spatrick isPseudo = 1, isMoveImm = 1, AddedComplexity = 10 in 32209467b48Spatrickdef MOV32r0 : I<0, Pseudo, (outs GR32:$dst), (ins), "", 32309467b48Spatrick [(set GR32:$dst, 0)]>, Sched<[WriteZero]>; 32409467b48Spatrick 32509467b48Spatrick// Other widths can also make use of the 32-bit xor, which may have a smaller 32609467b48Spatrick// encoding and avoid partial register updates. 32709467b48Spatricklet AddedComplexity = 10 in { 32809467b48Spatrickdef : Pat<(i8 0), (EXTRACT_SUBREG (MOV32r0), sub_8bit)>; 32909467b48Spatrickdef : Pat<(i16 0), (EXTRACT_SUBREG (MOV32r0), sub_16bit)>; 33009467b48Spatrickdef : Pat<(i64 0), (SUBREG_TO_REG (i64 0), (MOV32r0), sub_32bit)>; 33109467b48Spatrick} 33209467b48Spatrick 33309467b48Spatricklet Predicates = [OptForSize, Not64BitMode], 33409467b48Spatrick AddedComplexity = 10 in { 33509467b48Spatrick let SchedRW = [WriteALU] in { 33609467b48Spatrick // Pseudo instructions for materializing 1 and -1 using XOR+INC/DEC, 33709467b48Spatrick // which only require 3 bytes compared to MOV32ri which requires 5. 33809467b48Spatrick let Defs = [EFLAGS], isReMaterializable = 1, isPseudo = 1 in { 33909467b48Spatrick def MOV32r1 : I<0, Pseudo, (outs GR32:$dst), (ins), "", 34009467b48Spatrick [(set GR32:$dst, 1)]>; 34109467b48Spatrick def MOV32r_1 : I<0, Pseudo, (outs GR32:$dst), (ins), "", 34209467b48Spatrick [(set GR32:$dst, -1)]>; 34309467b48Spatrick } 34409467b48Spatrick } // SchedRW 34509467b48Spatrick 34609467b48Spatrick // MOV16ri is 4 bytes, so the instructions above are smaller. 34709467b48Spatrick def : Pat<(i16 1), (EXTRACT_SUBREG (MOV32r1), sub_16bit)>; 34809467b48Spatrick def : Pat<(i16 -1), (EXTRACT_SUBREG (MOV32r_1), sub_16bit)>; 34909467b48Spatrick} 35009467b48Spatrick 35109467b48Spatricklet isReMaterializable = 1, isPseudo = 1, AddedComplexity = 5, 35209467b48Spatrick SchedRW = [WriteALU] in { 35309467b48Spatrick// AddedComplexity higher than MOV64ri but lower than MOV32r0 and MOV32r1. 35409467b48Spatrickdef MOV32ImmSExti8 : I<0, Pseudo, (outs GR32:$dst), (ins i32i8imm:$src), "", 35509467b48Spatrick [(set GR32:$dst, i32immSExt8:$src)]>, 35609467b48Spatrick Requires<[OptForMinSize, NotWin64WithoutFP]>; 35709467b48Spatrickdef MOV64ImmSExti8 : I<0, Pseudo, (outs GR64:$dst), (ins i64i8imm:$src), "", 35809467b48Spatrick [(set GR64:$dst, i64immSExt8:$src)]>, 35909467b48Spatrick Requires<[OptForMinSize, NotWin64WithoutFP]>; 36009467b48Spatrick} 36109467b48Spatrick 36209467b48Spatrick// Materialize i64 constant where top 32-bits are zero. This could theoretically 36309467b48Spatrick// use MOV32ri with a SUBREG_TO_REG to represent the zero-extension, however 36409467b48Spatrick// that would make it more difficult to rematerialize. 365097a140dSpatricklet AddedComplexity = 1, isReMaterializable = 1, isAsCheapAsAMove = 1, 366097a140dSpatrick isPseudo = 1, SchedRW = [WriteMove] in 367097a140dSpatrickdef MOV32ri64 : I<0, Pseudo, (outs GR64:$dst), (ins i64i32imm:$src), "", 368097a140dSpatrick [(set GR64:$dst, i64immZExt32:$src)]>; 36909467b48Spatrick 370097a140dSpatrick// This 64-bit pseudo-move can also be used for labels in the x86-64 small code 371097a140dSpatrick// model. 372097a140dSpatrickdef mov64imm32 : ComplexPattern<i64, 1, "selectMOV64Imm32", [X86Wrapper]>; 37309467b48Spatrickdef : Pat<(i64 mov64imm32:$src), (MOV32ri64 mov64imm32:$src)>; 37409467b48Spatrick 37509467b48Spatrick// Use sbb to materialize carry bit. 376097a140dSpatricklet Uses = [EFLAGS], Defs = [EFLAGS], isPseudo = 1, SchedRW = [WriteADC], 377097a140dSpatrick hasSideEffects = 0 in { 37809467b48Spatrick// FIXME: These are pseudo ops that should be replaced with Pat<> patterns. 37909467b48Spatrick// However, Pat<> can't replicate the destination reg into the inputs of the 38009467b48Spatrick// result. 381097a140dSpatrickdef SETB_C32r : I<0, Pseudo, (outs GR32:$dst), (ins), "", []>; 382097a140dSpatrickdef SETB_C64r : I<0, Pseudo, (outs GR64:$dst), (ins), "", []>; 38309467b48Spatrick} // isCodeGenOnly 38409467b48Spatrick 38509467b48Spatrick//===----------------------------------------------------------------------===// 38609467b48Spatrick// String Pseudo Instructions 38709467b48Spatrick// 38809467b48Spatricklet SchedRW = [WriteMicrocoded] in { 38909467b48Spatricklet Defs = [ECX,EDI,ESI], Uses = [ECX,EDI,ESI], isCodeGenOnly = 1 in { 39009467b48Spatrickdef REP_MOVSB_32 : I<0xA4, RawFrm, (outs), (ins), 39109467b48Spatrick "{rep;movsb (%esi), %es:(%edi)|rep movsb es:[edi], [esi]}", 39209467b48Spatrick [(X86rep_movs i8)]>, REP, AdSize32, 39309467b48Spatrick Requires<[NotLP64]>; 39409467b48Spatrickdef REP_MOVSW_32 : I<0xA5, RawFrm, (outs), (ins), 39509467b48Spatrick "{rep;movsw (%esi), %es:(%edi)|rep movsw es:[edi], [esi]}", 39609467b48Spatrick [(X86rep_movs i16)]>, REP, AdSize32, OpSize16, 39709467b48Spatrick Requires<[NotLP64]>; 39809467b48Spatrickdef REP_MOVSD_32 : I<0xA5, RawFrm, (outs), (ins), 39909467b48Spatrick "{rep;movsl (%esi), %es:(%edi)|rep movsd es:[edi], [esi]}", 40009467b48Spatrick [(X86rep_movs i32)]>, REP, AdSize32, OpSize32, 40109467b48Spatrick Requires<[NotLP64]>; 40209467b48Spatrickdef REP_MOVSQ_32 : RI<0xA5, RawFrm, (outs), (ins), 40309467b48Spatrick "{rep;movsq (%esi), %es:(%edi)|rep movsq es:[edi], [esi]}", 40409467b48Spatrick [(X86rep_movs i64)]>, REP, AdSize32, 40509467b48Spatrick Requires<[NotLP64, In64BitMode]>; 40609467b48Spatrick} 40709467b48Spatrick 40809467b48Spatricklet Defs = [RCX,RDI,RSI], Uses = [RCX,RDI,RSI], isCodeGenOnly = 1 in { 40909467b48Spatrickdef REP_MOVSB_64 : I<0xA4, RawFrm, (outs), (ins), 41009467b48Spatrick "{rep;movsb (%rsi), %es:(%rdi)|rep movsb es:[rdi], [rsi]}", 41109467b48Spatrick [(X86rep_movs i8)]>, REP, AdSize64, 41209467b48Spatrick Requires<[IsLP64]>; 41309467b48Spatrickdef REP_MOVSW_64 : I<0xA5, RawFrm, (outs), (ins), 41409467b48Spatrick "{rep;movsw (%rsi), %es:(%rdi)|rep movsw es:[rdi], [rsi]}", 41509467b48Spatrick [(X86rep_movs i16)]>, REP, AdSize64, OpSize16, 41609467b48Spatrick Requires<[IsLP64]>; 41709467b48Spatrickdef REP_MOVSD_64 : I<0xA5, RawFrm, (outs), (ins), 41809467b48Spatrick "{rep;movsl (%rsi), %es:(%rdi)|rep movsdi es:[rdi], [rsi]}", 41909467b48Spatrick [(X86rep_movs i32)]>, REP, AdSize64, OpSize32, 42009467b48Spatrick Requires<[IsLP64]>; 42109467b48Spatrickdef REP_MOVSQ_64 : RI<0xA5, RawFrm, (outs), (ins), 42209467b48Spatrick "{rep;movsq (%rsi), %es:(%rdi)|rep movsq es:[rdi], [rsi]}", 42309467b48Spatrick [(X86rep_movs i64)]>, REP, AdSize64, 42409467b48Spatrick Requires<[IsLP64]>; 42509467b48Spatrick} 42609467b48Spatrick 42709467b48Spatrick// FIXME: Should use "(X86rep_stos AL)" as the pattern. 42809467b48Spatricklet Defs = [ECX,EDI], isCodeGenOnly = 1 in { 42909467b48Spatrick let Uses = [AL,ECX,EDI] in 43009467b48Spatrick def REP_STOSB_32 : I<0xAA, RawFrm, (outs), (ins), 43109467b48Spatrick "{rep;stosb %al, %es:(%edi)|rep stosb es:[edi], al}", 43209467b48Spatrick [(X86rep_stos i8)]>, REP, AdSize32, 43309467b48Spatrick Requires<[NotLP64]>; 43409467b48Spatrick let Uses = [AX,ECX,EDI] in 43509467b48Spatrick def REP_STOSW_32 : I<0xAB, RawFrm, (outs), (ins), 43609467b48Spatrick "{rep;stosw %ax, %es:(%edi)|rep stosw es:[edi], ax}", 43709467b48Spatrick [(X86rep_stos i16)]>, REP, AdSize32, OpSize16, 43809467b48Spatrick Requires<[NotLP64]>; 43909467b48Spatrick let Uses = [EAX,ECX,EDI] in 44009467b48Spatrick def REP_STOSD_32 : I<0xAB, RawFrm, (outs), (ins), 44109467b48Spatrick "{rep;stosl %eax, %es:(%edi)|rep stosd es:[edi], eax}", 44209467b48Spatrick [(X86rep_stos i32)]>, REP, AdSize32, OpSize32, 44309467b48Spatrick Requires<[NotLP64]>; 44409467b48Spatrick let Uses = [RAX,RCX,RDI] in 44509467b48Spatrick def REP_STOSQ_32 : RI<0xAB, RawFrm, (outs), (ins), 44609467b48Spatrick "{rep;stosq %rax, %es:(%edi)|rep stosq es:[edi], rax}", 44709467b48Spatrick [(X86rep_stos i64)]>, REP, AdSize32, 44809467b48Spatrick Requires<[NotLP64, In64BitMode]>; 44909467b48Spatrick} 45009467b48Spatrick 45109467b48Spatricklet Defs = [RCX,RDI], isCodeGenOnly = 1 in { 45209467b48Spatrick let Uses = [AL,RCX,RDI] in 45309467b48Spatrick def REP_STOSB_64 : I<0xAA, RawFrm, (outs), (ins), 45409467b48Spatrick "{rep;stosb %al, %es:(%rdi)|rep stosb es:[rdi], al}", 45509467b48Spatrick [(X86rep_stos i8)]>, REP, AdSize64, 45609467b48Spatrick Requires<[IsLP64]>; 45709467b48Spatrick let Uses = [AX,RCX,RDI] in 45809467b48Spatrick def REP_STOSW_64 : I<0xAB, RawFrm, (outs), (ins), 45909467b48Spatrick "{rep;stosw %ax, %es:(%rdi)|rep stosw es:[rdi], ax}", 46009467b48Spatrick [(X86rep_stos i16)]>, REP, AdSize64, OpSize16, 46109467b48Spatrick Requires<[IsLP64]>; 46209467b48Spatrick let Uses = [RAX,RCX,RDI] in 46309467b48Spatrick def REP_STOSD_64 : I<0xAB, RawFrm, (outs), (ins), 46409467b48Spatrick "{rep;stosl %eax, %es:(%rdi)|rep stosd es:[rdi], eax}", 46509467b48Spatrick [(X86rep_stos i32)]>, REP, AdSize64, OpSize32, 46609467b48Spatrick Requires<[IsLP64]>; 46709467b48Spatrick 46809467b48Spatrick let Uses = [RAX,RCX,RDI] in 46909467b48Spatrick def REP_STOSQ_64 : RI<0xAB, RawFrm, (outs), (ins), 47009467b48Spatrick "{rep;stosq %rax, %es:(%rdi)|rep stosq es:[rdi], rax}", 47109467b48Spatrick [(X86rep_stos i64)]>, REP, AdSize64, 47209467b48Spatrick Requires<[IsLP64]>; 47309467b48Spatrick} 47409467b48Spatrick} // SchedRW 47509467b48Spatrick 47609467b48Spatrick//===----------------------------------------------------------------------===// 47709467b48Spatrick// Thread Local Storage Instructions 47809467b48Spatrick// 47909467b48Spatricklet SchedRW = [WriteSystem] in { 48009467b48Spatrick 48109467b48Spatrick// ELF TLS Support 48209467b48Spatrick// All calls clobber the non-callee saved registers. ESP is marked as 48309467b48Spatrick// a use to prevent stack-pointer assignments that appear immediately 48409467b48Spatrick// before calls from potentially appearing dead. 48509467b48Spatricklet Defs = [EAX, ECX, EDX, FP0, FP1, FP2, FP3, FP4, FP5, FP6, FP7, 48609467b48Spatrick ST0, ST1, ST2, ST3, ST4, ST5, ST6, ST7, 48709467b48Spatrick MM0, MM1, MM2, MM3, MM4, MM5, MM6, MM7, 48809467b48Spatrick XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7, 48909467b48Spatrick XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15, EFLAGS, DF], 49009467b48Spatrick usesCustomInserter = 1, Uses = [ESP, SSP] in { 49109467b48Spatrickdef TLS_addr32 : I<0, Pseudo, (outs), (ins i32mem:$sym), 49209467b48Spatrick "# TLS_addr32", 49309467b48Spatrick [(X86tlsaddr tls32addr:$sym)]>, 49409467b48Spatrick Requires<[Not64BitMode]>; 49509467b48Spatrickdef TLS_base_addr32 : I<0, Pseudo, (outs), (ins i32mem:$sym), 49609467b48Spatrick "# TLS_base_addr32", 49709467b48Spatrick [(X86tlsbaseaddr tls32baseaddr:$sym)]>, 49809467b48Spatrick Requires<[Not64BitMode]>; 49909467b48Spatrick} 50009467b48Spatrick 50109467b48Spatrick// All calls clobber the non-callee saved registers. RSP is marked as 50209467b48Spatrick// a use to prevent stack-pointer assignments that appear immediately 50309467b48Spatrick// before calls from potentially appearing dead. 50409467b48Spatricklet Defs = [RAX, RCX, RDX, RSI, RDI, R8, R9, R10, R11, 50509467b48Spatrick FP0, FP1, FP2, FP3, FP4, FP5, FP6, FP7, 50609467b48Spatrick ST0, ST1, ST2, ST3, ST4, ST5, ST6, ST7, 50709467b48Spatrick MM0, MM1, MM2, MM3, MM4, MM5, MM6, MM7, 50809467b48Spatrick XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7, 50909467b48Spatrick XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15, EFLAGS, DF], 51009467b48Spatrick usesCustomInserter = 1, Uses = [RSP, SSP] in { 51109467b48Spatrickdef TLS_addr64 : I<0, Pseudo, (outs), (ins i64mem:$sym), 51209467b48Spatrick "# TLS_addr64", 51309467b48Spatrick [(X86tlsaddr tls64addr:$sym)]>, 514a0747c9fSpatrick Requires<[In64BitMode, IsLP64]>; 51509467b48Spatrickdef TLS_base_addr64 : I<0, Pseudo, (outs), (ins i64mem:$sym), 51609467b48Spatrick "# TLS_base_addr64", 51709467b48Spatrick [(X86tlsbaseaddr tls64baseaddr:$sym)]>, 518a0747c9fSpatrick Requires<[In64BitMode, IsLP64]>; 519a0747c9fSpatrickdef TLS_addrX32 : I<0, Pseudo, (outs), (ins i32mem:$sym), 520a0747c9fSpatrick "# TLS_addrX32", 521a0747c9fSpatrick [(X86tlsaddr tls32addr:$sym)]>, 522a0747c9fSpatrick Requires<[In64BitMode, NotLP64]>; 523a0747c9fSpatrickdef TLS_base_addrX32 : I<0, Pseudo, (outs), (ins i32mem:$sym), 524a0747c9fSpatrick "# TLS_base_addrX32", 525a0747c9fSpatrick [(X86tlsbaseaddr tls32baseaddr:$sym)]>, 526a0747c9fSpatrick Requires<[In64BitMode, NotLP64]>; 52709467b48Spatrick} 52809467b48Spatrick 52909467b48Spatrick// Darwin TLS Support 53009467b48Spatrick// For i386, the address of the thunk is passed on the stack, on return the 53109467b48Spatrick// address of the variable is in %eax. %ecx is trashed during the function 53209467b48Spatrick// call. All other registers are preserved. 53309467b48Spatricklet Defs = [EAX, ECX, EFLAGS, DF], 53409467b48Spatrick Uses = [ESP, SSP], 53509467b48Spatrick usesCustomInserter = 1 in 53609467b48Spatrickdef TLSCall_32 : I<0, Pseudo, (outs), (ins i32mem:$sym), 53709467b48Spatrick "# TLSCall_32", 53809467b48Spatrick [(X86TLSCall addr:$sym)]>, 53909467b48Spatrick Requires<[Not64BitMode]>; 54009467b48Spatrick 54109467b48Spatrick// For x86_64, the address of the thunk is passed in %rdi, but the 54209467b48Spatrick// pseudo directly use the symbol, so do not add an implicit use of 54309467b48Spatrick// %rdi. The lowering will do the right thing with RDI. 54409467b48Spatrick// On return the address of the variable is in %rax. All other 54509467b48Spatrick// registers are preserved. 54609467b48Spatricklet Defs = [RAX, EFLAGS, DF], 54709467b48Spatrick Uses = [RSP, SSP], 54809467b48Spatrick usesCustomInserter = 1 in 54909467b48Spatrickdef TLSCall_64 : I<0, Pseudo, (outs), (ins i64mem:$sym), 55009467b48Spatrick "# TLSCall_64", 55109467b48Spatrick [(X86TLSCall addr:$sym)]>, 55209467b48Spatrick Requires<[In64BitMode]>; 55309467b48Spatrick} // SchedRW 55409467b48Spatrick 55509467b48Spatrick//===----------------------------------------------------------------------===// 55609467b48Spatrick// Conditional Move Pseudo Instructions 55709467b48Spatrick 55809467b48Spatrick// CMOV* - Used to implement the SELECT DAG operation. Expanded after 55909467b48Spatrick// instruction selection into a branch sequence. 56009467b48Spatrickmulticlass CMOVrr_PSEUDO<RegisterClass RC, ValueType VT> { 56109467b48Spatrick def CMOV#NAME : I<0, Pseudo, 56209467b48Spatrick (outs RC:$dst), (ins RC:$t, RC:$f, i8imm:$cond), 56309467b48Spatrick "#CMOV_"#NAME#" PSEUDO!", 56409467b48Spatrick [(set RC:$dst, (VT (X86cmov RC:$t, RC:$f, timm:$cond, 56509467b48Spatrick EFLAGS)))]>; 56609467b48Spatrick} 56709467b48Spatrick 56809467b48Spatricklet usesCustomInserter = 1, hasNoSchedulingInfo = 1, Uses = [EFLAGS] in { 56909467b48Spatrick // X86 doesn't have 8-bit conditional moves. Use a customInserter to 57009467b48Spatrick // emit control flow. An alternative to this is to mark i8 SELECT as Promote, 57109467b48Spatrick // however that requires promoting the operands, and can induce additional 57209467b48Spatrick // i8 register pressure. 57309467b48Spatrick defm _GR8 : CMOVrr_PSEUDO<GR8, i8>; 57409467b48Spatrick 575*a96b3639Srobert let Predicates = [NoCMOV] in { 57609467b48Spatrick defm _GR32 : CMOVrr_PSEUDO<GR32, i32>; 57709467b48Spatrick defm _GR16 : CMOVrr_PSEUDO<GR16, i16>; 578*a96b3639Srobert } // Predicates = [NoCMOV] 57909467b48Spatrick 58009467b48Spatrick // fcmov doesn't handle all possible EFLAGS, provide a fallback if there is no 58109467b48Spatrick // SSE1/SSE2. 58209467b48Spatrick let Predicates = [FPStackf32] in 58309467b48Spatrick defm _RFP32 : CMOVrr_PSEUDO<RFP32, f32>; 58409467b48Spatrick 58509467b48Spatrick let Predicates = [FPStackf64] in 58609467b48Spatrick defm _RFP64 : CMOVrr_PSEUDO<RFP64, f64>; 58709467b48Spatrick 58809467b48Spatrick defm _RFP80 : CMOVrr_PSEUDO<RFP80, f80>; 58909467b48Spatrick 590097a140dSpatrick let Predicates = [HasMMX] in 591097a140dSpatrick defm _VR64 : CMOVrr_PSEUDO<VR64, x86mmx>; 592097a140dSpatrick 593097a140dSpatrick let Predicates = [HasSSE1,NoAVX512] in 59409467b48Spatrick defm _FR32 : CMOVrr_PSEUDO<FR32, f32>; 595*a96b3639Srobert let Predicates = [HasSSE2,NoAVX512] in { 596*a96b3639Srobert defm _FR16 : CMOVrr_PSEUDO<FR16, f16>; 59709467b48Spatrick defm _FR64 : CMOVrr_PSEUDO<FR64, f64>; 598*a96b3639Srobert } 59909467b48Spatrick let Predicates = [HasAVX512] in { 600*a96b3639Srobert defm _FR16X : CMOVrr_PSEUDO<FR16X, f16>; 60109467b48Spatrick defm _FR32X : CMOVrr_PSEUDO<FR32X, f32>; 60209467b48Spatrick defm _FR64X : CMOVrr_PSEUDO<FR64X, f64>; 60309467b48Spatrick } 60409467b48Spatrick let Predicates = [NoVLX] in { 60509467b48Spatrick defm _VR128 : CMOVrr_PSEUDO<VR128, v2i64>; 60609467b48Spatrick defm _VR256 : CMOVrr_PSEUDO<VR256, v4i64>; 60709467b48Spatrick } 60809467b48Spatrick let Predicates = [HasVLX] in { 60909467b48Spatrick defm _VR128X : CMOVrr_PSEUDO<VR128X, v2i64>; 61009467b48Spatrick defm _VR256X : CMOVrr_PSEUDO<VR256X, v4i64>; 61109467b48Spatrick } 61209467b48Spatrick defm _VR512 : CMOVrr_PSEUDO<VR512, v8i64>; 613097a140dSpatrick defm _VK1 : CMOVrr_PSEUDO<VK1, v1i1>; 61409467b48Spatrick defm _VK2 : CMOVrr_PSEUDO<VK2, v2i1>; 61509467b48Spatrick defm _VK4 : CMOVrr_PSEUDO<VK4, v4i1>; 61609467b48Spatrick defm _VK8 : CMOVrr_PSEUDO<VK8, v8i1>; 61709467b48Spatrick defm _VK16 : CMOVrr_PSEUDO<VK16, v16i1>; 61809467b48Spatrick defm _VK32 : CMOVrr_PSEUDO<VK32, v32i1>; 61909467b48Spatrick defm _VK64 : CMOVrr_PSEUDO<VK64, v64i1>; 62009467b48Spatrick} // usesCustomInserter = 1, hasNoSchedulingInfo = 1, Uses = [EFLAGS] 62109467b48Spatrick 62209467b48Spatrickdef : Pat<(f128 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 62309467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 62409467b48Spatrick 62509467b48Spatricklet Predicates = [NoVLX] in { 62609467b48Spatrick def : Pat<(v16i8 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 62709467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 62809467b48Spatrick def : Pat<(v8i16 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 62909467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 63009467b48Spatrick def : Pat<(v4i32 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 63109467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 63209467b48Spatrick def : Pat<(v4f32 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 63309467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 63409467b48Spatrick def : Pat<(v2f64 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)), 63509467b48Spatrick (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>; 63609467b48Spatrick 63709467b48Spatrick def : Pat<(v32i8 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)), 63809467b48Spatrick (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>; 63909467b48Spatrick def : Pat<(v16i16 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)), 64009467b48Spatrick (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>; 64109467b48Spatrick def : Pat<(v8i32 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)), 64209467b48Spatrick (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>; 64309467b48Spatrick def : Pat<(v8f32 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)), 64409467b48Spatrick (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>; 64509467b48Spatrick def : Pat<(v4f64 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)), 64609467b48Spatrick (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>; 64709467b48Spatrick} 64809467b48Spatricklet Predicates = [HasVLX] in { 64909467b48Spatrick def : Pat<(v16i8 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)), 65009467b48Spatrick (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>; 65109467b48Spatrick def : Pat<(v8i16 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)), 65209467b48Spatrick (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>; 653*a96b3639Srobert def : Pat<(v8f16 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)), 654*a96b3639Srobert (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>; 65509467b48Spatrick def : Pat<(v4i32 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)), 65609467b48Spatrick (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>; 65709467b48Spatrick def : Pat<(v4f32 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)), 65809467b48Spatrick (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>; 65909467b48Spatrick def : Pat<(v2f64 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)), 66009467b48Spatrick (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>; 66109467b48Spatrick 66209467b48Spatrick def : Pat<(v32i8 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)), 66309467b48Spatrick (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>; 66409467b48Spatrick def : Pat<(v16i16 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)), 66509467b48Spatrick (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>; 666*a96b3639Srobert def : Pat<(v16f16 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)), 667*a96b3639Srobert (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>; 66809467b48Spatrick def : Pat<(v8i32 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)), 66909467b48Spatrick (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>; 67009467b48Spatrick def : Pat<(v8f32 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)), 67109467b48Spatrick (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>; 67209467b48Spatrick def : Pat<(v4f64 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)), 67309467b48Spatrick (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>; 67409467b48Spatrick} 67509467b48Spatrick 67609467b48Spatrickdef : Pat<(v64i8 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)), 67709467b48Spatrick (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>; 67809467b48Spatrickdef : Pat<(v32i16 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)), 67909467b48Spatrick (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>; 680*a96b3639Srobertdef : Pat<(v32f16 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)), 681*a96b3639Srobert (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>; 68209467b48Spatrickdef : Pat<(v16i32 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)), 68309467b48Spatrick (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>; 68409467b48Spatrickdef : Pat<(v16f32 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)), 68509467b48Spatrick (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>; 68609467b48Spatrickdef : Pat<(v8f64 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)), 68709467b48Spatrick (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>; 68809467b48Spatrick 68909467b48Spatrick//===----------------------------------------------------------------------===// 69009467b48Spatrick// Normal-Instructions-With-Lock-Prefix Pseudo Instructions 69109467b48Spatrick//===----------------------------------------------------------------------===// 69209467b48Spatrick 69309467b48Spatrick// FIXME: Use normal instructions and add lock prefix dynamically. 69409467b48Spatrick 69509467b48Spatrick// Memory barriers 69609467b48Spatrick 69709467b48Spatricklet isCodeGenOnly = 1, Defs = [EFLAGS] in 69809467b48Spatrickdef OR32mi8Locked : Ii8<0x83, MRM1m, (outs), (ins i32mem:$dst, i32i8imm:$zero), 69909467b48Spatrick "or{l}\t{$zero, $dst|$dst, $zero}", []>, 70009467b48Spatrick Requires<[Not64BitMode]>, OpSize32, LOCK, 70109467b48Spatrick Sched<[WriteALURMW]>; 70209467b48Spatrick 70309467b48Spatrick// RegOpc corresponds to the mr version of the instruction 70409467b48Spatrick// ImmOpc corresponds to the mi version of the instruction 70509467b48Spatrick// ImmOpc8 corresponds to the mi8 version of the instruction 70609467b48Spatrick// ImmMod corresponds to the instruction format of the mi and mi8 versions 70709467b48Spatrickmulticlass LOCK_ArithBinOp<bits<8> RegOpc, bits<8> ImmOpc, bits<8> ImmOpc8, 70809467b48Spatrick Format ImmMod, SDNode Op, string mnemonic> { 70909467b48Spatricklet Defs = [EFLAGS], mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, 71009467b48Spatrick SchedRW = [WriteALURMW] in { 71109467b48Spatrick 71209467b48Spatrickdef NAME#8mr : I<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4}, 71309467b48Spatrick RegOpc{3}, RegOpc{2}, RegOpc{1}, 0 }, 71409467b48Spatrick MRMDestMem, (outs), (ins i8mem:$dst, GR8:$src2), 71509467b48Spatrick !strconcat(mnemonic, "{b}\t", 71609467b48Spatrick "{$src2, $dst|$dst, $src2}"), 71709467b48Spatrick [(set EFLAGS, (Op addr:$dst, GR8:$src2))]>, LOCK; 71809467b48Spatrick 71909467b48Spatrickdef NAME#16mr : I<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4}, 72009467b48Spatrick RegOpc{3}, RegOpc{2}, RegOpc{1}, 1 }, 72109467b48Spatrick MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src2), 72209467b48Spatrick !strconcat(mnemonic, "{w}\t", 72309467b48Spatrick "{$src2, $dst|$dst, $src2}"), 72409467b48Spatrick [(set EFLAGS, (Op addr:$dst, GR16:$src2))]>, 72509467b48Spatrick OpSize16, LOCK; 72609467b48Spatrick 72709467b48Spatrickdef NAME#32mr : I<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4}, 72809467b48Spatrick RegOpc{3}, RegOpc{2}, RegOpc{1}, 1 }, 72909467b48Spatrick MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src2), 73009467b48Spatrick !strconcat(mnemonic, "{l}\t", 73109467b48Spatrick "{$src2, $dst|$dst, $src2}"), 73209467b48Spatrick [(set EFLAGS, (Op addr:$dst, GR32:$src2))]>, 73309467b48Spatrick OpSize32, LOCK; 73409467b48Spatrick 73509467b48Spatrickdef NAME#64mr : RI<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4}, 73609467b48Spatrick RegOpc{3}, RegOpc{2}, RegOpc{1}, 1 }, 73709467b48Spatrick MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src2), 73809467b48Spatrick !strconcat(mnemonic, "{q}\t", 73909467b48Spatrick "{$src2, $dst|$dst, $src2}"), 74009467b48Spatrick [(set EFLAGS, (Op addr:$dst, GR64:$src2))]>, LOCK; 74109467b48Spatrick 74209467b48Spatrick// NOTE: These are order specific, we want the mi8 forms to be listed 74309467b48Spatrick// first so that they are slightly preferred to the mi forms. 74409467b48Spatrickdef NAME#16mi8 : Ii8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4}, 74509467b48Spatrick ImmOpc8{3}, ImmOpc8{2}, ImmOpc8{1}, 1 }, 74609467b48Spatrick ImmMod, (outs), (ins i16mem :$dst, i16i8imm :$src2), 74709467b48Spatrick !strconcat(mnemonic, "{w}\t", 74809467b48Spatrick "{$src2, $dst|$dst, $src2}"), 74909467b48Spatrick [(set EFLAGS, (Op addr:$dst, i16immSExt8:$src2))]>, 75009467b48Spatrick OpSize16, LOCK; 75109467b48Spatrick 75209467b48Spatrickdef NAME#32mi8 : Ii8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4}, 75309467b48Spatrick ImmOpc8{3}, ImmOpc8{2}, ImmOpc8{1}, 1 }, 75409467b48Spatrick ImmMod, (outs), (ins i32mem :$dst, i32i8imm :$src2), 75509467b48Spatrick !strconcat(mnemonic, "{l}\t", 75609467b48Spatrick "{$src2, $dst|$dst, $src2}"), 75709467b48Spatrick [(set EFLAGS, (Op addr:$dst, i32immSExt8:$src2))]>, 75809467b48Spatrick OpSize32, LOCK; 75909467b48Spatrick 76009467b48Spatrickdef NAME#64mi8 : RIi8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4}, 76109467b48Spatrick ImmOpc8{3}, ImmOpc8{2}, ImmOpc8{1}, 1 }, 76209467b48Spatrick ImmMod, (outs), (ins i64mem :$dst, i64i8imm :$src2), 76309467b48Spatrick !strconcat(mnemonic, "{q}\t", 76409467b48Spatrick "{$src2, $dst|$dst, $src2}"), 76509467b48Spatrick [(set EFLAGS, (Op addr:$dst, i64immSExt8:$src2))]>, 76609467b48Spatrick LOCK; 76709467b48Spatrick 76809467b48Spatrickdef NAME#8mi : Ii8<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4}, 76909467b48Spatrick ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 0 }, 77009467b48Spatrick ImmMod, (outs), (ins i8mem :$dst, i8imm :$src2), 77109467b48Spatrick !strconcat(mnemonic, "{b}\t", 77209467b48Spatrick "{$src2, $dst|$dst, $src2}"), 77309467b48Spatrick [(set EFLAGS, (Op addr:$dst, (i8 imm:$src2)))]>, LOCK; 77409467b48Spatrick 77509467b48Spatrickdef NAME#16mi : Ii16<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4}, 77609467b48Spatrick ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 1 }, 77709467b48Spatrick ImmMod, (outs), (ins i16mem :$dst, i16imm :$src2), 77809467b48Spatrick !strconcat(mnemonic, "{w}\t", 77909467b48Spatrick "{$src2, $dst|$dst, $src2}"), 78009467b48Spatrick [(set EFLAGS, (Op addr:$dst, (i16 imm:$src2)))]>, 78109467b48Spatrick OpSize16, LOCK; 78209467b48Spatrick 78309467b48Spatrickdef NAME#32mi : Ii32<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4}, 78409467b48Spatrick ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 1 }, 78509467b48Spatrick ImmMod, (outs), (ins i32mem :$dst, i32imm :$src2), 78609467b48Spatrick !strconcat(mnemonic, "{l}\t", 78709467b48Spatrick "{$src2, $dst|$dst, $src2}"), 78809467b48Spatrick [(set EFLAGS, (Op addr:$dst, (i32 imm:$src2)))]>, 78909467b48Spatrick OpSize32, LOCK; 79009467b48Spatrick 79109467b48Spatrickdef NAME#64mi32 : RIi32S<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4}, 79209467b48Spatrick ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 1 }, 79309467b48Spatrick ImmMod, (outs), (ins i64mem :$dst, i64i32imm :$src2), 79409467b48Spatrick !strconcat(mnemonic, "{q}\t", 79509467b48Spatrick "{$src2, $dst|$dst, $src2}"), 79609467b48Spatrick [(set EFLAGS, (Op addr:$dst, i64immSExt32:$src2))]>, 79709467b48Spatrick LOCK; 79809467b48Spatrick} 79909467b48Spatrick 80009467b48Spatrick} 80109467b48Spatrick 80209467b48Spatrickdefm LOCK_ADD : LOCK_ArithBinOp<0x00, 0x80, 0x83, MRM0m, X86lock_add, "add">; 80309467b48Spatrickdefm LOCK_SUB : LOCK_ArithBinOp<0x28, 0x80, 0x83, MRM5m, X86lock_sub, "sub">; 80409467b48Spatrickdefm LOCK_OR : LOCK_ArithBinOp<0x08, 0x80, 0x83, MRM1m, X86lock_or , "or">; 80509467b48Spatrickdefm LOCK_AND : LOCK_ArithBinOp<0x20, 0x80, 0x83, MRM4m, X86lock_and, "and">; 80609467b48Spatrickdefm LOCK_XOR : LOCK_ArithBinOp<0x30, 0x80, 0x83, MRM6m, X86lock_xor, "xor">; 80709467b48Spatrick 80809467b48Spatrickdef X86lock_add_nocf : PatFrag<(ops node:$lhs, node:$rhs), 80909467b48Spatrick (X86lock_add node:$lhs, node:$rhs), [{ 81009467b48Spatrick return hasNoCarryFlagUses(SDValue(N, 0)); 81109467b48Spatrick}]>; 81209467b48Spatrick 81309467b48Spatrickdef X86lock_sub_nocf : PatFrag<(ops node:$lhs, node:$rhs), 81409467b48Spatrick (X86lock_sub node:$lhs, node:$rhs), [{ 81509467b48Spatrick return hasNoCarryFlagUses(SDValue(N, 0)); 81609467b48Spatrick}]>; 81709467b48Spatrick 81809467b48Spatricklet Defs = [EFLAGS], mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, 81909467b48Spatrick SchedRW = [WriteALURMW] in { 820*a96b3639Srobert let Predicates = [UseIncDec] in { 82109467b48Spatrick def LOCK_INC8m : I<0xFE, MRM0m, (outs), (ins i8mem :$dst), 82209467b48Spatrick "inc{b}\t$dst", 82309467b48Spatrick [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i8 1)))]>, 82409467b48Spatrick LOCK; 82509467b48Spatrick def LOCK_INC16m : I<0xFF, MRM0m, (outs), (ins i16mem:$dst), 82609467b48Spatrick "inc{w}\t$dst", 82709467b48Spatrick [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i16 1)))]>, 82809467b48Spatrick OpSize16, LOCK; 82909467b48Spatrick def LOCK_INC32m : I<0xFF, MRM0m, (outs), (ins i32mem:$dst), 83009467b48Spatrick "inc{l}\t$dst", 83109467b48Spatrick [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i32 1)))]>, 83209467b48Spatrick OpSize32, LOCK; 83309467b48Spatrick 83409467b48Spatrick def LOCK_DEC8m : I<0xFE, MRM1m, (outs), (ins i8mem :$dst), 83509467b48Spatrick "dec{b}\t$dst", 83609467b48Spatrick [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i8 1)))]>, 83709467b48Spatrick LOCK; 83809467b48Spatrick def LOCK_DEC16m : I<0xFF, MRM1m, (outs), (ins i16mem:$dst), 83909467b48Spatrick "dec{w}\t$dst", 84009467b48Spatrick [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i16 1)))]>, 84109467b48Spatrick OpSize16, LOCK; 84209467b48Spatrick def LOCK_DEC32m : I<0xFF, MRM1m, (outs), (ins i32mem:$dst), 84309467b48Spatrick "dec{l}\t$dst", 84409467b48Spatrick [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i32 1)))]>, 84509467b48Spatrick OpSize32, LOCK; 846*a96b3639Srobert } 847*a96b3639Srobert 848*a96b3639Srobert let Predicates = [UseIncDec, In64BitMode] in { 849*a96b3639Srobert def LOCK_INC64m : RI<0xFF, MRM0m, (outs), (ins i64mem:$dst), 850*a96b3639Srobert "inc{q}\t$dst", 851*a96b3639Srobert [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i64 1)))]>, 852*a96b3639Srobert LOCK; 85309467b48Spatrick def LOCK_DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), 85409467b48Spatrick "dec{q}\t$dst", 85509467b48Spatrick [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i64 1)))]>, 85609467b48Spatrick LOCK; 85709467b48Spatrick } 858*a96b3639Srobert} 85909467b48Spatrick 860*a96b3639Srobertlet Predicates = [UseIncDec] in { 86109467b48Spatrick // Additional patterns for -1 constant. 86209467b48Spatrick def : Pat<(X86lock_add addr:$dst, (i8 -1)), (LOCK_DEC8m addr:$dst)>; 86309467b48Spatrick def : Pat<(X86lock_add addr:$dst, (i16 -1)), (LOCK_DEC16m addr:$dst)>; 86409467b48Spatrick def : Pat<(X86lock_add addr:$dst, (i32 -1)), (LOCK_DEC32m addr:$dst)>; 86509467b48Spatrick def : Pat<(X86lock_sub addr:$dst, (i8 -1)), (LOCK_INC8m addr:$dst)>; 86609467b48Spatrick def : Pat<(X86lock_sub addr:$dst, (i16 -1)), (LOCK_INC16m addr:$dst)>; 86709467b48Spatrick def : Pat<(X86lock_sub addr:$dst, (i32 -1)), (LOCK_INC32m addr:$dst)>; 868*a96b3639Srobert} 869*a96b3639Srobert 870*a96b3639Srobertlet Predicates = [UseIncDec, In64BitMode] in { 871*a96b3639Srobert // Additional patterns for -1 constant. 872*a96b3639Srobert def : Pat<(X86lock_add addr:$dst, (i64 -1)), (LOCK_DEC64m addr:$dst)>; 87309467b48Spatrick def : Pat<(X86lock_sub addr:$dst, (i64 -1)), (LOCK_INC64m addr:$dst)>; 87409467b48Spatrick} 87509467b48Spatrick 876*a96b3639Srobert// Atomic bit test. 877*a96b3639Srobertdef X86LBTest : SDTypeProfile<1, 3, [SDTCisVT<0, i32>, SDTCisPtrTy<1>, 878*a96b3639Srobert SDTCisVT<2, i8>, SDTCisVT<3, i32>]>; 879*a96b3639Srobertdef x86bts : SDNode<"X86ISD::LBTS", X86LBTest, 880*a96b3639Srobert [SDNPHasChain, SDNPMayLoad, SDNPMayStore, SDNPMemOperand]>; 881*a96b3639Srobertdef x86btc : SDNode<"X86ISD::LBTC", X86LBTest, 882*a96b3639Srobert [SDNPHasChain, SDNPMayLoad, SDNPMayStore, SDNPMemOperand]>; 883*a96b3639Srobertdef x86btr : SDNode<"X86ISD::LBTR", X86LBTest, 884*a96b3639Srobert [SDNPHasChain, SDNPMayLoad, SDNPMayStore, SDNPMemOperand]>; 885*a96b3639Srobert 886*a96b3639Srobertdef X86LBTestRM : SDTypeProfile<1, 2, [SDTCisVT<0, i32>, SDTCisPtrTy<1>, 887*a96b3639Srobert SDTCisInt<2>]>; 888*a96b3639Srobert 889*a96b3639Srobertdef x86_rm_bts : SDNode<"X86ISD::LBTS_RM", X86LBTestRM, 890*a96b3639Srobert [SDNPHasChain, SDNPMayLoad, SDNPMayStore, SDNPMemOperand]>; 891*a96b3639Srobertdef x86_rm_btc : SDNode<"X86ISD::LBTC_RM", X86LBTestRM, 892*a96b3639Srobert [SDNPHasChain, SDNPMayLoad, SDNPMayStore, SDNPMemOperand]>; 893*a96b3639Srobertdef x86_rm_btr : SDNode<"X86ISD::LBTR_RM", X86LBTestRM, 894*a96b3639Srobert [SDNPHasChain, SDNPMayLoad, SDNPMayStore, SDNPMemOperand]>; 895*a96b3639Srobert 896*a96b3639Srobert 897*a96b3639Srobertmulticlass ATOMIC_LOGIC_OP<Format Form, string s> { 898*a96b3639Srobert let Defs = [EFLAGS], mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, 899*a96b3639Srobert SchedRW = [WriteBitTestSetRegRMW] in { 900*a96b3639Srobert def 16m : Ii8<0xBA, Form, (outs), (ins i16mem:$src1, i8imm:$src2), 901*a96b3639Srobert !strconcat(s, "{w}\t{$src2, $src1|$src1, $src2}"), 902*a96b3639Srobert [(set EFLAGS, (!cast<SDNode>("x86" # s) addr:$src1, timm:$src2, (i32 16)))]>, 903*a96b3639Srobert OpSize16, TB, LOCK; 904*a96b3639Srobert def 32m : Ii8<0xBA, Form, (outs), (ins i32mem:$src1, i8imm:$src2), 905*a96b3639Srobert !strconcat(s, "{l}\t{$src2, $src1|$src1, $src2}"), 906*a96b3639Srobert [(set EFLAGS, (!cast<SDNode>("x86" # s) addr:$src1, timm:$src2, (i32 32)))]>, 907*a96b3639Srobert OpSize32, TB, LOCK; 908*a96b3639Srobert def 64m : RIi8<0xBA, Form, (outs), (ins i64mem:$src1, i8imm:$src2), 909*a96b3639Srobert !strconcat(s, "{q}\t{$src2, $src1|$src1, $src2}"), 910*a96b3639Srobert [(set EFLAGS, (!cast<SDNode>("x86" # s) addr:$src1, timm:$src2, (i32 64)))]>, 911*a96b3639Srobert TB, LOCK; 912*a96b3639Srobert } 913*a96b3639Srobert} 914*a96b3639Srobert 915*a96b3639Srobertmulticlass ATOMIC_LOGIC_OP_RM<bits<8> Opc8, string s> { 916*a96b3639Srobert let Defs = [EFLAGS], mayLoad = 1, mayStore = 1, isCodeGenOnly = 1, 917*a96b3639Srobert SchedRW = [WriteBitTestSetRegRMW] in { 918*a96b3639Srobert def 16rm : I<Opc8, MRMDestMem, (outs), (ins i16mem:$src1, GR16:$src2), 919*a96b3639Srobert !strconcat(s, "{w}\t{$src2, $src1|$src1, $src2}"), 920*a96b3639Srobert [(set EFLAGS, (!cast<SDNode>("x86_rm_" # s) addr:$src1, GR16:$src2))]>, 921*a96b3639Srobert OpSize16, TB, LOCK; 922*a96b3639Srobert def 32rm : I<Opc8, MRMDestMem, (outs), (ins i32mem:$src1, GR32:$src2), 923*a96b3639Srobert !strconcat(s, "{l}\t{$src2, $src1|$src1, $src2}"), 924*a96b3639Srobert [(set EFLAGS, (!cast<SDNode>("x86_rm_" # s) addr:$src1, GR32:$src2))]>, 925*a96b3639Srobert OpSize32, TB, LOCK; 926*a96b3639Srobert def 64rm : RI<Opc8, MRMDestMem, (outs), (ins i64mem:$src1, GR64:$src2), 927*a96b3639Srobert !strconcat(s, "{q}\t{$src2, $src1|$src1, $src2}"), 928*a96b3639Srobert [(set EFLAGS, (!cast<SDNode>("x86_rm_" # s) addr:$src1, GR64:$src2))]>, 929*a96b3639Srobert TB, LOCK; 930*a96b3639Srobert } 931*a96b3639Srobert} 932*a96b3639Srobert 933*a96b3639Srobert 934*a96b3639Srobertdefm LOCK_BTS : ATOMIC_LOGIC_OP<MRM5m, "bts">; 935*a96b3639Srobertdefm LOCK_BTC : ATOMIC_LOGIC_OP<MRM7m, "btc">; 936*a96b3639Srobertdefm LOCK_BTR : ATOMIC_LOGIC_OP<MRM6m, "btr">; 937*a96b3639Srobert 938*a96b3639Srobertdefm LOCK_BTS_RM : ATOMIC_LOGIC_OP_RM<0xAB, "bts">; 939*a96b3639Srobertdefm LOCK_BTC_RM : ATOMIC_LOGIC_OP_RM<0xBB, "btc">; 940*a96b3639Srobertdefm LOCK_BTR_RM : ATOMIC_LOGIC_OP_RM<0xB3, "btr">; 941*a96b3639Srobert 94209467b48Spatrick// Atomic compare and swap. 94309467b48Spatrickmulticlass LCMPXCHG_BinOp<bits<8> Opc8, bits<8> Opc, Format Form, 94409467b48Spatrick string mnemonic, SDPatternOperator frag> { 94509467b48Spatricklet isCodeGenOnly = 1, SchedRW = [WriteCMPXCHGRMW] in { 94609467b48Spatrick let Defs = [AL, EFLAGS], Uses = [AL] in 94709467b48Spatrick def NAME#8 : I<Opc8, Form, (outs), (ins i8mem:$ptr, GR8:$swap), 94809467b48Spatrick !strconcat(mnemonic, "{b}\t{$swap, $ptr|$ptr, $swap}"), 94909467b48Spatrick [(frag addr:$ptr, GR8:$swap, 1)]>, TB, LOCK; 95009467b48Spatrick let Defs = [AX, EFLAGS], Uses = [AX] in 95109467b48Spatrick def NAME#16 : I<Opc, Form, (outs), (ins i16mem:$ptr, GR16:$swap), 95209467b48Spatrick !strconcat(mnemonic, "{w}\t{$swap, $ptr|$ptr, $swap}"), 95309467b48Spatrick [(frag addr:$ptr, GR16:$swap, 2)]>, TB, OpSize16, LOCK; 95409467b48Spatrick let Defs = [EAX, EFLAGS], Uses = [EAX] in 95509467b48Spatrick def NAME#32 : I<Opc, Form, (outs), (ins i32mem:$ptr, GR32:$swap), 95609467b48Spatrick !strconcat(mnemonic, "{l}\t{$swap, $ptr|$ptr, $swap}"), 95709467b48Spatrick [(frag addr:$ptr, GR32:$swap, 4)]>, TB, OpSize32, LOCK; 95809467b48Spatrick let Defs = [RAX, EFLAGS], Uses = [RAX] in 95909467b48Spatrick def NAME#64 : RI<Opc, Form, (outs), (ins i64mem:$ptr, GR64:$swap), 96009467b48Spatrick !strconcat(mnemonic, "{q}\t{$swap, $ptr|$ptr, $swap}"), 96109467b48Spatrick [(frag addr:$ptr, GR64:$swap, 8)]>, TB, LOCK; 96209467b48Spatrick} 96309467b48Spatrick} 96409467b48Spatrick 96509467b48Spatricklet Defs = [EAX, EDX, EFLAGS], Uses = [EAX, EBX, ECX, EDX], 966*a96b3639Srobert Predicates = [HasCX8], SchedRW = [WriteCMPXCHGRMW], 967a0747c9fSpatrick isCodeGenOnly = 1, usesCustomInserter = 1 in { 968a0747c9fSpatrickdef LCMPXCHG8B : I<0xC7, MRM1m, (outs), (ins i64mem:$ptr), 969a0747c9fSpatrick "cmpxchg8b\t$ptr", 970a0747c9fSpatrick [(X86cas8 addr:$ptr)]>, TB, LOCK; 971a0747c9fSpatrick} 972a0747c9fSpatrick 973a0747c9fSpatricklet Defs = [RAX, RDX, EFLAGS], Uses = [RAX, RBX, RCX, RDX], 974*a96b3639Srobert Predicates = [HasCX16,In64BitMode], SchedRW = [WriteCMPXCHGRMW], 975a0747c9fSpatrick isCodeGenOnly = 1, mayLoad = 1, mayStore = 1, hasSideEffects = 0 in { 976a0747c9fSpatrickdef LCMPXCHG16B : RI<0xC7, MRM1m, (outs), (ins i128mem:$ptr), 977a0747c9fSpatrick "cmpxchg16b\t$ptr", 978a0747c9fSpatrick []>, TB, LOCK; 97909467b48Spatrick} 98009467b48Spatrick 98109467b48Spatrick// This pseudo must be used when the frame uses RBX as 98209467b48Spatrick// the base pointer. Indeed, in such situation RBX is a reserved 98309467b48Spatrick// register and the register allocator will ignore any use/def of 98409467b48Spatrick// it. In other words, the register will not fix the clobbering of 98509467b48Spatrick// RBX that will happen when setting the arguments for the instrucion. 98609467b48Spatrick// 987097a140dSpatrick// Unlike the actual related instruction, we mark that this one 988a0747c9fSpatrick// defines RBX (instead of using RBX). 98909467b48Spatrick// The rationale is that we will define RBX during the expansion of 990a0747c9fSpatrick// the pseudo. The argument feeding RBX is rbx_input. 99109467b48Spatrick// 992a0747c9fSpatrick// The additional argument, $rbx_save, is a temporary register used to 99309467b48Spatrick// save the value of RBX across the actual instruction. 99409467b48Spatrick// 995a0747c9fSpatrick// To make sure the register assigned to $rbx_save does not interfere with 99609467b48Spatrick// the definition of the actual instruction, we use a definition $dst which 99709467b48Spatrick// is tied to $rbx_save. That way, the live-range of $rbx_save spans across 99809467b48Spatrick// the instruction and we are sure we will have a valid register to restore 99909467b48Spatrick// the value of RBX. 100009467b48Spatricklet Defs = [RAX, RDX, RBX, EFLAGS], Uses = [RAX, RCX, RDX], 1001*a96b3639Srobert Predicates = [HasCX16,In64BitMode], SchedRW = [WriteCMPXCHGRMW], 1002a0747c9fSpatrick isCodeGenOnly = 1, isPseudo = 1, 1003a0747c9fSpatrick mayLoad = 1, mayStore = 1, hasSideEffects = 0, 1004a0747c9fSpatrick Constraints = "$rbx_save = $dst" in { 100509467b48Spatrickdef LCMPXCHG16B_SAVE_RBX : 100609467b48Spatrick I<0, Pseudo, (outs GR64:$dst), 1007a0747c9fSpatrick (ins i128mem:$ptr, GR64:$rbx_input, GR64:$rbx_save), "", []>; 100809467b48Spatrick} 100909467b48Spatrick 1010a0747c9fSpatrick// Pseudo instruction that doesn't read/write RBX. Will be turned into either 1011a0747c9fSpatrick// LCMPXCHG16B_SAVE_RBX or LCMPXCHG16B via a custom inserter. 1012a0747c9fSpatricklet Defs = [RAX, RDX, EFLAGS], Uses = [RAX, RCX, RDX], 1013*a96b3639Srobert Predicates = [HasCX16,In64BitMode], SchedRW = [WriteCMPXCHGRMW], 1014a0747c9fSpatrick isCodeGenOnly = 1, isPseudo = 1, 1015a0747c9fSpatrick mayLoad = 1, mayStore = 1, hasSideEffects = 0, 1016a0747c9fSpatrick usesCustomInserter = 1 in { 1017a0747c9fSpatrickdef LCMPXCHG16B_NO_RBX : 1018a0747c9fSpatrick I<0, Pseudo, (outs), (ins i128mem:$ptr, GR64:$rbx_input), "", 1019a0747c9fSpatrick [(X86cas16 addr:$ptr, GR64:$rbx_input)]>; 1020a0747c9fSpatrick} 1021a0747c9fSpatrick 1022a0747c9fSpatrick// This pseudo must be used when the frame uses RBX/EBX as 1023a0747c9fSpatrick// the base pointer. 1024a0747c9fSpatrick// cf comment for LCMPXCHG16B_SAVE_RBX. 1025a0747c9fSpatricklet Defs = [EBX], Uses = [ECX, EAX], 1026a0747c9fSpatrick Predicates = [HasMWAITX], SchedRW = [WriteSystem], 1027a0747c9fSpatrick isCodeGenOnly = 1, isPseudo = 1, Constraints = "$rbx_save = $dst" in { 1028a0747c9fSpatrickdef MWAITX_SAVE_RBX : 1029a0747c9fSpatrick I<0, Pseudo, (outs GR64:$dst), 1030a0747c9fSpatrick (ins GR32:$ebx_input, GR64:$rbx_save), 1031a0747c9fSpatrick "mwaitx", 1032a0747c9fSpatrick []>; 1033a0747c9fSpatrick} 1034a0747c9fSpatrick 1035a0747c9fSpatrick// Pseudo mwaitx instruction to use for custom insertion. 1036a0747c9fSpatricklet Predicates = [HasMWAITX], SchedRW = [WriteSystem], 1037a0747c9fSpatrick isCodeGenOnly = 1, isPseudo = 1, 1038a0747c9fSpatrick usesCustomInserter = 1 in { 1039a0747c9fSpatrickdef MWAITX : 1040a0747c9fSpatrick I<0, Pseudo, (outs), (ins GR32:$ecx, GR32:$eax, GR32:$ebx), 1041a0747c9fSpatrick "mwaitx", 1042a0747c9fSpatrick [(int_x86_mwaitx GR32:$ecx, GR32:$eax, GR32:$ebx)]>; 1043a0747c9fSpatrick} 1044a0747c9fSpatrick 1045a0747c9fSpatrick 104609467b48Spatrickdefm LCMPXCHG : LCMPXCHG_BinOp<0xB0, 0xB1, MRMDestMem, "cmpxchg", X86cas>; 104709467b48Spatrick 104809467b48Spatrick// Atomic exchange and add 1049a0747c9fSpatrickmulticlass ATOMIC_RMW_BINOP<bits<8> opc8, bits<8> opc, string mnemonic, 105009467b48Spatrick string frag> { 1051a0747c9fSpatrick let Constraints = "$val = $dst", Defs = [EFLAGS], mayLoad = 1, mayStore = 1, 1052a0747c9fSpatrick isCodeGenOnly = 1, SchedRW = [WriteALURMW] in { 105309467b48Spatrick def NAME#8 : I<opc8, MRMSrcMem, (outs GR8:$dst), 105409467b48Spatrick (ins GR8:$val, i8mem:$ptr), 105509467b48Spatrick !strconcat(mnemonic, "{b}\t{$val, $ptr|$ptr, $val}"), 105609467b48Spatrick [(set GR8:$dst, 105709467b48Spatrick (!cast<PatFrag>(frag # "_8") addr:$ptr, GR8:$val))]>; 105809467b48Spatrick def NAME#16 : I<opc, MRMSrcMem, (outs GR16:$dst), 105909467b48Spatrick (ins GR16:$val, i16mem:$ptr), 106009467b48Spatrick !strconcat(mnemonic, "{w}\t{$val, $ptr|$ptr, $val}"), 106109467b48Spatrick [(set 106209467b48Spatrick GR16:$dst, 106309467b48Spatrick (!cast<PatFrag>(frag # "_16") addr:$ptr, GR16:$val))]>, 106409467b48Spatrick OpSize16; 106509467b48Spatrick def NAME#32 : I<opc, MRMSrcMem, (outs GR32:$dst), 106609467b48Spatrick (ins GR32:$val, i32mem:$ptr), 106709467b48Spatrick !strconcat(mnemonic, "{l}\t{$val, $ptr|$ptr, $val}"), 106809467b48Spatrick [(set 106909467b48Spatrick GR32:$dst, 107009467b48Spatrick (!cast<PatFrag>(frag # "_32") addr:$ptr, GR32:$val))]>, 107109467b48Spatrick OpSize32; 107209467b48Spatrick def NAME#64 : RI<opc, MRMSrcMem, (outs GR64:$dst), 107309467b48Spatrick (ins GR64:$val, i64mem:$ptr), 107409467b48Spatrick !strconcat(mnemonic, "{q}\t{$val, $ptr|$ptr, $val}"), 107509467b48Spatrick [(set 107609467b48Spatrick GR64:$dst, 107709467b48Spatrick (!cast<PatFrag>(frag # "_64") addr:$ptr, GR64:$val))]>; 107809467b48Spatrick } 107909467b48Spatrick} 108009467b48Spatrick 1081a0747c9fSpatrickdefm LXADD : ATOMIC_RMW_BINOP<0xc0, 0xc1, "xadd", "atomic_load_add">, TB, LOCK; 108209467b48Spatrick 108309467b48Spatrick/* The following multiclass tries to make sure that in code like 108409467b48Spatrick * x.store (immediate op x.load(acquire), release) 108509467b48Spatrick * and 108609467b48Spatrick * x.store (register op x.load(acquire), release) 108709467b48Spatrick * an operation directly on memory is generated instead of wasting a register. 108809467b48Spatrick * It is not automatic as atomic_store/load are only lowered to MOV instructions 108909467b48Spatrick * extremely late to prevent them from being accidentally reordered in the backend 109009467b48Spatrick * (see below the RELEASE_MOV* / ACQUIRE_MOV* pseudo-instructions) 109109467b48Spatrick */ 109209467b48Spatrickmulticlass RELEASE_BINOP_MI<string Name, SDNode op> { 109309467b48Spatrick def : Pat<(atomic_store_8 addr:$dst, 109409467b48Spatrick (op (atomic_load_8 addr:$dst), (i8 imm:$src))), 109509467b48Spatrick (!cast<Instruction>(Name#"8mi") addr:$dst, imm:$src)>; 109609467b48Spatrick def : Pat<(atomic_store_16 addr:$dst, 109709467b48Spatrick (op (atomic_load_16 addr:$dst), (i16 imm:$src))), 109809467b48Spatrick (!cast<Instruction>(Name#"16mi") addr:$dst, imm:$src)>; 109909467b48Spatrick def : Pat<(atomic_store_32 addr:$dst, 110009467b48Spatrick (op (atomic_load_32 addr:$dst), (i32 imm:$src))), 110109467b48Spatrick (!cast<Instruction>(Name#"32mi") addr:$dst, imm:$src)>; 110209467b48Spatrick def : Pat<(atomic_store_64 addr:$dst, 110309467b48Spatrick (op (atomic_load_64 addr:$dst), (i64immSExt32:$src))), 110409467b48Spatrick (!cast<Instruction>(Name#"64mi32") addr:$dst, (i64immSExt32:$src))>; 110509467b48Spatrick 110609467b48Spatrick def : Pat<(atomic_store_8 addr:$dst, 110709467b48Spatrick (op (atomic_load_8 addr:$dst), (i8 GR8:$src))), 110809467b48Spatrick (!cast<Instruction>(Name#"8mr") addr:$dst, GR8:$src)>; 110909467b48Spatrick def : Pat<(atomic_store_16 addr:$dst, 111009467b48Spatrick (op (atomic_load_16 addr:$dst), (i16 GR16:$src))), 111109467b48Spatrick (!cast<Instruction>(Name#"16mr") addr:$dst, GR16:$src)>; 111209467b48Spatrick def : Pat<(atomic_store_32 addr:$dst, 111309467b48Spatrick (op (atomic_load_32 addr:$dst), (i32 GR32:$src))), 111409467b48Spatrick (!cast<Instruction>(Name#"32mr") addr:$dst, GR32:$src)>; 111509467b48Spatrick def : Pat<(atomic_store_64 addr:$dst, 111609467b48Spatrick (op (atomic_load_64 addr:$dst), (i64 GR64:$src))), 111709467b48Spatrick (!cast<Instruction>(Name#"64mr") addr:$dst, GR64:$src)>; 111809467b48Spatrick} 111909467b48Spatrickdefm : RELEASE_BINOP_MI<"ADD", add>; 112009467b48Spatrickdefm : RELEASE_BINOP_MI<"AND", and>; 112109467b48Spatrickdefm : RELEASE_BINOP_MI<"OR", or>; 112209467b48Spatrickdefm : RELEASE_BINOP_MI<"XOR", xor>; 112309467b48Spatrickdefm : RELEASE_BINOP_MI<"SUB", sub>; 112409467b48Spatrick 112509467b48Spatrick// Atomic load + floating point patterns. 112609467b48Spatrick// FIXME: This could also handle SIMD operations with *ps and *pd instructions. 112709467b48Spatrickmulticlass ATOMIC_LOAD_FP_BINOP_MI<string Name, SDNode op> { 112809467b48Spatrick def : Pat<(op FR32:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))), 112909467b48Spatrick (!cast<Instruction>(Name#"SSrm") FR32:$src1, addr:$src2)>, 113009467b48Spatrick Requires<[UseSSE1]>; 113109467b48Spatrick def : Pat<(op FR32:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))), 113209467b48Spatrick (!cast<Instruction>("V"#Name#"SSrm") FR32:$src1, addr:$src2)>, 113309467b48Spatrick Requires<[UseAVX]>; 113409467b48Spatrick def : Pat<(op FR32X:$src1, (bitconvert (i32 (atomic_load_32 addr:$src2)))), 113509467b48Spatrick (!cast<Instruction>("V"#Name#"SSZrm") FR32X:$src1, addr:$src2)>, 113609467b48Spatrick Requires<[HasAVX512]>; 113709467b48Spatrick 113809467b48Spatrick def : Pat<(op FR64:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))), 113909467b48Spatrick (!cast<Instruction>(Name#"SDrm") FR64:$src1, addr:$src2)>, 114009467b48Spatrick Requires<[UseSSE1]>; 114109467b48Spatrick def : Pat<(op FR64:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))), 114209467b48Spatrick (!cast<Instruction>("V"#Name#"SDrm") FR64:$src1, addr:$src2)>, 114309467b48Spatrick Requires<[UseAVX]>; 114409467b48Spatrick def : Pat<(op FR64X:$src1, (bitconvert (i64 (atomic_load_64 addr:$src2)))), 114509467b48Spatrick (!cast<Instruction>("V"#Name#"SDZrm") FR64X:$src1, addr:$src2)>, 114609467b48Spatrick Requires<[HasAVX512]>; 114709467b48Spatrick} 114809467b48Spatrickdefm : ATOMIC_LOAD_FP_BINOP_MI<"ADD", fadd>; 114909467b48Spatrick// FIXME: Add fsub, fmul, fdiv, ... 115009467b48Spatrick 115109467b48Spatrickmulticlass RELEASE_UNOP<string Name, dag dag8, dag dag16, dag dag32, 115209467b48Spatrick dag dag64> { 115309467b48Spatrick def : Pat<(atomic_store_8 addr:$dst, dag8), 115409467b48Spatrick (!cast<Instruction>(Name#8m) addr:$dst)>; 115509467b48Spatrick def : Pat<(atomic_store_16 addr:$dst, dag16), 115609467b48Spatrick (!cast<Instruction>(Name#16m) addr:$dst)>; 115709467b48Spatrick def : Pat<(atomic_store_32 addr:$dst, dag32), 115809467b48Spatrick (!cast<Instruction>(Name#32m) addr:$dst)>; 115909467b48Spatrick def : Pat<(atomic_store_64 addr:$dst, dag64), 116009467b48Spatrick (!cast<Instruction>(Name#64m) addr:$dst)>; 116109467b48Spatrick} 116209467b48Spatrick 116309467b48Spatricklet Predicates = [UseIncDec] in { 116409467b48Spatrick defm : RELEASE_UNOP<"INC", 116509467b48Spatrick (add (atomic_load_8 addr:$dst), (i8 1)), 116609467b48Spatrick (add (atomic_load_16 addr:$dst), (i16 1)), 116709467b48Spatrick (add (atomic_load_32 addr:$dst), (i32 1)), 116809467b48Spatrick (add (atomic_load_64 addr:$dst), (i64 1))>; 116909467b48Spatrick defm : RELEASE_UNOP<"DEC", 117009467b48Spatrick (add (atomic_load_8 addr:$dst), (i8 -1)), 117109467b48Spatrick (add (atomic_load_16 addr:$dst), (i16 -1)), 117209467b48Spatrick (add (atomic_load_32 addr:$dst), (i32 -1)), 117309467b48Spatrick (add (atomic_load_64 addr:$dst), (i64 -1))>; 117409467b48Spatrick} 117509467b48Spatrick 117609467b48Spatrickdefm : RELEASE_UNOP<"NEG", 117709467b48Spatrick (ineg (i8 (atomic_load_8 addr:$dst))), 117809467b48Spatrick (ineg (i16 (atomic_load_16 addr:$dst))), 117909467b48Spatrick (ineg (i32 (atomic_load_32 addr:$dst))), 118009467b48Spatrick (ineg (i64 (atomic_load_64 addr:$dst)))>; 118109467b48Spatrickdefm : RELEASE_UNOP<"NOT", 118209467b48Spatrick (not (i8 (atomic_load_8 addr:$dst))), 118309467b48Spatrick (not (i16 (atomic_load_16 addr:$dst))), 118409467b48Spatrick (not (i32 (atomic_load_32 addr:$dst))), 118509467b48Spatrick (not (i64 (atomic_load_64 addr:$dst)))>; 118609467b48Spatrick 118709467b48Spatrickdef : Pat<(atomic_store_8 addr:$dst, (i8 imm:$src)), 118809467b48Spatrick (MOV8mi addr:$dst, imm:$src)>; 118909467b48Spatrickdef : Pat<(atomic_store_16 addr:$dst, (i16 imm:$src)), 119009467b48Spatrick (MOV16mi addr:$dst, imm:$src)>; 119109467b48Spatrickdef : Pat<(atomic_store_32 addr:$dst, (i32 imm:$src)), 119209467b48Spatrick (MOV32mi addr:$dst, imm:$src)>; 119309467b48Spatrickdef : Pat<(atomic_store_64 addr:$dst, (i64immSExt32:$src)), 119409467b48Spatrick (MOV64mi32 addr:$dst, i64immSExt32:$src)>; 119509467b48Spatrick 119609467b48Spatrickdef : Pat<(atomic_store_8 addr:$dst, GR8:$src), 119709467b48Spatrick (MOV8mr addr:$dst, GR8:$src)>; 119809467b48Spatrickdef : Pat<(atomic_store_16 addr:$dst, GR16:$src), 119909467b48Spatrick (MOV16mr addr:$dst, GR16:$src)>; 120009467b48Spatrickdef : Pat<(atomic_store_32 addr:$dst, GR32:$src), 120109467b48Spatrick (MOV32mr addr:$dst, GR32:$src)>; 120209467b48Spatrickdef : Pat<(atomic_store_64 addr:$dst, GR64:$src), 120309467b48Spatrick (MOV64mr addr:$dst, GR64:$src)>; 120409467b48Spatrick 120509467b48Spatrickdef : Pat<(i8 (atomic_load_8 addr:$src)), (MOV8rm addr:$src)>; 120609467b48Spatrickdef : Pat<(i16 (atomic_load_16 addr:$src)), (MOV16rm addr:$src)>; 120709467b48Spatrickdef : Pat<(i32 (atomic_load_32 addr:$src)), (MOV32rm addr:$src)>; 120809467b48Spatrickdef : Pat<(i64 (atomic_load_64 addr:$src)), (MOV64rm addr:$src)>; 120909467b48Spatrick 121009467b48Spatrick// Floating point loads/stores. 121109467b48Spatrickdef : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), 121209467b48Spatrick (MOVSSmr addr:$dst, FR32:$src)>, Requires<[UseSSE1]>; 121309467b48Spatrickdef : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), 121409467b48Spatrick (VMOVSSmr addr:$dst, FR32:$src)>, Requires<[UseAVX]>; 121509467b48Spatrickdef : Pat<(atomic_store_32 addr:$dst, (i32 (bitconvert (f32 FR32:$src)))), 121609467b48Spatrick (VMOVSSZmr addr:$dst, FR32:$src)>, Requires<[HasAVX512]>; 121709467b48Spatrick 121809467b48Spatrickdef : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), 121909467b48Spatrick (MOVSDmr addr:$dst, FR64:$src)>, Requires<[UseSSE2]>; 122009467b48Spatrickdef : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), 122109467b48Spatrick (VMOVSDmr addr:$dst, FR64:$src)>, Requires<[UseAVX]>; 122209467b48Spatrickdef : Pat<(atomic_store_64 addr:$dst, (i64 (bitconvert (f64 FR64:$src)))), 122309467b48Spatrick (VMOVSDmr addr:$dst, FR64:$src)>, Requires<[HasAVX512]>; 122409467b48Spatrick 122509467b48Spatrickdef : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))), 122609467b48Spatrick (MOVSSrm_alt addr:$src)>, Requires<[UseSSE1]>; 122709467b48Spatrickdef : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))), 122809467b48Spatrick (VMOVSSrm_alt addr:$src)>, Requires<[UseAVX]>; 122909467b48Spatrickdef : Pat<(f32 (bitconvert (i32 (atomic_load_32 addr:$src)))), 123009467b48Spatrick (VMOVSSZrm_alt addr:$src)>, Requires<[HasAVX512]>; 123109467b48Spatrick 123209467b48Spatrickdef : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))), 123309467b48Spatrick (MOVSDrm_alt addr:$src)>, Requires<[UseSSE2]>; 123409467b48Spatrickdef : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))), 123509467b48Spatrick (VMOVSDrm_alt addr:$src)>, Requires<[UseAVX]>; 123609467b48Spatrickdef : Pat<(f64 (bitconvert (i64 (atomic_load_64 addr:$src)))), 123709467b48Spatrick (VMOVSDZrm_alt addr:$src)>, Requires<[HasAVX512]>; 123809467b48Spatrick 123909467b48Spatrick//===----------------------------------------------------------------------===// 124009467b48Spatrick// DAG Pattern Matching Rules 124109467b48Spatrick//===----------------------------------------------------------------------===// 124209467b48Spatrick 124309467b48Spatrick// Use AND/OR to store 0/-1 in memory when optimizing for minsize. This saves 124409467b48Spatrick// binary size compared to a regular MOV, but it introduces an unnecessary 124509467b48Spatrick// load, so is not suitable for regular or optsize functions. 124609467b48Spatricklet Predicates = [OptForMinSize] in { 124709467b48Spatrickdef : Pat<(simple_store (i16 0), addr:$dst), (AND16mi8 addr:$dst, 0)>; 124809467b48Spatrickdef : Pat<(simple_store (i32 0), addr:$dst), (AND32mi8 addr:$dst, 0)>; 124909467b48Spatrickdef : Pat<(simple_store (i64 0), addr:$dst), (AND64mi8 addr:$dst, 0)>; 125009467b48Spatrickdef : Pat<(simple_store (i16 -1), addr:$dst), (OR16mi8 addr:$dst, -1)>; 125109467b48Spatrickdef : Pat<(simple_store (i32 -1), addr:$dst), (OR32mi8 addr:$dst, -1)>; 125209467b48Spatrickdef : Pat<(simple_store (i64 -1), addr:$dst), (OR64mi8 addr:$dst, -1)>; 125309467b48Spatrick} 125409467b48Spatrick 125509467b48Spatrick// In kernel code model, we can get the address of a label 125609467b48Spatrick// into a register with 'movq'. FIXME: This is a hack, the 'imm' predicate of 125709467b48Spatrick// the MOV64ri32 should accept these. 125809467b48Spatrickdef : Pat<(i64 (X86Wrapper tconstpool :$dst)), 125909467b48Spatrick (MOV64ri32 tconstpool :$dst)>, Requires<[KernelCode]>; 126009467b48Spatrickdef : Pat<(i64 (X86Wrapper tjumptable :$dst)), 126109467b48Spatrick (MOV64ri32 tjumptable :$dst)>, Requires<[KernelCode]>; 126209467b48Spatrickdef : Pat<(i64 (X86Wrapper tglobaladdr :$dst)), 126309467b48Spatrick (MOV64ri32 tglobaladdr :$dst)>, Requires<[KernelCode]>; 126409467b48Spatrickdef : Pat<(i64 (X86Wrapper texternalsym:$dst)), 126509467b48Spatrick (MOV64ri32 texternalsym:$dst)>, Requires<[KernelCode]>; 126609467b48Spatrickdef : Pat<(i64 (X86Wrapper mcsym:$dst)), 126709467b48Spatrick (MOV64ri32 mcsym:$dst)>, Requires<[KernelCode]>; 126809467b48Spatrickdef : Pat<(i64 (X86Wrapper tblockaddress:$dst)), 126909467b48Spatrick (MOV64ri32 tblockaddress:$dst)>, Requires<[KernelCode]>; 127009467b48Spatrick 127109467b48Spatrick// If we have small model and -static mode, it is safe to store global addresses 127209467b48Spatrick// directly as immediates. FIXME: This is really a hack, the 'imm' predicate 127309467b48Spatrick// for MOV64mi32 should handle this sort of thing. 127409467b48Spatrickdef : Pat<(store (i64 (X86Wrapper tconstpool:$src)), addr:$dst), 127509467b48Spatrick (MOV64mi32 addr:$dst, tconstpool:$src)>, 127609467b48Spatrick Requires<[NearData, IsNotPIC]>; 127709467b48Spatrickdef : Pat<(store (i64 (X86Wrapper tjumptable:$src)), addr:$dst), 127809467b48Spatrick (MOV64mi32 addr:$dst, tjumptable:$src)>, 127909467b48Spatrick Requires<[NearData, IsNotPIC]>; 128009467b48Spatrickdef : Pat<(store (i64 (X86Wrapper tglobaladdr:$src)), addr:$dst), 128109467b48Spatrick (MOV64mi32 addr:$dst, tglobaladdr:$src)>, 128209467b48Spatrick Requires<[NearData, IsNotPIC]>; 128309467b48Spatrickdef : Pat<(store (i64 (X86Wrapper texternalsym:$src)), addr:$dst), 128409467b48Spatrick (MOV64mi32 addr:$dst, texternalsym:$src)>, 128509467b48Spatrick Requires<[NearData, IsNotPIC]>; 128609467b48Spatrickdef : Pat<(store (i64 (X86Wrapper mcsym:$src)), addr:$dst), 128709467b48Spatrick (MOV64mi32 addr:$dst, mcsym:$src)>, 128809467b48Spatrick Requires<[NearData, IsNotPIC]>; 128909467b48Spatrickdef : Pat<(store (i64 (X86Wrapper tblockaddress:$src)), addr:$dst), 129009467b48Spatrick (MOV64mi32 addr:$dst, tblockaddress:$src)>, 129109467b48Spatrick Requires<[NearData, IsNotPIC]>; 129209467b48Spatrick 129309467b48Spatrickdef : Pat<(i32 (X86RecoverFrameAlloc mcsym:$dst)), (MOV32ri mcsym:$dst)>; 129409467b48Spatrickdef : Pat<(i64 (X86RecoverFrameAlloc mcsym:$dst)), (MOV64ri mcsym:$dst)>; 129509467b48Spatrick 129609467b48Spatrick// Calls 129709467b48Spatrick 129809467b48Spatrick// tls has some funny stuff here... 129909467b48Spatrick// This corresponds to movabs $foo@tpoff, %rax 130009467b48Spatrickdef : Pat<(i64 (X86Wrapper tglobaltlsaddr :$dst)), 130109467b48Spatrick (MOV64ri32 tglobaltlsaddr :$dst)>; 130209467b48Spatrick// This corresponds to add $foo@tpoff, %rax 130309467b48Spatrickdef : Pat<(add GR64:$src1, (X86Wrapper tglobaltlsaddr :$dst)), 130409467b48Spatrick (ADD64ri32 GR64:$src1, tglobaltlsaddr :$dst)>; 130509467b48Spatrick 130609467b48Spatrick 130709467b48Spatrick// Direct PC relative function call for small code model. 32-bit displacement 130809467b48Spatrick// sign extended to 64-bit. 130909467b48Spatrickdef : Pat<(X86call (i64 tglobaladdr:$dst)), 131009467b48Spatrick (CALL64pcrel32 tglobaladdr:$dst)>; 131109467b48Spatrickdef : Pat<(X86call (i64 texternalsym:$dst)), 131209467b48Spatrick (CALL64pcrel32 texternalsym:$dst)>; 131309467b48Spatrick 1314*a96b3639Srobertdef : Pat<(X86call_rvmarker (i64 tglobaladdr:$rvfunc), (i64 texternalsym:$dst)), 1315*a96b3639Srobert (CALL64pcrel32_RVMARKER tglobaladdr:$rvfunc, texternalsym:$dst)>; 1316*a96b3639Srobertdef : Pat<(X86call_rvmarker (i64 tglobaladdr:$rvfunc), (i64 tglobaladdr:$dst)), 1317*a96b3639Srobert (CALL64pcrel32_RVMARKER tglobaladdr:$rvfunc, tglobaladdr:$dst)>; 1318a0747c9fSpatrick 1319a0747c9fSpatrick 132009467b48Spatrick// Tailcall stuff. The TCRETURN instructions execute after the epilog, so they 132109467b48Spatrick// can never use callee-saved registers. That is the purpose of the GR64_TC 132209467b48Spatrick// register classes. 132309467b48Spatrick// 132409467b48Spatrick// The only volatile register that is never used by the calling convention is 132509467b48Spatrick// %r11. This happens when calling a vararg function with 6 arguments. 132609467b48Spatrick// 132709467b48Spatrick// Match an X86tcret that uses less than 7 volatile registers. 132809467b48Spatrickdef X86tcret_6regs : PatFrag<(ops node:$ptr, node:$off), 132909467b48Spatrick (X86tcret node:$ptr, node:$off), [{ 133009467b48Spatrick // X86tcret args: (*chain, ptr, imm, regs..., glue) 133109467b48Spatrick unsigned NumRegs = 0; 133209467b48Spatrick for (unsigned i = 3, e = N->getNumOperands(); i != e; ++i) 133309467b48Spatrick if (isa<RegisterSDNode>(N->getOperand(i)) && ++NumRegs > 6) 133409467b48Spatrick return false; 133509467b48Spatrick return true; 133609467b48Spatrick}]>; 133709467b48Spatrick 1338*a96b3639Srobertdef X86tcret_1reg : PatFrag<(ops node:$ptr, node:$off), 1339*a96b3639Srobert (X86tcret node:$ptr, node:$off), [{ 1340*a96b3639Srobert // X86tcret args: (*chain, ptr, imm, regs..., glue) 1341*a96b3639Srobert unsigned NumRegs = 1; 1342*a96b3639Srobert const SDValue& BasePtr = cast<LoadSDNode>(N->getOperand(1))->getBasePtr(); 1343*a96b3639Srobert if (isa<FrameIndexSDNode>(BasePtr)) 1344*a96b3639Srobert NumRegs = 3; 1345*a96b3639Srobert else if (BasePtr->getNumOperands() && isa<GlobalAddressSDNode>(BasePtr->getOperand(0))) 1346*a96b3639Srobert NumRegs = 3; 1347*a96b3639Srobert for (unsigned i = 3, e = N->getNumOperands(); i != e; ++i) 1348*a96b3639Srobert if (isa<RegisterSDNode>(N->getOperand(i)) && ( NumRegs-- == 0)) 1349*a96b3639Srobert return false; 1350*a96b3639Srobert return true; 1351*a96b3639Srobert}]>; 1352*a96b3639Srobert 1353a0747c9fSpatrickdef : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off), 1354a0747c9fSpatrick (TCRETURNri ptr_rc_tailcall:$dst, timm:$off)>, 13557299aa8dSpatrick Requires<[Not64BitMode, NotUseIndirectThunkCalls]>; 135609467b48Spatrick 135709467b48Spatrick// FIXME: This is disabled for 32-bit PIC mode because the global base 135809467b48Spatrick// register which is part of the address mode may be assigned a 135909467b48Spatrick// callee-saved register. 1360*a96b3639Srobert// Similar to X86tcret_6regs, here we only have 1 register left 1361*a96b3639Srobertdef : Pat<(X86tcret_1reg (load addr:$dst), timm:$off), 1362a0747c9fSpatrick (TCRETURNmi addr:$dst, timm:$off)>, 13637299aa8dSpatrick Requires<[Not64BitMode, IsNotPIC, NotUseIndirectThunkCalls]>; 136409467b48Spatrick 1365a0747c9fSpatrickdef : Pat<(X86tcret (i32 tglobaladdr:$dst), timm:$off), 1366a0747c9fSpatrick (TCRETURNdi tglobaladdr:$dst, timm:$off)>, 136709467b48Spatrick Requires<[NotLP64]>; 136809467b48Spatrick 1369a0747c9fSpatrickdef : Pat<(X86tcret (i32 texternalsym:$dst), timm:$off), 1370a0747c9fSpatrick (TCRETURNdi texternalsym:$dst, timm:$off)>, 137109467b48Spatrick Requires<[NotLP64]>; 137209467b48Spatrick 1373a0747c9fSpatrickdef : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off), 1374a0747c9fSpatrick (TCRETURNri64 ptr_rc_tailcall:$dst, timm:$off)>, 13757299aa8dSpatrick Requires<[In64BitMode, NotUseIndirectThunkCalls]>; 137609467b48Spatrick 137709467b48Spatrick// Don't fold loads into X86tcret requiring more than 6 regs. 137809467b48Spatrick// There wouldn't be enough scratch registers for base+index. 1379a0747c9fSpatrickdef : Pat<(X86tcret_6regs (load addr:$dst), timm:$off), 1380a0747c9fSpatrick (TCRETURNmi64 addr:$dst, timm:$off)>, 13817299aa8dSpatrick Requires<[In64BitMode, NotUseIndirectThunkCalls]>; 138209467b48Spatrick 1383a0747c9fSpatrickdef : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off), 1384a0747c9fSpatrick (INDIRECT_THUNK_TCRETURN64 ptr_rc_tailcall:$dst, timm:$off)>, 13857299aa8dSpatrick Requires<[In64BitMode, UseIndirectThunkCalls]>; 138609467b48Spatrick 1387a0747c9fSpatrickdef : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off), 1388a0747c9fSpatrick (INDIRECT_THUNK_TCRETURN32 ptr_rc_tailcall:$dst, timm:$off)>, 13897299aa8dSpatrick Requires<[Not64BitMode, UseIndirectThunkCalls]>; 139009467b48Spatrick 1391a0747c9fSpatrickdef : Pat<(X86tcret (i64 tglobaladdr:$dst), timm:$off), 1392a0747c9fSpatrick (TCRETURNdi64 tglobaladdr:$dst, timm:$off)>, 139309467b48Spatrick Requires<[IsLP64]>; 139409467b48Spatrick 1395a0747c9fSpatrickdef : Pat<(X86tcret (i64 texternalsym:$dst), timm:$off), 1396a0747c9fSpatrick (TCRETURNdi64 texternalsym:$dst, timm:$off)>, 139709467b48Spatrick Requires<[IsLP64]>; 139809467b48Spatrick 139909467b48Spatrick// Normal calls, with various flavors of addresses. 140009467b48Spatrickdef : Pat<(X86call (i32 tglobaladdr:$dst)), 140109467b48Spatrick (CALLpcrel32 tglobaladdr:$dst)>; 140209467b48Spatrickdef : Pat<(X86call (i32 texternalsym:$dst)), 140309467b48Spatrick (CALLpcrel32 texternalsym:$dst)>; 140409467b48Spatrickdef : Pat<(X86call (i32 imm:$dst)), 140509467b48Spatrick (CALLpcrel32 imm:$dst)>, Requires<[CallImmAddr]>; 140609467b48Spatrick 140709467b48Spatrick// Comparisons. 140809467b48Spatrick 140909467b48Spatrick// TEST R,R is smaller than CMP R,0 141009467b48Spatrickdef : Pat<(X86cmp GR8:$src1, 0), 141109467b48Spatrick (TEST8rr GR8:$src1, GR8:$src1)>; 141209467b48Spatrickdef : Pat<(X86cmp GR16:$src1, 0), 141309467b48Spatrick (TEST16rr GR16:$src1, GR16:$src1)>; 141409467b48Spatrickdef : Pat<(X86cmp GR32:$src1, 0), 141509467b48Spatrick (TEST32rr GR32:$src1, GR32:$src1)>; 141609467b48Spatrickdef : Pat<(X86cmp GR64:$src1, 0), 141709467b48Spatrick (TEST64rr GR64:$src1, GR64:$src1)>; 141809467b48Spatrick 141909467b48Spatrick// zextload bool -> zextload byte 142009467b48Spatrick// i1 stored in one byte in zero-extended form. 142109467b48Spatrick// Upper bits cleanup should be executed before Store. 142209467b48Spatrickdef : Pat<(zextloadi8i1 addr:$src), (MOV8rm addr:$src)>; 142309467b48Spatrickdef : Pat<(zextloadi16i1 addr:$src), 142409467b48Spatrick (EXTRACT_SUBREG (MOVZX32rm8 addr:$src), sub_16bit)>; 142509467b48Spatrickdef : Pat<(zextloadi32i1 addr:$src), (MOVZX32rm8 addr:$src)>; 142609467b48Spatrickdef : Pat<(zextloadi64i1 addr:$src), 142709467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rm8 addr:$src), sub_32bit)>; 142809467b48Spatrick 142909467b48Spatrick// extload bool -> extload byte 143009467b48Spatrick// When extloading from 16-bit and smaller memory locations into 64-bit 143109467b48Spatrick// registers, use zero-extending loads so that the entire 64-bit register is 143209467b48Spatrick// defined, avoiding partial-register updates. 143309467b48Spatrick 143409467b48Spatrickdef : Pat<(extloadi8i1 addr:$src), (MOV8rm addr:$src)>; 143509467b48Spatrickdef : Pat<(extloadi16i1 addr:$src), 143609467b48Spatrick (EXTRACT_SUBREG (MOVZX32rm8 addr:$src), sub_16bit)>; 143709467b48Spatrickdef : Pat<(extloadi32i1 addr:$src), (MOVZX32rm8 addr:$src)>; 143809467b48Spatrickdef : Pat<(extloadi16i8 addr:$src), 143909467b48Spatrick (EXTRACT_SUBREG (MOVZX32rm8 addr:$src), sub_16bit)>; 144009467b48Spatrickdef : Pat<(extloadi32i8 addr:$src), (MOVZX32rm8 addr:$src)>; 144109467b48Spatrickdef : Pat<(extloadi32i16 addr:$src), (MOVZX32rm16 addr:$src)>; 144209467b48Spatrick 144309467b48Spatrick// For other extloads, use subregs, since the high contents of the register are 144409467b48Spatrick// defined after an extload. 144509467b48Spatrick// NOTE: The extloadi64i32 pattern needs to be first as it will try to form 144609467b48Spatrick// 32-bit loads for 4 byte aligned i8/i16 loads. 144709467b48Spatrickdef : Pat<(extloadi64i32 addr:$src), 144809467b48Spatrick (SUBREG_TO_REG (i64 0), (MOV32rm addr:$src), sub_32bit)>; 144909467b48Spatrickdef : Pat<(extloadi64i1 addr:$src), 145009467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rm8 addr:$src), sub_32bit)>; 145109467b48Spatrickdef : Pat<(extloadi64i8 addr:$src), 145209467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rm8 addr:$src), sub_32bit)>; 145309467b48Spatrickdef : Pat<(extloadi64i16 addr:$src), 145409467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rm16 addr:$src), sub_32bit)>; 145509467b48Spatrick 145609467b48Spatrick// anyext. Define these to do an explicit zero-extend to 145709467b48Spatrick// avoid partial-register updates. 145809467b48Spatrickdef : Pat<(i16 (anyext GR8 :$src)), (EXTRACT_SUBREG 145909467b48Spatrick (MOVZX32rr8 GR8 :$src), sub_16bit)>; 146009467b48Spatrickdef : Pat<(i32 (anyext GR8 :$src)), (MOVZX32rr8 GR8 :$src)>; 146109467b48Spatrick 146209467b48Spatrick// Except for i16 -> i32 since isel expect i16 ops to be promoted to i32. 146309467b48Spatrickdef : Pat<(i32 (anyext GR16:$src)), 146409467b48Spatrick (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR16:$src, sub_16bit)>; 146509467b48Spatrick 146609467b48Spatrickdef : Pat<(i64 (anyext GR8 :$src)), 146709467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rr8 GR8 :$src), sub_32bit)>; 146809467b48Spatrickdef : Pat<(i64 (anyext GR16:$src)), 146909467b48Spatrick (SUBREG_TO_REG (i64 0), (MOVZX32rr16 GR16 :$src), sub_32bit)>; 147009467b48Spatrickdef : Pat<(i64 (anyext GR32:$src)), 147109467b48Spatrick (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GR32:$src, sub_32bit)>; 147209467b48Spatrick 147309467b48Spatrick// If this is an anyext of the remainder of an 8-bit sdivrem, use a MOVSX 147409467b48Spatrick// instead of a MOVZX. The sdivrem lowering will emit emit a MOVSX to move 147509467b48Spatrick// %ah to the lower byte of a register. By using a MOVSX here we allow a 147609467b48Spatrick// post-isel peephole to merge the two MOVSX instructions into one. 147709467b48Spatrickdef anyext_sdiv : PatFrag<(ops node:$lhs), (anyext node:$lhs),[{ 147809467b48Spatrick return (N->getOperand(0).getOpcode() == ISD::SDIVREM && 147909467b48Spatrick N->getOperand(0).getResNo() == 1); 148009467b48Spatrick}]>; 148109467b48Spatrickdef : Pat<(i32 (anyext_sdiv GR8:$src)), (MOVSX32rr8 GR8:$src)>; 148209467b48Spatrick 148309467b48Spatrick// Any instruction that defines a 32-bit result leaves the high half of the 148409467b48Spatrick// register. Truncate can be lowered to EXTRACT_SUBREG. CopyFromReg may 1485a0747c9fSpatrick// be copying from a truncate. AssertSext/AssertZext/AssertAlign aren't saying 1486a0747c9fSpatrick// anything about the upper 32 bits, they're probably just qualifying a 1487a0747c9fSpatrick// CopyFromReg. FREEZE may be coming from a a truncate. Any other 32-bit 1488a0747c9fSpatrick// operation will zero-extend up to 64 bits. 148909467b48Spatrickdef def32 : PatLeaf<(i32 GR32:$src), [{ 149009467b48Spatrick return N->getOpcode() != ISD::TRUNCATE && 149109467b48Spatrick N->getOpcode() != TargetOpcode::EXTRACT_SUBREG && 149209467b48Spatrick N->getOpcode() != ISD::CopyFromReg && 149309467b48Spatrick N->getOpcode() != ISD::AssertSext && 1494a0747c9fSpatrick N->getOpcode() != ISD::AssertZext && 1495a0747c9fSpatrick N->getOpcode() != ISD::AssertAlign && 1496a0747c9fSpatrick N->getOpcode() != ISD::FREEZE; 149709467b48Spatrick}]>; 149809467b48Spatrick 149909467b48Spatrick// In the case of a 32-bit def that is known to implicitly zero-extend, 150009467b48Spatrick// we can use a SUBREG_TO_REG. 150109467b48Spatrickdef : Pat<(i64 (zext def32:$src)), 150209467b48Spatrick (SUBREG_TO_REG (i64 0), GR32:$src, sub_32bit)>; 150309467b48Spatrickdef : Pat<(i64 (and (anyext def32:$src), 0x00000000FFFFFFFF)), 150409467b48Spatrick (SUBREG_TO_REG (i64 0), GR32:$src, sub_32bit)>; 150509467b48Spatrick 150609467b48Spatrick//===----------------------------------------------------------------------===// 150709467b48Spatrick// Pattern match OR as ADD 150809467b48Spatrick//===----------------------------------------------------------------------===// 150909467b48Spatrick 151009467b48Spatrick// If safe, we prefer to pattern match OR as ADD at isel time. ADD can be 151109467b48Spatrick// 3-addressified into an LEA instruction to avoid copies. However, we also 151209467b48Spatrick// want to finally emit these instructions as an or at the end of the code 151309467b48Spatrick// generator to make the generated code easier to read. To do this, we select 151409467b48Spatrick// into "disjoint bits" pseudo ops. 151509467b48Spatrick 151609467b48Spatrick// Treat an 'or' node is as an 'add' if the or'ed bits are known to be zero. 151709467b48Spatrickdef or_is_add : PatFrag<(ops node:$lhs, node:$rhs), (or node:$lhs, node:$rhs),[{ 151809467b48Spatrick if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) 151909467b48Spatrick return CurDAG->MaskedValueIsZero(N->getOperand(0), CN->getAPIntValue()); 152009467b48Spatrick 152109467b48Spatrick KnownBits Known0 = CurDAG->computeKnownBits(N->getOperand(0), 0); 152209467b48Spatrick KnownBits Known1 = CurDAG->computeKnownBits(N->getOperand(1), 0); 152309467b48Spatrick return (~Known0.Zero & ~Known1.Zero) == 0; 152409467b48Spatrick}]>; 152509467b48Spatrick 152609467b48Spatrick 152709467b48Spatrick// (or x1, x2) -> (add x1, x2) if two operands are known not to share bits. 152809467b48Spatrick// Try this before the selecting to OR. 152909467b48Spatricklet SchedRW = [WriteALU] in { 153009467b48Spatrick 153109467b48Spatricklet isConvertibleToThreeAddress = 1, isPseudo = 1, 153209467b48Spatrick Constraints = "$src1 = $dst", Defs = [EFLAGS] in { 153309467b48Spatricklet isCommutable = 1 in { 153409467b48Spatrickdef ADD8rr_DB : I<0, Pseudo, (outs GR8:$dst), (ins GR8:$src1, GR8:$src2), 153509467b48Spatrick "", // orb/addb REG, REG 153609467b48Spatrick [(set GR8:$dst, (or_is_add GR8:$src1, GR8:$src2))]>; 153709467b48Spatrickdef ADD16rr_DB : I<0, Pseudo, (outs GR16:$dst), (ins GR16:$src1, GR16:$src2), 153809467b48Spatrick "", // orw/addw REG, REG 153909467b48Spatrick [(set GR16:$dst, (or_is_add GR16:$src1, GR16:$src2))]>; 154009467b48Spatrickdef ADD32rr_DB : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$src1, GR32:$src2), 154109467b48Spatrick "", // orl/addl REG, REG 154209467b48Spatrick [(set GR32:$dst, (or_is_add GR32:$src1, GR32:$src2))]>; 154309467b48Spatrickdef ADD64rr_DB : I<0, Pseudo, (outs GR64:$dst), (ins GR64:$src1, GR64:$src2), 154409467b48Spatrick "", // orq/addq REG, REG 154509467b48Spatrick [(set GR64:$dst, (or_is_add GR64:$src1, GR64:$src2))]>; 154609467b48Spatrick} // isCommutable 154709467b48Spatrick 154809467b48Spatrick// NOTE: These are order specific, we want the ri8 forms to be listed 154909467b48Spatrick// first so that they are slightly preferred to the ri forms. 155009467b48Spatrick 155109467b48Spatrickdef ADD8ri_DB : I<0, Pseudo, 155209467b48Spatrick (outs GR8:$dst), (ins GR8:$src1, i8imm:$src2), 155309467b48Spatrick "", // orb/addb REG, imm8 155409467b48Spatrick [(set GR8:$dst, (or_is_add GR8:$src1, imm:$src2))]>; 155509467b48Spatrickdef ADD16ri8_DB : I<0, Pseudo, 155609467b48Spatrick (outs GR16:$dst), (ins GR16:$src1, i16i8imm:$src2), 155709467b48Spatrick "", // orw/addw REG, imm8 155809467b48Spatrick [(set GR16:$dst,(or_is_add GR16:$src1,i16immSExt8:$src2))]>; 155909467b48Spatrickdef ADD16ri_DB : I<0, Pseudo, (outs GR16:$dst), (ins GR16:$src1, i16imm:$src2), 156009467b48Spatrick "", // orw/addw REG, imm 156109467b48Spatrick [(set GR16:$dst, (or_is_add GR16:$src1, imm:$src2))]>; 156209467b48Spatrick 156309467b48Spatrickdef ADD32ri8_DB : I<0, Pseudo, 156409467b48Spatrick (outs GR32:$dst), (ins GR32:$src1, i32i8imm:$src2), 156509467b48Spatrick "", // orl/addl REG, imm8 156609467b48Spatrick [(set GR32:$dst,(or_is_add GR32:$src1,i32immSExt8:$src2))]>; 156709467b48Spatrickdef ADD32ri_DB : I<0, Pseudo, (outs GR32:$dst), (ins GR32:$src1, i32imm:$src2), 156809467b48Spatrick "", // orl/addl REG, imm 156909467b48Spatrick [(set GR32:$dst, (or_is_add GR32:$src1, imm:$src2))]>; 157009467b48Spatrick 157109467b48Spatrick 157209467b48Spatrickdef ADD64ri8_DB : I<0, Pseudo, 157309467b48Spatrick (outs GR64:$dst), (ins GR64:$src1, i64i8imm:$src2), 157409467b48Spatrick "", // orq/addq REG, imm8 157509467b48Spatrick [(set GR64:$dst, (or_is_add GR64:$src1, 157609467b48Spatrick i64immSExt8:$src2))]>; 157709467b48Spatrickdef ADD64ri32_DB : I<0, Pseudo, 157809467b48Spatrick (outs GR64:$dst), (ins GR64:$src1, i64i32imm:$src2), 157909467b48Spatrick "", // orq/addq REG, imm 158009467b48Spatrick [(set GR64:$dst, (or_is_add GR64:$src1, 158109467b48Spatrick i64immSExt32:$src2))]>; 158209467b48Spatrick} 158309467b48Spatrick} // AddedComplexity, SchedRW 158409467b48Spatrick 158509467b48Spatrick//===----------------------------------------------------------------------===// 1586*a96b3639Srobert// Pattern match XOR as ADD 158709467b48Spatrick//===----------------------------------------------------------------------===// 158809467b48Spatrick 1589*a96b3639Srobert// Prefer to pattern match XOR with min_signed_value as ADD at isel time. 1590*a96b3639Srobert// ADD can be 3-addressified into an LEA instruction to avoid copies. 159109467b48Spatricklet AddedComplexity = 5 in { 1592*a96b3639Srobertdef : Pat<(xor GR8:$src1, -128), 1593*a96b3639Srobert (ADD8ri GR8:$src1, -128)>; 1594*a96b3639Srobertdef : Pat<(xor GR16:$src1, -32768), 1595*a96b3639Srobert (ADD16ri GR16:$src1, -32768)>; 1596*a96b3639Srobertdef : Pat<(xor GR32:$src1, -2147483648), 1597*a96b3639Srobert (ADD32ri GR32:$src1, -2147483648)>; 159809467b48Spatrick} 159909467b48Spatrick 160009467b48Spatrick//===----------------------------------------------------------------------===// 160109467b48Spatrick// Some peepholes 160209467b48Spatrick//===----------------------------------------------------------------------===// 160309467b48Spatrick 160409467b48Spatrick// Odd encoding trick: -128 fits into an 8-bit immediate field while 160509467b48Spatrick// +128 doesn't, so in this special case use a sub instead of an add. 160609467b48Spatrickdef : Pat<(add GR16:$src1, 128), 160709467b48Spatrick (SUB16ri8 GR16:$src1, -128)>; 160809467b48Spatrickdef : Pat<(store (add (loadi16 addr:$dst), 128), addr:$dst), 160909467b48Spatrick (SUB16mi8 addr:$dst, -128)>; 161009467b48Spatrick 161109467b48Spatrickdef : Pat<(add GR32:$src1, 128), 161209467b48Spatrick (SUB32ri8 GR32:$src1, -128)>; 161309467b48Spatrickdef : Pat<(store (add (loadi32 addr:$dst), 128), addr:$dst), 161409467b48Spatrick (SUB32mi8 addr:$dst, -128)>; 161509467b48Spatrick 161609467b48Spatrickdef : Pat<(add GR64:$src1, 128), 161709467b48Spatrick (SUB64ri8 GR64:$src1, -128)>; 161809467b48Spatrickdef : Pat<(store (add (loadi64 addr:$dst), 128), addr:$dst), 161909467b48Spatrick (SUB64mi8 addr:$dst, -128)>; 162009467b48Spatrick 162109467b48Spatrickdef : Pat<(X86add_flag_nocf GR16:$src1, 128), 162209467b48Spatrick (SUB16ri8 GR16:$src1, -128)>; 162309467b48Spatrickdef : Pat<(X86add_flag_nocf GR32:$src1, 128), 162409467b48Spatrick (SUB32ri8 GR32:$src1, -128)>; 162509467b48Spatrickdef : Pat<(X86add_flag_nocf GR64:$src1, 128), 162609467b48Spatrick (SUB64ri8 GR64:$src1, -128)>; 162709467b48Spatrick 162809467b48Spatrick// The same trick applies for 32-bit immediate fields in 64-bit 162909467b48Spatrick// instructions. 163009467b48Spatrickdef : Pat<(add GR64:$src1, 0x0000000080000000), 163109467b48Spatrick (SUB64ri32 GR64:$src1, 0xffffffff80000000)>; 163209467b48Spatrickdef : Pat<(store (add (loadi64 addr:$dst), 0x0000000080000000), addr:$dst), 163309467b48Spatrick (SUB64mi32 addr:$dst, 0xffffffff80000000)>; 163409467b48Spatrick 163509467b48Spatrickdef : Pat<(X86add_flag_nocf GR64:$src1, 0x0000000080000000), 163609467b48Spatrick (SUB64ri32 GR64:$src1, 0xffffffff80000000)>; 163709467b48Spatrick 163809467b48Spatrick// To avoid needing to materialize an immediate in a register, use a 32-bit and 163909467b48Spatrick// with implicit zero-extension instead of a 64-bit and if the immediate has at 164009467b48Spatrick// least 32 bits of leading zeros. If in addition the last 32 bits can be 164109467b48Spatrick// represented with a sign extension of a 8 bit constant, use that. 164209467b48Spatrick// This can also reduce instruction size by eliminating the need for the REX 164309467b48Spatrick// prefix. 164409467b48Spatrick 164509467b48Spatrick// AddedComplexity is needed to give priority over i64immSExt8 and i64immSExt32. 164609467b48Spatricklet AddedComplexity = 1 in { 164709467b48Spatrickdef : Pat<(and GR64:$src, i64immZExt32SExt8:$imm), 164809467b48Spatrick (SUBREG_TO_REG 164909467b48Spatrick (i64 0), 165009467b48Spatrick (AND32ri8 165109467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_32bit), 165209467b48Spatrick (i32 (GetLo32XForm imm:$imm))), 165309467b48Spatrick sub_32bit)>; 165409467b48Spatrick 165509467b48Spatrickdef : Pat<(and GR64:$src, i64immZExt32:$imm), 165609467b48Spatrick (SUBREG_TO_REG 165709467b48Spatrick (i64 0), 165809467b48Spatrick (AND32ri 165909467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_32bit), 166009467b48Spatrick (i32 (GetLo32XForm imm:$imm))), 166109467b48Spatrick sub_32bit)>; 166209467b48Spatrick} // AddedComplexity = 1 166309467b48Spatrick 166409467b48Spatrick 166509467b48Spatrick// AddedComplexity is needed due to the increased complexity on the 166609467b48Spatrick// i64immZExt32SExt8 and i64immZExt32 patterns above. Applying this to all 166709467b48Spatrick// the MOVZX patterns keeps thems together in DAGIsel tables. 166809467b48Spatricklet AddedComplexity = 1 in { 166909467b48Spatrick// r & (2^16-1) ==> movz 167009467b48Spatrickdef : Pat<(and GR32:$src1, 0xffff), 167109467b48Spatrick (MOVZX32rr16 (EXTRACT_SUBREG GR32:$src1, sub_16bit))>; 167209467b48Spatrick// r & (2^8-1) ==> movz 167309467b48Spatrickdef : Pat<(and GR32:$src1, 0xff), 167409467b48Spatrick (MOVZX32rr8 (EXTRACT_SUBREG GR32:$src1, sub_8bit))>; 167509467b48Spatrick// r & (2^8-1) ==> movz 167609467b48Spatrickdef : Pat<(and GR16:$src1, 0xff), 167709467b48Spatrick (EXTRACT_SUBREG (MOVZX32rr8 (EXTRACT_SUBREG GR16:$src1, sub_8bit)), 167809467b48Spatrick sub_16bit)>; 167909467b48Spatrick 168009467b48Spatrick// r & (2^32-1) ==> movz 168109467b48Spatrickdef : Pat<(and GR64:$src, 0x00000000FFFFFFFF), 168209467b48Spatrick (SUBREG_TO_REG (i64 0), 168309467b48Spatrick (MOV32rr (EXTRACT_SUBREG GR64:$src, sub_32bit)), 168409467b48Spatrick sub_32bit)>; 168509467b48Spatrick// r & (2^16-1) ==> movz 168609467b48Spatrickdef : Pat<(and GR64:$src, 0xffff), 168709467b48Spatrick (SUBREG_TO_REG (i64 0), 168809467b48Spatrick (MOVZX32rr16 (i16 (EXTRACT_SUBREG GR64:$src, sub_16bit))), 168909467b48Spatrick sub_32bit)>; 169009467b48Spatrick// r & (2^8-1) ==> movz 169109467b48Spatrickdef : Pat<(and GR64:$src, 0xff), 169209467b48Spatrick (SUBREG_TO_REG (i64 0), 169309467b48Spatrick (MOVZX32rr8 (i8 (EXTRACT_SUBREG GR64:$src, sub_8bit))), 169409467b48Spatrick sub_32bit)>; 169509467b48Spatrick} // AddedComplexity = 1 169609467b48Spatrick 169709467b48Spatrick 169809467b48Spatrick// Try to use BTS/BTR/BTC for single bit operations on the upper 32-bits. 169909467b48Spatrick 170009467b48Spatrickdef BTRXForm : SDNodeXForm<imm, [{ 170109467b48Spatrick // Transformation function: Find the lowest 0. 170209467b48Spatrick return getI64Imm((uint8_t)N->getAPIntValue().countTrailingOnes(), SDLoc(N)); 170309467b48Spatrick}]>; 170409467b48Spatrick 170509467b48Spatrickdef BTCBTSXForm : SDNodeXForm<imm, [{ 170609467b48Spatrick // Transformation function: Find the lowest 1. 170709467b48Spatrick return getI64Imm((uint8_t)N->getAPIntValue().countTrailingZeros(), SDLoc(N)); 170809467b48Spatrick}]>; 170909467b48Spatrick 171009467b48Spatrickdef BTRMask64 : ImmLeaf<i64, [{ 171109467b48Spatrick return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm); 171209467b48Spatrick}]>; 171309467b48Spatrick 171409467b48Spatrickdef BTCBTSMask64 : ImmLeaf<i64, [{ 171509467b48Spatrick return !isInt<32>(Imm) && isPowerOf2_64(Imm); 171609467b48Spatrick}]>; 171709467b48Spatrick 171809467b48Spatrick// For now only do this for optsize. 171909467b48Spatricklet AddedComplexity = 1, Predicates=[OptForSize] in { 172009467b48Spatrick def : Pat<(and GR64:$src1, BTRMask64:$mask), 172109467b48Spatrick (BTR64ri8 GR64:$src1, (BTRXForm imm:$mask))>; 172209467b48Spatrick def : Pat<(or GR64:$src1, BTCBTSMask64:$mask), 172309467b48Spatrick (BTS64ri8 GR64:$src1, (BTCBTSXForm imm:$mask))>; 172409467b48Spatrick def : Pat<(xor GR64:$src1, BTCBTSMask64:$mask), 172509467b48Spatrick (BTC64ri8 GR64:$src1, (BTCBTSXForm imm:$mask))>; 172609467b48Spatrick} 172709467b48Spatrick 172809467b48Spatrick 172909467b48Spatrick// sext_inreg patterns 173009467b48Spatrickdef : Pat<(sext_inreg GR32:$src, i16), 173109467b48Spatrick (MOVSX32rr16 (EXTRACT_SUBREG GR32:$src, sub_16bit))>; 173209467b48Spatrickdef : Pat<(sext_inreg GR32:$src, i8), 173309467b48Spatrick (MOVSX32rr8 (EXTRACT_SUBREG GR32:$src, sub_8bit))>; 173409467b48Spatrick 173509467b48Spatrickdef : Pat<(sext_inreg GR16:$src, i8), 173609467b48Spatrick (EXTRACT_SUBREG (MOVSX32rr8 (EXTRACT_SUBREG GR16:$src, sub_8bit)), 173709467b48Spatrick sub_16bit)>; 173809467b48Spatrick 173909467b48Spatrickdef : Pat<(sext_inreg GR64:$src, i32), 174009467b48Spatrick (MOVSX64rr32 (EXTRACT_SUBREG GR64:$src, sub_32bit))>; 174109467b48Spatrickdef : Pat<(sext_inreg GR64:$src, i16), 174209467b48Spatrick (MOVSX64rr16 (EXTRACT_SUBREG GR64:$src, sub_16bit))>; 174309467b48Spatrickdef : Pat<(sext_inreg GR64:$src, i8), 174409467b48Spatrick (MOVSX64rr8 (EXTRACT_SUBREG GR64:$src, sub_8bit))>; 174509467b48Spatrick 174609467b48Spatrick// sext, sext_load, zext, zext_load 174709467b48Spatrickdef: Pat<(i16 (sext GR8:$src)), 174809467b48Spatrick (EXTRACT_SUBREG (MOVSX32rr8 GR8:$src), sub_16bit)>; 174909467b48Spatrickdef: Pat<(sextloadi16i8 addr:$src), 175009467b48Spatrick (EXTRACT_SUBREG (MOVSX32rm8 addr:$src), sub_16bit)>; 175109467b48Spatrickdef: Pat<(i16 (zext GR8:$src)), 175209467b48Spatrick (EXTRACT_SUBREG (MOVZX32rr8 GR8:$src), sub_16bit)>; 175309467b48Spatrickdef: Pat<(zextloadi16i8 addr:$src), 175409467b48Spatrick (EXTRACT_SUBREG (MOVZX32rm8 addr:$src), sub_16bit)>; 175509467b48Spatrick 175609467b48Spatrick// trunc patterns 175709467b48Spatrickdef : Pat<(i16 (trunc GR32:$src)), 175809467b48Spatrick (EXTRACT_SUBREG GR32:$src, sub_16bit)>; 175909467b48Spatrickdef : Pat<(i8 (trunc GR32:$src)), 176009467b48Spatrick (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS GR32:$src, GR32_ABCD)), 176109467b48Spatrick sub_8bit)>, 176209467b48Spatrick Requires<[Not64BitMode]>; 176309467b48Spatrickdef : Pat<(i8 (trunc GR16:$src)), 176409467b48Spatrick (EXTRACT_SUBREG (i16 (COPY_TO_REGCLASS GR16:$src, GR16_ABCD)), 176509467b48Spatrick sub_8bit)>, 176609467b48Spatrick Requires<[Not64BitMode]>; 176709467b48Spatrickdef : Pat<(i32 (trunc GR64:$src)), 176809467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_32bit)>; 176909467b48Spatrickdef : Pat<(i16 (trunc GR64:$src)), 177009467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_16bit)>; 177109467b48Spatrickdef : Pat<(i8 (trunc GR64:$src)), 177209467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_8bit)>; 177309467b48Spatrickdef : Pat<(i8 (trunc GR32:$src)), 177409467b48Spatrick (EXTRACT_SUBREG GR32:$src, sub_8bit)>, 177509467b48Spatrick Requires<[In64BitMode]>; 177609467b48Spatrickdef : Pat<(i8 (trunc GR16:$src)), 177709467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit)>, 177809467b48Spatrick Requires<[In64BitMode]>; 177909467b48Spatrick 178009467b48Spatrickdef immff00_ffff : ImmLeaf<i32, [{ 178109467b48Spatrick return Imm >= 0xff00 && Imm <= 0xffff; 178209467b48Spatrick}]>; 178309467b48Spatrick 178409467b48Spatrick// h-register tricks 178509467b48Spatrickdef : Pat<(i8 (trunc (srl_su GR16:$src, (i8 8)))), 178609467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit_hi)>, 178709467b48Spatrick Requires<[Not64BitMode]>; 178809467b48Spatrickdef : Pat<(i8 (trunc (srl_su (i32 (anyext GR16:$src)), (i8 8)))), 178909467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit_hi)>, 179009467b48Spatrick Requires<[Not64BitMode]>; 179109467b48Spatrickdef : Pat<(i8 (trunc (srl_su GR32:$src, (i8 8)))), 179209467b48Spatrick (EXTRACT_SUBREG GR32:$src, sub_8bit_hi)>, 179309467b48Spatrick Requires<[Not64BitMode]>; 179409467b48Spatrickdef : Pat<(srl GR16:$src, (i8 8)), 179509467b48Spatrick (EXTRACT_SUBREG 179609467b48Spatrick (MOVZX32rr8_NOREX (EXTRACT_SUBREG GR16:$src, sub_8bit_hi)), 179709467b48Spatrick sub_16bit)>; 179809467b48Spatrickdef : Pat<(i32 (zext (srl_su GR16:$src, (i8 8)))), 179909467b48Spatrick (MOVZX32rr8_NOREX (EXTRACT_SUBREG GR16:$src, sub_8bit_hi))>; 180009467b48Spatrickdef : Pat<(i32 (anyext (srl_su GR16:$src, (i8 8)))), 180109467b48Spatrick (MOVZX32rr8_NOREX (EXTRACT_SUBREG GR16:$src, sub_8bit_hi))>; 180209467b48Spatrickdef : Pat<(and (srl_su GR32:$src, (i8 8)), (i32 255)), 180309467b48Spatrick (MOVZX32rr8_NOREX (EXTRACT_SUBREG GR32:$src, sub_8bit_hi))>; 180409467b48Spatrickdef : Pat<(srl (and_su GR32:$src, immff00_ffff), (i8 8)), 180509467b48Spatrick (MOVZX32rr8_NOREX (EXTRACT_SUBREG GR32:$src, sub_8bit_hi))>; 180609467b48Spatrick 180709467b48Spatrick// h-register tricks. 180809467b48Spatrick// For now, be conservative on x86-64 and use an h-register extract only if the 180909467b48Spatrick// value is immediately zero-extended or stored, which are somewhat common 181009467b48Spatrick// cases. This uses a bunch of code to prevent a register requiring a REX prefix 181109467b48Spatrick// from being allocated in the same instruction as the h register, as there's 181209467b48Spatrick// currently no way to describe this requirement to the register allocator. 181309467b48Spatrick 181409467b48Spatrick// h-register extract and zero-extend. 181509467b48Spatrickdef : Pat<(and (srl_su GR64:$src, (i8 8)), (i64 255)), 181609467b48Spatrick (SUBREG_TO_REG 181709467b48Spatrick (i64 0), 181809467b48Spatrick (MOVZX32rr8_NOREX 181909467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_8bit_hi)), 182009467b48Spatrick sub_32bit)>; 182109467b48Spatrickdef : Pat<(i64 (zext (srl_su GR16:$src, (i8 8)))), 182209467b48Spatrick (SUBREG_TO_REG 182309467b48Spatrick (i64 0), 182409467b48Spatrick (MOVZX32rr8_NOREX 182509467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit_hi)), 182609467b48Spatrick sub_32bit)>; 182709467b48Spatrickdef : Pat<(i64 (anyext (srl_su GR16:$src, (i8 8)))), 182809467b48Spatrick (SUBREG_TO_REG 182909467b48Spatrick (i64 0), 183009467b48Spatrick (MOVZX32rr8_NOREX 183109467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit_hi)), 183209467b48Spatrick sub_32bit)>; 183309467b48Spatrick 183409467b48Spatrick// h-register extract and store. 183509467b48Spatrickdef : Pat<(store (i8 (trunc_su (srl_su GR64:$src, (i8 8)))), addr:$dst), 183609467b48Spatrick (MOV8mr_NOREX 183709467b48Spatrick addr:$dst, 183809467b48Spatrick (EXTRACT_SUBREG GR64:$src, sub_8bit_hi))>; 183909467b48Spatrickdef : Pat<(store (i8 (trunc_su (srl_su GR32:$src, (i8 8)))), addr:$dst), 184009467b48Spatrick (MOV8mr_NOREX 184109467b48Spatrick addr:$dst, 184209467b48Spatrick (EXTRACT_SUBREG GR32:$src, sub_8bit_hi))>, 184309467b48Spatrick Requires<[In64BitMode]>; 184409467b48Spatrickdef : Pat<(store (i8 (trunc_su (srl_su GR16:$src, (i8 8)))), addr:$dst), 184509467b48Spatrick (MOV8mr_NOREX 184609467b48Spatrick addr:$dst, 184709467b48Spatrick (EXTRACT_SUBREG GR16:$src, sub_8bit_hi))>, 184809467b48Spatrick Requires<[In64BitMode]>; 184909467b48Spatrick 1850a0747c9fSpatrick// Special pattern to catch the last step of __builtin_parity handling. Our 1851a0747c9fSpatrick// goal is to use an xor of an h-register with the corresponding l-register. 1852a0747c9fSpatrick// The above patterns would handle this on non 64-bit targets, but for 64-bit 1853a0747c9fSpatrick// we need to be more careful. We're using a NOREX instruction here in case 1854a0747c9fSpatrick// register allocation fails to keep the two registers together. So we need to 1855a0747c9fSpatrick// make sure we can't accidentally mix R8-R15 with an h-register. 1856a0747c9fSpatrickdef : Pat<(X86xor_flag (i8 (trunc GR32:$src)), 1857a0747c9fSpatrick (i8 (trunc (srl_su GR32:$src, (i8 8))))), 1858a0747c9fSpatrick (XOR8rr_NOREX (EXTRACT_SUBREG GR32:$src, sub_8bit), 1859a0747c9fSpatrick (EXTRACT_SUBREG GR32:$src, sub_8bit_hi))>; 186009467b48Spatrick 186109467b48Spatrick// (shl x, 1) ==> (add x, x) 186209467b48Spatrick// Note that if x is undef (immediate or otherwise), we could theoretically 186309467b48Spatrick// end up with the two uses of x getting different values, producing a result 186409467b48Spatrick// where the least significant bit is not 0. However, the probability of this 186509467b48Spatrick// happening is considered low enough that this is officially not a 186609467b48Spatrick// "real problem". 186709467b48Spatrickdef : Pat<(shl GR8 :$src1, (i8 1)), (ADD8rr GR8 :$src1, GR8 :$src1)>; 186809467b48Spatrickdef : Pat<(shl GR16:$src1, (i8 1)), (ADD16rr GR16:$src1, GR16:$src1)>; 186909467b48Spatrickdef : Pat<(shl GR32:$src1, (i8 1)), (ADD32rr GR32:$src1, GR32:$src1)>; 187009467b48Spatrickdef : Pat<(shl GR64:$src1, (i8 1)), (ADD64rr GR64:$src1, GR64:$src1)>; 187109467b48Spatrick 187209467b48Spatrickdef shiftMask8 : PatFrag<(ops node:$lhs), (and node:$lhs, imm), [{ 187309467b48Spatrick return isUnneededShiftMask(N, 3); 187409467b48Spatrick}]>; 187509467b48Spatrick 187609467b48Spatrickdef shiftMask16 : PatFrag<(ops node:$lhs), (and node:$lhs, imm), [{ 187709467b48Spatrick return isUnneededShiftMask(N, 4); 187809467b48Spatrick}]>; 187909467b48Spatrick 188009467b48Spatrickdef shiftMask32 : PatFrag<(ops node:$lhs), (and node:$lhs, imm), [{ 188109467b48Spatrick return isUnneededShiftMask(N, 5); 188209467b48Spatrick}]>; 188309467b48Spatrick 188409467b48Spatrickdef shiftMask64 : PatFrag<(ops node:$lhs), (and node:$lhs, imm), [{ 188509467b48Spatrick return isUnneededShiftMask(N, 6); 188609467b48Spatrick}]>; 188709467b48Spatrick 188809467b48Spatrick 188909467b48Spatrick// Shift amount is implicitly masked. 189009467b48Spatrickmulticlass MaskedShiftAmountPats<SDNode frag, string name> { 189109467b48Spatrick // (shift x (and y, 31)) ==> (shift x, y) 189209467b48Spatrick def : Pat<(frag GR8:$src1, (shiftMask32 CL)), 189309467b48Spatrick (!cast<Instruction>(name # "8rCL") GR8:$src1)>; 189409467b48Spatrick def : Pat<(frag GR16:$src1, (shiftMask32 CL)), 189509467b48Spatrick (!cast<Instruction>(name # "16rCL") GR16:$src1)>; 189609467b48Spatrick def : Pat<(frag GR32:$src1, (shiftMask32 CL)), 189709467b48Spatrick (!cast<Instruction>(name # "32rCL") GR32:$src1)>; 189809467b48Spatrick def : Pat<(store (frag (loadi8 addr:$dst), (shiftMask32 CL)), addr:$dst), 189909467b48Spatrick (!cast<Instruction>(name # "8mCL") addr:$dst)>; 190009467b48Spatrick def : Pat<(store (frag (loadi16 addr:$dst), (shiftMask32 CL)), addr:$dst), 190109467b48Spatrick (!cast<Instruction>(name # "16mCL") addr:$dst)>; 190209467b48Spatrick def : Pat<(store (frag (loadi32 addr:$dst), (shiftMask32 CL)), addr:$dst), 190309467b48Spatrick (!cast<Instruction>(name # "32mCL") addr:$dst)>; 190409467b48Spatrick 190509467b48Spatrick // (shift x (and y, 63)) ==> (shift x, y) 190609467b48Spatrick def : Pat<(frag GR64:$src1, (shiftMask64 CL)), 190709467b48Spatrick (!cast<Instruction>(name # "64rCL") GR64:$src1)>; 190809467b48Spatrick def : Pat<(store (frag (loadi64 addr:$dst), (shiftMask64 CL)), addr:$dst), 190909467b48Spatrick (!cast<Instruction>(name # "64mCL") addr:$dst)>; 191009467b48Spatrick} 191109467b48Spatrick 191209467b48Spatrickdefm : MaskedShiftAmountPats<shl, "SHL">; 191309467b48Spatrickdefm : MaskedShiftAmountPats<srl, "SHR">; 191409467b48Spatrickdefm : MaskedShiftAmountPats<sra, "SAR">; 191509467b48Spatrick 191609467b48Spatrick// ROL/ROR instructions allow a stronger mask optimization than shift for 8- and 191709467b48Spatrick// 16-bit. We can remove a mask of any (bitwidth - 1) on the rotation amount 191809467b48Spatrick// because over-rotating produces the same result. This is noted in the Intel 191909467b48Spatrick// docs with: "tempCOUNT <- (COUNT & COUNTMASK) MOD SIZE". Masking the rotation 192009467b48Spatrick// amount could affect EFLAGS results, but that does not matter because we are 192109467b48Spatrick// not tracking flags for these nodes. 192209467b48Spatrickmulticlass MaskedRotateAmountPats<SDNode frag, string name> { 192309467b48Spatrick // (rot x (and y, BitWidth - 1)) ==> (rot x, y) 192409467b48Spatrick def : Pat<(frag GR8:$src1, (shiftMask8 CL)), 192509467b48Spatrick (!cast<Instruction>(name # "8rCL") GR8:$src1)>; 192609467b48Spatrick def : Pat<(frag GR16:$src1, (shiftMask16 CL)), 192709467b48Spatrick (!cast<Instruction>(name # "16rCL") GR16:$src1)>; 192809467b48Spatrick def : Pat<(frag GR32:$src1, (shiftMask32 CL)), 192909467b48Spatrick (!cast<Instruction>(name # "32rCL") GR32:$src1)>; 193009467b48Spatrick def : Pat<(store (frag (loadi8 addr:$dst), (shiftMask8 CL)), addr:$dst), 193109467b48Spatrick (!cast<Instruction>(name # "8mCL") addr:$dst)>; 193209467b48Spatrick def : Pat<(store (frag (loadi16 addr:$dst), (shiftMask16 CL)), addr:$dst), 193309467b48Spatrick (!cast<Instruction>(name # "16mCL") addr:$dst)>; 193409467b48Spatrick def : Pat<(store (frag (loadi32 addr:$dst), (shiftMask32 CL)), addr:$dst), 193509467b48Spatrick (!cast<Instruction>(name # "32mCL") addr:$dst)>; 193609467b48Spatrick 193709467b48Spatrick // (rot x (and y, 63)) ==> (rot x, y) 193809467b48Spatrick def : Pat<(frag GR64:$src1, (shiftMask64 CL)), 193909467b48Spatrick (!cast<Instruction>(name # "64rCL") GR64:$src1)>; 194009467b48Spatrick def : Pat<(store (frag (loadi64 addr:$dst), (shiftMask64 CL)), addr:$dst), 194109467b48Spatrick (!cast<Instruction>(name # "64mCL") addr:$dst)>; 194209467b48Spatrick} 194309467b48Spatrick 194409467b48Spatrick 194509467b48Spatrickdefm : MaskedRotateAmountPats<rotl, "ROL">; 194609467b48Spatrickdefm : MaskedRotateAmountPats<rotr, "ROR">; 194709467b48Spatrick 1948097a140dSpatrick// Double "funnel" shift amount is implicitly masked. 1949097a140dSpatrick// (fshl/fshr x (and y, 31)) ==> (fshl/fshr x, y) (NOTE: modulo32) 1950097a140dSpatrickdef : Pat<(X86fshl GR16:$src1, GR16:$src2, (shiftMask32 CL)), 1951097a140dSpatrick (SHLD16rrCL GR16:$src1, GR16:$src2)>; 1952097a140dSpatrickdef : Pat<(X86fshr GR16:$src2, GR16:$src1, (shiftMask32 CL)), 1953097a140dSpatrick (SHRD16rrCL GR16:$src1, GR16:$src2)>; 195409467b48Spatrick 1955097a140dSpatrick// (fshl/fshr x (and y, 31)) ==> (fshl/fshr x, y) 1956097a140dSpatrickdef : Pat<(fshl GR32:$src1, GR32:$src2, (shiftMask32 CL)), 1957097a140dSpatrick (SHLD32rrCL GR32:$src1, GR32:$src2)>; 1958097a140dSpatrickdef : Pat<(fshr GR32:$src2, GR32:$src1, (shiftMask32 CL)), 1959097a140dSpatrick (SHRD32rrCL GR32:$src1, GR32:$src2)>; 196009467b48Spatrick 1961097a140dSpatrick// (fshl/fshr x (and y, 63)) ==> (fshl/fshr x, y) 1962097a140dSpatrickdef : Pat<(fshl GR64:$src1, GR64:$src2, (shiftMask64 CL)), 1963097a140dSpatrick (SHLD64rrCL GR64:$src1, GR64:$src2)>; 1964097a140dSpatrickdef : Pat<(fshr GR64:$src2, GR64:$src1, (shiftMask64 CL)), 1965097a140dSpatrick (SHRD64rrCL GR64:$src1, GR64:$src2)>; 196609467b48Spatrick 196709467b48Spatricklet Predicates = [HasBMI2] in { 196809467b48Spatrick let AddedComplexity = 1 in { 196909467b48Spatrick def : Pat<(sra GR32:$src1, (shiftMask32 GR8:$src2)), 197009467b48Spatrick (SARX32rr GR32:$src1, 197109467b48Spatrick (INSERT_SUBREG 197209467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 197309467b48Spatrick def : Pat<(sra GR64:$src1, (shiftMask64 GR8:$src2)), 197409467b48Spatrick (SARX64rr GR64:$src1, 197509467b48Spatrick (INSERT_SUBREG 197609467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 197709467b48Spatrick 197809467b48Spatrick def : Pat<(srl GR32:$src1, (shiftMask32 GR8:$src2)), 197909467b48Spatrick (SHRX32rr GR32:$src1, 198009467b48Spatrick (INSERT_SUBREG 198109467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 198209467b48Spatrick def : Pat<(srl GR64:$src1, (shiftMask64 GR8:$src2)), 198309467b48Spatrick (SHRX64rr GR64:$src1, 198409467b48Spatrick (INSERT_SUBREG 198509467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 198609467b48Spatrick 198709467b48Spatrick def : Pat<(shl GR32:$src1, (shiftMask32 GR8:$src2)), 198809467b48Spatrick (SHLX32rr GR32:$src1, 198909467b48Spatrick (INSERT_SUBREG 199009467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 199109467b48Spatrick def : Pat<(shl GR64:$src1, (shiftMask64 GR8:$src2)), 199209467b48Spatrick (SHLX64rr GR64:$src1, 199309467b48Spatrick (INSERT_SUBREG 199409467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 199509467b48Spatrick } 199609467b48Spatrick 199709467b48Spatrick def : Pat<(sra (loadi32 addr:$src1), (shiftMask32 GR8:$src2)), 199809467b48Spatrick (SARX32rm addr:$src1, 199909467b48Spatrick (INSERT_SUBREG 200009467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 200109467b48Spatrick def : Pat<(sra (loadi64 addr:$src1), (shiftMask64 GR8:$src2)), 200209467b48Spatrick (SARX64rm addr:$src1, 200309467b48Spatrick (INSERT_SUBREG 200409467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 200509467b48Spatrick 200609467b48Spatrick def : Pat<(srl (loadi32 addr:$src1), (shiftMask32 GR8:$src2)), 200709467b48Spatrick (SHRX32rm addr:$src1, 200809467b48Spatrick (INSERT_SUBREG 200909467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 201009467b48Spatrick def : Pat<(srl (loadi64 addr:$src1), (shiftMask64 GR8:$src2)), 201109467b48Spatrick (SHRX64rm addr:$src1, 201209467b48Spatrick (INSERT_SUBREG 201309467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 201409467b48Spatrick 201509467b48Spatrick def : Pat<(shl (loadi32 addr:$src1), (shiftMask32 GR8:$src2)), 201609467b48Spatrick (SHLX32rm addr:$src1, 201709467b48Spatrick (INSERT_SUBREG 201809467b48Spatrick (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 201909467b48Spatrick def : Pat<(shl (loadi64 addr:$src1), (shiftMask64 GR8:$src2)), 202009467b48Spatrick (SHLX64rm addr:$src1, 202109467b48Spatrick (INSERT_SUBREG 202209467b48Spatrick (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 202309467b48Spatrick} 202409467b48Spatrick 202509467b48Spatrick// Use BTR/BTS/BTC for clearing/setting/toggling a bit in a variable location. 202609467b48Spatrickmulticlass one_bit_patterns<RegisterClass RC, ValueType VT, Instruction BTR, 202709467b48Spatrick Instruction BTS, Instruction BTC, 202809467b48Spatrick PatFrag ShiftMask> { 202909467b48Spatrick def : Pat<(and RC:$src1, (rotl -2, GR8:$src2)), 203009467b48Spatrick (BTR RC:$src1, 203109467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 203209467b48Spatrick def : Pat<(or RC:$src1, (shl 1, GR8:$src2)), 203309467b48Spatrick (BTS RC:$src1, 203409467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 203509467b48Spatrick def : Pat<(xor RC:$src1, (shl 1, GR8:$src2)), 203609467b48Spatrick (BTC RC:$src1, 203709467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 203809467b48Spatrick 203909467b48Spatrick // Similar to above, but removing unneeded masking of the shift amount. 204009467b48Spatrick def : Pat<(and RC:$src1, (rotl -2, (ShiftMask GR8:$src2))), 204109467b48Spatrick (BTR RC:$src1, 204209467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 204309467b48Spatrick def : Pat<(or RC:$src1, (shl 1, (ShiftMask GR8:$src2))), 204409467b48Spatrick (BTS RC:$src1, 204509467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 204609467b48Spatrick def : Pat<(xor RC:$src1, (shl 1, (ShiftMask GR8:$src2))), 204709467b48Spatrick (BTC RC:$src1, 204809467b48Spatrick (INSERT_SUBREG (VT (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>; 204909467b48Spatrick} 205009467b48Spatrick 205109467b48Spatrickdefm : one_bit_patterns<GR16, i16, BTR16rr, BTS16rr, BTC16rr, shiftMask16>; 205209467b48Spatrickdefm : one_bit_patterns<GR32, i32, BTR32rr, BTS32rr, BTC32rr, shiftMask32>; 205309467b48Spatrickdefm : one_bit_patterns<GR64, i64, BTR64rr, BTS64rr, BTC64rr, shiftMask64>; 205409467b48Spatrick 205509467b48Spatrick//===----------------------------------------------------------------------===// 205609467b48Spatrick// EFLAGS-defining Patterns 205709467b48Spatrick//===----------------------------------------------------------------------===// 205809467b48Spatrick 205909467b48Spatrick// add reg, reg 206009467b48Spatrickdef : Pat<(add GR8 :$src1, GR8 :$src2), (ADD8rr GR8 :$src1, GR8 :$src2)>; 206109467b48Spatrickdef : Pat<(add GR16:$src1, GR16:$src2), (ADD16rr GR16:$src1, GR16:$src2)>; 206209467b48Spatrickdef : Pat<(add GR32:$src1, GR32:$src2), (ADD32rr GR32:$src1, GR32:$src2)>; 206309467b48Spatrickdef : Pat<(add GR64:$src1, GR64:$src2), (ADD64rr GR64:$src1, GR64:$src2)>; 206409467b48Spatrick 206509467b48Spatrick// add reg, mem 206609467b48Spatrickdef : Pat<(add GR8:$src1, (loadi8 addr:$src2)), 206709467b48Spatrick (ADD8rm GR8:$src1, addr:$src2)>; 206809467b48Spatrickdef : Pat<(add GR16:$src1, (loadi16 addr:$src2)), 206909467b48Spatrick (ADD16rm GR16:$src1, addr:$src2)>; 207009467b48Spatrickdef : Pat<(add GR32:$src1, (loadi32 addr:$src2)), 207109467b48Spatrick (ADD32rm GR32:$src1, addr:$src2)>; 207209467b48Spatrickdef : Pat<(add GR64:$src1, (loadi64 addr:$src2)), 207309467b48Spatrick (ADD64rm GR64:$src1, addr:$src2)>; 207409467b48Spatrick 207509467b48Spatrick// add reg, imm 207609467b48Spatrickdef : Pat<(add GR8 :$src1, imm:$src2), (ADD8ri GR8:$src1 , imm:$src2)>; 207709467b48Spatrickdef : Pat<(add GR16:$src1, imm:$src2), (ADD16ri GR16:$src1, imm:$src2)>; 207809467b48Spatrickdef : Pat<(add GR32:$src1, imm:$src2), (ADD32ri GR32:$src1, imm:$src2)>; 207909467b48Spatrickdef : Pat<(add GR16:$src1, i16immSExt8:$src2), 208009467b48Spatrick (ADD16ri8 GR16:$src1, i16immSExt8:$src2)>; 208109467b48Spatrickdef : Pat<(add GR32:$src1, i32immSExt8:$src2), 208209467b48Spatrick (ADD32ri8 GR32:$src1, i32immSExt8:$src2)>; 208309467b48Spatrickdef : Pat<(add GR64:$src1, i64immSExt8:$src2), 208409467b48Spatrick (ADD64ri8 GR64:$src1, i64immSExt8:$src2)>; 208509467b48Spatrickdef : Pat<(add GR64:$src1, i64immSExt32:$src2), 208609467b48Spatrick (ADD64ri32 GR64:$src1, i64immSExt32:$src2)>; 208709467b48Spatrick 208809467b48Spatrick// sub reg, reg 208909467b48Spatrickdef : Pat<(sub GR8 :$src1, GR8 :$src2), (SUB8rr GR8 :$src1, GR8 :$src2)>; 209009467b48Spatrickdef : Pat<(sub GR16:$src1, GR16:$src2), (SUB16rr GR16:$src1, GR16:$src2)>; 209109467b48Spatrickdef : Pat<(sub GR32:$src1, GR32:$src2), (SUB32rr GR32:$src1, GR32:$src2)>; 209209467b48Spatrickdef : Pat<(sub GR64:$src1, GR64:$src2), (SUB64rr GR64:$src1, GR64:$src2)>; 209309467b48Spatrick 209409467b48Spatrick// sub reg, mem 209509467b48Spatrickdef : Pat<(sub GR8:$src1, (loadi8 addr:$src2)), 209609467b48Spatrick (SUB8rm GR8:$src1, addr:$src2)>; 209709467b48Spatrickdef : Pat<(sub GR16:$src1, (loadi16 addr:$src2)), 209809467b48Spatrick (SUB16rm GR16:$src1, addr:$src2)>; 209909467b48Spatrickdef : Pat<(sub GR32:$src1, (loadi32 addr:$src2)), 210009467b48Spatrick (SUB32rm GR32:$src1, addr:$src2)>; 210109467b48Spatrickdef : Pat<(sub GR64:$src1, (loadi64 addr:$src2)), 210209467b48Spatrick (SUB64rm GR64:$src1, addr:$src2)>; 210309467b48Spatrick 210409467b48Spatrick// sub reg, imm 210509467b48Spatrickdef : Pat<(sub GR8:$src1, imm:$src2), 210609467b48Spatrick (SUB8ri GR8:$src1, imm:$src2)>; 210709467b48Spatrickdef : Pat<(sub GR16:$src1, imm:$src2), 210809467b48Spatrick (SUB16ri GR16:$src1, imm:$src2)>; 210909467b48Spatrickdef : Pat<(sub GR32:$src1, imm:$src2), 211009467b48Spatrick (SUB32ri GR32:$src1, imm:$src2)>; 211109467b48Spatrickdef : Pat<(sub GR16:$src1, i16immSExt8:$src2), 211209467b48Spatrick (SUB16ri8 GR16:$src1, i16immSExt8:$src2)>; 211309467b48Spatrickdef : Pat<(sub GR32:$src1, i32immSExt8:$src2), 211409467b48Spatrick (SUB32ri8 GR32:$src1, i32immSExt8:$src2)>; 211509467b48Spatrickdef : Pat<(sub GR64:$src1, i64immSExt8:$src2), 211609467b48Spatrick (SUB64ri8 GR64:$src1, i64immSExt8:$src2)>; 211709467b48Spatrickdef : Pat<(sub GR64:$src1, i64immSExt32:$src2), 211809467b48Spatrick (SUB64ri32 GR64:$src1, i64immSExt32:$src2)>; 211909467b48Spatrick 212009467b48Spatrick// sub 0, reg 212109467b48Spatrickdef : Pat<(X86sub_flag 0, GR8 :$src), (NEG8r GR8 :$src)>; 212209467b48Spatrickdef : Pat<(X86sub_flag 0, GR16:$src), (NEG16r GR16:$src)>; 212309467b48Spatrickdef : Pat<(X86sub_flag 0, GR32:$src), (NEG32r GR32:$src)>; 212409467b48Spatrickdef : Pat<(X86sub_flag 0, GR64:$src), (NEG64r GR64:$src)>; 212509467b48Spatrick 212609467b48Spatrick// mul reg, reg 212709467b48Spatrickdef : Pat<(mul GR16:$src1, GR16:$src2), 212809467b48Spatrick (IMUL16rr GR16:$src1, GR16:$src2)>; 212909467b48Spatrickdef : Pat<(mul GR32:$src1, GR32:$src2), 213009467b48Spatrick (IMUL32rr GR32:$src1, GR32:$src2)>; 213109467b48Spatrickdef : Pat<(mul GR64:$src1, GR64:$src2), 213209467b48Spatrick (IMUL64rr GR64:$src1, GR64:$src2)>; 213309467b48Spatrick 213409467b48Spatrick// mul reg, mem 213509467b48Spatrickdef : Pat<(mul GR16:$src1, (loadi16 addr:$src2)), 213609467b48Spatrick (IMUL16rm GR16:$src1, addr:$src2)>; 213709467b48Spatrickdef : Pat<(mul GR32:$src1, (loadi32 addr:$src2)), 213809467b48Spatrick (IMUL32rm GR32:$src1, addr:$src2)>; 213909467b48Spatrickdef : Pat<(mul GR64:$src1, (loadi64 addr:$src2)), 214009467b48Spatrick (IMUL64rm GR64:$src1, addr:$src2)>; 214109467b48Spatrick 214209467b48Spatrick// mul reg, imm 214309467b48Spatrickdef : Pat<(mul GR16:$src1, imm:$src2), 214409467b48Spatrick (IMUL16rri GR16:$src1, imm:$src2)>; 214509467b48Spatrickdef : Pat<(mul GR32:$src1, imm:$src2), 214609467b48Spatrick (IMUL32rri GR32:$src1, imm:$src2)>; 214709467b48Spatrickdef : Pat<(mul GR16:$src1, i16immSExt8:$src2), 214809467b48Spatrick (IMUL16rri8 GR16:$src1, i16immSExt8:$src2)>; 214909467b48Spatrickdef : Pat<(mul GR32:$src1, i32immSExt8:$src2), 215009467b48Spatrick (IMUL32rri8 GR32:$src1, i32immSExt8:$src2)>; 215109467b48Spatrickdef : Pat<(mul GR64:$src1, i64immSExt8:$src2), 215209467b48Spatrick (IMUL64rri8 GR64:$src1, i64immSExt8:$src2)>; 215309467b48Spatrickdef : Pat<(mul GR64:$src1, i64immSExt32:$src2), 215409467b48Spatrick (IMUL64rri32 GR64:$src1, i64immSExt32:$src2)>; 215509467b48Spatrick 215609467b48Spatrick// reg = mul mem, imm 215709467b48Spatrickdef : Pat<(mul (loadi16 addr:$src1), imm:$src2), 215809467b48Spatrick (IMUL16rmi addr:$src1, imm:$src2)>; 215909467b48Spatrickdef : Pat<(mul (loadi32 addr:$src1), imm:$src2), 216009467b48Spatrick (IMUL32rmi addr:$src1, imm:$src2)>; 216109467b48Spatrickdef : Pat<(mul (loadi16 addr:$src1), i16immSExt8:$src2), 216209467b48Spatrick (IMUL16rmi8 addr:$src1, i16immSExt8:$src2)>; 216309467b48Spatrickdef : Pat<(mul (loadi32 addr:$src1), i32immSExt8:$src2), 216409467b48Spatrick (IMUL32rmi8 addr:$src1, i32immSExt8:$src2)>; 216509467b48Spatrickdef : Pat<(mul (loadi64 addr:$src1), i64immSExt8:$src2), 216609467b48Spatrick (IMUL64rmi8 addr:$src1, i64immSExt8:$src2)>; 216709467b48Spatrickdef : Pat<(mul (loadi64 addr:$src1), i64immSExt32:$src2), 216809467b48Spatrick (IMUL64rmi32 addr:$src1, i64immSExt32:$src2)>; 216909467b48Spatrick 217009467b48Spatrick// Increment/Decrement reg. 217109467b48Spatrick// Do not make INC/DEC if it is slow 217209467b48Spatricklet Predicates = [UseIncDec] in { 217309467b48Spatrick def : Pat<(add GR8:$src, 1), (INC8r GR8:$src)>; 217409467b48Spatrick def : Pat<(add GR16:$src, 1), (INC16r GR16:$src)>; 217509467b48Spatrick def : Pat<(add GR32:$src, 1), (INC32r GR32:$src)>; 217609467b48Spatrick def : Pat<(add GR64:$src, 1), (INC64r GR64:$src)>; 217709467b48Spatrick def : Pat<(add GR8:$src, -1), (DEC8r GR8:$src)>; 217809467b48Spatrick def : Pat<(add GR16:$src, -1), (DEC16r GR16:$src)>; 217909467b48Spatrick def : Pat<(add GR32:$src, -1), (DEC32r GR32:$src)>; 218009467b48Spatrick def : Pat<(add GR64:$src, -1), (DEC64r GR64:$src)>; 218109467b48Spatrick 218209467b48Spatrick def : Pat<(X86add_flag_nocf GR8:$src, -1), (DEC8r GR8:$src)>; 218309467b48Spatrick def : Pat<(X86add_flag_nocf GR16:$src, -1), (DEC16r GR16:$src)>; 218409467b48Spatrick def : Pat<(X86add_flag_nocf GR32:$src, -1), (DEC32r GR32:$src)>; 218509467b48Spatrick def : Pat<(X86add_flag_nocf GR64:$src, -1), (DEC64r GR64:$src)>; 218609467b48Spatrick def : Pat<(X86sub_flag_nocf GR8:$src, -1), (INC8r GR8:$src)>; 218709467b48Spatrick def : Pat<(X86sub_flag_nocf GR16:$src, -1), (INC16r GR16:$src)>; 218809467b48Spatrick def : Pat<(X86sub_flag_nocf GR32:$src, -1), (INC32r GR32:$src)>; 218909467b48Spatrick def : Pat<(X86sub_flag_nocf GR64:$src, -1), (INC64r GR64:$src)>; 219009467b48Spatrick} 219109467b48Spatrick 219209467b48Spatrick// or reg/reg. 219309467b48Spatrickdef : Pat<(or GR8 :$src1, GR8 :$src2), (OR8rr GR8 :$src1, GR8 :$src2)>; 219409467b48Spatrickdef : Pat<(or GR16:$src1, GR16:$src2), (OR16rr GR16:$src1, GR16:$src2)>; 219509467b48Spatrickdef : Pat<(or GR32:$src1, GR32:$src2), (OR32rr GR32:$src1, GR32:$src2)>; 219609467b48Spatrickdef : Pat<(or GR64:$src1, GR64:$src2), (OR64rr GR64:$src1, GR64:$src2)>; 219709467b48Spatrick 219809467b48Spatrick// or reg/mem 219909467b48Spatrickdef : Pat<(or GR8:$src1, (loadi8 addr:$src2)), 220009467b48Spatrick (OR8rm GR8:$src1, addr:$src2)>; 220109467b48Spatrickdef : Pat<(or GR16:$src1, (loadi16 addr:$src2)), 220209467b48Spatrick (OR16rm GR16:$src1, addr:$src2)>; 220309467b48Spatrickdef : Pat<(or GR32:$src1, (loadi32 addr:$src2)), 220409467b48Spatrick (OR32rm GR32:$src1, addr:$src2)>; 220509467b48Spatrickdef : Pat<(or GR64:$src1, (loadi64 addr:$src2)), 220609467b48Spatrick (OR64rm GR64:$src1, addr:$src2)>; 220709467b48Spatrick 220809467b48Spatrick// or reg/imm 220909467b48Spatrickdef : Pat<(or GR8:$src1 , imm:$src2), (OR8ri GR8 :$src1, imm:$src2)>; 221009467b48Spatrickdef : Pat<(or GR16:$src1, imm:$src2), (OR16ri GR16:$src1, imm:$src2)>; 221109467b48Spatrickdef : Pat<(or GR32:$src1, imm:$src2), (OR32ri GR32:$src1, imm:$src2)>; 221209467b48Spatrickdef : Pat<(or GR16:$src1, i16immSExt8:$src2), 221309467b48Spatrick (OR16ri8 GR16:$src1, i16immSExt8:$src2)>; 221409467b48Spatrickdef : Pat<(or GR32:$src1, i32immSExt8:$src2), 221509467b48Spatrick (OR32ri8 GR32:$src1, i32immSExt8:$src2)>; 221609467b48Spatrickdef : Pat<(or GR64:$src1, i64immSExt8:$src2), 221709467b48Spatrick (OR64ri8 GR64:$src1, i64immSExt8:$src2)>; 221809467b48Spatrickdef : Pat<(or GR64:$src1, i64immSExt32:$src2), 221909467b48Spatrick (OR64ri32 GR64:$src1, i64immSExt32:$src2)>; 222009467b48Spatrick 222109467b48Spatrick// xor reg/reg 222209467b48Spatrickdef : Pat<(xor GR8 :$src1, GR8 :$src2), (XOR8rr GR8 :$src1, GR8 :$src2)>; 222309467b48Spatrickdef : Pat<(xor GR16:$src1, GR16:$src2), (XOR16rr GR16:$src1, GR16:$src2)>; 222409467b48Spatrickdef : Pat<(xor GR32:$src1, GR32:$src2), (XOR32rr GR32:$src1, GR32:$src2)>; 222509467b48Spatrickdef : Pat<(xor GR64:$src1, GR64:$src2), (XOR64rr GR64:$src1, GR64:$src2)>; 222609467b48Spatrick 222709467b48Spatrick// xor reg/mem 222809467b48Spatrickdef : Pat<(xor GR8:$src1, (loadi8 addr:$src2)), 222909467b48Spatrick (XOR8rm GR8:$src1, addr:$src2)>; 223009467b48Spatrickdef : Pat<(xor GR16:$src1, (loadi16 addr:$src2)), 223109467b48Spatrick (XOR16rm GR16:$src1, addr:$src2)>; 223209467b48Spatrickdef : Pat<(xor GR32:$src1, (loadi32 addr:$src2)), 223309467b48Spatrick (XOR32rm GR32:$src1, addr:$src2)>; 223409467b48Spatrickdef : Pat<(xor GR64:$src1, (loadi64 addr:$src2)), 223509467b48Spatrick (XOR64rm GR64:$src1, addr:$src2)>; 223609467b48Spatrick 223709467b48Spatrick// xor reg/imm 223809467b48Spatrickdef : Pat<(xor GR8:$src1, imm:$src2), 223909467b48Spatrick (XOR8ri GR8:$src1, imm:$src2)>; 224009467b48Spatrickdef : Pat<(xor GR16:$src1, imm:$src2), 224109467b48Spatrick (XOR16ri GR16:$src1, imm:$src2)>; 224209467b48Spatrickdef : Pat<(xor GR32:$src1, imm:$src2), 224309467b48Spatrick (XOR32ri GR32:$src1, imm:$src2)>; 224409467b48Spatrickdef : Pat<(xor GR16:$src1, i16immSExt8:$src2), 224509467b48Spatrick (XOR16ri8 GR16:$src1, i16immSExt8:$src2)>; 224609467b48Spatrickdef : Pat<(xor GR32:$src1, i32immSExt8:$src2), 224709467b48Spatrick (XOR32ri8 GR32:$src1, i32immSExt8:$src2)>; 224809467b48Spatrickdef : Pat<(xor GR64:$src1, i64immSExt8:$src2), 224909467b48Spatrick (XOR64ri8 GR64:$src1, i64immSExt8:$src2)>; 225009467b48Spatrickdef : Pat<(xor GR64:$src1, i64immSExt32:$src2), 225109467b48Spatrick (XOR64ri32 GR64:$src1, i64immSExt32:$src2)>; 225209467b48Spatrick 225309467b48Spatrick// and reg/reg 225409467b48Spatrickdef : Pat<(and GR8 :$src1, GR8 :$src2), (AND8rr GR8 :$src1, GR8 :$src2)>; 225509467b48Spatrickdef : Pat<(and GR16:$src1, GR16:$src2), (AND16rr GR16:$src1, GR16:$src2)>; 225609467b48Spatrickdef : Pat<(and GR32:$src1, GR32:$src2), (AND32rr GR32:$src1, GR32:$src2)>; 225709467b48Spatrickdef : Pat<(and GR64:$src1, GR64:$src2), (AND64rr GR64:$src1, GR64:$src2)>; 225809467b48Spatrick 225909467b48Spatrick// and reg/mem 226009467b48Spatrickdef : Pat<(and GR8:$src1, (loadi8 addr:$src2)), 226109467b48Spatrick (AND8rm GR8:$src1, addr:$src2)>; 226209467b48Spatrickdef : Pat<(and GR16:$src1, (loadi16 addr:$src2)), 226309467b48Spatrick (AND16rm GR16:$src1, addr:$src2)>; 226409467b48Spatrickdef : Pat<(and GR32:$src1, (loadi32 addr:$src2)), 226509467b48Spatrick (AND32rm GR32:$src1, addr:$src2)>; 226609467b48Spatrickdef : Pat<(and GR64:$src1, (loadi64 addr:$src2)), 226709467b48Spatrick (AND64rm GR64:$src1, addr:$src2)>; 226809467b48Spatrick 226909467b48Spatrick// and reg/imm 227009467b48Spatrickdef : Pat<(and GR8:$src1, imm:$src2), 227109467b48Spatrick (AND8ri GR8:$src1, imm:$src2)>; 227209467b48Spatrickdef : Pat<(and GR16:$src1, imm:$src2), 227309467b48Spatrick (AND16ri GR16:$src1, imm:$src2)>; 227409467b48Spatrickdef : Pat<(and GR32:$src1, imm:$src2), 227509467b48Spatrick (AND32ri GR32:$src1, imm:$src2)>; 227609467b48Spatrickdef : Pat<(and GR16:$src1, i16immSExt8:$src2), 227709467b48Spatrick (AND16ri8 GR16:$src1, i16immSExt8:$src2)>; 227809467b48Spatrickdef : Pat<(and GR32:$src1, i32immSExt8:$src2), 227909467b48Spatrick (AND32ri8 GR32:$src1, i32immSExt8:$src2)>; 228009467b48Spatrickdef : Pat<(and GR64:$src1, i64immSExt8:$src2), 228109467b48Spatrick (AND64ri8 GR64:$src1, i64immSExt8:$src2)>; 228209467b48Spatrickdef : Pat<(and GR64:$src1, i64immSExt32:$src2), 228309467b48Spatrick (AND64ri32 GR64:$src1, i64immSExt32:$src2)>; 228409467b48Spatrick 228509467b48Spatrick// Bit scan instruction patterns to match explicit zero-undef behavior. 228609467b48Spatrickdef : Pat<(cttz_zero_undef GR16:$src), (BSF16rr GR16:$src)>; 228709467b48Spatrickdef : Pat<(cttz_zero_undef GR32:$src), (BSF32rr GR32:$src)>; 228809467b48Spatrickdef : Pat<(cttz_zero_undef GR64:$src), (BSF64rr GR64:$src)>; 228909467b48Spatrickdef : Pat<(cttz_zero_undef (loadi16 addr:$src)), (BSF16rm addr:$src)>; 229009467b48Spatrickdef : Pat<(cttz_zero_undef (loadi32 addr:$src)), (BSF32rm addr:$src)>; 229109467b48Spatrickdef : Pat<(cttz_zero_undef (loadi64 addr:$src)), (BSF64rm addr:$src)>; 229209467b48Spatrick 229309467b48Spatrick// When HasMOVBE is enabled it is possible to get a non-legalized 229409467b48Spatrick// register-register 16 bit bswap. This maps it to a ROL instruction. 229509467b48Spatricklet Predicates = [HasMOVBE] in { 229609467b48Spatrick def : Pat<(bswap GR16:$src), (ROL16ri GR16:$src, (i8 8))>; 229709467b48Spatrick} 2298