1 //===- X86InstructionSelector.cpp -----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the InstructionSelector class for
10 /// X86.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13 
14 #include "MCTargetDesc/X86BaseInfo.h"
15 #include "X86.h"
16 #include "X86InstrBuilder.h"
17 #include "X86InstrInfo.h"
18 #include "X86RegisterBankInfo.h"
19 #include "X86RegisterInfo.h"
20 #include "X86Subtarget.h"
21 #include "X86TargetMachine.h"
22 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
23 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
24 #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
25 #include "llvm/CodeGen/GlobalISel/Utils.h"
26 #include "llvm/CodeGen/MachineBasicBlock.h"
27 #include "llvm/CodeGen/MachineConstantPool.h"
28 #include "llvm/CodeGen/MachineFunction.h"
29 #include "llvm/CodeGen/MachineInstr.h"
30 #include "llvm/CodeGen/MachineInstrBuilder.h"
31 #include "llvm/CodeGen/MachineMemOperand.h"
32 #include "llvm/CodeGen/MachineOperand.h"
33 #include "llvm/CodeGen/MachineRegisterInfo.h"
34 #include "llvm/CodeGen/TargetOpcodes.h"
35 #include "llvm/CodeGen/TargetRegisterInfo.h"
36 #include "llvm/IR/DataLayout.h"
37 #include "llvm/IR/InstrTypes.h"
38 #include "llvm/IR/IntrinsicsX86.h"
39 #include "llvm/Support/AtomicOrdering.h"
40 #include "llvm/Support/CodeGen.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Support/LowLevelTypeImpl.h"
44 #include "llvm/Support/MathExtras.h"
45 #include "llvm/Support/raw_ostream.h"
46 #include <cassert>
47 #include <cstdint>
48 #include <tuple>
49 
50 #define DEBUG_TYPE "X86-isel"
51 
52 using namespace llvm;
53 
54 namespace {
55 
56 #define GET_GLOBALISEL_PREDICATE_BITSET
57 #include "X86GenGlobalISel.inc"
58 #undef GET_GLOBALISEL_PREDICATE_BITSET
59 
60 class X86InstructionSelector : public InstructionSelector {
61 public:
62   X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
63                          const X86RegisterBankInfo &RBI);
64 
65   bool select(MachineInstr &I) override;
getName()66   static const char *getName() { return DEBUG_TYPE; }
67 
68 private:
69   /// tblgen-erated 'select' implementation, used as the initial selector for
70   /// the patterns that don't require complex C++.
71   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
72 
73   // TODO: remove after supported by Tablegen-erated instruction selection.
74   unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
75                           Align Alignment) const;
76 
77   bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
78                          MachineFunction &MF) const;
79   bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
80                              MachineFunction &MF) const;
81   bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
82                          MachineFunction &MF) const;
83   bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
84                       MachineFunction &MF) const;
85   bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
86                              MachineFunction &MF) const;
87   bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
88                   MachineFunction &MF) const;
89   bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
90                     MachineFunction &MF) const;
91   bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
92                  MachineFunction &MF) const;
93   bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
94                   MachineFunction &MF) const;
95   bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI,
96                    MachineFunction &MF) const;
97   bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
98   bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
99                            MachineFunction &MF);
100   bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
101                          MachineFunction &MF);
102   bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
103                     MachineFunction &MF) const;
104   bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
105                      MachineFunction &MF) const;
106   bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
107                         MachineFunction &MF) const;
108   bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
109                           const unsigned DstReg,
110                           const TargetRegisterClass *DstRC,
111                           const unsigned SrcReg,
112                           const TargetRegisterClass *SrcRC) const;
113   bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
114                      MachineFunction &MF) const;
115   bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
116   bool selectDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
117                     MachineFunction &MF) const;
118   bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI,
119                                    MachineFunction &MF) const;
120 
121   // emit insert subreg instruction and insert it before MachineInstr &I
122   bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
123                         MachineRegisterInfo &MRI, MachineFunction &MF) const;
124   // emit extract subreg instruction and insert it before MachineInstr &I
125   bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
126                          MachineRegisterInfo &MRI, MachineFunction &MF) const;
127 
128   const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
129   const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg,
130                                          MachineRegisterInfo &MRI) const;
131 
132   const X86TargetMachine &TM;
133   const X86Subtarget &STI;
134   const X86InstrInfo &TII;
135   const X86RegisterInfo &TRI;
136   const X86RegisterBankInfo &RBI;
137 
138 #define GET_GLOBALISEL_PREDICATES_DECL
139 #include "X86GenGlobalISel.inc"
140 #undef GET_GLOBALISEL_PREDICATES_DECL
141 
142 #define GET_GLOBALISEL_TEMPORARIES_DECL
143 #include "X86GenGlobalISel.inc"
144 #undef GET_GLOBALISEL_TEMPORARIES_DECL
145 };
146 
147 } // end anonymous namespace
148 
149 #define GET_GLOBALISEL_IMPL
150 #include "X86GenGlobalISel.inc"
151 #undef GET_GLOBALISEL_IMPL
152 
X86InstructionSelector(const X86TargetMachine & TM,const X86Subtarget & STI,const X86RegisterBankInfo & RBI)153 X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
154                                                const X86Subtarget &STI,
155                                                const X86RegisterBankInfo &RBI)
156     : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
157       TRI(*STI.getRegisterInfo()), RBI(RBI),
158 #define GET_GLOBALISEL_PREDICATES_INIT
159 #include "X86GenGlobalISel.inc"
160 #undef GET_GLOBALISEL_PREDICATES_INIT
161 #define GET_GLOBALISEL_TEMPORARIES_INIT
162 #include "X86GenGlobalISel.inc"
163 #undef GET_GLOBALISEL_TEMPORARIES_INIT
164 {
165 }
166 
167 // FIXME: This should be target-independent, inferred from the types declared
168 // for each class in the bank.
169 const TargetRegisterClass *
getRegClass(LLT Ty,const RegisterBank & RB) const170 X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
171   if (RB.getID() == X86::GPRRegBankID) {
172     if (Ty.getSizeInBits() <= 8)
173       return &X86::GR8RegClass;
174     if (Ty.getSizeInBits() == 16)
175       return &X86::GR16RegClass;
176     if (Ty.getSizeInBits() == 32)
177       return &X86::GR32RegClass;
178     if (Ty.getSizeInBits() == 64)
179       return &X86::GR64RegClass;
180   }
181   if (RB.getID() == X86::VECRRegBankID) {
182     if (Ty.getSizeInBits() == 32)
183       return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
184     if (Ty.getSizeInBits() == 64)
185       return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
186     if (Ty.getSizeInBits() == 128)
187       return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
188     if (Ty.getSizeInBits() == 256)
189       return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
190     if (Ty.getSizeInBits() == 512)
191       return &X86::VR512RegClass;
192   }
193 
194   llvm_unreachable("Unknown RegBank!");
195 }
196 
197 const TargetRegisterClass *
getRegClass(LLT Ty,unsigned Reg,MachineRegisterInfo & MRI) const198 X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg,
199                                     MachineRegisterInfo &MRI) const {
200   const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
201   return getRegClass(Ty, RegBank);
202 }
203 
getSubRegIndex(const TargetRegisterClass * RC)204 static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
205   unsigned SubIdx = X86::NoSubRegister;
206   if (RC == &X86::GR32RegClass) {
207     SubIdx = X86::sub_32bit;
208   } else if (RC == &X86::GR16RegClass) {
209     SubIdx = X86::sub_16bit;
210   } else if (RC == &X86::GR8RegClass) {
211     SubIdx = X86::sub_8bit;
212   }
213 
214   return SubIdx;
215 }
216 
getRegClassFromGRPhysReg(unsigned Reg)217 static const TargetRegisterClass *getRegClassFromGRPhysReg(unsigned Reg) {
218   assert(Register::isPhysicalRegister(Reg));
219   if (X86::GR64RegClass.contains(Reg))
220     return &X86::GR64RegClass;
221   if (X86::GR32RegClass.contains(Reg))
222     return &X86::GR32RegClass;
223   if (X86::GR16RegClass.contains(Reg))
224     return &X86::GR16RegClass;
225   if (X86::GR8RegClass.contains(Reg))
226     return &X86::GR8RegClass;
227 
228   llvm_unreachable("Unknown RegClass for PhysReg!");
229 }
230 
231 // Set X86 Opcode and constrain DestReg.
selectCopy(MachineInstr & I,MachineRegisterInfo & MRI) const232 bool X86InstructionSelector::selectCopy(MachineInstr &I,
233                                         MachineRegisterInfo &MRI) const {
234   Register DstReg = I.getOperand(0).getReg();
235   const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
236   const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
237 
238   Register SrcReg = I.getOperand(1).getReg();
239   const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
240   const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
241 
242   if (Register::isPhysicalRegister(DstReg)) {
243     assert(I.isCopy() && "Generic operators do not allow physical registers");
244 
245     if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
246         DstRegBank.getID() == X86::GPRRegBankID) {
247 
248       const TargetRegisterClass *SrcRC =
249           getRegClass(MRI.getType(SrcReg), SrcRegBank);
250       const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg);
251 
252       if (SrcRC != DstRC) {
253         // This case can be generated by ABI lowering, performe anyext
254         Register ExtSrc = MRI.createVirtualRegister(DstRC);
255         BuildMI(*I.getParent(), I, I.getDebugLoc(),
256                 TII.get(TargetOpcode::SUBREG_TO_REG))
257             .addDef(ExtSrc)
258             .addImm(0)
259             .addReg(SrcReg)
260             .addImm(getSubRegIndex(SrcRC));
261 
262         I.getOperand(1).setReg(ExtSrc);
263       }
264     }
265 
266     return true;
267   }
268 
269   assert((!Register::isPhysicalRegister(SrcReg) || I.isCopy()) &&
270          "No phys reg on generic operators");
271   assert((DstSize == SrcSize ||
272           // Copies are a mean to setup initial types, the number of
273           // bits may not exactly match.
274           (Register::isPhysicalRegister(SrcReg) &&
275            DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
276          "Copy with different width?!");
277 
278   const TargetRegisterClass *DstRC =
279       getRegClass(MRI.getType(DstReg), DstRegBank);
280 
281   if (SrcRegBank.getID() == X86::GPRRegBankID &&
282       DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
283       Register::isPhysicalRegister(SrcReg)) {
284     // Change the physical register to performe truncate.
285 
286     const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg);
287 
288     if (DstRC != SrcRC) {
289       I.getOperand(1).setSubReg(getSubRegIndex(DstRC));
290       I.getOperand(1).substPhysReg(SrcReg, TRI);
291     }
292   }
293 
294   // No need to constrain SrcReg. It will get constrained when
295   // we hit another of its use or its defs.
296   // Copies do not have constraints.
297   const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg);
298   if (!OldRC || !DstRC->hasSubClassEq(OldRC)) {
299     if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
300       LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
301                         << " operand\n");
302       return false;
303     }
304   }
305   I.setDesc(TII.get(X86::COPY));
306   return true;
307 }
308 
select(MachineInstr & I)309 bool X86InstructionSelector::select(MachineInstr &I) {
310   assert(I.getParent() && "Instruction should be in a basic block!");
311   assert(I.getParent()->getParent() && "Instruction should be in a function!");
312 
313   MachineBasicBlock &MBB = *I.getParent();
314   MachineFunction &MF = *MBB.getParent();
315   MachineRegisterInfo &MRI = MF.getRegInfo();
316 
317   unsigned Opcode = I.getOpcode();
318   if (!isPreISelGenericOpcode(Opcode)) {
319     // Certain non-generic instructions also need some special handling.
320 
321     if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
322       return false;
323 
324     if (I.isCopy())
325       return selectCopy(I, MRI);
326 
327     return true;
328   }
329 
330   assert(I.getNumOperands() == I.getNumExplicitOperands() &&
331          "Generic instruction has unexpected implicit operands\n");
332 
333   if (selectImpl(I, *CoverageInfo))
334     return true;
335 
336   LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
337 
338   // TODO: This should be implemented by tblgen.
339   switch (I.getOpcode()) {
340   default:
341     return false;
342   case TargetOpcode::G_STORE:
343   case TargetOpcode::G_LOAD:
344     return selectLoadStoreOp(I, MRI, MF);
345   case TargetOpcode::G_PTR_ADD:
346   case TargetOpcode::G_FRAME_INDEX:
347     return selectFrameIndexOrGep(I, MRI, MF);
348   case TargetOpcode::G_GLOBAL_VALUE:
349     return selectGlobalValue(I, MRI, MF);
350   case TargetOpcode::G_CONSTANT:
351     return selectConstant(I, MRI, MF);
352   case TargetOpcode::G_FCONSTANT:
353     return materializeFP(I, MRI, MF);
354   case TargetOpcode::G_PTRTOINT:
355   case TargetOpcode::G_TRUNC:
356     return selectTruncOrPtrToInt(I, MRI, MF);
357   case TargetOpcode::G_INTTOPTR:
358     return selectCopy(I, MRI);
359   case TargetOpcode::G_ZEXT:
360     return selectZext(I, MRI, MF);
361   case TargetOpcode::G_ANYEXT:
362     return selectAnyext(I, MRI, MF);
363   case TargetOpcode::G_ICMP:
364     return selectCmp(I, MRI, MF);
365   case TargetOpcode::G_FCMP:
366     return selectFCmp(I, MRI, MF);
367   case TargetOpcode::G_UADDE:
368     return selectUadde(I, MRI, MF);
369   case TargetOpcode::G_UNMERGE_VALUES:
370     return selectUnmergeValues(I, MRI, MF);
371   case TargetOpcode::G_MERGE_VALUES:
372   case TargetOpcode::G_CONCAT_VECTORS:
373     return selectMergeValues(I, MRI, MF);
374   case TargetOpcode::G_EXTRACT:
375     return selectExtract(I, MRI, MF);
376   case TargetOpcode::G_INSERT:
377     return selectInsert(I, MRI, MF);
378   case TargetOpcode::G_BRCOND:
379     return selectCondBranch(I, MRI, MF);
380   case TargetOpcode::G_IMPLICIT_DEF:
381   case TargetOpcode::G_PHI:
382     return selectImplicitDefOrPHI(I, MRI);
383   case TargetOpcode::G_SDIV:
384   case TargetOpcode::G_UDIV:
385   case TargetOpcode::G_SREM:
386   case TargetOpcode::G_UREM:
387     return selectDivRem(I, MRI, MF);
388   case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
389     return selectIntrinsicWSideEffects(I, MRI, MF);
390   }
391 
392   return false;
393 }
394 
getLoadStoreOp(const LLT & Ty,const RegisterBank & RB,unsigned Opc,Align Alignment) const395 unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
396                                                 const RegisterBank &RB,
397                                                 unsigned Opc,
398                                                 Align Alignment) const {
399   bool Isload = (Opc == TargetOpcode::G_LOAD);
400   bool HasAVX = STI.hasAVX();
401   bool HasAVX512 = STI.hasAVX512();
402   bool HasVLX = STI.hasVLX();
403 
404   if (Ty == LLT::scalar(8)) {
405     if (X86::GPRRegBankID == RB.getID())
406       return Isload ? X86::MOV8rm : X86::MOV8mr;
407   } else if (Ty == LLT::scalar(16)) {
408     if (X86::GPRRegBankID == RB.getID())
409       return Isload ? X86::MOV16rm : X86::MOV16mr;
410   } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
411     if (X86::GPRRegBankID == RB.getID())
412       return Isload ? X86::MOV32rm : X86::MOV32mr;
413     if (X86::VECRRegBankID == RB.getID())
414       return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt :
415                        HasAVX    ? X86::VMOVSSrm_alt :
416                                    X86::MOVSSrm_alt)
417                     : (HasAVX512 ? X86::VMOVSSZmr :
418                        HasAVX    ? X86::VMOVSSmr :
419                                    X86::MOVSSmr);
420   } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
421     if (X86::GPRRegBankID == RB.getID())
422       return Isload ? X86::MOV64rm : X86::MOV64mr;
423     if (X86::VECRRegBankID == RB.getID())
424       return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt :
425                        HasAVX    ? X86::VMOVSDrm_alt :
426                                    X86::MOVSDrm_alt)
427                     : (HasAVX512 ? X86::VMOVSDZmr :
428                        HasAVX    ? X86::VMOVSDmr :
429                                    X86::MOVSDmr);
430   } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
431     if (Alignment >= Align(16))
432       return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
433                               : HasAVX512
434                                     ? X86::VMOVAPSZ128rm_NOVLX
435                                     : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
436                     : (HasVLX ? X86::VMOVAPSZ128mr
437                               : HasAVX512
438                                     ? X86::VMOVAPSZ128mr_NOVLX
439                                     : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
440     else
441       return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
442                               : HasAVX512
443                                     ? X86::VMOVUPSZ128rm_NOVLX
444                                     : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
445                     : (HasVLX ? X86::VMOVUPSZ128mr
446                               : HasAVX512
447                                     ? X86::VMOVUPSZ128mr_NOVLX
448                                     : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
449   } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
450     if (Alignment >= Align(32))
451       return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
452                               : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
453                                           : X86::VMOVAPSYrm)
454                     : (HasVLX ? X86::VMOVAPSZ256mr
455                               : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
456                                           : X86::VMOVAPSYmr);
457     else
458       return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
459                               : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
460                                           : X86::VMOVUPSYrm)
461                     : (HasVLX ? X86::VMOVUPSZ256mr
462                               : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
463                                           : X86::VMOVUPSYmr);
464   } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
465     if (Alignment >= Align(64))
466       return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
467     else
468       return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
469   }
470   return Opc;
471 }
472 
473 // Fill in an address from the given instruction.
X86SelectAddress(const MachineInstr & I,const MachineRegisterInfo & MRI,X86AddressMode & AM)474 static void X86SelectAddress(const MachineInstr &I,
475                              const MachineRegisterInfo &MRI,
476                              X86AddressMode &AM) {
477   assert(I.getOperand(0).isReg() && "unsupported opperand.");
478   assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
479          "unsupported type.");
480 
481   if (I.getOpcode() == TargetOpcode::G_PTR_ADD) {
482     if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) {
483       int64_t Imm = *COff;
484       if (isInt<32>(Imm)) { // Check for displacement overflow.
485         AM.Disp = static_cast<int32_t>(Imm);
486         AM.Base.Reg = I.getOperand(1).getReg();
487         return;
488       }
489     }
490   } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
491     AM.Base.FrameIndex = I.getOperand(1).getIndex();
492     AM.BaseType = X86AddressMode::FrameIndexBase;
493     return;
494   }
495 
496   // Default behavior.
497   AM.Base.Reg = I.getOperand(0).getReg();
498 }
499 
selectLoadStoreOp(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const500 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
501                                                MachineRegisterInfo &MRI,
502                                                MachineFunction &MF) const {
503   unsigned Opc = I.getOpcode();
504 
505   assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
506          "unexpected instruction");
507 
508   const Register DefReg = I.getOperand(0).getReg();
509   LLT Ty = MRI.getType(DefReg);
510   const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
511 
512   assert(I.hasOneMemOperand());
513   auto &MemOp = **I.memoperands_begin();
514   if (MemOp.isAtomic()) {
515     // Note: for unordered operations, we rely on the fact the appropriate MMO
516     // is already on the instruction we're mutating, and thus we don't need to
517     // make any changes.  So long as we select an opcode which is capable of
518     // loading or storing the appropriate size atomically, the rest of the
519     // backend is required to respect the MMO state.
520     if (!MemOp.isUnordered()) {
521       LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
522       return false;
523     }
524     if (MemOp.getAlign() < Ty.getSizeInBits() / 8) {
525       LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
526       return false;
527     }
528   }
529 
530   unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlign());
531   if (NewOpc == Opc)
532     return false;
533 
534   X86AddressMode AM;
535   X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM);
536 
537   I.setDesc(TII.get(NewOpc));
538   MachineInstrBuilder MIB(MF, I);
539   if (Opc == TargetOpcode::G_LOAD) {
540     I.RemoveOperand(1);
541     addFullAddress(MIB, AM);
542   } else {
543     // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
544     I.RemoveOperand(1);
545     I.RemoveOperand(0);
546     addFullAddress(MIB, AM).addUse(DefReg);
547   }
548   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
549 }
550 
getLeaOP(LLT Ty,const X86Subtarget & STI)551 static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
552   if (Ty == LLT::pointer(0, 64))
553     return X86::LEA64r;
554   else if (Ty == LLT::pointer(0, 32))
555     return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
556   else
557     llvm_unreachable("Can't get LEA opcode. Unsupported type.");
558 }
559 
selectFrameIndexOrGep(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const560 bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
561                                                    MachineRegisterInfo &MRI,
562                                                    MachineFunction &MF) const {
563   unsigned Opc = I.getOpcode();
564 
565   assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) &&
566          "unexpected instruction");
567 
568   const Register DefReg = I.getOperand(0).getReg();
569   LLT Ty = MRI.getType(DefReg);
570 
571   // Use LEA to calculate frame index and GEP
572   unsigned NewOpc = getLeaOP(Ty, STI);
573   I.setDesc(TII.get(NewOpc));
574   MachineInstrBuilder MIB(MF, I);
575 
576   if (Opc == TargetOpcode::G_FRAME_INDEX) {
577     addOffset(MIB, 0);
578   } else {
579     MachineOperand &InxOp = I.getOperand(2);
580     I.addOperand(InxOp);        // set IndexReg
581     InxOp.ChangeToImmediate(1); // set Scale
582     MIB.addImm(0).addReg(0);
583   }
584 
585   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
586 }
587 
selectGlobalValue(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const588 bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
589                                                MachineRegisterInfo &MRI,
590                                                MachineFunction &MF) const {
591   assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
592          "unexpected instruction");
593 
594   auto GV = I.getOperand(1).getGlobal();
595   if (GV->isThreadLocal()) {
596     return false; // TODO: we don't support TLS yet.
597   }
598 
599   // Can't handle alternate code models yet.
600   if (TM.getCodeModel() != CodeModel::Small)
601     return false;
602 
603   X86AddressMode AM;
604   AM.GV = GV;
605   AM.GVOpFlags = STI.classifyGlobalReference(GV);
606 
607   // TODO: The ABI requires an extra load. not supported yet.
608   if (isGlobalStubReference(AM.GVOpFlags))
609     return false;
610 
611   // TODO: This reference is relative to the pic base. not supported yet.
612   if (isGlobalRelativeToPICBase(AM.GVOpFlags))
613     return false;
614 
615   if (STI.isPICStyleRIPRel()) {
616     // Use rip-relative addressing.
617     assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
618     AM.Base.Reg = X86::RIP;
619   }
620 
621   const Register DefReg = I.getOperand(0).getReg();
622   LLT Ty = MRI.getType(DefReg);
623   unsigned NewOpc = getLeaOP(Ty, STI);
624 
625   I.setDesc(TII.get(NewOpc));
626   MachineInstrBuilder MIB(MF, I);
627 
628   I.RemoveOperand(1);
629   addFullAddress(MIB, AM);
630 
631   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
632 }
633 
selectConstant(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const634 bool X86InstructionSelector::selectConstant(MachineInstr &I,
635                                             MachineRegisterInfo &MRI,
636                                             MachineFunction &MF) const {
637   assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
638          "unexpected instruction");
639 
640   const Register DefReg = I.getOperand(0).getReg();
641   LLT Ty = MRI.getType(DefReg);
642 
643   if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
644     return false;
645 
646   uint64_t Val = 0;
647   if (I.getOperand(1).isCImm()) {
648     Val = I.getOperand(1).getCImm()->getZExtValue();
649     I.getOperand(1).ChangeToImmediate(Val);
650   } else if (I.getOperand(1).isImm()) {
651     Val = I.getOperand(1).getImm();
652   } else
653     llvm_unreachable("Unsupported operand type.");
654 
655   unsigned NewOpc;
656   switch (Ty.getSizeInBits()) {
657   case 8:
658     NewOpc = X86::MOV8ri;
659     break;
660   case 16:
661     NewOpc = X86::MOV16ri;
662     break;
663   case 32:
664     NewOpc = X86::MOV32ri;
665     break;
666   case 64:
667     // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used
668     if (isInt<32>(Val))
669       NewOpc = X86::MOV64ri32;
670     else
671       NewOpc = X86::MOV64ri;
672     break;
673   default:
674     llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
675   }
676 
677   I.setDesc(TII.get(NewOpc));
678   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
679 }
680 
681 // Helper function for selectTruncOrPtrToInt and selectAnyext.
682 // Returns true if DstRC lives on a floating register class and
683 // SrcRC lives on a 128-bit vector class.
canTurnIntoCOPY(const TargetRegisterClass * DstRC,const TargetRegisterClass * SrcRC)684 static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
685                             const TargetRegisterClass *SrcRC) {
686   return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass ||
687           DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) &&
688          (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass);
689 }
690 
selectTurnIntoCOPY(MachineInstr & I,MachineRegisterInfo & MRI,const unsigned DstReg,const TargetRegisterClass * DstRC,const unsigned SrcReg,const TargetRegisterClass * SrcRC) const691 bool X86InstructionSelector::selectTurnIntoCOPY(
692     MachineInstr &I, MachineRegisterInfo &MRI, const unsigned DstReg,
693     const TargetRegisterClass *DstRC, const unsigned SrcReg,
694     const TargetRegisterClass *SrcRC) const {
695 
696   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
697       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
698     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
699                       << " operand\n");
700     return false;
701   }
702   I.setDesc(TII.get(X86::COPY));
703   return true;
704 }
705 
selectTruncOrPtrToInt(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const706 bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
707                                                    MachineRegisterInfo &MRI,
708                                                    MachineFunction &MF) const {
709   assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
710           I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
711          "unexpected instruction");
712 
713   const Register DstReg = I.getOperand(0).getReg();
714   const Register SrcReg = I.getOperand(1).getReg();
715 
716   const LLT DstTy = MRI.getType(DstReg);
717   const LLT SrcTy = MRI.getType(SrcReg);
718 
719   const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
720   const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
721 
722   if (DstRB.getID() != SrcRB.getID()) {
723     LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
724                       << " input/output on different banks\n");
725     return false;
726   }
727 
728   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
729   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
730 
731   if (!DstRC || !SrcRC)
732     return false;
733 
734   // If that's truncation of the value that lives on the vector class and goes
735   // into the floating class, just replace it with copy, as we are able to
736   // select it as a regular move.
737   if (canTurnIntoCOPY(DstRC, SrcRC))
738     return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
739 
740   if (DstRB.getID() != X86::GPRRegBankID)
741     return false;
742 
743   unsigned SubIdx;
744   if (DstRC == SrcRC) {
745     // Nothing to be done
746     SubIdx = X86::NoSubRegister;
747   } else if (DstRC == &X86::GR32RegClass) {
748     SubIdx = X86::sub_32bit;
749   } else if (DstRC == &X86::GR16RegClass) {
750     SubIdx = X86::sub_16bit;
751   } else if (DstRC == &X86::GR8RegClass) {
752     SubIdx = X86::sub_8bit;
753   } else {
754     return false;
755   }
756 
757   SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
758 
759   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
760       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
761     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
762                       << "\n");
763     return false;
764   }
765 
766   I.getOperand(1).setSubReg(SubIdx);
767 
768   I.setDesc(TII.get(X86::COPY));
769   return true;
770 }
771 
selectZext(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const772 bool X86InstructionSelector::selectZext(MachineInstr &I,
773                                         MachineRegisterInfo &MRI,
774                                         MachineFunction &MF) const {
775   assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
776 
777   const Register DstReg = I.getOperand(0).getReg();
778   const Register SrcReg = I.getOperand(1).getReg();
779 
780   const LLT DstTy = MRI.getType(DstReg);
781   const LLT SrcTy = MRI.getType(SrcReg);
782 
783   assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
784          "8=>32 Zext is handled by tablegen");
785   assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
786          "16=>32 Zext is handled by tablegen");
787 
788   const static struct ZextEntry {
789     LLT SrcTy;
790     LLT DstTy;
791     unsigned MovOp;
792     bool NeedSubregToReg;
793   } OpTable[] = {
794       {LLT::scalar(8), LLT::scalar(16), X86::MOVZX16rr8, false},  // i8  => i16
795       {LLT::scalar(8), LLT::scalar(64), X86::MOVZX32rr8, true},   // i8  => i64
796       {LLT::scalar(16), LLT::scalar(64), X86::MOVZX32rr16, true}, // i16 => i64
797       {LLT::scalar(32), LLT::scalar(64), 0, true}                 // i32 => i64
798   };
799 
800   auto ZextEntryIt =
801       std::find_if(std::begin(OpTable), std::end(OpTable),
802                    [SrcTy, DstTy](const ZextEntry &El) {
803                      return El.DstTy == DstTy && El.SrcTy == SrcTy;
804                    });
805 
806   // Here we try to select Zext into a MOVZ and/or SUBREG_TO_REG instruction.
807   if (ZextEntryIt != std::end(OpTable)) {
808     const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
809     const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
810     const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
811     const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
812 
813     if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
814         !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
815       LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
816                         << " operand\n");
817       return false;
818     }
819 
820     unsigned TransitRegTo = DstReg;
821     unsigned TransitRegFrom = SrcReg;
822     if (ZextEntryIt->MovOp) {
823       // If we select Zext into MOVZ + SUBREG_TO_REG, we need to have
824       // a transit register in between: create it here.
825       if (ZextEntryIt->NeedSubregToReg) {
826         TransitRegFrom = MRI.createVirtualRegister(
827             getRegClass(LLT::scalar(32), DstReg, MRI));
828         TransitRegTo = TransitRegFrom;
829       }
830 
831       BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(ZextEntryIt->MovOp))
832           .addDef(TransitRegTo)
833           .addReg(SrcReg);
834     }
835     if (ZextEntryIt->NeedSubregToReg) {
836       BuildMI(*I.getParent(), I, I.getDebugLoc(),
837               TII.get(TargetOpcode::SUBREG_TO_REG))
838           .addDef(DstReg)
839           .addImm(0)
840           .addReg(TransitRegFrom)
841           .addImm(X86::sub_32bit);
842     }
843     I.eraseFromParent();
844     return true;
845   }
846 
847   if (SrcTy != LLT::scalar(1))
848     return false;
849 
850   unsigned AndOpc;
851   if (DstTy == LLT::scalar(8))
852     AndOpc = X86::AND8ri;
853   else if (DstTy == LLT::scalar(16))
854     AndOpc = X86::AND16ri8;
855   else if (DstTy == LLT::scalar(32))
856     AndOpc = X86::AND32ri8;
857   else if (DstTy == LLT::scalar(64))
858     AndOpc = X86::AND64ri8;
859   else
860     return false;
861 
862   unsigned DefReg = SrcReg;
863   if (DstTy != LLT::scalar(8)) {
864     DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI));
865     BuildMI(*I.getParent(), I, I.getDebugLoc(),
866             TII.get(TargetOpcode::SUBREG_TO_REG), DefReg)
867         .addImm(0)
868         .addReg(SrcReg)
869         .addImm(X86::sub_8bit);
870   }
871 
872   MachineInstr &AndInst =
873       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg)
874            .addReg(DefReg)
875            .addImm(1);
876 
877   constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI);
878 
879   I.eraseFromParent();
880   return true;
881 }
882 
selectAnyext(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const883 bool X86InstructionSelector::selectAnyext(MachineInstr &I,
884                                           MachineRegisterInfo &MRI,
885                                           MachineFunction &MF) const {
886   assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
887 
888   const Register DstReg = I.getOperand(0).getReg();
889   const Register SrcReg = I.getOperand(1).getReg();
890 
891   const LLT DstTy = MRI.getType(DstReg);
892   const LLT SrcTy = MRI.getType(SrcReg);
893 
894   const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
895   const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
896 
897   assert(DstRB.getID() == SrcRB.getID() &&
898          "G_ANYEXT input/output on different banks\n");
899 
900   assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
901          "G_ANYEXT incorrect operand size");
902 
903   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
904   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
905 
906   // If that's ANY_EXT of the value that lives on the floating class and goes
907   // into the vector class, just replace it with copy, as we are able to select
908   // it as a regular move.
909   if (canTurnIntoCOPY(SrcRC, DstRC))
910     return selectTurnIntoCOPY(I, MRI, SrcReg, SrcRC, DstReg, DstRC);
911 
912   if (DstRB.getID() != X86::GPRRegBankID)
913     return false;
914 
915   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
916       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
917     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
918                       << " operand\n");
919     return false;
920   }
921 
922   if (SrcRC == DstRC) {
923     I.setDesc(TII.get(X86::COPY));
924     return true;
925   }
926 
927   BuildMI(*I.getParent(), I, I.getDebugLoc(),
928           TII.get(TargetOpcode::SUBREG_TO_REG))
929       .addDef(DstReg)
930       .addImm(0)
931       .addReg(SrcReg)
932       .addImm(getSubRegIndex(SrcRC));
933 
934   I.eraseFromParent();
935   return true;
936 }
937 
selectCmp(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const938 bool X86InstructionSelector::selectCmp(MachineInstr &I,
939                                        MachineRegisterInfo &MRI,
940                                        MachineFunction &MF) const {
941   assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
942 
943   X86::CondCode CC;
944   bool SwapArgs;
945   std::tie(CC, SwapArgs) = X86::getX86ConditionCode(
946       (CmpInst::Predicate)I.getOperand(1).getPredicate());
947 
948   Register LHS = I.getOperand(2).getReg();
949   Register RHS = I.getOperand(3).getReg();
950 
951   if (SwapArgs)
952     std::swap(LHS, RHS);
953 
954   unsigned OpCmp;
955   LLT Ty = MRI.getType(LHS);
956 
957   switch (Ty.getSizeInBits()) {
958   default:
959     return false;
960   case 8:
961     OpCmp = X86::CMP8rr;
962     break;
963   case 16:
964     OpCmp = X86::CMP16rr;
965     break;
966   case 32:
967     OpCmp = X86::CMP32rr;
968     break;
969   case 64:
970     OpCmp = X86::CMP64rr;
971     break;
972   }
973 
974   MachineInstr &CmpInst =
975       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
976            .addReg(LHS)
977            .addReg(RHS);
978 
979   MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
980                                    TII.get(X86::SETCCr), I.getOperand(0).getReg()).addImm(CC);
981 
982   constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
983   constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI);
984 
985   I.eraseFromParent();
986   return true;
987 }
988 
selectFCmp(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const989 bool X86InstructionSelector::selectFCmp(MachineInstr &I,
990                                         MachineRegisterInfo &MRI,
991                                         MachineFunction &MF) const {
992   assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
993 
994   Register LhsReg = I.getOperand(2).getReg();
995   Register RhsReg = I.getOperand(3).getReg();
996   CmpInst::Predicate Predicate =
997       (CmpInst::Predicate)I.getOperand(1).getPredicate();
998 
999   // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1000   static const uint16_t SETFOpcTable[2][3] = {
1001       {X86::COND_E, X86::COND_NP, X86::AND8rr},
1002       {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1003   const uint16_t *SETFOpc = nullptr;
1004   switch (Predicate) {
1005   default:
1006     break;
1007   case CmpInst::FCMP_OEQ:
1008     SETFOpc = &SETFOpcTable[0][0];
1009     break;
1010   case CmpInst::FCMP_UNE:
1011     SETFOpc = &SETFOpcTable[1][0];
1012     break;
1013   }
1014 
1015   // Compute the opcode for the CMP instruction.
1016   unsigned OpCmp;
1017   LLT Ty = MRI.getType(LhsReg);
1018   switch (Ty.getSizeInBits()) {
1019   default:
1020     return false;
1021   case 32:
1022     OpCmp = X86::UCOMISSrr;
1023     break;
1024   case 64:
1025     OpCmp = X86::UCOMISDrr;
1026     break;
1027   }
1028 
1029   Register ResultReg = I.getOperand(0).getReg();
1030   RBI.constrainGenericRegister(
1031       ResultReg,
1032       *getRegClass(LLT::scalar(8), *RBI.getRegBank(ResultReg, MRI, TRI)), MRI);
1033   if (SETFOpc) {
1034     MachineInstr &CmpInst =
1035         *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1036              .addReg(LhsReg)
1037              .addReg(RhsReg);
1038 
1039     Register FlagReg1 = MRI.createVirtualRegister(&X86::GR8RegClass);
1040     Register FlagReg2 = MRI.createVirtualRegister(&X86::GR8RegClass);
1041     MachineInstr &Set1 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1042                                   TII.get(X86::SETCCr), FlagReg1).addImm(SETFOpc[0]);
1043     MachineInstr &Set2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1044                                   TII.get(X86::SETCCr), FlagReg2).addImm(SETFOpc[1]);
1045     MachineInstr &Set3 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1046                                   TII.get(SETFOpc[2]), ResultReg)
1047                               .addReg(FlagReg1)
1048                               .addReg(FlagReg2);
1049     constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1050     constrainSelectedInstRegOperands(Set1, TII, TRI, RBI);
1051     constrainSelectedInstRegOperands(Set2, TII, TRI, RBI);
1052     constrainSelectedInstRegOperands(Set3, TII, TRI, RBI);
1053 
1054     I.eraseFromParent();
1055     return true;
1056   }
1057 
1058   X86::CondCode CC;
1059   bool SwapArgs;
1060   std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate);
1061   assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1062 
1063   if (SwapArgs)
1064     std::swap(LhsReg, RhsReg);
1065 
1066   // Emit a compare of LHS/RHS.
1067   MachineInstr &CmpInst =
1068       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1069            .addReg(LhsReg)
1070            .addReg(RhsReg);
1071 
1072   MachineInstr &Set =
1073       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SETCCr), ResultReg).addImm(CC);
1074   constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1075   constrainSelectedInstRegOperands(Set, TII, TRI, RBI);
1076   I.eraseFromParent();
1077   return true;
1078 }
1079 
selectUadde(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1080 bool X86InstructionSelector::selectUadde(MachineInstr &I,
1081                                          MachineRegisterInfo &MRI,
1082                                          MachineFunction &MF) const {
1083   assert((I.getOpcode() == TargetOpcode::G_UADDE) && "unexpected instruction");
1084 
1085   const Register DstReg = I.getOperand(0).getReg();
1086   const Register CarryOutReg = I.getOperand(1).getReg();
1087   const Register Op0Reg = I.getOperand(2).getReg();
1088   const Register Op1Reg = I.getOperand(3).getReg();
1089   Register CarryInReg = I.getOperand(4).getReg();
1090 
1091   const LLT DstTy = MRI.getType(DstReg);
1092 
1093   if (DstTy != LLT::scalar(32))
1094     return false;
1095 
1096   // find CarryIn def instruction.
1097   MachineInstr *Def = MRI.getVRegDef(CarryInReg);
1098   while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1099     CarryInReg = Def->getOperand(1).getReg();
1100     Def = MRI.getVRegDef(CarryInReg);
1101   }
1102 
1103   unsigned Opcode;
1104   if (Def->getOpcode() == TargetOpcode::G_UADDE) {
1105     // carry set by prev ADD.
1106 
1107     BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS)
1108         .addReg(CarryInReg);
1109 
1110     if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI))
1111       return false;
1112 
1113     Opcode = X86::ADC32rr;
1114   } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) {
1115     // carry is constant, support only 0.
1116     if (*val != 0)
1117       return false;
1118 
1119     Opcode = X86::ADD32rr;
1120   } else
1121     return false;
1122 
1123   MachineInstr &AddInst =
1124       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
1125            .addReg(Op0Reg)
1126            .addReg(Op1Reg);
1127 
1128   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg)
1129       .addReg(X86::EFLAGS);
1130 
1131   if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) ||
1132       !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI))
1133     return false;
1134 
1135   I.eraseFromParent();
1136   return true;
1137 }
1138 
selectExtract(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1139 bool X86InstructionSelector::selectExtract(MachineInstr &I,
1140                                            MachineRegisterInfo &MRI,
1141                                            MachineFunction &MF) const {
1142   assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1143          "unexpected instruction");
1144 
1145   const Register DstReg = I.getOperand(0).getReg();
1146   const Register SrcReg = I.getOperand(1).getReg();
1147   int64_t Index = I.getOperand(2).getImm();
1148 
1149   const LLT DstTy = MRI.getType(DstReg);
1150   const LLT SrcTy = MRI.getType(SrcReg);
1151 
1152   // Meanwile handle vector type only.
1153   if (!DstTy.isVector())
1154     return false;
1155 
1156   if (Index % DstTy.getSizeInBits() != 0)
1157     return false; // Not extract subvector.
1158 
1159   if (Index == 0) {
1160     // Replace by extract subreg copy.
1161     if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1162       return false;
1163 
1164     I.eraseFromParent();
1165     return true;
1166   }
1167 
1168   bool HasAVX = STI.hasAVX();
1169   bool HasAVX512 = STI.hasAVX512();
1170   bool HasVLX = STI.hasVLX();
1171 
1172   if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
1173     if (HasVLX)
1174       I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr));
1175     else if (HasAVX)
1176       I.setDesc(TII.get(X86::VEXTRACTF128rr));
1177     else
1178       return false;
1179   } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1180     if (DstTy.getSizeInBits() == 128)
1181       I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr));
1182     else if (DstTy.getSizeInBits() == 256)
1183       I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr));
1184     else
1185       return false;
1186   } else
1187     return false;
1188 
1189   // Convert to X86 VEXTRACT immediate.
1190   Index = Index / DstTy.getSizeInBits();
1191   I.getOperand(2).setImm(Index);
1192 
1193   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1194 }
1195 
emitExtractSubreg(unsigned DstReg,unsigned SrcReg,MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1196 bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg,
1197                                                MachineInstr &I,
1198                                                MachineRegisterInfo &MRI,
1199                                                MachineFunction &MF) const {
1200   const LLT DstTy = MRI.getType(DstReg);
1201   const LLT SrcTy = MRI.getType(SrcReg);
1202   unsigned SubIdx = X86::NoSubRegister;
1203 
1204   if (!DstTy.isVector() || !SrcTy.isVector())
1205     return false;
1206 
1207   assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1208          "Incorrect Src/Dst register size");
1209 
1210   if (DstTy.getSizeInBits() == 128)
1211     SubIdx = X86::sub_xmm;
1212   else if (DstTy.getSizeInBits() == 256)
1213     SubIdx = X86::sub_ymm;
1214   else
1215     return false;
1216 
1217   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1218   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1219 
1220   SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
1221 
1222   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1223       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1224     LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n");
1225     return false;
1226   }
1227 
1228   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg)
1229       .addReg(SrcReg, 0, SubIdx);
1230 
1231   return true;
1232 }
1233 
emitInsertSubreg(unsigned DstReg,unsigned SrcReg,MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1234 bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg,
1235                                               MachineInstr &I,
1236                                               MachineRegisterInfo &MRI,
1237                                               MachineFunction &MF) const {
1238   const LLT DstTy = MRI.getType(DstReg);
1239   const LLT SrcTy = MRI.getType(SrcReg);
1240   unsigned SubIdx = X86::NoSubRegister;
1241 
1242   // TODO: support scalar types
1243   if (!DstTy.isVector() || !SrcTy.isVector())
1244     return false;
1245 
1246   assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1247          "Incorrect Src/Dst register size");
1248 
1249   if (SrcTy.getSizeInBits() == 128)
1250     SubIdx = X86::sub_xmm;
1251   else if (SrcTy.getSizeInBits() == 256)
1252     SubIdx = X86::sub_ymm;
1253   else
1254     return false;
1255 
1256   const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1257   const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1258 
1259   if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1260       !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1261     LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1262     return false;
1263   }
1264 
1265   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY))
1266       .addReg(DstReg, RegState::DefineNoRead, SubIdx)
1267       .addReg(SrcReg);
1268 
1269   return true;
1270 }
1271 
selectInsert(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1272 bool X86InstructionSelector::selectInsert(MachineInstr &I,
1273                                           MachineRegisterInfo &MRI,
1274                                           MachineFunction &MF) const {
1275   assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1276 
1277   const Register DstReg = I.getOperand(0).getReg();
1278   const Register SrcReg = I.getOperand(1).getReg();
1279   const Register InsertReg = I.getOperand(2).getReg();
1280   int64_t Index = I.getOperand(3).getImm();
1281 
1282   const LLT DstTy = MRI.getType(DstReg);
1283   const LLT InsertRegTy = MRI.getType(InsertReg);
1284 
1285   // Meanwile handle vector type only.
1286   if (!DstTy.isVector())
1287     return false;
1288 
1289   if (Index % InsertRegTy.getSizeInBits() != 0)
1290     return false; // Not insert subvector.
1291 
1292   if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) {
1293     // Replace by subreg copy.
1294     if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF))
1295       return false;
1296 
1297     I.eraseFromParent();
1298     return true;
1299   }
1300 
1301   bool HasAVX = STI.hasAVX();
1302   bool HasAVX512 = STI.hasAVX512();
1303   bool HasVLX = STI.hasVLX();
1304 
1305   if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1306     if (HasVLX)
1307       I.setDesc(TII.get(X86::VINSERTF32x4Z256rr));
1308     else if (HasAVX)
1309       I.setDesc(TII.get(X86::VINSERTF128rr));
1310     else
1311       return false;
1312   } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1313     if (InsertRegTy.getSizeInBits() == 128)
1314       I.setDesc(TII.get(X86::VINSERTF32x4Zrr));
1315     else if (InsertRegTy.getSizeInBits() == 256)
1316       I.setDesc(TII.get(X86::VINSERTF64x4Zrr));
1317     else
1318       return false;
1319   } else
1320     return false;
1321 
1322   // Convert to X86 VINSERT immediate.
1323   Index = Index / InsertRegTy.getSizeInBits();
1324 
1325   I.getOperand(3).setImm(Index);
1326 
1327   return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1328 }
1329 
selectUnmergeValues(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF)1330 bool X86InstructionSelector::selectUnmergeValues(
1331     MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1332   assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1333          "unexpected instruction");
1334 
1335   // Split to extracts.
1336   unsigned NumDefs = I.getNumOperands() - 1;
1337   Register SrcReg = I.getOperand(NumDefs).getReg();
1338   unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits();
1339 
1340   for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1341     MachineInstr &ExtrInst =
1342         *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1343                  TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg())
1344              .addReg(SrcReg)
1345              .addImm(Idx * DefSize);
1346 
1347     if (!select(ExtrInst))
1348       return false;
1349   }
1350 
1351   I.eraseFromParent();
1352   return true;
1353 }
1354 
selectMergeValues(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF)1355 bool X86InstructionSelector::selectMergeValues(
1356     MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1357   assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1358           I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1359          "unexpected instruction");
1360 
1361   // Split to inserts.
1362   Register DstReg = I.getOperand(0).getReg();
1363   Register SrcReg0 = I.getOperand(1).getReg();
1364 
1365   const LLT DstTy = MRI.getType(DstReg);
1366   const LLT SrcTy = MRI.getType(SrcReg0);
1367   unsigned SrcSize = SrcTy.getSizeInBits();
1368 
1369   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1370 
1371   // For the first src use insertSubReg.
1372   Register DefReg = MRI.createGenericVirtualRegister(DstTy);
1373   MRI.setRegBank(DefReg, RegBank);
1374   if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF))
1375     return false;
1376 
1377   for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1378     Register Tmp = MRI.createGenericVirtualRegister(DstTy);
1379     MRI.setRegBank(Tmp, RegBank);
1380 
1381     MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1382                                         TII.get(TargetOpcode::G_INSERT), Tmp)
1383                                     .addReg(DefReg)
1384                                     .addReg(I.getOperand(Idx).getReg())
1385                                     .addImm((Idx - 1) * SrcSize);
1386 
1387     DefReg = Tmp;
1388 
1389     if (!select(InsertInst))
1390       return false;
1391   }
1392 
1393   MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1394                                     TII.get(TargetOpcode::COPY), DstReg)
1395                                 .addReg(DefReg);
1396 
1397   if (!select(CopyInst))
1398     return false;
1399 
1400   I.eraseFromParent();
1401   return true;
1402 }
1403 
selectCondBranch(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1404 bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1405                                               MachineRegisterInfo &MRI,
1406                                               MachineFunction &MF) const {
1407   assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1408 
1409   const Register CondReg = I.getOperand(0).getReg();
1410   MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1411 
1412   MachineInstr &TestInst =
1413       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri))
1414            .addReg(CondReg)
1415            .addImm(1);
1416   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JCC_1))
1417       .addMBB(DestMBB).addImm(X86::COND_NE);
1418 
1419   constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI);
1420 
1421   I.eraseFromParent();
1422   return true;
1423 }
1424 
materializeFP(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1425 bool X86InstructionSelector::materializeFP(MachineInstr &I,
1426                                            MachineRegisterInfo &MRI,
1427                                            MachineFunction &MF) const {
1428   assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1429          "unexpected instruction");
1430 
1431   // Can't handle alternate code models yet.
1432   CodeModel::Model CM = TM.getCodeModel();
1433   if (CM != CodeModel::Small && CM != CodeModel::Large)
1434     return false;
1435 
1436   const Register DstReg = I.getOperand(0).getReg();
1437   const LLT DstTy = MRI.getType(DstReg);
1438   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1439   Align Alignment = Align(DstTy.getSizeInBytes());
1440   const DebugLoc &DbgLoc = I.getDebugLoc();
1441 
1442   unsigned Opc =
1443       getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Alignment);
1444 
1445   // Create the load from the constant pool.
1446   const ConstantFP *CFP = I.getOperand(1).getFPImm();
1447   unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Alignment);
1448   MachineInstr *LoadInst = nullptr;
1449   unsigned char OpFlag = STI.classifyLocalReference(nullptr);
1450 
1451   if (CM == CodeModel::Large && STI.is64Bit()) {
1452     // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1453     // they cannot be folded into immediate fields.
1454 
1455     Register AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass);
1456     BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg)
1457         .addConstantPoolIndex(CPI, 0, OpFlag);
1458 
1459     MachineMemOperand *MMO = MF.getMachineMemOperand(
1460         MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
1461         MF.getDataLayout().getPointerSize(), Alignment);
1462 
1463     LoadInst =
1464         addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg),
1465                      AddrReg)
1466             .addMemOperand(MMO);
1467 
1468   } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1469     // Handle the case when globals fit in our immediate field.
1470     // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1471 
1472     // x86-32 PIC requires a PIC base register for constant pools.
1473     unsigned PICBase = 0;
1474     if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1475       // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1476       // In DAGISEL the code that initialize it generated by the CGBR pass.
1477       return false; // TODO support the mode.
1478     } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1479       PICBase = X86::RIP;
1480 
1481     LoadInst = addConstantPoolReference(
1482         BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase,
1483         OpFlag);
1484   } else
1485     return false;
1486 
1487   constrainSelectedInstRegOperands(*LoadInst, TII, TRI, RBI);
1488   I.eraseFromParent();
1489   return true;
1490 }
1491 
selectImplicitDefOrPHI(MachineInstr & I,MachineRegisterInfo & MRI) const1492 bool X86InstructionSelector::selectImplicitDefOrPHI(
1493     MachineInstr &I, MachineRegisterInfo &MRI) const {
1494   assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1495           I.getOpcode() == TargetOpcode::G_PHI) &&
1496          "unexpected instruction");
1497 
1498   Register DstReg = I.getOperand(0).getReg();
1499 
1500   if (!MRI.getRegClassOrNull(DstReg)) {
1501     const LLT DstTy = MRI.getType(DstReg);
1502     const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI);
1503 
1504     if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
1505       LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1506                         << " operand\n");
1507       return false;
1508     }
1509   }
1510 
1511   if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1512     I.setDesc(TII.get(X86::IMPLICIT_DEF));
1513   else
1514     I.setDesc(TII.get(X86::PHI));
1515 
1516   return true;
1517 }
1518 
selectDivRem(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1519 bool X86InstructionSelector::selectDivRem(MachineInstr &I,
1520                                           MachineRegisterInfo &MRI,
1521                                           MachineFunction &MF) const {
1522   // The implementation of this function is taken from X86FastISel.
1523   assert((I.getOpcode() == TargetOpcode::G_SDIV ||
1524           I.getOpcode() == TargetOpcode::G_SREM ||
1525           I.getOpcode() == TargetOpcode::G_UDIV ||
1526           I.getOpcode() == TargetOpcode::G_UREM) &&
1527          "unexpected instruction");
1528 
1529   const Register DstReg = I.getOperand(0).getReg();
1530   const Register Op1Reg = I.getOperand(1).getReg();
1531   const Register Op2Reg = I.getOperand(2).getReg();
1532 
1533   const LLT RegTy = MRI.getType(DstReg);
1534   assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1535          "Arguments and return value types must match");
1536 
1537   const RegisterBank *RegRB = RBI.getRegBank(DstReg, MRI, TRI);
1538   if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1539     return false;
1540 
1541   const static unsigned NumTypes = 4; // i8, i16, i32, i64
1542   const static unsigned NumOps = 4;   // SDiv, SRem, UDiv, URem
1543   const static bool S = true;         // IsSigned
1544   const static bool U = false;        // !IsSigned
1545   const static unsigned Copy = TargetOpcode::COPY;
1546   // For the X86 IDIV instruction, in most cases the dividend
1547   // (numerator) must be in a specific register pair highreg:lowreg,
1548   // producing the quotient in lowreg and the remainder in highreg.
1549   // For most data types, to set up the instruction, the dividend is
1550   // copied into lowreg, and lowreg is sign-extended into highreg.  The
1551   // exception is i8, where the dividend is defined as a single register rather
1552   // than a register pair, and we therefore directly sign-extend the dividend
1553   // into lowreg, instead of copying, and ignore the highreg.
1554   const static struct DivRemEntry {
1555     // The following portion depends only on the data type.
1556     unsigned SizeInBits;
1557     unsigned LowInReg;  // low part of the register pair
1558     unsigned HighInReg; // high part of the register pair
1559     // The following portion depends on both the data type and the operation.
1560     struct DivRemResult {
1561       unsigned OpDivRem;        // The specific DIV/IDIV opcode to use.
1562       unsigned OpSignExtend;    // Opcode for sign-extending lowreg into
1563                                 // highreg, or copying a zero into highreg.
1564       unsigned OpCopy;          // Opcode for copying dividend into lowreg, or
1565                                 // zero/sign-extending into lowreg for i8.
1566       unsigned DivRemResultReg; // Register containing the desired result.
1567       bool IsOpSigned;          // Whether to use signed or unsigned form.
1568     } ResultTable[NumOps];
1569   } OpTable[NumTypes] = {
1570       {8,
1571        X86::AX,
1572        0,
1573        {
1574            {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S}, // SDiv
1575            {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S}, // SRem
1576            {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U},  // UDiv
1577            {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U},  // URem
1578        }},                                                // i8
1579       {16,
1580        X86::AX,
1581        X86::DX,
1582        {
1583            {X86::IDIV16r, X86::CWD, Copy, X86::AX, S},    // SDiv
1584            {X86::IDIV16r, X86::CWD, Copy, X86::DX, S},    // SRem
1585            {X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U}, // UDiv
1586            {X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U}, // URem
1587        }},                                                // i16
1588       {32,
1589        X86::EAX,
1590        X86::EDX,
1591        {
1592            {X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S},    // SDiv
1593            {X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S},    // SRem
1594            {X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U}, // UDiv
1595            {X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U}, // URem
1596        }},                                                 // i32
1597       {64,
1598        X86::RAX,
1599        X86::RDX,
1600        {
1601            {X86::IDIV64r, X86::CQO, Copy, X86::RAX, S},    // SDiv
1602            {X86::IDIV64r, X86::CQO, Copy, X86::RDX, S},    // SRem
1603            {X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U}, // UDiv
1604            {X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U}, // URem
1605        }},                                                 // i64
1606   };
1607 
1608   auto OpEntryIt = std::find_if(std::begin(OpTable), std::end(OpTable),
1609                                 [RegTy](const DivRemEntry &El) {
1610                                   return El.SizeInBits == RegTy.getSizeInBits();
1611                                 });
1612   if (OpEntryIt == std::end(OpTable))
1613     return false;
1614 
1615   unsigned OpIndex;
1616   switch (I.getOpcode()) {
1617   default:
1618     llvm_unreachable("Unexpected div/rem opcode");
1619   case TargetOpcode::G_SDIV:
1620     OpIndex = 0;
1621     break;
1622   case TargetOpcode::G_SREM:
1623     OpIndex = 1;
1624     break;
1625   case TargetOpcode::G_UDIV:
1626     OpIndex = 2;
1627     break;
1628   case TargetOpcode::G_UREM:
1629     OpIndex = 3;
1630     break;
1631   }
1632 
1633   const DivRemEntry &TypeEntry = *OpEntryIt;
1634   const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1635 
1636   const TargetRegisterClass *RegRC = getRegClass(RegTy, *RegRB);
1637   if (!RBI.constrainGenericRegister(Op1Reg, *RegRC, MRI) ||
1638       !RBI.constrainGenericRegister(Op2Reg, *RegRC, MRI) ||
1639       !RBI.constrainGenericRegister(DstReg, *RegRC, MRI)) {
1640     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1641                       << " operand\n");
1642     return false;
1643   }
1644 
1645   // Move op1 into low-order input register.
1646   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpCopy),
1647           TypeEntry.LowInReg)
1648       .addReg(Op1Reg);
1649   // Zero-extend or sign-extend into high-order input register.
1650   if (OpEntry.OpSignExtend) {
1651     if (OpEntry.IsOpSigned)
1652       BuildMI(*I.getParent(), I, I.getDebugLoc(),
1653               TII.get(OpEntry.OpSignExtend));
1654     else {
1655       Register Zero32 = MRI.createVirtualRegister(&X86::GR32RegClass);
1656       BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::MOV32r0),
1657               Zero32);
1658 
1659       // Copy the zero into the appropriate sub/super/identical physical
1660       // register. Unfortunately the operations needed are not uniform enough
1661       // to fit neatly into the table above.
1662       if (RegTy.getSizeInBits() == 16) {
1663         BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1664                 TypeEntry.HighInReg)
1665             .addReg(Zero32, 0, X86::sub_16bit);
1666       } else if (RegTy.getSizeInBits() == 32) {
1667         BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1668                 TypeEntry.HighInReg)
1669             .addReg(Zero32);
1670       } else if (RegTy.getSizeInBits() == 64) {
1671         BuildMI(*I.getParent(), I, I.getDebugLoc(),
1672                 TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
1673             .addImm(0)
1674             .addReg(Zero32)
1675             .addImm(X86::sub_32bit);
1676       }
1677     }
1678   }
1679   // Generate the DIV/IDIV instruction.
1680   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpDivRem))
1681       .addReg(Op2Reg);
1682   // For i8 remainder, we can't reference ah directly, as we'll end
1683   // up with bogus copies like %r9b = COPY %ah. Reference ax
1684   // instead to prevent ah references in a rex instruction.
1685   //
1686   // The current assumption of the fast register allocator is that isel
1687   // won't generate explicit references to the GR8_NOREX registers. If
1688   // the allocator and/or the backend get enhanced to be more robust in
1689   // that regard, this can be, and should be, removed.
1690   if ((I.getOpcode() == Instruction::SRem ||
1691        I.getOpcode() == Instruction::URem) &&
1692       OpEntry.DivRemResultReg == X86::AH && STI.is64Bit()) {
1693     Register SourceSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1694     Register ResultSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1695     BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), SourceSuperReg)
1696         .addReg(X86::AX);
1697 
1698     // Shift AX right by 8 bits instead of using AH.
1699     BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SHR16ri),
1700             ResultSuperReg)
1701         .addReg(SourceSuperReg)
1702         .addImm(8);
1703 
1704     // Now reference the 8-bit subreg of the result.
1705     BuildMI(*I.getParent(), I, I.getDebugLoc(),
1706             TII.get(TargetOpcode::SUBREG_TO_REG))
1707         .addDef(DstReg)
1708         .addImm(0)
1709         .addReg(ResultSuperReg)
1710         .addImm(X86::sub_8bit);
1711   } else {
1712     BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
1713             DstReg)
1714         .addReg(OpEntry.DivRemResultReg);
1715   }
1716   I.eraseFromParent();
1717   return true;
1718 }
1719 
selectIntrinsicWSideEffects(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1720 bool X86InstructionSelector::selectIntrinsicWSideEffects(
1721     MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const {
1722 
1723   assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
1724          "unexpected instruction");
1725 
1726   if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap)
1727     return false;
1728 
1729   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP));
1730 
1731   I.eraseFromParent();
1732   return true;
1733 }
1734 
1735 InstructionSelector *
createX86InstructionSelector(const X86TargetMachine & TM,X86Subtarget & Subtarget,X86RegisterBankInfo & RBI)1736 llvm::createX86InstructionSelector(const X86TargetMachine &TM,
1737                                    X86Subtarget &Subtarget,
1738                                    X86RegisterBankInfo &RBI) {
1739   return new X86InstructionSelector(TM, Subtarget, RBI);
1740 }
1741