1 //===- X86InstructionSelector.cpp -----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the InstructionSelector class for
10 /// X86.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13
14 #include "MCTargetDesc/X86BaseInfo.h"
15 #include "X86.h"
16 #include "X86InstrBuilder.h"
17 #include "X86InstrInfo.h"
18 #include "X86RegisterBankInfo.h"
19 #include "X86RegisterInfo.h"
20 #include "X86Subtarget.h"
21 #include "X86TargetMachine.h"
22 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
23 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
24 #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
25 #include "llvm/CodeGen/GlobalISel/Utils.h"
26 #include "llvm/CodeGen/MachineBasicBlock.h"
27 #include "llvm/CodeGen/MachineConstantPool.h"
28 #include "llvm/CodeGen/MachineFunction.h"
29 #include "llvm/CodeGen/MachineInstr.h"
30 #include "llvm/CodeGen/MachineInstrBuilder.h"
31 #include "llvm/CodeGen/MachineMemOperand.h"
32 #include "llvm/CodeGen/MachineOperand.h"
33 #include "llvm/CodeGen/MachineRegisterInfo.h"
34 #include "llvm/CodeGen/TargetOpcodes.h"
35 #include "llvm/CodeGen/TargetRegisterInfo.h"
36 #include "llvm/IR/DataLayout.h"
37 #include "llvm/IR/InstrTypes.h"
38 #include "llvm/IR/IntrinsicsX86.h"
39 #include "llvm/Support/AtomicOrdering.h"
40 #include "llvm/Support/CodeGen.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Support/LowLevelTypeImpl.h"
44 #include "llvm/Support/MathExtras.h"
45 #include "llvm/Support/raw_ostream.h"
46 #include <cassert>
47 #include <cstdint>
48 #include <tuple>
49
50 #define DEBUG_TYPE "X86-isel"
51
52 using namespace llvm;
53
54 namespace {
55
56 #define GET_GLOBALISEL_PREDICATE_BITSET
57 #include "X86GenGlobalISel.inc"
58 #undef GET_GLOBALISEL_PREDICATE_BITSET
59
60 class X86InstructionSelector : public InstructionSelector {
61 public:
62 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
63 const X86RegisterBankInfo &RBI);
64
65 bool select(MachineInstr &I) override;
getName()66 static const char *getName() { return DEBUG_TYPE; }
67
68 private:
69 /// tblgen-erated 'select' implementation, used as the initial selector for
70 /// the patterns that don't require complex C++.
71 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
72
73 // TODO: remove after supported by Tablegen-erated instruction selection.
74 unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
75 Align Alignment) const;
76
77 bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
78 MachineFunction &MF) const;
79 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
80 MachineFunction &MF) const;
81 bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
82 MachineFunction &MF) const;
83 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
84 MachineFunction &MF) const;
85 bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
86 MachineFunction &MF) const;
87 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
88 MachineFunction &MF) const;
89 bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
90 MachineFunction &MF) const;
91 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
92 MachineFunction &MF) const;
93 bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
94 MachineFunction &MF) const;
95 bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI,
96 MachineFunction &MF) const;
97 bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
98 bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
99 MachineFunction &MF);
100 bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
101 MachineFunction &MF);
102 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
103 MachineFunction &MF) const;
104 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
105 MachineFunction &MF) const;
106 bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
107 MachineFunction &MF) const;
108 bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
109 const unsigned DstReg,
110 const TargetRegisterClass *DstRC,
111 const unsigned SrcReg,
112 const TargetRegisterClass *SrcRC) const;
113 bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
114 MachineFunction &MF) const;
115 bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
116 bool selectDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
117 MachineFunction &MF) const;
118 bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI,
119 MachineFunction &MF) const;
120
121 // emit insert subreg instruction and insert it before MachineInstr &I
122 bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
123 MachineRegisterInfo &MRI, MachineFunction &MF) const;
124 // emit extract subreg instruction and insert it before MachineInstr &I
125 bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
126 MachineRegisterInfo &MRI, MachineFunction &MF) const;
127
128 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
129 const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg,
130 MachineRegisterInfo &MRI) const;
131
132 const X86TargetMachine &TM;
133 const X86Subtarget &STI;
134 const X86InstrInfo &TII;
135 const X86RegisterInfo &TRI;
136 const X86RegisterBankInfo &RBI;
137
138 #define GET_GLOBALISEL_PREDICATES_DECL
139 #include "X86GenGlobalISel.inc"
140 #undef GET_GLOBALISEL_PREDICATES_DECL
141
142 #define GET_GLOBALISEL_TEMPORARIES_DECL
143 #include "X86GenGlobalISel.inc"
144 #undef GET_GLOBALISEL_TEMPORARIES_DECL
145 };
146
147 } // end anonymous namespace
148
149 #define GET_GLOBALISEL_IMPL
150 #include "X86GenGlobalISel.inc"
151 #undef GET_GLOBALISEL_IMPL
152
X86InstructionSelector(const X86TargetMachine & TM,const X86Subtarget & STI,const X86RegisterBankInfo & RBI)153 X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
154 const X86Subtarget &STI,
155 const X86RegisterBankInfo &RBI)
156 : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
157 TRI(*STI.getRegisterInfo()), RBI(RBI),
158 #define GET_GLOBALISEL_PREDICATES_INIT
159 #include "X86GenGlobalISel.inc"
160 #undef GET_GLOBALISEL_PREDICATES_INIT
161 #define GET_GLOBALISEL_TEMPORARIES_INIT
162 #include "X86GenGlobalISel.inc"
163 #undef GET_GLOBALISEL_TEMPORARIES_INIT
164 {
165 }
166
167 // FIXME: This should be target-independent, inferred from the types declared
168 // for each class in the bank.
169 const TargetRegisterClass *
getRegClass(LLT Ty,const RegisterBank & RB) const170 X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
171 if (RB.getID() == X86::GPRRegBankID) {
172 if (Ty.getSizeInBits() <= 8)
173 return &X86::GR8RegClass;
174 if (Ty.getSizeInBits() == 16)
175 return &X86::GR16RegClass;
176 if (Ty.getSizeInBits() == 32)
177 return &X86::GR32RegClass;
178 if (Ty.getSizeInBits() == 64)
179 return &X86::GR64RegClass;
180 }
181 if (RB.getID() == X86::VECRRegBankID) {
182 if (Ty.getSizeInBits() == 32)
183 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
184 if (Ty.getSizeInBits() == 64)
185 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
186 if (Ty.getSizeInBits() == 128)
187 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
188 if (Ty.getSizeInBits() == 256)
189 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
190 if (Ty.getSizeInBits() == 512)
191 return &X86::VR512RegClass;
192 }
193
194 llvm_unreachable("Unknown RegBank!");
195 }
196
197 const TargetRegisterClass *
getRegClass(LLT Ty,unsigned Reg,MachineRegisterInfo & MRI) const198 X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg,
199 MachineRegisterInfo &MRI) const {
200 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
201 return getRegClass(Ty, RegBank);
202 }
203
getSubRegIndex(const TargetRegisterClass * RC)204 static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
205 unsigned SubIdx = X86::NoSubRegister;
206 if (RC == &X86::GR32RegClass) {
207 SubIdx = X86::sub_32bit;
208 } else if (RC == &X86::GR16RegClass) {
209 SubIdx = X86::sub_16bit;
210 } else if (RC == &X86::GR8RegClass) {
211 SubIdx = X86::sub_8bit;
212 }
213
214 return SubIdx;
215 }
216
getRegClassFromGRPhysReg(unsigned Reg)217 static const TargetRegisterClass *getRegClassFromGRPhysReg(unsigned Reg) {
218 assert(Register::isPhysicalRegister(Reg));
219 if (X86::GR64RegClass.contains(Reg))
220 return &X86::GR64RegClass;
221 if (X86::GR32RegClass.contains(Reg))
222 return &X86::GR32RegClass;
223 if (X86::GR16RegClass.contains(Reg))
224 return &X86::GR16RegClass;
225 if (X86::GR8RegClass.contains(Reg))
226 return &X86::GR8RegClass;
227
228 llvm_unreachable("Unknown RegClass for PhysReg!");
229 }
230
231 // Set X86 Opcode and constrain DestReg.
selectCopy(MachineInstr & I,MachineRegisterInfo & MRI) const232 bool X86InstructionSelector::selectCopy(MachineInstr &I,
233 MachineRegisterInfo &MRI) const {
234 Register DstReg = I.getOperand(0).getReg();
235 const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
236 const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
237
238 Register SrcReg = I.getOperand(1).getReg();
239 const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
240 const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
241
242 if (Register::isPhysicalRegister(DstReg)) {
243 assert(I.isCopy() && "Generic operators do not allow physical registers");
244
245 if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
246 DstRegBank.getID() == X86::GPRRegBankID) {
247
248 const TargetRegisterClass *SrcRC =
249 getRegClass(MRI.getType(SrcReg), SrcRegBank);
250 const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg);
251
252 if (SrcRC != DstRC) {
253 // This case can be generated by ABI lowering, performe anyext
254 Register ExtSrc = MRI.createVirtualRegister(DstRC);
255 BuildMI(*I.getParent(), I, I.getDebugLoc(),
256 TII.get(TargetOpcode::SUBREG_TO_REG))
257 .addDef(ExtSrc)
258 .addImm(0)
259 .addReg(SrcReg)
260 .addImm(getSubRegIndex(SrcRC));
261
262 I.getOperand(1).setReg(ExtSrc);
263 }
264 }
265
266 return true;
267 }
268
269 assert((!Register::isPhysicalRegister(SrcReg) || I.isCopy()) &&
270 "No phys reg on generic operators");
271 assert((DstSize == SrcSize ||
272 // Copies are a mean to setup initial types, the number of
273 // bits may not exactly match.
274 (Register::isPhysicalRegister(SrcReg) &&
275 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
276 "Copy with different width?!");
277
278 const TargetRegisterClass *DstRC =
279 getRegClass(MRI.getType(DstReg), DstRegBank);
280
281 if (SrcRegBank.getID() == X86::GPRRegBankID &&
282 DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
283 Register::isPhysicalRegister(SrcReg)) {
284 // Change the physical register to performe truncate.
285
286 const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg);
287
288 if (DstRC != SrcRC) {
289 I.getOperand(1).setSubReg(getSubRegIndex(DstRC));
290 I.getOperand(1).substPhysReg(SrcReg, TRI);
291 }
292 }
293
294 // No need to constrain SrcReg. It will get constrained when
295 // we hit another of its use or its defs.
296 // Copies do not have constraints.
297 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg);
298 if (!OldRC || !DstRC->hasSubClassEq(OldRC)) {
299 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
300 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
301 << " operand\n");
302 return false;
303 }
304 }
305 I.setDesc(TII.get(X86::COPY));
306 return true;
307 }
308
select(MachineInstr & I)309 bool X86InstructionSelector::select(MachineInstr &I) {
310 assert(I.getParent() && "Instruction should be in a basic block!");
311 assert(I.getParent()->getParent() && "Instruction should be in a function!");
312
313 MachineBasicBlock &MBB = *I.getParent();
314 MachineFunction &MF = *MBB.getParent();
315 MachineRegisterInfo &MRI = MF.getRegInfo();
316
317 unsigned Opcode = I.getOpcode();
318 if (!isPreISelGenericOpcode(Opcode)) {
319 // Certain non-generic instructions also need some special handling.
320
321 if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
322 return false;
323
324 if (I.isCopy())
325 return selectCopy(I, MRI);
326
327 return true;
328 }
329
330 assert(I.getNumOperands() == I.getNumExplicitOperands() &&
331 "Generic instruction has unexpected implicit operands\n");
332
333 if (selectImpl(I, *CoverageInfo))
334 return true;
335
336 LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
337
338 // TODO: This should be implemented by tblgen.
339 switch (I.getOpcode()) {
340 default:
341 return false;
342 case TargetOpcode::G_STORE:
343 case TargetOpcode::G_LOAD:
344 return selectLoadStoreOp(I, MRI, MF);
345 case TargetOpcode::G_PTR_ADD:
346 case TargetOpcode::G_FRAME_INDEX:
347 return selectFrameIndexOrGep(I, MRI, MF);
348 case TargetOpcode::G_GLOBAL_VALUE:
349 return selectGlobalValue(I, MRI, MF);
350 case TargetOpcode::G_CONSTANT:
351 return selectConstant(I, MRI, MF);
352 case TargetOpcode::G_FCONSTANT:
353 return materializeFP(I, MRI, MF);
354 case TargetOpcode::G_PTRTOINT:
355 case TargetOpcode::G_TRUNC:
356 return selectTruncOrPtrToInt(I, MRI, MF);
357 case TargetOpcode::G_INTTOPTR:
358 return selectCopy(I, MRI);
359 case TargetOpcode::G_ZEXT:
360 return selectZext(I, MRI, MF);
361 case TargetOpcode::G_ANYEXT:
362 return selectAnyext(I, MRI, MF);
363 case TargetOpcode::G_ICMP:
364 return selectCmp(I, MRI, MF);
365 case TargetOpcode::G_FCMP:
366 return selectFCmp(I, MRI, MF);
367 case TargetOpcode::G_UADDE:
368 return selectUadde(I, MRI, MF);
369 case TargetOpcode::G_UNMERGE_VALUES:
370 return selectUnmergeValues(I, MRI, MF);
371 case TargetOpcode::G_MERGE_VALUES:
372 case TargetOpcode::G_CONCAT_VECTORS:
373 return selectMergeValues(I, MRI, MF);
374 case TargetOpcode::G_EXTRACT:
375 return selectExtract(I, MRI, MF);
376 case TargetOpcode::G_INSERT:
377 return selectInsert(I, MRI, MF);
378 case TargetOpcode::G_BRCOND:
379 return selectCondBranch(I, MRI, MF);
380 case TargetOpcode::G_IMPLICIT_DEF:
381 case TargetOpcode::G_PHI:
382 return selectImplicitDefOrPHI(I, MRI);
383 case TargetOpcode::G_SDIV:
384 case TargetOpcode::G_UDIV:
385 case TargetOpcode::G_SREM:
386 case TargetOpcode::G_UREM:
387 return selectDivRem(I, MRI, MF);
388 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
389 return selectIntrinsicWSideEffects(I, MRI, MF);
390 }
391
392 return false;
393 }
394
getLoadStoreOp(const LLT & Ty,const RegisterBank & RB,unsigned Opc,Align Alignment) const395 unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
396 const RegisterBank &RB,
397 unsigned Opc,
398 Align Alignment) const {
399 bool Isload = (Opc == TargetOpcode::G_LOAD);
400 bool HasAVX = STI.hasAVX();
401 bool HasAVX512 = STI.hasAVX512();
402 bool HasVLX = STI.hasVLX();
403
404 if (Ty == LLT::scalar(8)) {
405 if (X86::GPRRegBankID == RB.getID())
406 return Isload ? X86::MOV8rm : X86::MOV8mr;
407 } else if (Ty == LLT::scalar(16)) {
408 if (X86::GPRRegBankID == RB.getID())
409 return Isload ? X86::MOV16rm : X86::MOV16mr;
410 } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
411 if (X86::GPRRegBankID == RB.getID())
412 return Isload ? X86::MOV32rm : X86::MOV32mr;
413 if (X86::VECRRegBankID == RB.getID())
414 return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt :
415 HasAVX ? X86::VMOVSSrm_alt :
416 X86::MOVSSrm_alt)
417 : (HasAVX512 ? X86::VMOVSSZmr :
418 HasAVX ? X86::VMOVSSmr :
419 X86::MOVSSmr);
420 } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
421 if (X86::GPRRegBankID == RB.getID())
422 return Isload ? X86::MOV64rm : X86::MOV64mr;
423 if (X86::VECRRegBankID == RB.getID())
424 return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt :
425 HasAVX ? X86::VMOVSDrm_alt :
426 X86::MOVSDrm_alt)
427 : (HasAVX512 ? X86::VMOVSDZmr :
428 HasAVX ? X86::VMOVSDmr :
429 X86::MOVSDmr);
430 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
431 if (Alignment >= Align(16))
432 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
433 : HasAVX512
434 ? X86::VMOVAPSZ128rm_NOVLX
435 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
436 : (HasVLX ? X86::VMOVAPSZ128mr
437 : HasAVX512
438 ? X86::VMOVAPSZ128mr_NOVLX
439 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
440 else
441 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
442 : HasAVX512
443 ? X86::VMOVUPSZ128rm_NOVLX
444 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
445 : (HasVLX ? X86::VMOVUPSZ128mr
446 : HasAVX512
447 ? X86::VMOVUPSZ128mr_NOVLX
448 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
449 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
450 if (Alignment >= Align(32))
451 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
452 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
453 : X86::VMOVAPSYrm)
454 : (HasVLX ? X86::VMOVAPSZ256mr
455 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
456 : X86::VMOVAPSYmr);
457 else
458 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
459 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
460 : X86::VMOVUPSYrm)
461 : (HasVLX ? X86::VMOVUPSZ256mr
462 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
463 : X86::VMOVUPSYmr);
464 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
465 if (Alignment >= Align(64))
466 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
467 else
468 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
469 }
470 return Opc;
471 }
472
473 // Fill in an address from the given instruction.
X86SelectAddress(const MachineInstr & I,const MachineRegisterInfo & MRI,X86AddressMode & AM)474 static void X86SelectAddress(const MachineInstr &I,
475 const MachineRegisterInfo &MRI,
476 X86AddressMode &AM) {
477 assert(I.getOperand(0).isReg() && "unsupported opperand.");
478 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
479 "unsupported type.");
480
481 if (I.getOpcode() == TargetOpcode::G_PTR_ADD) {
482 if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) {
483 int64_t Imm = *COff;
484 if (isInt<32>(Imm)) { // Check for displacement overflow.
485 AM.Disp = static_cast<int32_t>(Imm);
486 AM.Base.Reg = I.getOperand(1).getReg();
487 return;
488 }
489 }
490 } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
491 AM.Base.FrameIndex = I.getOperand(1).getIndex();
492 AM.BaseType = X86AddressMode::FrameIndexBase;
493 return;
494 }
495
496 // Default behavior.
497 AM.Base.Reg = I.getOperand(0).getReg();
498 }
499
selectLoadStoreOp(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const500 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
501 MachineRegisterInfo &MRI,
502 MachineFunction &MF) const {
503 unsigned Opc = I.getOpcode();
504
505 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
506 "unexpected instruction");
507
508 const Register DefReg = I.getOperand(0).getReg();
509 LLT Ty = MRI.getType(DefReg);
510 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
511
512 assert(I.hasOneMemOperand());
513 auto &MemOp = **I.memoperands_begin();
514 if (MemOp.isAtomic()) {
515 // Note: for unordered operations, we rely on the fact the appropriate MMO
516 // is already on the instruction we're mutating, and thus we don't need to
517 // make any changes. So long as we select an opcode which is capable of
518 // loading or storing the appropriate size atomically, the rest of the
519 // backend is required to respect the MMO state.
520 if (!MemOp.isUnordered()) {
521 LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
522 return false;
523 }
524 if (MemOp.getAlign() < Ty.getSizeInBits() / 8) {
525 LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
526 return false;
527 }
528 }
529
530 unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlign());
531 if (NewOpc == Opc)
532 return false;
533
534 X86AddressMode AM;
535 X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM);
536
537 I.setDesc(TII.get(NewOpc));
538 MachineInstrBuilder MIB(MF, I);
539 if (Opc == TargetOpcode::G_LOAD) {
540 I.RemoveOperand(1);
541 addFullAddress(MIB, AM);
542 } else {
543 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
544 I.RemoveOperand(1);
545 I.RemoveOperand(0);
546 addFullAddress(MIB, AM).addUse(DefReg);
547 }
548 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
549 }
550
getLeaOP(LLT Ty,const X86Subtarget & STI)551 static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
552 if (Ty == LLT::pointer(0, 64))
553 return X86::LEA64r;
554 else if (Ty == LLT::pointer(0, 32))
555 return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
556 else
557 llvm_unreachable("Can't get LEA opcode. Unsupported type.");
558 }
559
selectFrameIndexOrGep(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const560 bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
561 MachineRegisterInfo &MRI,
562 MachineFunction &MF) const {
563 unsigned Opc = I.getOpcode();
564
565 assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) &&
566 "unexpected instruction");
567
568 const Register DefReg = I.getOperand(0).getReg();
569 LLT Ty = MRI.getType(DefReg);
570
571 // Use LEA to calculate frame index and GEP
572 unsigned NewOpc = getLeaOP(Ty, STI);
573 I.setDesc(TII.get(NewOpc));
574 MachineInstrBuilder MIB(MF, I);
575
576 if (Opc == TargetOpcode::G_FRAME_INDEX) {
577 addOffset(MIB, 0);
578 } else {
579 MachineOperand &InxOp = I.getOperand(2);
580 I.addOperand(InxOp); // set IndexReg
581 InxOp.ChangeToImmediate(1); // set Scale
582 MIB.addImm(0).addReg(0);
583 }
584
585 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
586 }
587
selectGlobalValue(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const588 bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
589 MachineRegisterInfo &MRI,
590 MachineFunction &MF) const {
591 assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
592 "unexpected instruction");
593
594 auto GV = I.getOperand(1).getGlobal();
595 if (GV->isThreadLocal()) {
596 return false; // TODO: we don't support TLS yet.
597 }
598
599 // Can't handle alternate code models yet.
600 if (TM.getCodeModel() != CodeModel::Small)
601 return false;
602
603 X86AddressMode AM;
604 AM.GV = GV;
605 AM.GVOpFlags = STI.classifyGlobalReference(GV);
606
607 // TODO: The ABI requires an extra load. not supported yet.
608 if (isGlobalStubReference(AM.GVOpFlags))
609 return false;
610
611 // TODO: This reference is relative to the pic base. not supported yet.
612 if (isGlobalRelativeToPICBase(AM.GVOpFlags))
613 return false;
614
615 if (STI.isPICStyleRIPRel()) {
616 // Use rip-relative addressing.
617 assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
618 AM.Base.Reg = X86::RIP;
619 }
620
621 const Register DefReg = I.getOperand(0).getReg();
622 LLT Ty = MRI.getType(DefReg);
623 unsigned NewOpc = getLeaOP(Ty, STI);
624
625 I.setDesc(TII.get(NewOpc));
626 MachineInstrBuilder MIB(MF, I);
627
628 I.RemoveOperand(1);
629 addFullAddress(MIB, AM);
630
631 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
632 }
633
selectConstant(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const634 bool X86InstructionSelector::selectConstant(MachineInstr &I,
635 MachineRegisterInfo &MRI,
636 MachineFunction &MF) const {
637 assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
638 "unexpected instruction");
639
640 const Register DefReg = I.getOperand(0).getReg();
641 LLT Ty = MRI.getType(DefReg);
642
643 if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
644 return false;
645
646 uint64_t Val = 0;
647 if (I.getOperand(1).isCImm()) {
648 Val = I.getOperand(1).getCImm()->getZExtValue();
649 I.getOperand(1).ChangeToImmediate(Val);
650 } else if (I.getOperand(1).isImm()) {
651 Val = I.getOperand(1).getImm();
652 } else
653 llvm_unreachable("Unsupported operand type.");
654
655 unsigned NewOpc;
656 switch (Ty.getSizeInBits()) {
657 case 8:
658 NewOpc = X86::MOV8ri;
659 break;
660 case 16:
661 NewOpc = X86::MOV16ri;
662 break;
663 case 32:
664 NewOpc = X86::MOV32ri;
665 break;
666 case 64:
667 // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used
668 if (isInt<32>(Val))
669 NewOpc = X86::MOV64ri32;
670 else
671 NewOpc = X86::MOV64ri;
672 break;
673 default:
674 llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
675 }
676
677 I.setDesc(TII.get(NewOpc));
678 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
679 }
680
681 // Helper function for selectTruncOrPtrToInt and selectAnyext.
682 // Returns true if DstRC lives on a floating register class and
683 // SrcRC lives on a 128-bit vector class.
canTurnIntoCOPY(const TargetRegisterClass * DstRC,const TargetRegisterClass * SrcRC)684 static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
685 const TargetRegisterClass *SrcRC) {
686 return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass ||
687 DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) &&
688 (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass);
689 }
690
selectTurnIntoCOPY(MachineInstr & I,MachineRegisterInfo & MRI,const unsigned DstReg,const TargetRegisterClass * DstRC,const unsigned SrcReg,const TargetRegisterClass * SrcRC) const691 bool X86InstructionSelector::selectTurnIntoCOPY(
692 MachineInstr &I, MachineRegisterInfo &MRI, const unsigned DstReg,
693 const TargetRegisterClass *DstRC, const unsigned SrcReg,
694 const TargetRegisterClass *SrcRC) const {
695
696 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
697 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
698 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
699 << " operand\n");
700 return false;
701 }
702 I.setDesc(TII.get(X86::COPY));
703 return true;
704 }
705
selectTruncOrPtrToInt(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const706 bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
707 MachineRegisterInfo &MRI,
708 MachineFunction &MF) const {
709 assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
710 I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
711 "unexpected instruction");
712
713 const Register DstReg = I.getOperand(0).getReg();
714 const Register SrcReg = I.getOperand(1).getReg();
715
716 const LLT DstTy = MRI.getType(DstReg);
717 const LLT SrcTy = MRI.getType(SrcReg);
718
719 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
720 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
721
722 if (DstRB.getID() != SrcRB.getID()) {
723 LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
724 << " input/output on different banks\n");
725 return false;
726 }
727
728 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
729 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
730
731 if (!DstRC || !SrcRC)
732 return false;
733
734 // If that's truncation of the value that lives on the vector class and goes
735 // into the floating class, just replace it with copy, as we are able to
736 // select it as a regular move.
737 if (canTurnIntoCOPY(DstRC, SrcRC))
738 return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
739
740 if (DstRB.getID() != X86::GPRRegBankID)
741 return false;
742
743 unsigned SubIdx;
744 if (DstRC == SrcRC) {
745 // Nothing to be done
746 SubIdx = X86::NoSubRegister;
747 } else if (DstRC == &X86::GR32RegClass) {
748 SubIdx = X86::sub_32bit;
749 } else if (DstRC == &X86::GR16RegClass) {
750 SubIdx = X86::sub_16bit;
751 } else if (DstRC == &X86::GR8RegClass) {
752 SubIdx = X86::sub_8bit;
753 } else {
754 return false;
755 }
756
757 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
758
759 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
760 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
761 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
762 << "\n");
763 return false;
764 }
765
766 I.getOperand(1).setSubReg(SubIdx);
767
768 I.setDesc(TII.get(X86::COPY));
769 return true;
770 }
771
selectZext(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const772 bool X86InstructionSelector::selectZext(MachineInstr &I,
773 MachineRegisterInfo &MRI,
774 MachineFunction &MF) const {
775 assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
776
777 const Register DstReg = I.getOperand(0).getReg();
778 const Register SrcReg = I.getOperand(1).getReg();
779
780 const LLT DstTy = MRI.getType(DstReg);
781 const LLT SrcTy = MRI.getType(SrcReg);
782
783 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
784 "8=>32 Zext is handled by tablegen");
785 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
786 "16=>32 Zext is handled by tablegen");
787
788 const static struct ZextEntry {
789 LLT SrcTy;
790 LLT DstTy;
791 unsigned MovOp;
792 bool NeedSubregToReg;
793 } OpTable[] = {
794 {LLT::scalar(8), LLT::scalar(16), X86::MOVZX16rr8, false}, // i8 => i16
795 {LLT::scalar(8), LLT::scalar(64), X86::MOVZX32rr8, true}, // i8 => i64
796 {LLT::scalar(16), LLT::scalar(64), X86::MOVZX32rr16, true}, // i16 => i64
797 {LLT::scalar(32), LLT::scalar(64), 0, true} // i32 => i64
798 };
799
800 auto ZextEntryIt =
801 std::find_if(std::begin(OpTable), std::end(OpTable),
802 [SrcTy, DstTy](const ZextEntry &El) {
803 return El.DstTy == DstTy && El.SrcTy == SrcTy;
804 });
805
806 // Here we try to select Zext into a MOVZ and/or SUBREG_TO_REG instruction.
807 if (ZextEntryIt != std::end(OpTable)) {
808 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
809 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
810 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
811 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
812
813 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
814 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
815 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
816 << " operand\n");
817 return false;
818 }
819
820 unsigned TransitRegTo = DstReg;
821 unsigned TransitRegFrom = SrcReg;
822 if (ZextEntryIt->MovOp) {
823 // If we select Zext into MOVZ + SUBREG_TO_REG, we need to have
824 // a transit register in between: create it here.
825 if (ZextEntryIt->NeedSubregToReg) {
826 TransitRegFrom = MRI.createVirtualRegister(
827 getRegClass(LLT::scalar(32), DstReg, MRI));
828 TransitRegTo = TransitRegFrom;
829 }
830
831 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(ZextEntryIt->MovOp))
832 .addDef(TransitRegTo)
833 .addReg(SrcReg);
834 }
835 if (ZextEntryIt->NeedSubregToReg) {
836 BuildMI(*I.getParent(), I, I.getDebugLoc(),
837 TII.get(TargetOpcode::SUBREG_TO_REG))
838 .addDef(DstReg)
839 .addImm(0)
840 .addReg(TransitRegFrom)
841 .addImm(X86::sub_32bit);
842 }
843 I.eraseFromParent();
844 return true;
845 }
846
847 if (SrcTy != LLT::scalar(1))
848 return false;
849
850 unsigned AndOpc;
851 if (DstTy == LLT::scalar(8))
852 AndOpc = X86::AND8ri;
853 else if (DstTy == LLT::scalar(16))
854 AndOpc = X86::AND16ri8;
855 else if (DstTy == LLT::scalar(32))
856 AndOpc = X86::AND32ri8;
857 else if (DstTy == LLT::scalar(64))
858 AndOpc = X86::AND64ri8;
859 else
860 return false;
861
862 unsigned DefReg = SrcReg;
863 if (DstTy != LLT::scalar(8)) {
864 DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI));
865 BuildMI(*I.getParent(), I, I.getDebugLoc(),
866 TII.get(TargetOpcode::SUBREG_TO_REG), DefReg)
867 .addImm(0)
868 .addReg(SrcReg)
869 .addImm(X86::sub_8bit);
870 }
871
872 MachineInstr &AndInst =
873 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg)
874 .addReg(DefReg)
875 .addImm(1);
876
877 constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI);
878
879 I.eraseFromParent();
880 return true;
881 }
882
selectAnyext(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const883 bool X86InstructionSelector::selectAnyext(MachineInstr &I,
884 MachineRegisterInfo &MRI,
885 MachineFunction &MF) const {
886 assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
887
888 const Register DstReg = I.getOperand(0).getReg();
889 const Register SrcReg = I.getOperand(1).getReg();
890
891 const LLT DstTy = MRI.getType(DstReg);
892 const LLT SrcTy = MRI.getType(SrcReg);
893
894 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
895 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
896
897 assert(DstRB.getID() == SrcRB.getID() &&
898 "G_ANYEXT input/output on different banks\n");
899
900 assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
901 "G_ANYEXT incorrect operand size");
902
903 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
904 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
905
906 // If that's ANY_EXT of the value that lives on the floating class and goes
907 // into the vector class, just replace it with copy, as we are able to select
908 // it as a regular move.
909 if (canTurnIntoCOPY(SrcRC, DstRC))
910 return selectTurnIntoCOPY(I, MRI, SrcReg, SrcRC, DstReg, DstRC);
911
912 if (DstRB.getID() != X86::GPRRegBankID)
913 return false;
914
915 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
916 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
917 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
918 << " operand\n");
919 return false;
920 }
921
922 if (SrcRC == DstRC) {
923 I.setDesc(TII.get(X86::COPY));
924 return true;
925 }
926
927 BuildMI(*I.getParent(), I, I.getDebugLoc(),
928 TII.get(TargetOpcode::SUBREG_TO_REG))
929 .addDef(DstReg)
930 .addImm(0)
931 .addReg(SrcReg)
932 .addImm(getSubRegIndex(SrcRC));
933
934 I.eraseFromParent();
935 return true;
936 }
937
selectCmp(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const938 bool X86InstructionSelector::selectCmp(MachineInstr &I,
939 MachineRegisterInfo &MRI,
940 MachineFunction &MF) const {
941 assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
942
943 X86::CondCode CC;
944 bool SwapArgs;
945 std::tie(CC, SwapArgs) = X86::getX86ConditionCode(
946 (CmpInst::Predicate)I.getOperand(1).getPredicate());
947
948 Register LHS = I.getOperand(2).getReg();
949 Register RHS = I.getOperand(3).getReg();
950
951 if (SwapArgs)
952 std::swap(LHS, RHS);
953
954 unsigned OpCmp;
955 LLT Ty = MRI.getType(LHS);
956
957 switch (Ty.getSizeInBits()) {
958 default:
959 return false;
960 case 8:
961 OpCmp = X86::CMP8rr;
962 break;
963 case 16:
964 OpCmp = X86::CMP16rr;
965 break;
966 case 32:
967 OpCmp = X86::CMP32rr;
968 break;
969 case 64:
970 OpCmp = X86::CMP64rr;
971 break;
972 }
973
974 MachineInstr &CmpInst =
975 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
976 .addReg(LHS)
977 .addReg(RHS);
978
979 MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
980 TII.get(X86::SETCCr), I.getOperand(0).getReg()).addImm(CC);
981
982 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
983 constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI);
984
985 I.eraseFromParent();
986 return true;
987 }
988
selectFCmp(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const989 bool X86InstructionSelector::selectFCmp(MachineInstr &I,
990 MachineRegisterInfo &MRI,
991 MachineFunction &MF) const {
992 assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
993
994 Register LhsReg = I.getOperand(2).getReg();
995 Register RhsReg = I.getOperand(3).getReg();
996 CmpInst::Predicate Predicate =
997 (CmpInst::Predicate)I.getOperand(1).getPredicate();
998
999 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1000 static const uint16_t SETFOpcTable[2][3] = {
1001 {X86::COND_E, X86::COND_NP, X86::AND8rr},
1002 {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1003 const uint16_t *SETFOpc = nullptr;
1004 switch (Predicate) {
1005 default:
1006 break;
1007 case CmpInst::FCMP_OEQ:
1008 SETFOpc = &SETFOpcTable[0][0];
1009 break;
1010 case CmpInst::FCMP_UNE:
1011 SETFOpc = &SETFOpcTable[1][0];
1012 break;
1013 }
1014
1015 // Compute the opcode for the CMP instruction.
1016 unsigned OpCmp;
1017 LLT Ty = MRI.getType(LhsReg);
1018 switch (Ty.getSizeInBits()) {
1019 default:
1020 return false;
1021 case 32:
1022 OpCmp = X86::UCOMISSrr;
1023 break;
1024 case 64:
1025 OpCmp = X86::UCOMISDrr;
1026 break;
1027 }
1028
1029 Register ResultReg = I.getOperand(0).getReg();
1030 RBI.constrainGenericRegister(
1031 ResultReg,
1032 *getRegClass(LLT::scalar(8), *RBI.getRegBank(ResultReg, MRI, TRI)), MRI);
1033 if (SETFOpc) {
1034 MachineInstr &CmpInst =
1035 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1036 .addReg(LhsReg)
1037 .addReg(RhsReg);
1038
1039 Register FlagReg1 = MRI.createVirtualRegister(&X86::GR8RegClass);
1040 Register FlagReg2 = MRI.createVirtualRegister(&X86::GR8RegClass);
1041 MachineInstr &Set1 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1042 TII.get(X86::SETCCr), FlagReg1).addImm(SETFOpc[0]);
1043 MachineInstr &Set2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1044 TII.get(X86::SETCCr), FlagReg2).addImm(SETFOpc[1]);
1045 MachineInstr &Set3 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1046 TII.get(SETFOpc[2]), ResultReg)
1047 .addReg(FlagReg1)
1048 .addReg(FlagReg2);
1049 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1050 constrainSelectedInstRegOperands(Set1, TII, TRI, RBI);
1051 constrainSelectedInstRegOperands(Set2, TII, TRI, RBI);
1052 constrainSelectedInstRegOperands(Set3, TII, TRI, RBI);
1053
1054 I.eraseFromParent();
1055 return true;
1056 }
1057
1058 X86::CondCode CC;
1059 bool SwapArgs;
1060 std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate);
1061 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1062
1063 if (SwapArgs)
1064 std::swap(LhsReg, RhsReg);
1065
1066 // Emit a compare of LHS/RHS.
1067 MachineInstr &CmpInst =
1068 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1069 .addReg(LhsReg)
1070 .addReg(RhsReg);
1071
1072 MachineInstr &Set =
1073 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SETCCr), ResultReg).addImm(CC);
1074 constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1075 constrainSelectedInstRegOperands(Set, TII, TRI, RBI);
1076 I.eraseFromParent();
1077 return true;
1078 }
1079
selectUadde(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1080 bool X86InstructionSelector::selectUadde(MachineInstr &I,
1081 MachineRegisterInfo &MRI,
1082 MachineFunction &MF) const {
1083 assert((I.getOpcode() == TargetOpcode::G_UADDE) && "unexpected instruction");
1084
1085 const Register DstReg = I.getOperand(0).getReg();
1086 const Register CarryOutReg = I.getOperand(1).getReg();
1087 const Register Op0Reg = I.getOperand(2).getReg();
1088 const Register Op1Reg = I.getOperand(3).getReg();
1089 Register CarryInReg = I.getOperand(4).getReg();
1090
1091 const LLT DstTy = MRI.getType(DstReg);
1092
1093 if (DstTy != LLT::scalar(32))
1094 return false;
1095
1096 // find CarryIn def instruction.
1097 MachineInstr *Def = MRI.getVRegDef(CarryInReg);
1098 while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1099 CarryInReg = Def->getOperand(1).getReg();
1100 Def = MRI.getVRegDef(CarryInReg);
1101 }
1102
1103 unsigned Opcode;
1104 if (Def->getOpcode() == TargetOpcode::G_UADDE) {
1105 // carry set by prev ADD.
1106
1107 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS)
1108 .addReg(CarryInReg);
1109
1110 if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI))
1111 return false;
1112
1113 Opcode = X86::ADC32rr;
1114 } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) {
1115 // carry is constant, support only 0.
1116 if (*val != 0)
1117 return false;
1118
1119 Opcode = X86::ADD32rr;
1120 } else
1121 return false;
1122
1123 MachineInstr &AddInst =
1124 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
1125 .addReg(Op0Reg)
1126 .addReg(Op1Reg);
1127
1128 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg)
1129 .addReg(X86::EFLAGS);
1130
1131 if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) ||
1132 !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI))
1133 return false;
1134
1135 I.eraseFromParent();
1136 return true;
1137 }
1138
selectExtract(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1139 bool X86InstructionSelector::selectExtract(MachineInstr &I,
1140 MachineRegisterInfo &MRI,
1141 MachineFunction &MF) const {
1142 assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1143 "unexpected instruction");
1144
1145 const Register DstReg = I.getOperand(0).getReg();
1146 const Register SrcReg = I.getOperand(1).getReg();
1147 int64_t Index = I.getOperand(2).getImm();
1148
1149 const LLT DstTy = MRI.getType(DstReg);
1150 const LLT SrcTy = MRI.getType(SrcReg);
1151
1152 // Meanwile handle vector type only.
1153 if (!DstTy.isVector())
1154 return false;
1155
1156 if (Index % DstTy.getSizeInBits() != 0)
1157 return false; // Not extract subvector.
1158
1159 if (Index == 0) {
1160 // Replace by extract subreg copy.
1161 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1162 return false;
1163
1164 I.eraseFromParent();
1165 return true;
1166 }
1167
1168 bool HasAVX = STI.hasAVX();
1169 bool HasAVX512 = STI.hasAVX512();
1170 bool HasVLX = STI.hasVLX();
1171
1172 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
1173 if (HasVLX)
1174 I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr));
1175 else if (HasAVX)
1176 I.setDesc(TII.get(X86::VEXTRACTF128rr));
1177 else
1178 return false;
1179 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1180 if (DstTy.getSizeInBits() == 128)
1181 I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr));
1182 else if (DstTy.getSizeInBits() == 256)
1183 I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr));
1184 else
1185 return false;
1186 } else
1187 return false;
1188
1189 // Convert to X86 VEXTRACT immediate.
1190 Index = Index / DstTy.getSizeInBits();
1191 I.getOperand(2).setImm(Index);
1192
1193 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1194 }
1195
emitExtractSubreg(unsigned DstReg,unsigned SrcReg,MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1196 bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg,
1197 MachineInstr &I,
1198 MachineRegisterInfo &MRI,
1199 MachineFunction &MF) const {
1200 const LLT DstTy = MRI.getType(DstReg);
1201 const LLT SrcTy = MRI.getType(SrcReg);
1202 unsigned SubIdx = X86::NoSubRegister;
1203
1204 if (!DstTy.isVector() || !SrcTy.isVector())
1205 return false;
1206
1207 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1208 "Incorrect Src/Dst register size");
1209
1210 if (DstTy.getSizeInBits() == 128)
1211 SubIdx = X86::sub_xmm;
1212 else if (DstTy.getSizeInBits() == 256)
1213 SubIdx = X86::sub_ymm;
1214 else
1215 return false;
1216
1217 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1218 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1219
1220 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
1221
1222 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1223 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1224 LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n");
1225 return false;
1226 }
1227
1228 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg)
1229 .addReg(SrcReg, 0, SubIdx);
1230
1231 return true;
1232 }
1233
emitInsertSubreg(unsigned DstReg,unsigned SrcReg,MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1234 bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg,
1235 MachineInstr &I,
1236 MachineRegisterInfo &MRI,
1237 MachineFunction &MF) const {
1238 const LLT DstTy = MRI.getType(DstReg);
1239 const LLT SrcTy = MRI.getType(SrcReg);
1240 unsigned SubIdx = X86::NoSubRegister;
1241
1242 // TODO: support scalar types
1243 if (!DstTy.isVector() || !SrcTy.isVector())
1244 return false;
1245
1246 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1247 "Incorrect Src/Dst register size");
1248
1249 if (SrcTy.getSizeInBits() == 128)
1250 SubIdx = X86::sub_xmm;
1251 else if (SrcTy.getSizeInBits() == 256)
1252 SubIdx = X86::sub_ymm;
1253 else
1254 return false;
1255
1256 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1257 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1258
1259 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1260 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1261 LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1262 return false;
1263 }
1264
1265 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY))
1266 .addReg(DstReg, RegState::DefineNoRead, SubIdx)
1267 .addReg(SrcReg);
1268
1269 return true;
1270 }
1271
selectInsert(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1272 bool X86InstructionSelector::selectInsert(MachineInstr &I,
1273 MachineRegisterInfo &MRI,
1274 MachineFunction &MF) const {
1275 assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1276
1277 const Register DstReg = I.getOperand(0).getReg();
1278 const Register SrcReg = I.getOperand(1).getReg();
1279 const Register InsertReg = I.getOperand(2).getReg();
1280 int64_t Index = I.getOperand(3).getImm();
1281
1282 const LLT DstTy = MRI.getType(DstReg);
1283 const LLT InsertRegTy = MRI.getType(InsertReg);
1284
1285 // Meanwile handle vector type only.
1286 if (!DstTy.isVector())
1287 return false;
1288
1289 if (Index % InsertRegTy.getSizeInBits() != 0)
1290 return false; // Not insert subvector.
1291
1292 if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) {
1293 // Replace by subreg copy.
1294 if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF))
1295 return false;
1296
1297 I.eraseFromParent();
1298 return true;
1299 }
1300
1301 bool HasAVX = STI.hasAVX();
1302 bool HasAVX512 = STI.hasAVX512();
1303 bool HasVLX = STI.hasVLX();
1304
1305 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1306 if (HasVLX)
1307 I.setDesc(TII.get(X86::VINSERTF32x4Z256rr));
1308 else if (HasAVX)
1309 I.setDesc(TII.get(X86::VINSERTF128rr));
1310 else
1311 return false;
1312 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1313 if (InsertRegTy.getSizeInBits() == 128)
1314 I.setDesc(TII.get(X86::VINSERTF32x4Zrr));
1315 else if (InsertRegTy.getSizeInBits() == 256)
1316 I.setDesc(TII.get(X86::VINSERTF64x4Zrr));
1317 else
1318 return false;
1319 } else
1320 return false;
1321
1322 // Convert to X86 VINSERT immediate.
1323 Index = Index / InsertRegTy.getSizeInBits();
1324
1325 I.getOperand(3).setImm(Index);
1326
1327 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1328 }
1329
selectUnmergeValues(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF)1330 bool X86InstructionSelector::selectUnmergeValues(
1331 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1332 assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1333 "unexpected instruction");
1334
1335 // Split to extracts.
1336 unsigned NumDefs = I.getNumOperands() - 1;
1337 Register SrcReg = I.getOperand(NumDefs).getReg();
1338 unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits();
1339
1340 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1341 MachineInstr &ExtrInst =
1342 *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1343 TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg())
1344 .addReg(SrcReg)
1345 .addImm(Idx * DefSize);
1346
1347 if (!select(ExtrInst))
1348 return false;
1349 }
1350
1351 I.eraseFromParent();
1352 return true;
1353 }
1354
selectMergeValues(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF)1355 bool X86InstructionSelector::selectMergeValues(
1356 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) {
1357 assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1358 I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1359 "unexpected instruction");
1360
1361 // Split to inserts.
1362 Register DstReg = I.getOperand(0).getReg();
1363 Register SrcReg0 = I.getOperand(1).getReg();
1364
1365 const LLT DstTy = MRI.getType(DstReg);
1366 const LLT SrcTy = MRI.getType(SrcReg0);
1367 unsigned SrcSize = SrcTy.getSizeInBits();
1368
1369 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1370
1371 // For the first src use insertSubReg.
1372 Register DefReg = MRI.createGenericVirtualRegister(DstTy);
1373 MRI.setRegBank(DefReg, RegBank);
1374 if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF))
1375 return false;
1376
1377 for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1378 Register Tmp = MRI.createGenericVirtualRegister(DstTy);
1379 MRI.setRegBank(Tmp, RegBank);
1380
1381 MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1382 TII.get(TargetOpcode::G_INSERT), Tmp)
1383 .addReg(DefReg)
1384 .addReg(I.getOperand(Idx).getReg())
1385 .addImm((Idx - 1) * SrcSize);
1386
1387 DefReg = Tmp;
1388
1389 if (!select(InsertInst))
1390 return false;
1391 }
1392
1393 MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1394 TII.get(TargetOpcode::COPY), DstReg)
1395 .addReg(DefReg);
1396
1397 if (!select(CopyInst))
1398 return false;
1399
1400 I.eraseFromParent();
1401 return true;
1402 }
1403
selectCondBranch(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1404 bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1405 MachineRegisterInfo &MRI,
1406 MachineFunction &MF) const {
1407 assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1408
1409 const Register CondReg = I.getOperand(0).getReg();
1410 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1411
1412 MachineInstr &TestInst =
1413 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri))
1414 .addReg(CondReg)
1415 .addImm(1);
1416 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JCC_1))
1417 .addMBB(DestMBB).addImm(X86::COND_NE);
1418
1419 constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI);
1420
1421 I.eraseFromParent();
1422 return true;
1423 }
1424
materializeFP(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1425 bool X86InstructionSelector::materializeFP(MachineInstr &I,
1426 MachineRegisterInfo &MRI,
1427 MachineFunction &MF) const {
1428 assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1429 "unexpected instruction");
1430
1431 // Can't handle alternate code models yet.
1432 CodeModel::Model CM = TM.getCodeModel();
1433 if (CM != CodeModel::Small && CM != CodeModel::Large)
1434 return false;
1435
1436 const Register DstReg = I.getOperand(0).getReg();
1437 const LLT DstTy = MRI.getType(DstReg);
1438 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1439 Align Alignment = Align(DstTy.getSizeInBytes());
1440 const DebugLoc &DbgLoc = I.getDebugLoc();
1441
1442 unsigned Opc =
1443 getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Alignment);
1444
1445 // Create the load from the constant pool.
1446 const ConstantFP *CFP = I.getOperand(1).getFPImm();
1447 unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Alignment);
1448 MachineInstr *LoadInst = nullptr;
1449 unsigned char OpFlag = STI.classifyLocalReference(nullptr);
1450
1451 if (CM == CodeModel::Large && STI.is64Bit()) {
1452 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1453 // they cannot be folded into immediate fields.
1454
1455 Register AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass);
1456 BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg)
1457 .addConstantPoolIndex(CPI, 0, OpFlag);
1458
1459 MachineMemOperand *MMO = MF.getMachineMemOperand(
1460 MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
1461 MF.getDataLayout().getPointerSize(), Alignment);
1462
1463 LoadInst =
1464 addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg),
1465 AddrReg)
1466 .addMemOperand(MMO);
1467
1468 } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1469 // Handle the case when globals fit in our immediate field.
1470 // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1471
1472 // x86-32 PIC requires a PIC base register for constant pools.
1473 unsigned PICBase = 0;
1474 if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1475 // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1476 // In DAGISEL the code that initialize it generated by the CGBR pass.
1477 return false; // TODO support the mode.
1478 } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1479 PICBase = X86::RIP;
1480
1481 LoadInst = addConstantPoolReference(
1482 BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase,
1483 OpFlag);
1484 } else
1485 return false;
1486
1487 constrainSelectedInstRegOperands(*LoadInst, TII, TRI, RBI);
1488 I.eraseFromParent();
1489 return true;
1490 }
1491
selectImplicitDefOrPHI(MachineInstr & I,MachineRegisterInfo & MRI) const1492 bool X86InstructionSelector::selectImplicitDefOrPHI(
1493 MachineInstr &I, MachineRegisterInfo &MRI) const {
1494 assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1495 I.getOpcode() == TargetOpcode::G_PHI) &&
1496 "unexpected instruction");
1497
1498 Register DstReg = I.getOperand(0).getReg();
1499
1500 if (!MRI.getRegClassOrNull(DstReg)) {
1501 const LLT DstTy = MRI.getType(DstReg);
1502 const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI);
1503
1504 if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
1505 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1506 << " operand\n");
1507 return false;
1508 }
1509 }
1510
1511 if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1512 I.setDesc(TII.get(X86::IMPLICIT_DEF));
1513 else
1514 I.setDesc(TII.get(X86::PHI));
1515
1516 return true;
1517 }
1518
selectDivRem(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1519 bool X86InstructionSelector::selectDivRem(MachineInstr &I,
1520 MachineRegisterInfo &MRI,
1521 MachineFunction &MF) const {
1522 // The implementation of this function is taken from X86FastISel.
1523 assert((I.getOpcode() == TargetOpcode::G_SDIV ||
1524 I.getOpcode() == TargetOpcode::G_SREM ||
1525 I.getOpcode() == TargetOpcode::G_UDIV ||
1526 I.getOpcode() == TargetOpcode::G_UREM) &&
1527 "unexpected instruction");
1528
1529 const Register DstReg = I.getOperand(0).getReg();
1530 const Register Op1Reg = I.getOperand(1).getReg();
1531 const Register Op2Reg = I.getOperand(2).getReg();
1532
1533 const LLT RegTy = MRI.getType(DstReg);
1534 assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1535 "Arguments and return value types must match");
1536
1537 const RegisterBank *RegRB = RBI.getRegBank(DstReg, MRI, TRI);
1538 if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1539 return false;
1540
1541 const static unsigned NumTypes = 4; // i8, i16, i32, i64
1542 const static unsigned NumOps = 4; // SDiv, SRem, UDiv, URem
1543 const static bool S = true; // IsSigned
1544 const static bool U = false; // !IsSigned
1545 const static unsigned Copy = TargetOpcode::COPY;
1546 // For the X86 IDIV instruction, in most cases the dividend
1547 // (numerator) must be in a specific register pair highreg:lowreg,
1548 // producing the quotient in lowreg and the remainder in highreg.
1549 // For most data types, to set up the instruction, the dividend is
1550 // copied into lowreg, and lowreg is sign-extended into highreg. The
1551 // exception is i8, where the dividend is defined as a single register rather
1552 // than a register pair, and we therefore directly sign-extend the dividend
1553 // into lowreg, instead of copying, and ignore the highreg.
1554 const static struct DivRemEntry {
1555 // The following portion depends only on the data type.
1556 unsigned SizeInBits;
1557 unsigned LowInReg; // low part of the register pair
1558 unsigned HighInReg; // high part of the register pair
1559 // The following portion depends on both the data type and the operation.
1560 struct DivRemResult {
1561 unsigned OpDivRem; // The specific DIV/IDIV opcode to use.
1562 unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1563 // highreg, or copying a zero into highreg.
1564 unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1565 // zero/sign-extending into lowreg for i8.
1566 unsigned DivRemResultReg; // Register containing the desired result.
1567 bool IsOpSigned; // Whether to use signed or unsigned form.
1568 } ResultTable[NumOps];
1569 } OpTable[NumTypes] = {
1570 {8,
1571 X86::AX,
1572 0,
1573 {
1574 {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S}, // SDiv
1575 {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S}, // SRem
1576 {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U}, // UDiv
1577 {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U}, // URem
1578 }}, // i8
1579 {16,
1580 X86::AX,
1581 X86::DX,
1582 {
1583 {X86::IDIV16r, X86::CWD, Copy, X86::AX, S}, // SDiv
1584 {X86::IDIV16r, X86::CWD, Copy, X86::DX, S}, // SRem
1585 {X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U}, // UDiv
1586 {X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U}, // URem
1587 }}, // i16
1588 {32,
1589 X86::EAX,
1590 X86::EDX,
1591 {
1592 {X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S}, // SDiv
1593 {X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S}, // SRem
1594 {X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U}, // UDiv
1595 {X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U}, // URem
1596 }}, // i32
1597 {64,
1598 X86::RAX,
1599 X86::RDX,
1600 {
1601 {X86::IDIV64r, X86::CQO, Copy, X86::RAX, S}, // SDiv
1602 {X86::IDIV64r, X86::CQO, Copy, X86::RDX, S}, // SRem
1603 {X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U}, // UDiv
1604 {X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U}, // URem
1605 }}, // i64
1606 };
1607
1608 auto OpEntryIt = std::find_if(std::begin(OpTable), std::end(OpTable),
1609 [RegTy](const DivRemEntry &El) {
1610 return El.SizeInBits == RegTy.getSizeInBits();
1611 });
1612 if (OpEntryIt == std::end(OpTable))
1613 return false;
1614
1615 unsigned OpIndex;
1616 switch (I.getOpcode()) {
1617 default:
1618 llvm_unreachable("Unexpected div/rem opcode");
1619 case TargetOpcode::G_SDIV:
1620 OpIndex = 0;
1621 break;
1622 case TargetOpcode::G_SREM:
1623 OpIndex = 1;
1624 break;
1625 case TargetOpcode::G_UDIV:
1626 OpIndex = 2;
1627 break;
1628 case TargetOpcode::G_UREM:
1629 OpIndex = 3;
1630 break;
1631 }
1632
1633 const DivRemEntry &TypeEntry = *OpEntryIt;
1634 const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1635
1636 const TargetRegisterClass *RegRC = getRegClass(RegTy, *RegRB);
1637 if (!RBI.constrainGenericRegister(Op1Reg, *RegRC, MRI) ||
1638 !RBI.constrainGenericRegister(Op2Reg, *RegRC, MRI) ||
1639 !RBI.constrainGenericRegister(DstReg, *RegRC, MRI)) {
1640 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1641 << " operand\n");
1642 return false;
1643 }
1644
1645 // Move op1 into low-order input register.
1646 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpCopy),
1647 TypeEntry.LowInReg)
1648 .addReg(Op1Reg);
1649 // Zero-extend or sign-extend into high-order input register.
1650 if (OpEntry.OpSignExtend) {
1651 if (OpEntry.IsOpSigned)
1652 BuildMI(*I.getParent(), I, I.getDebugLoc(),
1653 TII.get(OpEntry.OpSignExtend));
1654 else {
1655 Register Zero32 = MRI.createVirtualRegister(&X86::GR32RegClass);
1656 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::MOV32r0),
1657 Zero32);
1658
1659 // Copy the zero into the appropriate sub/super/identical physical
1660 // register. Unfortunately the operations needed are not uniform enough
1661 // to fit neatly into the table above.
1662 if (RegTy.getSizeInBits() == 16) {
1663 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1664 TypeEntry.HighInReg)
1665 .addReg(Zero32, 0, X86::sub_16bit);
1666 } else if (RegTy.getSizeInBits() == 32) {
1667 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1668 TypeEntry.HighInReg)
1669 .addReg(Zero32);
1670 } else if (RegTy.getSizeInBits() == 64) {
1671 BuildMI(*I.getParent(), I, I.getDebugLoc(),
1672 TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
1673 .addImm(0)
1674 .addReg(Zero32)
1675 .addImm(X86::sub_32bit);
1676 }
1677 }
1678 }
1679 // Generate the DIV/IDIV instruction.
1680 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpDivRem))
1681 .addReg(Op2Reg);
1682 // For i8 remainder, we can't reference ah directly, as we'll end
1683 // up with bogus copies like %r9b = COPY %ah. Reference ax
1684 // instead to prevent ah references in a rex instruction.
1685 //
1686 // The current assumption of the fast register allocator is that isel
1687 // won't generate explicit references to the GR8_NOREX registers. If
1688 // the allocator and/or the backend get enhanced to be more robust in
1689 // that regard, this can be, and should be, removed.
1690 if ((I.getOpcode() == Instruction::SRem ||
1691 I.getOpcode() == Instruction::URem) &&
1692 OpEntry.DivRemResultReg == X86::AH && STI.is64Bit()) {
1693 Register SourceSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1694 Register ResultSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1695 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), SourceSuperReg)
1696 .addReg(X86::AX);
1697
1698 // Shift AX right by 8 bits instead of using AH.
1699 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SHR16ri),
1700 ResultSuperReg)
1701 .addReg(SourceSuperReg)
1702 .addImm(8);
1703
1704 // Now reference the 8-bit subreg of the result.
1705 BuildMI(*I.getParent(), I, I.getDebugLoc(),
1706 TII.get(TargetOpcode::SUBREG_TO_REG))
1707 .addDef(DstReg)
1708 .addImm(0)
1709 .addReg(ResultSuperReg)
1710 .addImm(X86::sub_8bit);
1711 } else {
1712 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
1713 DstReg)
1714 .addReg(OpEntry.DivRemResultReg);
1715 }
1716 I.eraseFromParent();
1717 return true;
1718 }
1719
selectIntrinsicWSideEffects(MachineInstr & I,MachineRegisterInfo & MRI,MachineFunction & MF) const1720 bool X86InstructionSelector::selectIntrinsicWSideEffects(
1721 MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const {
1722
1723 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
1724 "unexpected instruction");
1725
1726 if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap)
1727 return false;
1728
1729 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP));
1730
1731 I.eraseFromParent();
1732 return true;
1733 }
1734
1735 InstructionSelector *
createX86InstructionSelector(const X86TargetMachine & TM,X86Subtarget & Subtarget,X86RegisterBankInfo & RBI)1736 llvm::createX86InstructionSelector(const X86TargetMachine &TM,
1737 X86Subtarget &Subtarget,
1738 X86RegisterBankInfo &RBI) {
1739 return new X86InstructionSelector(TM, Subtarget, RBI);
1740 }
1741