1 //==--- InstrEmitter.cpp - Emit MachineInstrs for the SelectionDAG class ---==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the Emit routines for the SelectionDAG class, which creates
10 // MachineInstrs based on the decisions of the SelectionDAG instruction
11 // selection.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "InstrEmitter.h"
16 #include "SDNodeDbgValue.h"
17 #include "llvm/ADT/Statistic.h"
18 #include "llvm/CodeGen/MachineConstantPool.h"
19 #include "llvm/CodeGen/MachineFunction.h"
20 #include "llvm/CodeGen/MachineInstrBuilder.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/CodeGen/SelectionDAG.h"
23 #include "llvm/CodeGen/StackMaps.h"
24 #include "llvm/CodeGen/TargetInstrInfo.h"
25 #include "llvm/CodeGen/TargetLowering.h"
26 #include "llvm/CodeGen/TargetSubtargetInfo.h"
27 #include "llvm/IR/DataLayout.h"
28 #include "llvm/IR/DebugInfo.h"
29 #include "llvm/IR/PseudoProbe.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include "llvm/Support/MathExtras.h"
33 #include "llvm/Target/TargetMachine.h"
34 using namespace llvm;
35 
36 #define DEBUG_TYPE "instr-emitter"
37 
38 /// MinRCSize - Smallest register class we allow when constraining virtual
39 /// registers.  If satisfying all register class constraints would require
40 /// using a smaller register class, emit a COPY to a new virtual register
41 /// instead.
42 const unsigned MinRCSize = 4;
43 
44 /// CountResults - The results of target nodes have register or immediate
45 /// operands first, then an optional chain, and optional glue operands (which do
46 /// not go into the resulting MachineInstr).
CountResults(SDNode * Node)47 unsigned InstrEmitter::CountResults(SDNode *Node) {
48   unsigned N = Node->getNumValues();
49   while (N && Node->getValueType(N - 1) == MVT::Glue)
50     --N;
51   if (N && Node->getValueType(N - 1) == MVT::Other)
52     --N;    // Skip over chain result.
53   return N;
54 }
55 
56 /// countOperands - The inputs to target nodes have any actual inputs first,
57 /// followed by an optional chain operand, then an optional glue operand.
58 /// Compute the number of actual operands that will go into the resulting
59 /// MachineInstr.
60 ///
61 /// Also count physreg RegisterSDNode and RegisterMaskSDNode operands preceding
62 /// the chain and glue. These operands may be implicit on the machine instr.
countOperands(SDNode * Node,unsigned NumExpUses,unsigned & NumImpUses)63 static unsigned countOperands(SDNode *Node, unsigned NumExpUses,
64                               unsigned &NumImpUses) {
65   unsigned N = Node->getNumOperands();
66   while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
67     --N;
68   if (N && Node->getOperand(N - 1).getValueType() == MVT::Other)
69     --N; // Ignore chain if it exists.
70 
71   // Count RegisterSDNode and RegisterMaskSDNode operands for NumImpUses.
72   NumImpUses = N - NumExpUses;
73   for (unsigned I = N; I > NumExpUses; --I) {
74     if (isa<RegisterMaskSDNode>(Node->getOperand(I - 1)))
75       continue;
76     if (RegisterSDNode *RN = dyn_cast<RegisterSDNode>(Node->getOperand(I - 1)))
77       if (Register::isPhysicalRegister(RN->getReg()))
78         continue;
79     NumImpUses = N - I;
80     break;
81   }
82 
83   return N;
84 }
85 
86 /// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an
87 /// implicit physical register output.
88 void InstrEmitter::
EmitCopyFromReg(SDNode * Node,unsigned ResNo,bool IsClone,bool IsCloned,Register SrcReg,DenseMap<SDValue,Register> & VRBaseMap)89 EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned,
90                 Register SrcReg, DenseMap<SDValue, Register> &VRBaseMap) {
91   Register VRBase;
92   if (SrcReg.isVirtual()) {
93     // Just use the input register directly!
94     SDValue Op(Node, ResNo);
95     if (IsClone)
96       VRBaseMap.erase(Op);
97     bool isNew = VRBaseMap.insert(std::make_pair(Op, SrcReg)).second;
98     (void)isNew; // Silence compiler warning.
99     assert(isNew && "Node emitted out of order - early");
100     return;
101   }
102 
103   // If the node is only used by a CopyToReg and the dest reg is a vreg, use
104   // the CopyToReg'd destination register instead of creating a new vreg.
105   bool MatchReg = true;
106   const TargetRegisterClass *UseRC = nullptr;
107   MVT VT = Node->getSimpleValueType(ResNo);
108 
109   // Stick to the preferred register classes for legal types.
110   if (TLI->isTypeLegal(VT))
111     UseRC = TLI->getRegClassFor(VT, Node->isDivergent());
112 
113   if (!IsClone && !IsCloned)
114     for (SDNode *User : Node->uses()) {
115       bool Match = true;
116       if (User->getOpcode() == ISD::CopyToReg &&
117           User->getOperand(2).getNode() == Node &&
118           User->getOperand(2).getResNo() == ResNo) {
119         Register DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
120         if (DestReg.isVirtual()) {
121           VRBase = DestReg;
122           Match = false;
123         } else if (DestReg != SrcReg)
124           Match = false;
125       } else {
126         for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
127           SDValue Op = User->getOperand(i);
128           if (Op.getNode() != Node || Op.getResNo() != ResNo)
129             continue;
130           MVT VT = Node->getSimpleValueType(Op.getResNo());
131           if (VT == MVT::Other || VT == MVT::Glue)
132             continue;
133           Match = false;
134           if (User->isMachineOpcode()) {
135             const MCInstrDesc &II = TII->get(User->getMachineOpcode());
136             const TargetRegisterClass *RC = nullptr;
137             if (i+II.getNumDefs() < II.getNumOperands()) {
138               RC = TRI->getAllocatableClass(
139                 TII->getRegClass(II, i+II.getNumDefs(), TRI, *MF));
140             }
141             if (!UseRC)
142               UseRC = RC;
143             else if (RC) {
144               const TargetRegisterClass *ComRC =
145                 TRI->getCommonSubClass(UseRC, RC);
146               // If multiple uses expect disjoint register classes, we emit
147               // copies in AddRegisterOperand.
148               if (ComRC)
149                 UseRC = ComRC;
150             }
151           }
152         }
153       }
154       MatchReg &= Match;
155       if (VRBase)
156         break;
157     }
158 
159   const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
160   SrcRC = TRI->getMinimalPhysRegClass(SrcReg, VT);
161 
162   // Figure out the register class to create for the destreg.
163   if (VRBase) {
164     DstRC = MRI->getRegClass(VRBase);
165   } else if (UseRC) {
166     assert(TRI->isTypeLegalForClass(*UseRC, VT) &&
167            "Incompatible phys register def and uses!");
168     DstRC = UseRC;
169   } else
170     DstRC = SrcRC;
171 
172   // If all uses are reading from the src physical register and copying the
173   // register is either impossible or very expensive, then don't create a copy.
174   if (MatchReg && SrcRC->getCopyCost() < 0) {
175     VRBase = SrcReg;
176   } else {
177     // Create the reg, emit the copy.
178     VRBase = MRI->createVirtualRegister(DstRC);
179     BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TargetOpcode::COPY),
180             VRBase).addReg(SrcReg);
181   }
182 
183   SDValue Op(Node, ResNo);
184   if (IsClone)
185     VRBaseMap.erase(Op);
186   bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second;
187   (void)isNew; // Silence compiler warning.
188   assert(isNew && "Node emitted out of order - early");
189 }
190 
CreateVirtualRegisters(SDNode * Node,MachineInstrBuilder & MIB,const MCInstrDesc & II,bool IsClone,bool IsCloned,DenseMap<SDValue,Register> & VRBaseMap)191 void InstrEmitter::CreateVirtualRegisters(SDNode *Node,
192                                        MachineInstrBuilder &MIB,
193                                        const MCInstrDesc &II,
194                                        bool IsClone, bool IsCloned,
195                                        DenseMap<SDValue, Register> &VRBaseMap) {
196   assert(Node->getMachineOpcode() != TargetOpcode::IMPLICIT_DEF &&
197          "IMPLICIT_DEF should have been handled as a special case elsewhere!");
198 
199   unsigned NumResults = CountResults(Node);
200   bool HasVRegVariadicDefs = !MF->getTarget().usesPhysRegsForValues() &&
201                              II.isVariadic() && II.variadicOpsAreDefs();
202   unsigned NumVRegs = HasVRegVariadicDefs ? NumResults : II.getNumDefs();
203   if (Node->getMachineOpcode() == TargetOpcode::STATEPOINT)
204     NumVRegs = NumResults;
205   for (unsigned i = 0; i < NumVRegs; ++i) {
206     // If the specific node value is only used by a CopyToReg and the dest reg
207     // is a vreg in the same register class, use the CopyToReg'd destination
208     // register instead of creating a new vreg.
209     Register VRBase;
210     const TargetRegisterClass *RC =
211       TRI->getAllocatableClass(TII->getRegClass(II, i, TRI, *MF));
212     // Always let the value type influence the used register class. The
213     // constraints on the instruction may be too lax to represent the value
214     // type correctly. For example, a 64-bit float (X86::FR64) can't live in
215     // the 32-bit float super-class (X86::FR32).
216     if (i < NumResults && TLI->isTypeLegal(Node->getSimpleValueType(i))) {
217       const TargetRegisterClass *VTRC = TLI->getRegClassFor(
218           Node->getSimpleValueType(i),
219           (Node->isDivergent() || (RC && TRI->isDivergentRegClass(RC))));
220       if (RC)
221         VTRC = TRI->getCommonSubClass(RC, VTRC);
222       if (VTRC)
223         RC = VTRC;
224     }
225 
226     if (II.OpInfo != nullptr && II.OpInfo[i].isOptionalDef()) {
227       // Optional def must be a physical register.
228       VRBase = cast<RegisterSDNode>(Node->getOperand(i-NumResults))->getReg();
229       assert(VRBase.isPhysical());
230       MIB.addReg(VRBase, RegState::Define);
231     }
232 
233     if (!VRBase && !IsClone && !IsCloned)
234       for (SDNode *User : Node->uses()) {
235         if (User->getOpcode() == ISD::CopyToReg &&
236             User->getOperand(2).getNode() == Node &&
237             User->getOperand(2).getResNo() == i) {
238           unsigned Reg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
239           if (Register::isVirtualRegister(Reg)) {
240             const TargetRegisterClass *RegRC = MRI->getRegClass(Reg);
241             if (RegRC == RC) {
242               VRBase = Reg;
243               MIB.addReg(VRBase, RegState::Define);
244               break;
245             }
246           }
247         }
248       }
249 
250     // Create the result registers for this node and add the result regs to
251     // the machine instruction.
252     if (VRBase == 0) {
253       assert(RC && "Isn't a register operand!");
254       VRBase = MRI->createVirtualRegister(RC);
255       MIB.addReg(VRBase, RegState::Define);
256     }
257 
258     // If this def corresponds to a result of the SDNode insert the VRBase into
259     // the lookup map.
260     if (i < NumResults) {
261       SDValue Op(Node, i);
262       if (IsClone)
263         VRBaseMap.erase(Op);
264       bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second;
265       (void)isNew; // Silence compiler warning.
266       assert(isNew && "Node emitted out of order - early");
267     }
268   }
269 }
270 
271 /// getVR - Return the virtual register corresponding to the specified result
272 /// of the specified node.
getVR(SDValue Op,DenseMap<SDValue,Register> & VRBaseMap)273 Register InstrEmitter::getVR(SDValue Op,
274                              DenseMap<SDValue, Register> &VRBaseMap) {
275   if (Op.isMachineOpcode() &&
276       Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) {
277     // Add an IMPLICIT_DEF instruction before every use.
278     // IMPLICIT_DEF can produce any type of result so its MCInstrDesc
279     // does not include operand register class info.
280     const TargetRegisterClass *RC = TLI->getRegClassFor(
281         Op.getSimpleValueType(), Op.getNode()->isDivergent());
282     Register VReg = MRI->createVirtualRegister(RC);
283     BuildMI(*MBB, InsertPos, Op.getDebugLoc(),
284             TII->get(TargetOpcode::IMPLICIT_DEF), VReg);
285     return VReg;
286   }
287 
288   DenseMap<SDValue, Register>::iterator I = VRBaseMap.find(Op);
289   assert(I != VRBaseMap.end() && "Node emitted out of order - late");
290   return I->second;
291 }
292 
293 
294 /// AddRegisterOperand - Add the specified register as an operand to the
295 /// specified machine instr. Insert register copies if the register is
296 /// not in the required register class.
297 void
AddRegisterOperand(MachineInstrBuilder & MIB,SDValue Op,unsigned IIOpNum,const MCInstrDesc * II,DenseMap<SDValue,Register> & VRBaseMap,bool IsDebug,bool IsClone,bool IsCloned)298 InstrEmitter::AddRegisterOperand(MachineInstrBuilder &MIB,
299                                  SDValue Op,
300                                  unsigned IIOpNum,
301                                  const MCInstrDesc *II,
302                                  DenseMap<SDValue, Register> &VRBaseMap,
303                                  bool IsDebug, bool IsClone, bool IsCloned) {
304   assert(Op.getValueType() != MVT::Other &&
305          Op.getValueType() != MVT::Glue &&
306          "Chain and glue operands should occur at end of operand list!");
307   // Get/emit the operand.
308   Register VReg = getVR(Op, VRBaseMap);
309 
310   const MCInstrDesc &MCID = MIB->getDesc();
311   bool isOptDef = IIOpNum < MCID.getNumOperands() &&
312     MCID.OpInfo[IIOpNum].isOptionalDef();
313 
314   // If the instruction requires a register in a different class, create
315   // a new virtual register and copy the value into it, but first attempt to
316   // shrink VReg's register class within reason.  For example, if VReg == GR32
317   // and II requires a GR32_NOSP, just constrain VReg to GR32_NOSP.
318   if (II) {
319     const TargetRegisterClass *OpRC = nullptr;
320     if (IIOpNum < II->getNumOperands())
321       OpRC = TII->getRegClass(*II, IIOpNum, TRI, *MF);
322 
323     if (OpRC) {
324       const TargetRegisterClass *ConstrainedRC
325         = MRI->constrainRegClass(VReg, OpRC, MinRCSize);
326       if (!ConstrainedRC) {
327         OpRC = TRI->getAllocatableClass(OpRC);
328         assert(OpRC && "Constraints cannot be fulfilled for allocation");
329         Register NewVReg = MRI->createVirtualRegister(OpRC);
330         BuildMI(*MBB, InsertPos, Op.getNode()->getDebugLoc(),
331                 TII->get(TargetOpcode::COPY), NewVReg).addReg(VReg);
332         VReg = NewVReg;
333       } else {
334         assert(ConstrainedRC->isAllocatable() &&
335            "Constraining an allocatable VReg produced an unallocatable class?");
336       }
337     }
338   }
339 
340   // If this value has only one use, that use is a kill. This is a
341   // conservative approximation. InstrEmitter does trivial coalescing
342   // with CopyFromReg nodes, so don't emit kill flags for them.
343   // Avoid kill flags on Schedule cloned nodes, since there will be
344   // multiple uses.
345   // Tied operands are never killed, so we need to check that. And that
346   // means we need to determine the index of the operand.
347   bool isKill = Op.hasOneUse() &&
348                 Op.getNode()->getOpcode() != ISD::CopyFromReg &&
349                 !IsDebug &&
350                 !(IsClone || IsCloned);
351   if (isKill) {
352     unsigned Idx = MIB->getNumOperands();
353     while (Idx > 0 &&
354            MIB->getOperand(Idx-1).isReg() &&
355            MIB->getOperand(Idx-1).isImplicit())
356       --Idx;
357     bool isTied = MCID.getOperandConstraint(Idx, MCOI::TIED_TO) != -1;
358     if (isTied)
359       isKill = false;
360   }
361 
362   MIB.addReg(VReg, getDefRegState(isOptDef) | getKillRegState(isKill) |
363              getDebugRegState(IsDebug));
364 }
365 
366 /// AddOperand - Add the specified operand to the specified machine instr.  II
367 /// specifies the instruction information for the node, and IIOpNum is the
368 /// operand number (in the II) that we are adding.
AddOperand(MachineInstrBuilder & MIB,SDValue Op,unsigned IIOpNum,const MCInstrDesc * II,DenseMap<SDValue,Register> & VRBaseMap,bool IsDebug,bool IsClone,bool IsCloned)369 void InstrEmitter::AddOperand(MachineInstrBuilder &MIB,
370                               SDValue Op,
371                               unsigned IIOpNum,
372                               const MCInstrDesc *II,
373                               DenseMap<SDValue, Register> &VRBaseMap,
374                               bool IsDebug, bool IsClone, bool IsCloned) {
375   if (Op.isMachineOpcode()) {
376     AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
377                        IsDebug, IsClone, IsCloned);
378   } else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
379     MIB.addImm(C->getSExtValue());
380   } else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Op)) {
381     MIB.addFPImm(F->getConstantFPValue());
382   } else if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Op)) {
383     Register VReg = R->getReg();
384     MVT OpVT = Op.getSimpleValueType();
385     const TargetRegisterClass *IIRC =
386         II ? TRI->getAllocatableClass(TII->getRegClass(*II, IIOpNum, TRI, *MF))
387            : nullptr;
388     const TargetRegisterClass *OpRC =
389         TLI->isTypeLegal(OpVT)
390             ? TLI->getRegClassFor(OpVT,
391                                   Op.getNode()->isDivergent() ||
392                                       (IIRC && TRI->isDivergentRegClass(IIRC)))
393             : nullptr;
394 
395     if (OpRC && IIRC && OpRC != IIRC && Register::isVirtualRegister(VReg)) {
396       Register NewVReg = MRI->createVirtualRegister(IIRC);
397       BuildMI(*MBB, InsertPos, Op.getNode()->getDebugLoc(),
398                TII->get(TargetOpcode::COPY), NewVReg).addReg(VReg);
399       VReg = NewVReg;
400     }
401     // Turn additional physreg operands into implicit uses on non-variadic
402     // instructions. This is used by call and return instructions passing
403     // arguments in registers.
404     bool Imp = II && (IIOpNum >= II->getNumOperands() && !II->isVariadic());
405     MIB.addReg(VReg, getImplRegState(Imp));
406   } else if (RegisterMaskSDNode *RM = dyn_cast<RegisterMaskSDNode>(Op)) {
407     MIB.addRegMask(RM->getRegMask());
408   } else if (GlobalAddressSDNode *TGA = dyn_cast<GlobalAddressSDNode>(Op)) {
409     MIB.addGlobalAddress(TGA->getGlobal(), TGA->getOffset(),
410                          TGA->getTargetFlags());
411   } else if (BasicBlockSDNode *BBNode = dyn_cast<BasicBlockSDNode>(Op)) {
412     MIB.addMBB(BBNode->getBasicBlock());
413   } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
414     MIB.addFrameIndex(FI->getIndex());
415   } else if (JumpTableSDNode *JT = dyn_cast<JumpTableSDNode>(Op)) {
416     MIB.addJumpTableIndex(JT->getIndex(), JT->getTargetFlags());
417   } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op)) {
418     int Offset = CP->getOffset();
419     Align Alignment = CP->getAlign();
420 
421     unsigned Idx;
422     MachineConstantPool *MCP = MF->getConstantPool();
423     if (CP->isMachineConstantPoolEntry())
424       Idx = MCP->getConstantPoolIndex(CP->getMachineCPVal(), Alignment);
425     else
426       Idx = MCP->getConstantPoolIndex(CP->getConstVal(), Alignment);
427     MIB.addConstantPoolIndex(Idx, Offset, CP->getTargetFlags());
428   } else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op)) {
429     MIB.addExternalSymbol(ES->getSymbol(), ES->getTargetFlags());
430   } else if (auto *SymNode = dyn_cast<MCSymbolSDNode>(Op)) {
431     MIB.addSym(SymNode->getMCSymbol());
432   } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Op)) {
433     MIB.addBlockAddress(BA->getBlockAddress(),
434                         BA->getOffset(),
435                         BA->getTargetFlags());
436   } else if (TargetIndexSDNode *TI = dyn_cast<TargetIndexSDNode>(Op)) {
437     MIB.addTargetIndex(TI->getIndex(), TI->getOffset(), TI->getTargetFlags());
438   } else {
439     assert(Op.getValueType() != MVT::Other &&
440            Op.getValueType() != MVT::Glue &&
441            "Chain and glue operands should occur at end of operand list!");
442     AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
443                        IsDebug, IsClone, IsCloned);
444   }
445 }
446 
ConstrainForSubReg(Register VReg,unsigned SubIdx,MVT VT,bool isDivergent,const DebugLoc & DL)447 Register InstrEmitter::ConstrainForSubReg(Register VReg, unsigned SubIdx,
448                                           MVT VT, bool isDivergent, const DebugLoc &DL) {
449   const TargetRegisterClass *VRC = MRI->getRegClass(VReg);
450   const TargetRegisterClass *RC = TRI->getSubClassWithSubReg(VRC, SubIdx);
451 
452   // RC is a sub-class of VRC that supports SubIdx.  Try to constrain VReg
453   // within reason.
454   if (RC && RC != VRC)
455     RC = MRI->constrainRegClass(VReg, RC, MinRCSize);
456 
457   // VReg has been adjusted.  It can be used with SubIdx operands now.
458   if (RC)
459     return VReg;
460 
461   // VReg couldn't be reasonably constrained.  Emit a COPY to a new virtual
462   // register instead.
463   RC = TRI->getSubClassWithSubReg(TLI->getRegClassFor(VT, isDivergent), SubIdx);
464   assert(RC && "No legal register class for VT supports that SubIdx");
465   Register NewReg = MRI->createVirtualRegister(RC);
466   BuildMI(*MBB, InsertPos, DL, TII->get(TargetOpcode::COPY), NewReg)
467     .addReg(VReg);
468   return NewReg;
469 }
470 
471 /// EmitSubregNode - Generate machine code for subreg nodes.
472 ///
EmitSubregNode(SDNode * Node,DenseMap<SDValue,Register> & VRBaseMap,bool IsClone,bool IsCloned)473 void InstrEmitter::EmitSubregNode(SDNode *Node,
474                                   DenseMap<SDValue, Register> &VRBaseMap,
475                                   bool IsClone, bool IsCloned) {
476   Register VRBase;
477   unsigned Opc = Node->getMachineOpcode();
478 
479   // If the node is only used by a CopyToReg and the dest reg is a vreg, use
480   // the CopyToReg'd destination register instead of creating a new vreg.
481   for (SDNode *User : Node->uses()) {
482     if (User->getOpcode() == ISD::CopyToReg &&
483         User->getOperand(2).getNode() == Node) {
484       Register DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
485       if (DestReg.isVirtual()) {
486         VRBase = DestReg;
487         break;
488       }
489     }
490   }
491 
492   if (Opc == TargetOpcode::EXTRACT_SUBREG) {
493     // EXTRACT_SUBREG is lowered as %dst = COPY %src:sub.  There are no
494     // constraints on the %dst register, COPY can target all legal register
495     // classes.
496     unsigned SubIdx = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
497     const TargetRegisterClass *TRC =
498       TLI->getRegClassFor(Node->getSimpleValueType(0), Node->isDivergent());
499 
500     Register Reg;
501     MachineInstr *DefMI;
502     RegisterSDNode *R = dyn_cast<RegisterSDNode>(Node->getOperand(0));
503     if (R && Register::isPhysicalRegister(R->getReg())) {
504       Reg = R->getReg();
505       DefMI = nullptr;
506     } else {
507       Reg = R ? R->getReg() : getVR(Node->getOperand(0), VRBaseMap);
508       DefMI = MRI->getVRegDef(Reg);
509     }
510 
511     Register SrcReg, DstReg;
512     unsigned DefSubIdx;
513     if (DefMI &&
514         TII->isCoalescableExtInstr(*DefMI, SrcReg, DstReg, DefSubIdx) &&
515         SubIdx == DefSubIdx &&
516         TRC == MRI->getRegClass(SrcReg)) {
517       // Optimize these:
518       // r1025 = s/zext r1024, 4
519       // r1026 = extract_subreg r1025, 4
520       // to a copy
521       // r1026 = copy r1024
522       VRBase = MRI->createVirtualRegister(TRC);
523       BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
524               TII->get(TargetOpcode::COPY), VRBase).addReg(SrcReg);
525       MRI->clearKillFlags(SrcReg);
526     } else {
527       // Reg may not support a SubIdx sub-register, and we may need to
528       // constrain its register class or issue a COPY to a compatible register
529       // class.
530       if (Reg.isVirtual())
531         Reg = ConstrainForSubReg(Reg, SubIdx,
532                                  Node->getOperand(0).getSimpleValueType(),
533                                  Node->isDivergent(), Node->getDebugLoc());
534       // Create the destreg if it is missing.
535       if (!VRBase)
536         VRBase = MRI->createVirtualRegister(TRC);
537 
538       // Create the extract_subreg machine instruction.
539       MachineInstrBuilder CopyMI =
540           BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
541                   TII->get(TargetOpcode::COPY), VRBase);
542       if (Reg.isVirtual())
543         CopyMI.addReg(Reg, 0, SubIdx);
544       else
545         CopyMI.addReg(TRI->getSubReg(Reg, SubIdx));
546     }
547   } else if (Opc == TargetOpcode::INSERT_SUBREG ||
548              Opc == TargetOpcode::SUBREG_TO_REG) {
549     SDValue N0 = Node->getOperand(0);
550     SDValue N1 = Node->getOperand(1);
551     SDValue N2 = Node->getOperand(2);
552     unsigned SubIdx = cast<ConstantSDNode>(N2)->getZExtValue();
553 
554     // Figure out the register class to create for the destreg.  It should be
555     // the largest legal register class supporting SubIdx sub-registers.
556     // RegisterCoalescer will constrain it further if it decides to eliminate
557     // the INSERT_SUBREG instruction.
558     //
559     //   %dst = INSERT_SUBREG %src, %sub, SubIdx
560     //
561     // is lowered by TwoAddressInstructionPass to:
562     //
563     //   %dst = COPY %src
564     //   %dst:SubIdx = COPY %sub
565     //
566     // There is no constraint on the %src register class.
567     //
568     const TargetRegisterClass *SRC =
569         TLI->getRegClassFor(Node->getSimpleValueType(0), Node->isDivergent());
570     SRC = TRI->getSubClassWithSubReg(SRC, SubIdx);
571     assert(SRC && "No register class supports VT and SubIdx for INSERT_SUBREG");
572 
573     if (VRBase == 0 || !SRC->hasSubClassEq(MRI->getRegClass(VRBase)))
574       VRBase = MRI->createVirtualRegister(SRC);
575 
576     // Create the insert_subreg or subreg_to_reg machine instruction.
577     MachineInstrBuilder MIB =
578       BuildMI(*MF, Node->getDebugLoc(), TII->get(Opc), VRBase);
579 
580     // If creating a subreg_to_reg, then the first input operand
581     // is an implicit value immediate, otherwise it's a register
582     if (Opc == TargetOpcode::SUBREG_TO_REG) {
583       const ConstantSDNode *SD = cast<ConstantSDNode>(N0);
584       MIB.addImm(SD->getZExtValue());
585     } else
586       AddOperand(MIB, N0, 0, nullptr, VRBaseMap, /*IsDebug=*/false,
587                  IsClone, IsCloned);
588     // Add the subregister being inserted
589     AddOperand(MIB, N1, 0, nullptr, VRBaseMap, /*IsDebug=*/false,
590                IsClone, IsCloned);
591     MIB.addImm(SubIdx);
592     MBB->insert(InsertPos, MIB);
593   } else
594     llvm_unreachable("Node is not insert_subreg, extract_subreg, or subreg_to_reg");
595 
596   SDValue Op(Node, 0);
597   bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second;
598   (void)isNew; // Silence compiler warning.
599   assert(isNew && "Node emitted out of order - early");
600 }
601 
602 /// EmitCopyToRegClassNode - Generate machine code for COPY_TO_REGCLASS nodes.
603 /// COPY_TO_REGCLASS is just a normal copy, except that the destination
604 /// register is constrained to be in a particular register class.
605 ///
606 void
EmitCopyToRegClassNode(SDNode * Node,DenseMap<SDValue,Register> & VRBaseMap)607 InstrEmitter::EmitCopyToRegClassNode(SDNode *Node,
608                                      DenseMap<SDValue, Register> &VRBaseMap) {
609   unsigned VReg = getVR(Node->getOperand(0), VRBaseMap);
610 
611   // Create the new VReg in the destination class and emit a copy.
612   unsigned DstRCIdx = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
613   const TargetRegisterClass *DstRC =
614     TRI->getAllocatableClass(TRI->getRegClass(DstRCIdx));
615   Register NewVReg = MRI->createVirtualRegister(DstRC);
616   BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TargetOpcode::COPY),
617     NewVReg).addReg(VReg);
618 
619   SDValue Op(Node, 0);
620   bool isNew = VRBaseMap.insert(std::make_pair(Op, NewVReg)).second;
621   (void)isNew; // Silence compiler warning.
622   assert(isNew && "Node emitted out of order - early");
623 }
624 
625 /// EmitRegSequence - Generate machine code for REG_SEQUENCE nodes.
626 ///
EmitRegSequence(SDNode * Node,DenseMap<SDValue,Register> & VRBaseMap,bool IsClone,bool IsCloned)627 void InstrEmitter::EmitRegSequence(SDNode *Node,
628                                   DenseMap<SDValue, Register> &VRBaseMap,
629                                   bool IsClone, bool IsCloned) {
630   unsigned DstRCIdx = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue();
631   const TargetRegisterClass *RC = TRI->getRegClass(DstRCIdx);
632   Register NewVReg = MRI->createVirtualRegister(TRI->getAllocatableClass(RC));
633   const MCInstrDesc &II = TII->get(TargetOpcode::REG_SEQUENCE);
634   MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II, NewVReg);
635   unsigned NumOps = Node->getNumOperands();
636   // If the input pattern has a chain, then the root of the corresponding
637   // output pattern will get a chain as well. This can happen to be a
638   // REG_SEQUENCE (which is not "guarded" by countOperands/CountResults).
639   if (NumOps && Node->getOperand(NumOps-1).getValueType() == MVT::Other)
640     --NumOps; // Ignore chain if it exists.
641 
642   assert((NumOps & 1) == 1 &&
643          "REG_SEQUENCE must have an odd number of operands!");
644   for (unsigned i = 1; i != NumOps; ++i) {
645     SDValue Op = Node->getOperand(i);
646     if ((i & 1) == 0) {
647       RegisterSDNode *R = dyn_cast<RegisterSDNode>(Node->getOperand(i-1));
648       // Skip physical registers as they don't have a vreg to get and we'll
649       // insert copies for them in TwoAddressInstructionPass anyway.
650       if (!R || !Register::isPhysicalRegister(R->getReg())) {
651         unsigned SubIdx = cast<ConstantSDNode>(Op)->getZExtValue();
652         unsigned SubReg = getVR(Node->getOperand(i-1), VRBaseMap);
653         const TargetRegisterClass *TRC = MRI->getRegClass(SubReg);
654         const TargetRegisterClass *SRC =
655         TRI->getMatchingSuperRegClass(RC, TRC, SubIdx);
656         if (SRC && SRC != RC) {
657           MRI->setRegClass(NewVReg, SRC);
658           RC = SRC;
659         }
660       }
661     }
662     AddOperand(MIB, Op, i+1, &II, VRBaseMap, /*IsDebug=*/false,
663                IsClone, IsCloned);
664   }
665 
666   MBB->insert(InsertPos, MIB);
667   SDValue Op(Node, 0);
668   bool isNew = VRBaseMap.insert(std::make_pair(Op, NewVReg)).second;
669   (void)isNew; // Silence compiler warning.
670   assert(isNew && "Node emitted out of order - early");
671 }
672 
673 /// EmitDbgValue - Generate machine instruction for a dbg_value node.
674 ///
675 MachineInstr *
EmitDbgValue(SDDbgValue * SD,DenseMap<SDValue,Register> & VRBaseMap)676 InstrEmitter::EmitDbgValue(SDDbgValue *SD,
677                            DenseMap<SDValue, Register> &VRBaseMap) {
678   MDNode *Var = SD->getVariable();
679   MDNode *Expr = SD->getExpression();
680   DebugLoc DL = SD->getDebugLoc();
681   assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
682          "Expected inlined-at fields to agree");
683 
684   SD->setIsEmitted();
685 
686   ArrayRef<SDDbgOperand> LocationOps = SD->getLocationOps();
687   assert(!LocationOps.empty() && "dbg_value with no location operands?");
688 
689   if (SD->isInvalidated())
690     return EmitDbgNoLocation(SD);
691 
692   // Emit variadic dbg_value nodes as DBG_VALUE_LIST.
693   if (SD->isVariadic()) {
694     // DBG_VALUE_LIST := "DBG_VALUE_LIST" var, expression, loc (, loc)*
695     const MCInstrDesc &DbgValDesc = TII->get(TargetOpcode::DBG_VALUE_LIST);
696     // Build the DBG_VALUE_LIST instruction base.
697     auto MIB = BuildMI(*MF, DL, DbgValDesc);
698     MIB.addMetadata(Var);
699     MIB.addMetadata(Expr);
700     AddDbgValueLocationOps(MIB, DbgValDesc, LocationOps, VRBaseMap);
701     return &*MIB;
702   }
703 
704   // Attempt to produce a DBG_INSTR_REF if we've been asked to.
705   // We currently exclude the possibility of instruction references for
706   // variadic nodes; if at some point we enable them, this should be moved
707   // above the variadic block.
708   if (EmitDebugInstrRefs)
709     if (auto *InstrRef = EmitDbgInstrRef(SD, VRBaseMap))
710       return InstrRef;
711 
712   return EmitDbgValueFromSingleOp(SD, VRBaseMap);
713 }
714 
AddDbgValueLocationOps(MachineInstrBuilder & MIB,const MCInstrDesc & DbgValDesc,ArrayRef<SDDbgOperand> LocationOps,DenseMap<SDValue,Register> & VRBaseMap)715 void InstrEmitter::AddDbgValueLocationOps(
716     MachineInstrBuilder &MIB, const MCInstrDesc &DbgValDesc,
717     ArrayRef<SDDbgOperand> LocationOps,
718     DenseMap<SDValue, Register> &VRBaseMap) {
719   for (const SDDbgOperand &Op : LocationOps) {
720     switch (Op.getKind()) {
721     case SDDbgOperand::FRAMEIX:
722       MIB.addFrameIndex(Op.getFrameIx());
723       break;
724     case SDDbgOperand::VREG:
725       MIB.addReg(Op.getVReg(), RegState::Debug);
726       break;
727     case SDDbgOperand::SDNODE: {
728       SDValue V = SDValue(Op.getSDNode(), Op.getResNo());
729       // It's possible we replaced this SDNode with other(s) and therefore
730       // didn't generate code for it. It's better to catch these cases where
731       // they happen and transfer the debug info, but trying to guarantee that
732       // in all cases would be very fragile; this is a safeguard for any
733       // that were missed.
734       if (VRBaseMap.count(V) == 0)
735         MIB.addReg(0U); // undef
736       else
737         AddOperand(MIB, V, (*MIB).getNumOperands(), &DbgValDesc, VRBaseMap,
738                    /*IsDebug=*/true, /*IsClone=*/false, /*IsCloned=*/false);
739     } break;
740     case SDDbgOperand::CONST: {
741       const Value *V = Op.getConst();
742       if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) {
743         if (CI->getBitWidth() > 64)
744           MIB.addCImm(CI);
745         else
746           MIB.addImm(CI->getSExtValue());
747       } else if (const ConstantFP *CF = dyn_cast<ConstantFP>(V)) {
748         MIB.addFPImm(CF);
749       } else if (isa<ConstantPointerNull>(V)) {
750         // Note: This assumes that all nullptr constants are zero-valued.
751         MIB.addImm(0);
752       } else {
753         // Could be an Undef. In any case insert an Undef so we can see what we
754         // dropped.
755         MIB.addReg(0U);
756       }
757     } break;
758     }
759   }
760 }
761 
762 MachineInstr *
EmitDbgInstrRef(SDDbgValue * SD,DenseMap<SDValue,Register> & VRBaseMap)763 InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
764                               DenseMap<SDValue, Register> &VRBaseMap) {
765   assert(!SD->isVariadic());
766   SDDbgOperand DbgOperand = SD->getLocationOps()[0];
767   MDNode *Var = SD->getVariable();
768   MDNode *Expr = SD->getExpression();
769   DebugLoc DL = SD->getDebugLoc();
770   const MCInstrDesc &RefII = TII->get(TargetOpcode::DBG_INSTR_REF);
771 
772   // Handle variable locations that don't actually depend on the instructions
773   // in the program: constants and stack locations.
774   if (DbgOperand.getKind() == SDDbgOperand::FRAMEIX ||
775       DbgOperand.getKind() == SDDbgOperand::CONST)
776     return EmitDbgValueFromSingleOp(SD, VRBaseMap);
777 
778   // It may not be immediately possible to identify the MachineInstr that
779   // defines a VReg, it can depend for example on the order blocks are
780   // emitted in. When this happens, or when further analysis is needed later,
781   // produce an instruction like this:
782   //
783   //    DBG_INSTR_REF %0:gr64, 0, !123, !456
784   //
785   // i.e., point the instruction at the vreg, and patch it up later in
786   // MachineFunction::finalizeDebugInstrRefs.
787   auto EmitHalfDoneInstrRef = [&](unsigned VReg) -> MachineInstr * {
788     auto MIB = BuildMI(*MF, DL, RefII);
789     MIB.addReg(VReg);
790     MIB.addImm(0);
791     MIB.addMetadata(Var);
792     MIB.addMetadata(Expr);
793     return MIB;
794   };
795 
796   // Try to find both the defined register and the instruction defining it.
797   MachineInstr *DefMI = nullptr;
798   unsigned VReg;
799 
800   if (DbgOperand.getKind() == SDDbgOperand::VREG) {
801     VReg = DbgOperand.getVReg();
802 
803     // No definition means that block hasn't been emitted yet. Leave a vreg
804     // reference to be fixed later.
805     if (!MRI->hasOneDef(VReg))
806       return EmitHalfDoneInstrRef(VReg);
807 
808     DefMI = &*MRI->def_instr_begin(VReg);
809   } else {
810     assert(DbgOperand.getKind() == SDDbgOperand::SDNODE);
811     // Look up the corresponding VReg for the given SDNode, if any.
812     SDNode *Node = DbgOperand.getSDNode();
813     SDValue Op = SDValue(Node, DbgOperand.getResNo());
814     DenseMap<SDValue, Register>::iterator I = VRBaseMap.find(Op);
815     // No VReg -> produce a DBG_VALUE $noreg instead.
816     if (I==VRBaseMap.end())
817       return EmitDbgNoLocation(SD);
818 
819     // Try to pick out a defining instruction at this point.
820     VReg = getVR(Op, VRBaseMap);
821 
822     // Again, if there's no instruction defining the VReg right now, fix it up
823     // later.
824     if (!MRI->hasOneDef(VReg))
825       return EmitHalfDoneInstrRef(VReg);
826 
827     DefMI = &*MRI->def_instr_begin(VReg);
828   }
829 
830   // Avoid copy like instructions: they don't define values, only move them.
831   // Leave a virtual-register reference until it can be fixed up later, to find
832   // the underlying value definition.
833   if (DefMI->isCopyLike() || TII->isCopyInstr(*DefMI))
834     return EmitHalfDoneInstrRef(VReg);
835 
836   auto MIB = BuildMI(*MF, DL, RefII);
837 
838   // Find the operand number which defines the specified VReg.
839   unsigned OperandIdx = 0;
840   for (const auto &MO : DefMI->operands()) {
841     if (MO.isReg() && MO.isDef() && MO.getReg() == VReg)
842       break;
843     ++OperandIdx;
844   }
845   assert(OperandIdx < DefMI->getNumOperands());
846 
847   // Make the DBG_INSTR_REF refer to that instruction, and that operand.
848   unsigned InstrNum = DefMI->getDebugInstrNum();
849   MIB.addImm(InstrNum);
850   MIB.addImm(OperandIdx);
851   MIB.addMetadata(Var);
852   MIB.addMetadata(Expr);
853   return &*MIB;
854 }
855 
EmitDbgNoLocation(SDDbgValue * SD)856 MachineInstr *InstrEmitter::EmitDbgNoLocation(SDDbgValue *SD) {
857   // An invalidated SDNode must generate an undef DBG_VALUE: although the
858   // original value is no longer computed, earlier DBG_VALUEs live ranges
859   // must not leak into later code.
860   MDNode *Var = SD->getVariable();
861   MDNode *Expr = SD->getExpression();
862   DebugLoc DL = SD->getDebugLoc();
863   auto MIB = BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE));
864   MIB.addReg(0U);
865   MIB.addReg(0U, RegState::Debug);
866   MIB.addMetadata(Var);
867   MIB.addMetadata(Expr);
868   return &*MIB;
869 }
870 
871 MachineInstr *
EmitDbgValueFromSingleOp(SDDbgValue * SD,DenseMap<SDValue,Register> & VRBaseMap)872 InstrEmitter::EmitDbgValueFromSingleOp(SDDbgValue *SD,
873                                        DenseMap<SDValue, Register> &VRBaseMap) {
874   MDNode *Var = SD->getVariable();
875   MDNode *Expr = SD->getExpression();
876   DebugLoc DL = SD->getDebugLoc();
877   const MCInstrDesc &II = TII->get(TargetOpcode::DBG_VALUE);
878 
879   assert(SD->getLocationOps().size() == 1 &&
880          "Non variadic dbg_value should have only one location op");
881 
882   // Emit non-variadic dbg_value nodes as DBG_VALUE.
883   // DBG_VALUE := "DBG_VALUE" loc, isIndirect, var, expr
884   auto MIB = BuildMI(*MF, DL, II);
885   AddDbgValueLocationOps(MIB, II, SD->getLocationOps(), VRBaseMap);
886 
887   if (SD->isIndirect())
888     MIB.addImm(0U);
889   else
890     MIB.addReg(0U, RegState::Debug);
891 
892   return MIB.addMetadata(Var).addMetadata(Expr);
893 }
894 
895 MachineInstr *
EmitDbgLabel(SDDbgLabel * SD)896 InstrEmitter::EmitDbgLabel(SDDbgLabel *SD) {
897   MDNode *Label = SD->getLabel();
898   DebugLoc DL = SD->getDebugLoc();
899   assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
900          "Expected inlined-at fields to agree");
901 
902   const MCInstrDesc &II = TII->get(TargetOpcode::DBG_LABEL);
903   MachineInstrBuilder MIB = BuildMI(*MF, DL, II);
904   MIB.addMetadata(Label);
905 
906   return &*MIB;
907 }
908 
909 /// EmitMachineNode - Generate machine code for a target-specific node and
910 /// needed dependencies.
911 ///
912 void InstrEmitter::
EmitMachineNode(SDNode * Node,bool IsClone,bool IsCloned,DenseMap<SDValue,Register> & VRBaseMap)913 EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
914                 DenseMap<SDValue, Register> &VRBaseMap) {
915   unsigned Opc = Node->getMachineOpcode();
916 
917   // Handle subreg insert/extract specially
918   if (Opc == TargetOpcode::EXTRACT_SUBREG ||
919       Opc == TargetOpcode::INSERT_SUBREG ||
920       Opc == TargetOpcode::SUBREG_TO_REG) {
921     EmitSubregNode(Node, VRBaseMap, IsClone, IsCloned);
922     return;
923   }
924 
925   // Handle COPY_TO_REGCLASS specially.
926   if (Opc == TargetOpcode::COPY_TO_REGCLASS) {
927     EmitCopyToRegClassNode(Node, VRBaseMap);
928     return;
929   }
930 
931   // Handle REG_SEQUENCE specially.
932   if (Opc == TargetOpcode::REG_SEQUENCE) {
933     EmitRegSequence(Node, VRBaseMap, IsClone, IsCloned);
934     return;
935   }
936 
937   if (Opc == TargetOpcode::IMPLICIT_DEF)
938     // We want a unique VR for each IMPLICIT_DEF use.
939     return;
940 
941   const MCInstrDesc &II = TII->get(Opc);
942   unsigned NumResults = CountResults(Node);
943   unsigned NumDefs = II.getNumDefs();
944   const MCPhysReg *ScratchRegs = nullptr;
945 
946   // Handle STACKMAP and PATCHPOINT specially and then use the generic code.
947   if (Opc == TargetOpcode::STACKMAP || Opc == TargetOpcode::PATCHPOINT) {
948     // Stackmaps do not have arguments and do not preserve their calling
949     // convention. However, to simplify runtime support, they clobber the same
950     // scratch registers as AnyRegCC.
951     unsigned CC = CallingConv::AnyReg;
952     if (Opc == TargetOpcode::PATCHPOINT) {
953       CC = Node->getConstantOperandVal(PatchPointOpers::CCPos);
954       NumDefs = NumResults;
955     }
956     ScratchRegs = TLI->getScratchRegisters((CallingConv::ID) CC);
957   } else if (Opc == TargetOpcode::STATEPOINT) {
958     NumDefs = NumResults;
959   }
960 
961   unsigned NumImpUses = 0;
962   unsigned NodeOperands =
963     countOperands(Node, II.getNumOperands() - NumDefs, NumImpUses);
964   bool HasVRegVariadicDefs = !MF->getTarget().usesPhysRegsForValues() &&
965                              II.isVariadic() && II.variadicOpsAreDefs();
966   bool HasPhysRegOuts = NumResults > NumDefs &&
967                         II.getImplicitDefs() != nullptr && !HasVRegVariadicDefs;
968 #ifndef NDEBUG
969   unsigned NumMIOperands = NodeOperands + NumResults;
970   if (II.isVariadic())
971     assert(NumMIOperands >= II.getNumOperands() &&
972            "Too few operands for a variadic node!");
973   else
974     assert(NumMIOperands >= II.getNumOperands() &&
975            NumMIOperands <= II.getNumOperands() + II.getNumImplicitDefs() +
976                             NumImpUses &&
977            "#operands for dag node doesn't match .td file!");
978 #endif
979 
980   // Create the new machine instruction.
981   MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II);
982 
983   // Add result register values for things that are defined by this
984   // instruction.
985   if (NumResults) {
986     CreateVirtualRegisters(Node, MIB, II, IsClone, IsCloned, VRBaseMap);
987 
988     // Transfer any IR flags from the SDNode to the MachineInstr
989     MachineInstr *MI = MIB.getInstr();
990     const SDNodeFlags Flags = Node->getFlags();
991     if (Flags.hasNoSignedZeros())
992       MI->setFlag(MachineInstr::MIFlag::FmNsz);
993 
994     if (Flags.hasAllowReciprocal())
995       MI->setFlag(MachineInstr::MIFlag::FmArcp);
996 
997     if (Flags.hasNoNaNs())
998       MI->setFlag(MachineInstr::MIFlag::FmNoNans);
999 
1000     if (Flags.hasNoInfs())
1001       MI->setFlag(MachineInstr::MIFlag::FmNoInfs);
1002 
1003     if (Flags.hasAllowContract())
1004       MI->setFlag(MachineInstr::MIFlag::FmContract);
1005 
1006     if (Flags.hasApproximateFuncs())
1007       MI->setFlag(MachineInstr::MIFlag::FmAfn);
1008 
1009     if (Flags.hasAllowReassociation())
1010       MI->setFlag(MachineInstr::MIFlag::FmReassoc);
1011 
1012     if (Flags.hasNoUnsignedWrap())
1013       MI->setFlag(MachineInstr::MIFlag::NoUWrap);
1014 
1015     if (Flags.hasNoSignedWrap())
1016       MI->setFlag(MachineInstr::MIFlag::NoSWrap);
1017 
1018     if (Flags.hasExact())
1019       MI->setFlag(MachineInstr::MIFlag::IsExact);
1020 
1021     if (Flags.hasNoFPExcept())
1022       MI->setFlag(MachineInstr::MIFlag::NoFPExcept);
1023   }
1024 
1025   // Emit all of the actual operands of this instruction, adding them to the
1026   // instruction as appropriate.
1027   bool HasOptPRefs = NumDefs > NumResults;
1028   assert((!HasOptPRefs || !HasPhysRegOuts) &&
1029          "Unable to cope with optional defs and phys regs defs!");
1030   unsigned NumSkip = HasOptPRefs ? NumDefs - NumResults : 0;
1031   for (unsigned i = NumSkip; i != NodeOperands; ++i)
1032     AddOperand(MIB, Node->getOperand(i), i-NumSkip+NumDefs, &II,
1033                VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned);
1034 
1035   // Add scratch registers as implicit def and early clobber
1036   if (ScratchRegs)
1037     for (unsigned i = 0; ScratchRegs[i]; ++i)
1038       MIB.addReg(ScratchRegs[i], RegState::ImplicitDefine |
1039                                  RegState::EarlyClobber);
1040 
1041   // Set the memory reference descriptions of this instruction now that it is
1042   // part of the function.
1043   MIB.setMemRefs(cast<MachineSDNode>(Node)->memoperands());
1044 
1045   // Insert the instruction into position in the block. This needs to
1046   // happen before any custom inserter hook is called so that the
1047   // hook knows where in the block to insert the replacement code.
1048   MBB->insert(InsertPos, MIB);
1049 
1050   // The MachineInstr may also define physregs instead of virtregs.  These
1051   // physreg values can reach other instructions in different ways:
1052   //
1053   // 1. When there is a use of a Node value beyond the explicitly defined
1054   //    virtual registers, we emit a CopyFromReg for one of the implicitly
1055   //    defined physregs.  This only happens when HasPhysRegOuts is true.
1056   //
1057   // 2. A CopyFromReg reading a physreg may be glued to this instruction.
1058   //
1059   // 3. A glued instruction may implicitly use a physreg.
1060   //
1061   // 4. A glued instruction may use a RegisterSDNode operand.
1062   //
1063   // Collect all the used physreg defs, and make sure that any unused physreg
1064   // defs are marked as dead.
1065   SmallVector<Register, 8> UsedRegs;
1066 
1067   // Additional results must be physical register defs.
1068   if (HasPhysRegOuts) {
1069     for (unsigned i = NumDefs; i < NumResults; ++i) {
1070       Register Reg = II.getImplicitDefs()[i - NumDefs];
1071       if (!Node->hasAnyUseOfValue(i))
1072         continue;
1073       // This implicitly defined physreg has a use.
1074       UsedRegs.push_back(Reg);
1075       EmitCopyFromReg(Node, i, IsClone, IsCloned, Reg, VRBaseMap);
1076     }
1077   }
1078 
1079   // Scan the glue chain for any used physregs.
1080   if (Node->getValueType(Node->getNumValues()-1) == MVT::Glue) {
1081     for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) {
1082       if (F->getOpcode() == ISD::CopyFromReg) {
1083         UsedRegs.push_back(cast<RegisterSDNode>(F->getOperand(1))->getReg());
1084         continue;
1085       } else if (F->getOpcode() == ISD::CopyToReg) {
1086         // Skip CopyToReg nodes that are internal to the glue chain.
1087         continue;
1088       }
1089       // Collect declared implicit uses.
1090       const MCInstrDesc &MCID = TII->get(F->getMachineOpcode());
1091       UsedRegs.append(MCID.getImplicitUses(),
1092                       MCID.getImplicitUses() + MCID.getNumImplicitUses());
1093       // In addition to declared implicit uses, we must also check for
1094       // direct RegisterSDNode operands.
1095       for (unsigned i = 0, e = F->getNumOperands(); i != e; ++i)
1096         if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(F->getOperand(i))) {
1097           Register Reg = R->getReg();
1098           if (Reg.isPhysical())
1099             UsedRegs.push_back(Reg);
1100         }
1101     }
1102   }
1103 
1104   // Finally mark unused registers as dead.
1105   if (!UsedRegs.empty() || II.getImplicitDefs() || II.hasOptionalDef())
1106     MIB->setPhysRegsDeadExcept(UsedRegs, *TRI);
1107 
1108   // STATEPOINT is too 'dynamic' to have meaningful machine description.
1109   // We have to manually tie operands.
1110   if (Opc == TargetOpcode::STATEPOINT && NumDefs > 0) {
1111     assert(!HasPhysRegOuts && "STATEPOINT mishandled");
1112     MachineInstr *MI = MIB;
1113     unsigned Def = 0;
1114     int First = StatepointOpers(MI).getFirstGCPtrIdx();
1115     assert(First > 0 && "Statepoint has Defs but no GC ptr list");
1116     unsigned Use = (unsigned)First;
1117     while (Def < NumDefs) {
1118       if (MI->getOperand(Use).isReg())
1119         MI->tieOperands(Def++, Use);
1120       Use = StackMaps::getNextMetaArgIdx(MI, Use);
1121     }
1122   }
1123 
1124   // Run post-isel target hook to adjust this instruction if needed.
1125   if (II.hasPostISelHook())
1126     TLI->AdjustInstrPostInstrSelection(*MIB, Node);
1127 }
1128 
1129 /// EmitSpecialNode - Generate machine code for a target-independent node and
1130 /// needed dependencies.
1131 void InstrEmitter::
EmitSpecialNode(SDNode * Node,bool IsClone,bool IsCloned,DenseMap<SDValue,Register> & VRBaseMap)1132 EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
1133                 DenseMap<SDValue, Register> &VRBaseMap) {
1134   switch (Node->getOpcode()) {
1135   default:
1136 #ifndef NDEBUG
1137     Node->dump();
1138 #endif
1139     llvm_unreachable("This target-independent node should have been selected!");
1140   case ISD::EntryToken:
1141     llvm_unreachable("EntryToken should have been excluded from the schedule!");
1142   case ISD::MERGE_VALUES:
1143   case ISD::TokenFactor: // fall thru
1144     break;
1145   case ISD::CopyToReg: {
1146     Register DestReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
1147     SDValue SrcVal = Node->getOperand(2);
1148     if (Register::isVirtualRegister(DestReg) && SrcVal.isMachineOpcode() &&
1149         SrcVal.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) {
1150       // Instead building a COPY to that vreg destination, build an
1151       // IMPLICIT_DEF instruction instead.
1152       BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
1153               TII->get(TargetOpcode::IMPLICIT_DEF), DestReg);
1154       break;
1155     }
1156     Register SrcReg;
1157     if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(SrcVal))
1158       SrcReg = R->getReg();
1159     else
1160       SrcReg = getVR(SrcVal, VRBaseMap);
1161 
1162     if (SrcReg == DestReg) // Coalesced away the copy? Ignore.
1163       break;
1164 
1165     BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TargetOpcode::COPY),
1166             DestReg).addReg(SrcReg);
1167     break;
1168   }
1169   case ISD::CopyFromReg: {
1170     unsigned SrcReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
1171     EmitCopyFromReg(Node, 0, IsClone, IsCloned, SrcReg, VRBaseMap);
1172     break;
1173   }
1174   case ISD::EH_LABEL:
1175   case ISD::ANNOTATION_LABEL: {
1176     unsigned Opc = (Node->getOpcode() == ISD::EH_LABEL)
1177                        ? TargetOpcode::EH_LABEL
1178                        : TargetOpcode::ANNOTATION_LABEL;
1179     MCSymbol *S = cast<LabelSDNode>(Node)->getLabel();
1180     BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
1181             TII->get(Opc)).addSym(S);
1182     break;
1183   }
1184 
1185   case ISD::LIFETIME_START:
1186   case ISD::LIFETIME_END: {
1187     unsigned TarOp = (Node->getOpcode() == ISD::LIFETIME_START)
1188                          ? TargetOpcode::LIFETIME_START
1189                          : TargetOpcode::LIFETIME_END;
1190     auto *FI = cast<FrameIndexSDNode>(Node->getOperand(1));
1191     BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TarOp))
1192     .addFrameIndex(FI->getIndex());
1193     break;
1194   }
1195 
1196   case ISD::PSEUDO_PROBE: {
1197     unsigned TarOp = TargetOpcode::PSEUDO_PROBE;
1198     auto Guid = cast<PseudoProbeSDNode>(Node)->getGuid();
1199     auto Index = cast<PseudoProbeSDNode>(Node)->getIndex();
1200     auto Attr = cast<PseudoProbeSDNode>(Node)->getAttributes();
1201 
1202     BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TarOp))
1203         .addImm(Guid)
1204         .addImm(Index)
1205         .addImm((uint8_t)PseudoProbeType::Block)
1206         .addImm(Attr);
1207     break;
1208   }
1209 
1210   case ISD::INLINEASM:
1211   case ISD::INLINEASM_BR: {
1212     unsigned NumOps = Node->getNumOperands();
1213     if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
1214       --NumOps;  // Ignore the glue operand.
1215 
1216     // Create the inline asm machine instruction.
1217     unsigned TgtOpc = Node->getOpcode() == ISD::INLINEASM_BR
1218                           ? TargetOpcode::INLINEASM_BR
1219                           : TargetOpcode::INLINEASM;
1220     MachineInstrBuilder MIB =
1221         BuildMI(*MF, Node->getDebugLoc(), TII->get(TgtOpc));
1222 
1223     // Add the asm string as an external symbol operand.
1224     SDValue AsmStrV = Node->getOperand(InlineAsm::Op_AsmString);
1225     const char *AsmStr = cast<ExternalSymbolSDNode>(AsmStrV)->getSymbol();
1226     MIB.addExternalSymbol(AsmStr);
1227 
1228     // Add the HasSideEffect, isAlignStack, AsmDialect, MayLoad and MayStore
1229     // bits.
1230     int64_t ExtraInfo =
1231       cast<ConstantSDNode>(Node->getOperand(InlineAsm::Op_ExtraInfo))->
1232                           getZExtValue();
1233     MIB.addImm(ExtraInfo);
1234 
1235     // Remember to operand index of the group flags.
1236     SmallVector<unsigned, 8> GroupIdx;
1237 
1238     // Remember registers that are part of early-clobber defs.
1239     SmallVector<unsigned, 8> ECRegs;
1240 
1241     // Add all of the operand registers to the instruction.
1242     for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
1243       unsigned Flags =
1244         cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
1245       const unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
1246 
1247       GroupIdx.push_back(MIB->getNumOperands());
1248       MIB.addImm(Flags);
1249       ++i;  // Skip the ID value.
1250 
1251       switch (InlineAsm::getKind(Flags)) {
1252       default: llvm_unreachable("Bad flags!");
1253         case InlineAsm::Kind_RegDef:
1254         for (unsigned j = 0; j != NumVals; ++j, ++i) {
1255           unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
1256           // FIXME: Add dead flags for physical and virtual registers defined.
1257           // For now, mark physical register defs as implicit to help fast
1258           // regalloc. This makes inline asm look a lot like calls.
1259           MIB.addReg(Reg,
1260                      RegState::Define |
1261                          getImplRegState(Register::isPhysicalRegister(Reg)));
1262         }
1263         break;
1264       case InlineAsm::Kind_RegDefEarlyClobber:
1265       case InlineAsm::Kind_Clobber:
1266         for (unsigned j = 0; j != NumVals; ++j, ++i) {
1267           unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
1268           MIB.addReg(Reg,
1269                      RegState::Define | RegState::EarlyClobber |
1270                          getImplRegState(Register::isPhysicalRegister(Reg)));
1271           ECRegs.push_back(Reg);
1272         }
1273         break;
1274       case InlineAsm::Kind_RegUse:  // Use of register.
1275       case InlineAsm::Kind_Imm:  // Immediate.
1276       case InlineAsm::Kind_Mem:  // Addressing mode.
1277         // The addressing mode has been selected, just add all of the
1278         // operands to the machine instruction.
1279         for (unsigned j = 0; j != NumVals; ++j, ++i)
1280           AddOperand(MIB, Node->getOperand(i), 0, nullptr, VRBaseMap,
1281                      /*IsDebug=*/false, IsClone, IsCloned);
1282 
1283         // Manually set isTied bits.
1284         if (InlineAsm::getKind(Flags) == InlineAsm::Kind_RegUse) {
1285           unsigned DefGroup = 0;
1286           if (InlineAsm::isUseOperandTiedToDef(Flags, DefGroup)) {
1287             unsigned DefIdx = GroupIdx[DefGroup] + 1;
1288             unsigned UseIdx = GroupIdx.back() + 1;
1289             for (unsigned j = 0; j != NumVals; ++j)
1290               MIB->tieOperands(DefIdx + j, UseIdx + j);
1291           }
1292         }
1293         break;
1294       }
1295     }
1296 
1297     // GCC inline assembly allows input operands to also be early-clobber
1298     // output operands (so long as the operand is written only after it's
1299     // used), but this does not match the semantics of our early-clobber flag.
1300     // If an early-clobber operand register is also an input operand register,
1301     // then remove the early-clobber flag.
1302     for (unsigned Reg : ECRegs) {
1303       if (MIB->readsRegister(Reg, TRI)) {
1304         MachineOperand *MO =
1305             MIB->findRegisterDefOperand(Reg, false, false, TRI);
1306         assert(MO && "No def operand for clobbered register?");
1307         MO->setIsEarlyClobber(false);
1308       }
1309     }
1310 
1311     // Get the mdnode from the asm if it exists and add it to the instruction.
1312     SDValue MDV = Node->getOperand(InlineAsm::Op_MDNode);
1313     const MDNode *MD = cast<MDNodeSDNode>(MDV)->getMD();
1314     if (MD)
1315       MIB.addMetadata(MD);
1316 
1317     MBB->insert(InsertPos, MIB);
1318     break;
1319   }
1320   }
1321 }
1322 
1323 /// InstrEmitter - Construct an InstrEmitter and set it to start inserting
1324 /// at the given position in the given block.
InstrEmitter(const TargetMachine & TM,MachineBasicBlock * mbb,MachineBasicBlock::iterator insertpos)1325 InstrEmitter::InstrEmitter(const TargetMachine &TM, MachineBasicBlock *mbb,
1326                            MachineBasicBlock::iterator insertpos)
1327     : MF(mbb->getParent()), MRI(&MF->getRegInfo()),
1328       TII(MF->getSubtarget().getInstrInfo()),
1329       TRI(MF->getSubtarget().getRegisterInfo()),
1330       TLI(MF->getSubtarget().getTargetLowering()), MBB(mbb),
1331       InsertPos(insertpos) {
1332   EmitDebugInstrRefs = TM.Options.ValueTrackingVariableLocations;
1333 }
1334