1 //==--- InstrEmitter.cpp - Emit MachineInstrs for the SelectionDAG class ---==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the Emit routines for the SelectionDAG class, which creates
10 // MachineInstrs based on the decisions of the SelectionDAG instruction
11 // selection.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "InstrEmitter.h"
16 #include "SDNodeDbgValue.h"
17 #include "llvm/ADT/Statistic.h"
18 #include "llvm/CodeGen/MachineConstantPool.h"
19 #include "llvm/CodeGen/MachineFunction.h"
20 #include "llvm/CodeGen/MachineInstrBuilder.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/CodeGen/SelectionDAG.h"
23 #include "llvm/CodeGen/StackMaps.h"
24 #include "llvm/CodeGen/TargetInstrInfo.h"
25 #include "llvm/CodeGen/TargetLowering.h"
26 #include "llvm/CodeGen/TargetSubtargetInfo.h"
27 #include "llvm/IR/DataLayout.h"
28 #include "llvm/IR/DebugInfo.h"
29 #include "llvm/IR/PseudoProbe.h"
30 #include "llvm/Support/Debug.h"
31 #include "llvm/Support/ErrorHandling.h"
32 #include "llvm/Support/MathExtras.h"
33 #include "llvm/Target/TargetMachine.h"
34 using namespace llvm;
35
36 #define DEBUG_TYPE "instr-emitter"
37
38 /// MinRCSize - Smallest register class we allow when constraining virtual
39 /// registers. If satisfying all register class constraints would require
40 /// using a smaller register class, emit a COPY to a new virtual register
41 /// instead.
42 const unsigned MinRCSize = 4;
43
44 /// CountResults - The results of target nodes have register or immediate
45 /// operands first, then an optional chain, and optional glue operands (which do
46 /// not go into the resulting MachineInstr).
CountResults(SDNode * Node)47 unsigned InstrEmitter::CountResults(SDNode *Node) {
48 unsigned N = Node->getNumValues();
49 while (N && Node->getValueType(N - 1) == MVT::Glue)
50 --N;
51 if (N && Node->getValueType(N - 1) == MVT::Other)
52 --N; // Skip over chain result.
53 return N;
54 }
55
56 /// countOperands - The inputs to target nodes have any actual inputs first,
57 /// followed by an optional chain operand, then an optional glue operand.
58 /// Compute the number of actual operands that will go into the resulting
59 /// MachineInstr.
60 ///
61 /// Also count physreg RegisterSDNode and RegisterMaskSDNode operands preceding
62 /// the chain and glue. These operands may be implicit on the machine instr.
countOperands(SDNode * Node,unsigned NumExpUses,unsigned & NumImpUses)63 static unsigned countOperands(SDNode *Node, unsigned NumExpUses,
64 unsigned &NumImpUses) {
65 unsigned N = Node->getNumOperands();
66 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
67 --N;
68 if (N && Node->getOperand(N - 1).getValueType() == MVT::Other)
69 --N; // Ignore chain if it exists.
70
71 // Count RegisterSDNode and RegisterMaskSDNode operands for NumImpUses.
72 NumImpUses = N - NumExpUses;
73 for (unsigned I = N; I > NumExpUses; --I) {
74 if (isa<RegisterMaskSDNode>(Node->getOperand(I - 1)))
75 continue;
76 if (RegisterSDNode *RN = dyn_cast<RegisterSDNode>(Node->getOperand(I - 1)))
77 if (Register::isPhysicalRegister(RN->getReg()))
78 continue;
79 NumImpUses = N - I;
80 break;
81 }
82
83 return N;
84 }
85
86 /// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an
87 /// implicit physical register output.
88 void InstrEmitter::
EmitCopyFromReg(SDNode * Node,unsigned ResNo,bool IsClone,bool IsCloned,Register SrcReg,DenseMap<SDValue,Register> & VRBaseMap)89 EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned,
90 Register SrcReg, DenseMap<SDValue, Register> &VRBaseMap) {
91 Register VRBase;
92 if (SrcReg.isVirtual()) {
93 // Just use the input register directly!
94 SDValue Op(Node, ResNo);
95 if (IsClone)
96 VRBaseMap.erase(Op);
97 bool isNew = VRBaseMap.insert(std::make_pair(Op, SrcReg)).second;
98 (void)isNew; // Silence compiler warning.
99 assert(isNew && "Node emitted out of order - early");
100 return;
101 }
102
103 // If the node is only used by a CopyToReg and the dest reg is a vreg, use
104 // the CopyToReg'd destination register instead of creating a new vreg.
105 bool MatchReg = true;
106 const TargetRegisterClass *UseRC = nullptr;
107 MVT VT = Node->getSimpleValueType(ResNo);
108
109 // Stick to the preferred register classes for legal types.
110 if (TLI->isTypeLegal(VT))
111 UseRC = TLI->getRegClassFor(VT, Node->isDivergent());
112
113 if (!IsClone && !IsCloned)
114 for (SDNode *User : Node->uses()) {
115 bool Match = true;
116 if (User->getOpcode() == ISD::CopyToReg &&
117 User->getOperand(2).getNode() == Node &&
118 User->getOperand(2).getResNo() == ResNo) {
119 Register DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
120 if (DestReg.isVirtual()) {
121 VRBase = DestReg;
122 Match = false;
123 } else if (DestReg != SrcReg)
124 Match = false;
125 } else {
126 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
127 SDValue Op = User->getOperand(i);
128 if (Op.getNode() != Node || Op.getResNo() != ResNo)
129 continue;
130 MVT VT = Node->getSimpleValueType(Op.getResNo());
131 if (VT == MVT::Other || VT == MVT::Glue)
132 continue;
133 Match = false;
134 if (User->isMachineOpcode()) {
135 const MCInstrDesc &II = TII->get(User->getMachineOpcode());
136 const TargetRegisterClass *RC = nullptr;
137 if (i+II.getNumDefs() < II.getNumOperands()) {
138 RC = TRI->getAllocatableClass(
139 TII->getRegClass(II, i+II.getNumDefs(), TRI, *MF));
140 }
141 if (!UseRC)
142 UseRC = RC;
143 else if (RC) {
144 const TargetRegisterClass *ComRC =
145 TRI->getCommonSubClass(UseRC, RC);
146 // If multiple uses expect disjoint register classes, we emit
147 // copies in AddRegisterOperand.
148 if (ComRC)
149 UseRC = ComRC;
150 }
151 }
152 }
153 }
154 MatchReg &= Match;
155 if (VRBase)
156 break;
157 }
158
159 const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
160 SrcRC = TRI->getMinimalPhysRegClass(SrcReg, VT);
161
162 // Figure out the register class to create for the destreg.
163 if (VRBase) {
164 DstRC = MRI->getRegClass(VRBase);
165 } else if (UseRC) {
166 assert(TRI->isTypeLegalForClass(*UseRC, VT) &&
167 "Incompatible phys register def and uses!");
168 DstRC = UseRC;
169 } else {
170 DstRC = TLI->getRegClassFor(VT, Node->isDivergent());
171 }
172
173 // If all uses are reading from the src physical register and copying the
174 // register is either impossible or very expensive, then don't create a copy.
175 if (MatchReg && SrcRC->getCopyCost() < 0) {
176 VRBase = SrcReg;
177 } else {
178 // Create the reg, emit the copy.
179 VRBase = MRI->createVirtualRegister(DstRC);
180 BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TargetOpcode::COPY),
181 VRBase).addReg(SrcReg);
182 }
183
184 SDValue Op(Node, ResNo);
185 if (IsClone)
186 VRBaseMap.erase(Op);
187 bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second;
188 (void)isNew; // Silence compiler warning.
189 assert(isNew && "Node emitted out of order - early");
190 }
191
CreateVirtualRegisters(SDNode * Node,MachineInstrBuilder & MIB,const MCInstrDesc & II,bool IsClone,bool IsCloned,DenseMap<SDValue,Register> & VRBaseMap)192 void InstrEmitter::CreateVirtualRegisters(SDNode *Node,
193 MachineInstrBuilder &MIB,
194 const MCInstrDesc &II,
195 bool IsClone, bool IsCloned,
196 DenseMap<SDValue, Register> &VRBaseMap) {
197 assert(Node->getMachineOpcode() != TargetOpcode::IMPLICIT_DEF &&
198 "IMPLICIT_DEF should have been handled as a special case elsewhere!");
199
200 unsigned NumResults = CountResults(Node);
201 bool HasVRegVariadicDefs = !MF->getTarget().usesPhysRegsForValues() &&
202 II.isVariadic() && II.variadicOpsAreDefs();
203 unsigned NumVRegs = HasVRegVariadicDefs ? NumResults : II.getNumDefs();
204 if (Node->getMachineOpcode() == TargetOpcode::STATEPOINT)
205 NumVRegs = NumResults;
206 for (unsigned i = 0; i < NumVRegs; ++i) {
207 // If the specific node value is only used by a CopyToReg and the dest reg
208 // is a vreg in the same register class, use the CopyToReg'd destination
209 // register instead of creating a new vreg.
210 Register VRBase;
211 const TargetRegisterClass *RC =
212 TRI->getAllocatableClass(TII->getRegClass(II, i, TRI, *MF));
213 // Always let the value type influence the used register class. The
214 // constraints on the instruction may be too lax to represent the value
215 // type correctly. For example, a 64-bit float (X86::FR64) can't live in
216 // the 32-bit float super-class (X86::FR32).
217 if (i < NumResults && TLI->isTypeLegal(Node->getSimpleValueType(i))) {
218 const TargetRegisterClass *VTRC = TLI->getRegClassFor(
219 Node->getSimpleValueType(i),
220 (Node->isDivergent() || (RC && TRI->isDivergentRegClass(RC))));
221 if (RC)
222 VTRC = TRI->getCommonSubClass(RC, VTRC);
223 if (VTRC)
224 RC = VTRC;
225 }
226
227 if (II.OpInfo != nullptr && II.OpInfo[i].isOptionalDef()) {
228 // Optional def must be a physical register.
229 VRBase = cast<RegisterSDNode>(Node->getOperand(i-NumResults))->getReg();
230 assert(VRBase.isPhysical());
231 MIB.addReg(VRBase, RegState::Define);
232 }
233
234 if (!VRBase && !IsClone && !IsCloned)
235 for (SDNode *User : Node->uses()) {
236 if (User->getOpcode() == ISD::CopyToReg &&
237 User->getOperand(2).getNode() == Node &&
238 User->getOperand(2).getResNo() == i) {
239 unsigned Reg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
240 if (Register::isVirtualRegister(Reg)) {
241 const TargetRegisterClass *RegRC = MRI->getRegClass(Reg);
242 if (RegRC == RC) {
243 VRBase = Reg;
244 MIB.addReg(VRBase, RegState::Define);
245 break;
246 }
247 }
248 }
249 }
250
251 // Create the result registers for this node and add the result regs to
252 // the machine instruction.
253 if (VRBase == 0) {
254 assert(RC && "Isn't a register operand!");
255 VRBase = MRI->createVirtualRegister(RC);
256 MIB.addReg(VRBase, RegState::Define);
257 }
258
259 // If this def corresponds to a result of the SDNode insert the VRBase into
260 // the lookup map.
261 if (i < NumResults) {
262 SDValue Op(Node, i);
263 if (IsClone)
264 VRBaseMap.erase(Op);
265 bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second;
266 (void)isNew; // Silence compiler warning.
267 assert(isNew && "Node emitted out of order - early");
268 }
269 }
270 }
271
272 /// getVR - Return the virtual register corresponding to the specified result
273 /// of the specified node.
getVR(SDValue Op,DenseMap<SDValue,Register> & VRBaseMap)274 Register InstrEmitter::getVR(SDValue Op,
275 DenseMap<SDValue, Register> &VRBaseMap) {
276 if (Op.isMachineOpcode() &&
277 Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) {
278 // Add an IMPLICIT_DEF instruction before every use.
279 // IMPLICIT_DEF can produce any type of result so its MCInstrDesc
280 // does not include operand register class info.
281 const TargetRegisterClass *RC = TLI->getRegClassFor(
282 Op.getSimpleValueType(), Op.getNode()->isDivergent());
283 Register VReg = MRI->createVirtualRegister(RC);
284 BuildMI(*MBB, InsertPos, Op.getDebugLoc(),
285 TII->get(TargetOpcode::IMPLICIT_DEF), VReg);
286 return VReg;
287 }
288
289 DenseMap<SDValue, Register>::iterator I = VRBaseMap.find(Op);
290 assert(I != VRBaseMap.end() && "Node emitted out of order - late");
291 return I->second;
292 }
293
294
295 /// AddRegisterOperand - Add the specified register as an operand to the
296 /// specified machine instr. Insert register copies if the register is
297 /// not in the required register class.
298 void
AddRegisterOperand(MachineInstrBuilder & MIB,SDValue Op,unsigned IIOpNum,const MCInstrDesc * II,DenseMap<SDValue,Register> & VRBaseMap,bool IsDebug,bool IsClone,bool IsCloned)299 InstrEmitter::AddRegisterOperand(MachineInstrBuilder &MIB,
300 SDValue Op,
301 unsigned IIOpNum,
302 const MCInstrDesc *II,
303 DenseMap<SDValue, Register> &VRBaseMap,
304 bool IsDebug, bool IsClone, bool IsCloned) {
305 assert(Op.getValueType() != MVT::Other &&
306 Op.getValueType() != MVT::Glue &&
307 "Chain and glue operands should occur at end of operand list!");
308 // Get/emit the operand.
309 Register VReg = getVR(Op, VRBaseMap);
310
311 const MCInstrDesc &MCID = MIB->getDesc();
312 bool isOptDef = IIOpNum < MCID.getNumOperands() &&
313 MCID.OpInfo[IIOpNum].isOptionalDef();
314
315 // If the instruction requires a register in a different class, create
316 // a new virtual register and copy the value into it, but first attempt to
317 // shrink VReg's register class within reason. For example, if VReg == GR32
318 // and II requires a GR32_NOSP, just constrain VReg to GR32_NOSP.
319 if (II) {
320 const TargetRegisterClass *OpRC = nullptr;
321 if (IIOpNum < II->getNumOperands())
322 OpRC = TII->getRegClass(*II, IIOpNum, TRI, *MF);
323
324 if (OpRC) {
325 const TargetRegisterClass *ConstrainedRC
326 = MRI->constrainRegClass(VReg, OpRC, MinRCSize);
327 if (!ConstrainedRC) {
328 OpRC = TRI->getAllocatableClass(OpRC);
329 assert(OpRC && "Constraints cannot be fulfilled for allocation");
330 Register NewVReg = MRI->createVirtualRegister(OpRC);
331 BuildMI(*MBB, InsertPos, Op.getNode()->getDebugLoc(),
332 TII->get(TargetOpcode::COPY), NewVReg).addReg(VReg);
333 VReg = NewVReg;
334 } else {
335 assert(ConstrainedRC->isAllocatable() &&
336 "Constraining an allocatable VReg produced an unallocatable class?");
337 }
338 }
339 }
340
341 // If this value has only one use, that use is a kill. This is a
342 // conservative approximation. InstrEmitter does trivial coalescing
343 // with CopyFromReg nodes, so don't emit kill flags for them.
344 // Avoid kill flags on Schedule cloned nodes, since there will be
345 // multiple uses.
346 // Tied operands are never killed, so we need to check that. And that
347 // means we need to determine the index of the operand.
348 bool isKill = Op.hasOneUse() &&
349 Op.getNode()->getOpcode() != ISD::CopyFromReg &&
350 !IsDebug &&
351 !(IsClone || IsCloned);
352 if (isKill) {
353 unsigned Idx = MIB->getNumOperands();
354 while (Idx > 0 &&
355 MIB->getOperand(Idx-1).isReg() &&
356 MIB->getOperand(Idx-1).isImplicit())
357 --Idx;
358 bool isTied = MCID.getOperandConstraint(Idx, MCOI::TIED_TO) != -1;
359 if (isTied)
360 isKill = false;
361 }
362
363 MIB.addReg(VReg, getDefRegState(isOptDef) | getKillRegState(isKill) |
364 getDebugRegState(IsDebug));
365 }
366
367 /// AddOperand - Add the specified operand to the specified machine instr. II
368 /// specifies the instruction information for the node, and IIOpNum is the
369 /// operand number (in the II) that we are adding.
AddOperand(MachineInstrBuilder & MIB,SDValue Op,unsigned IIOpNum,const MCInstrDesc * II,DenseMap<SDValue,Register> & VRBaseMap,bool IsDebug,bool IsClone,bool IsCloned)370 void InstrEmitter::AddOperand(MachineInstrBuilder &MIB,
371 SDValue Op,
372 unsigned IIOpNum,
373 const MCInstrDesc *II,
374 DenseMap<SDValue, Register> &VRBaseMap,
375 bool IsDebug, bool IsClone, bool IsCloned) {
376 if (Op.isMachineOpcode()) {
377 AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
378 IsDebug, IsClone, IsCloned);
379 } else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
380 MIB.addImm(C->getSExtValue());
381 } else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Op)) {
382 MIB.addFPImm(F->getConstantFPValue());
383 } else if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Op)) {
384 Register VReg = R->getReg();
385 MVT OpVT = Op.getSimpleValueType();
386 const TargetRegisterClass *IIRC =
387 II ? TRI->getAllocatableClass(TII->getRegClass(*II, IIOpNum, TRI, *MF))
388 : nullptr;
389 const TargetRegisterClass *OpRC =
390 TLI->isTypeLegal(OpVT)
391 ? TLI->getRegClassFor(OpVT,
392 Op.getNode()->isDivergent() ||
393 (IIRC && TRI->isDivergentRegClass(IIRC)))
394 : nullptr;
395
396 if (OpRC && IIRC && OpRC != IIRC && Register::isVirtualRegister(VReg)) {
397 Register NewVReg = MRI->createVirtualRegister(IIRC);
398 BuildMI(*MBB, InsertPos, Op.getNode()->getDebugLoc(),
399 TII->get(TargetOpcode::COPY), NewVReg).addReg(VReg);
400 VReg = NewVReg;
401 }
402 // Turn additional physreg operands into implicit uses on non-variadic
403 // instructions. This is used by call and return instructions passing
404 // arguments in registers.
405 bool Imp = II && (IIOpNum >= II->getNumOperands() && !II->isVariadic());
406 MIB.addReg(VReg, getImplRegState(Imp));
407 } else if (RegisterMaskSDNode *RM = dyn_cast<RegisterMaskSDNode>(Op)) {
408 MIB.addRegMask(RM->getRegMask());
409 } else if (GlobalAddressSDNode *TGA = dyn_cast<GlobalAddressSDNode>(Op)) {
410 MIB.addGlobalAddress(TGA->getGlobal(), TGA->getOffset(),
411 TGA->getTargetFlags());
412 } else if (BasicBlockSDNode *BBNode = dyn_cast<BasicBlockSDNode>(Op)) {
413 MIB.addMBB(BBNode->getBasicBlock());
414 } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
415 MIB.addFrameIndex(FI->getIndex());
416 } else if (JumpTableSDNode *JT = dyn_cast<JumpTableSDNode>(Op)) {
417 MIB.addJumpTableIndex(JT->getIndex(), JT->getTargetFlags());
418 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op)) {
419 int Offset = CP->getOffset();
420 Align Alignment = CP->getAlign();
421
422 unsigned Idx;
423 MachineConstantPool *MCP = MF->getConstantPool();
424 if (CP->isMachineConstantPoolEntry())
425 Idx = MCP->getConstantPoolIndex(CP->getMachineCPVal(), Alignment);
426 else
427 Idx = MCP->getConstantPoolIndex(CP->getConstVal(), Alignment);
428 MIB.addConstantPoolIndex(Idx, Offset, CP->getTargetFlags());
429 } else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op)) {
430 MIB.addExternalSymbol(ES->getSymbol(), ES->getTargetFlags());
431 } else if (auto *SymNode = dyn_cast<MCSymbolSDNode>(Op)) {
432 MIB.addSym(SymNode->getMCSymbol());
433 } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Op)) {
434 MIB.addBlockAddress(BA->getBlockAddress(),
435 BA->getOffset(),
436 BA->getTargetFlags());
437 } else if (TargetIndexSDNode *TI = dyn_cast<TargetIndexSDNode>(Op)) {
438 MIB.addTargetIndex(TI->getIndex(), TI->getOffset(), TI->getTargetFlags());
439 } else {
440 assert(Op.getValueType() != MVT::Other &&
441 Op.getValueType() != MVT::Glue &&
442 "Chain and glue operands should occur at end of operand list!");
443 AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
444 IsDebug, IsClone, IsCloned);
445 }
446 }
447
ConstrainForSubReg(Register VReg,unsigned SubIdx,MVT VT,bool isDivergent,const DebugLoc & DL)448 Register InstrEmitter::ConstrainForSubReg(Register VReg, unsigned SubIdx,
449 MVT VT, bool isDivergent, const DebugLoc &DL) {
450 const TargetRegisterClass *VRC = MRI->getRegClass(VReg);
451 const TargetRegisterClass *RC = TRI->getSubClassWithSubReg(VRC, SubIdx);
452
453 // RC is a sub-class of VRC that supports SubIdx. Try to constrain VReg
454 // within reason.
455 if (RC && RC != VRC)
456 RC = MRI->constrainRegClass(VReg, RC, MinRCSize);
457
458 // VReg has been adjusted. It can be used with SubIdx operands now.
459 if (RC)
460 return VReg;
461
462 // VReg couldn't be reasonably constrained. Emit a COPY to a new virtual
463 // register instead.
464 RC = TRI->getSubClassWithSubReg(TLI->getRegClassFor(VT, isDivergent), SubIdx);
465 assert(RC && "No legal register class for VT supports that SubIdx");
466 Register NewReg = MRI->createVirtualRegister(RC);
467 BuildMI(*MBB, InsertPos, DL, TII->get(TargetOpcode::COPY), NewReg)
468 .addReg(VReg);
469 return NewReg;
470 }
471
472 /// EmitSubregNode - Generate machine code for subreg nodes.
473 ///
EmitSubregNode(SDNode * Node,DenseMap<SDValue,Register> & VRBaseMap,bool IsClone,bool IsCloned)474 void InstrEmitter::EmitSubregNode(SDNode *Node,
475 DenseMap<SDValue, Register> &VRBaseMap,
476 bool IsClone, bool IsCloned) {
477 Register VRBase;
478 unsigned Opc = Node->getMachineOpcode();
479
480 // If the node is only used by a CopyToReg and the dest reg is a vreg, use
481 // the CopyToReg'd destination register instead of creating a new vreg.
482 for (SDNode *User : Node->uses()) {
483 if (User->getOpcode() == ISD::CopyToReg &&
484 User->getOperand(2).getNode() == Node) {
485 Register DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
486 if (DestReg.isVirtual()) {
487 VRBase = DestReg;
488 break;
489 }
490 }
491 }
492
493 if (Opc == TargetOpcode::EXTRACT_SUBREG) {
494 // EXTRACT_SUBREG is lowered as %dst = COPY %src:sub. There are no
495 // constraints on the %dst register, COPY can target all legal register
496 // classes.
497 unsigned SubIdx = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
498 const TargetRegisterClass *TRC =
499 TLI->getRegClassFor(Node->getSimpleValueType(0), Node->isDivergent());
500
501 Register Reg;
502 MachineInstr *DefMI;
503 RegisterSDNode *R = dyn_cast<RegisterSDNode>(Node->getOperand(0));
504 if (R && Register::isPhysicalRegister(R->getReg())) {
505 Reg = R->getReg();
506 DefMI = nullptr;
507 } else {
508 Reg = R ? R->getReg() : getVR(Node->getOperand(0), VRBaseMap);
509 DefMI = MRI->getVRegDef(Reg);
510 }
511
512 Register SrcReg, DstReg;
513 unsigned DefSubIdx;
514 if (DefMI &&
515 TII->isCoalescableExtInstr(*DefMI, SrcReg, DstReg, DefSubIdx) &&
516 SubIdx == DefSubIdx &&
517 TRC == MRI->getRegClass(SrcReg)) {
518 // Optimize these:
519 // r1025 = s/zext r1024, 4
520 // r1026 = extract_subreg r1025, 4
521 // to a copy
522 // r1026 = copy r1024
523 VRBase = MRI->createVirtualRegister(TRC);
524 BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
525 TII->get(TargetOpcode::COPY), VRBase).addReg(SrcReg);
526 MRI->clearKillFlags(SrcReg);
527 } else {
528 // Reg may not support a SubIdx sub-register, and we may need to
529 // constrain its register class or issue a COPY to a compatible register
530 // class.
531 if (Reg.isVirtual())
532 Reg = ConstrainForSubReg(Reg, SubIdx,
533 Node->getOperand(0).getSimpleValueType(),
534 Node->isDivergent(), Node->getDebugLoc());
535 // Create the destreg if it is missing.
536 if (!VRBase)
537 VRBase = MRI->createVirtualRegister(TRC);
538
539 // Create the extract_subreg machine instruction.
540 MachineInstrBuilder CopyMI =
541 BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
542 TII->get(TargetOpcode::COPY), VRBase);
543 if (Reg.isVirtual())
544 CopyMI.addReg(Reg, 0, SubIdx);
545 else
546 CopyMI.addReg(TRI->getSubReg(Reg, SubIdx));
547 }
548 } else if (Opc == TargetOpcode::INSERT_SUBREG ||
549 Opc == TargetOpcode::SUBREG_TO_REG) {
550 SDValue N0 = Node->getOperand(0);
551 SDValue N1 = Node->getOperand(1);
552 SDValue N2 = Node->getOperand(2);
553 unsigned SubIdx = cast<ConstantSDNode>(N2)->getZExtValue();
554
555 // Figure out the register class to create for the destreg. It should be
556 // the largest legal register class supporting SubIdx sub-registers.
557 // RegisterCoalescer will constrain it further if it decides to eliminate
558 // the INSERT_SUBREG instruction.
559 //
560 // %dst = INSERT_SUBREG %src, %sub, SubIdx
561 //
562 // is lowered by TwoAddressInstructionPass to:
563 //
564 // %dst = COPY %src
565 // %dst:SubIdx = COPY %sub
566 //
567 // There is no constraint on the %src register class.
568 //
569 const TargetRegisterClass *SRC =
570 TLI->getRegClassFor(Node->getSimpleValueType(0), Node->isDivergent());
571 SRC = TRI->getSubClassWithSubReg(SRC, SubIdx);
572 assert(SRC && "No register class supports VT and SubIdx for INSERT_SUBREG");
573
574 if (VRBase == 0 || !SRC->hasSubClassEq(MRI->getRegClass(VRBase)))
575 VRBase = MRI->createVirtualRegister(SRC);
576
577 // Create the insert_subreg or subreg_to_reg machine instruction.
578 MachineInstrBuilder MIB =
579 BuildMI(*MF, Node->getDebugLoc(), TII->get(Opc), VRBase);
580
581 // If creating a subreg_to_reg, then the first input operand
582 // is an implicit value immediate, otherwise it's a register
583 if (Opc == TargetOpcode::SUBREG_TO_REG) {
584 const ConstantSDNode *SD = cast<ConstantSDNode>(N0);
585 MIB.addImm(SD->getZExtValue());
586 } else
587 AddOperand(MIB, N0, 0, nullptr, VRBaseMap, /*IsDebug=*/false,
588 IsClone, IsCloned);
589 // Add the subregister being inserted
590 AddOperand(MIB, N1, 0, nullptr, VRBaseMap, /*IsDebug=*/false,
591 IsClone, IsCloned);
592 MIB.addImm(SubIdx);
593 MBB->insert(InsertPos, MIB);
594 } else
595 llvm_unreachable("Node is not insert_subreg, extract_subreg, or subreg_to_reg");
596
597 SDValue Op(Node, 0);
598 bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second;
599 (void)isNew; // Silence compiler warning.
600 assert(isNew && "Node emitted out of order - early");
601 }
602
603 /// EmitCopyToRegClassNode - Generate machine code for COPY_TO_REGCLASS nodes.
604 /// COPY_TO_REGCLASS is just a normal copy, except that the destination
605 /// register is constrained to be in a particular register class.
606 ///
607 void
EmitCopyToRegClassNode(SDNode * Node,DenseMap<SDValue,Register> & VRBaseMap)608 InstrEmitter::EmitCopyToRegClassNode(SDNode *Node,
609 DenseMap<SDValue, Register> &VRBaseMap) {
610 unsigned VReg = getVR(Node->getOperand(0), VRBaseMap);
611
612 // Create the new VReg in the destination class and emit a copy.
613 unsigned DstRCIdx = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
614 const TargetRegisterClass *DstRC =
615 TRI->getAllocatableClass(TRI->getRegClass(DstRCIdx));
616 Register NewVReg = MRI->createVirtualRegister(DstRC);
617 BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TargetOpcode::COPY),
618 NewVReg).addReg(VReg);
619
620 SDValue Op(Node, 0);
621 bool isNew = VRBaseMap.insert(std::make_pair(Op, NewVReg)).second;
622 (void)isNew; // Silence compiler warning.
623 assert(isNew && "Node emitted out of order - early");
624 }
625
626 /// EmitRegSequence - Generate machine code for REG_SEQUENCE nodes.
627 ///
EmitRegSequence(SDNode * Node,DenseMap<SDValue,Register> & VRBaseMap,bool IsClone,bool IsCloned)628 void InstrEmitter::EmitRegSequence(SDNode *Node,
629 DenseMap<SDValue, Register> &VRBaseMap,
630 bool IsClone, bool IsCloned) {
631 unsigned DstRCIdx = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue();
632 const TargetRegisterClass *RC = TRI->getRegClass(DstRCIdx);
633 Register NewVReg = MRI->createVirtualRegister(TRI->getAllocatableClass(RC));
634 const MCInstrDesc &II = TII->get(TargetOpcode::REG_SEQUENCE);
635 MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II, NewVReg);
636 unsigned NumOps = Node->getNumOperands();
637 // If the input pattern has a chain, then the root of the corresponding
638 // output pattern will get a chain as well. This can happen to be a
639 // REG_SEQUENCE (which is not "guarded" by countOperands/CountResults).
640 if (NumOps && Node->getOperand(NumOps-1).getValueType() == MVT::Other)
641 --NumOps; // Ignore chain if it exists.
642
643 assert((NumOps & 1) == 1 &&
644 "REG_SEQUENCE must have an odd number of operands!");
645 for (unsigned i = 1; i != NumOps; ++i) {
646 SDValue Op = Node->getOperand(i);
647 if ((i & 1) == 0) {
648 RegisterSDNode *R = dyn_cast<RegisterSDNode>(Node->getOperand(i-1));
649 // Skip physical registers as they don't have a vreg to get and we'll
650 // insert copies for them in TwoAddressInstructionPass anyway.
651 if (!R || !Register::isPhysicalRegister(R->getReg())) {
652 unsigned SubIdx = cast<ConstantSDNode>(Op)->getZExtValue();
653 unsigned SubReg = getVR(Node->getOperand(i-1), VRBaseMap);
654 const TargetRegisterClass *TRC = MRI->getRegClass(SubReg);
655 const TargetRegisterClass *SRC =
656 TRI->getMatchingSuperRegClass(RC, TRC, SubIdx);
657 if (SRC && SRC != RC) {
658 MRI->setRegClass(NewVReg, SRC);
659 RC = SRC;
660 }
661 }
662 }
663 AddOperand(MIB, Op, i+1, &II, VRBaseMap, /*IsDebug=*/false,
664 IsClone, IsCloned);
665 }
666
667 MBB->insert(InsertPos, MIB);
668 SDValue Op(Node, 0);
669 bool isNew = VRBaseMap.insert(std::make_pair(Op, NewVReg)).second;
670 (void)isNew; // Silence compiler warning.
671 assert(isNew && "Node emitted out of order - early");
672 }
673
674 /// EmitDbgValue - Generate machine instruction for a dbg_value node.
675 ///
676 MachineInstr *
EmitDbgValue(SDDbgValue * SD,DenseMap<SDValue,Register> & VRBaseMap)677 InstrEmitter::EmitDbgValue(SDDbgValue *SD,
678 DenseMap<SDValue, Register> &VRBaseMap) {
679 MDNode *Var = SD->getVariable();
680 MDNode *Expr = SD->getExpression();
681 DebugLoc DL = SD->getDebugLoc();
682 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
683 "Expected inlined-at fields to agree");
684
685 SD->setIsEmitted();
686
687 ArrayRef<SDDbgOperand> LocationOps = SD->getLocationOps();
688 assert(!LocationOps.empty() && "dbg_value with no location operands?");
689
690 // Emit variadic dbg_value nodes as DBG_VALUE_LIST.
691 if (SD->isVariadic()) {
692 // DBG_VALUE_LIST := "DBG_VALUE_LIST" var, expression, loc (, loc)*
693 const MCInstrDesc &DbgValDesc = TII->get(TargetOpcode::DBG_VALUE_LIST);
694 // Build the DBG_VALUE_LIST instruction base.
695 auto MIB = BuildMI(*MF, DL, DbgValDesc);
696 MIB.addMetadata(Var);
697 MIB.addMetadata(Expr);
698 if (SD->isInvalidated()) {
699 // At least one node is no longer computed so we are unable to emit the
700 // location. Simply mark all of the operands as undef.
701 for (size_t i = 0; i < LocationOps.size(); ++i)
702 MIB.addReg(0U, RegState::Debug);
703 } else {
704 AddDbgValueLocationOps(MIB, DbgValDesc, LocationOps, VRBaseMap);
705 }
706 return &*MIB;
707 }
708
709 if (SD->isInvalidated()) {
710 // An invalidated SDNode must generate an undef DBG_VALUE: although the
711 // original value is no longer computed, earlier DBG_VALUEs live ranges
712 // must not leak into later code.
713 auto MIB = BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE));
714 MIB.addReg(0U);
715 MIB.addReg(0U, RegState::Debug);
716 MIB.addMetadata(Var);
717 MIB.addMetadata(Expr);
718 return &*MIB;
719 }
720
721 // Attempt to produce a DBG_INSTR_REF if we've been asked to.
722 // We currently exclude the possibility of instruction references for
723 // variadic nodes; if at some point we enable them, this should be moved
724 // above the variadic block.
725 if (EmitDebugInstrRefs)
726 if (auto *InstrRef = EmitDbgInstrRef(SD, VRBaseMap))
727 return InstrRef;
728
729 // Emit non-variadic dbg_value nodes as DBG_VALUE.
730 // DBG_VALUE := "DBG_VALUE" loc, isIndirect, var, expr
731 const MCInstrDesc &DbgValDesc = TII->get(TargetOpcode::DBG_VALUE);
732 // Build the DBG_VALUE instruction base.
733 auto MIB = BuildMI(*MF, DL, DbgValDesc);
734 // Add the single location componenet 'loc'.
735 assert(LocationOps.size() == 1 &&
736 "Non variadic dbg_value should have only one location op");
737 AddDbgValueLocationOps(MIB, DbgValDesc, LocationOps, VRBaseMap);
738 // Add 'isIndirect'.
739 if (SD->isIndirect())
740 MIB.addImm(0U);
741 else
742 MIB.addReg(0U, RegState::Debug);
743 MIB.addMetadata(Var);
744 MIB.addMetadata(Expr);
745 return &*MIB;
746 }
747
AddDbgValueLocationOps(MachineInstrBuilder & MIB,const MCInstrDesc & DbgValDesc,ArrayRef<SDDbgOperand> LocationOps,DenseMap<SDValue,Register> & VRBaseMap)748 void InstrEmitter::AddDbgValueLocationOps(
749 MachineInstrBuilder &MIB, const MCInstrDesc &DbgValDesc,
750 ArrayRef<SDDbgOperand> LocationOps,
751 DenseMap<SDValue, Register> &VRBaseMap) {
752 for (const SDDbgOperand &Op : LocationOps) {
753 switch (Op.getKind()) {
754 case SDDbgOperand::FRAMEIX:
755 MIB.addFrameIndex(Op.getFrameIx());
756 break;
757 case SDDbgOperand::VREG:
758 MIB.addReg(Op.getVReg(), RegState::Debug);
759 break;
760 case SDDbgOperand::SDNODE: {
761 SDValue V = SDValue(Op.getSDNode(), Op.getResNo());
762 // It's possible we replaced this SDNode with other(s) and therefore
763 // didn't generate code for it. It's better to catch these cases where
764 // they happen and transfer the debug info, but trying to guarantee that
765 // in all cases would be very fragile; this is a safeguard for any
766 // that were missed.
767 if (VRBaseMap.count(V) == 0)
768 MIB.addReg(0U); // undef
769 else
770 AddOperand(MIB, V, (*MIB).getNumOperands(), &DbgValDesc, VRBaseMap,
771 /*IsDebug=*/true, /*IsClone=*/false, /*IsCloned=*/false);
772 } break;
773 case SDDbgOperand::CONST: {
774 const Value *V = Op.getConst();
775 if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) {
776 if (CI->getBitWidth() > 64)
777 MIB.addCImm(CI);
778 else
779 MIB.addImm(CI->getSExtValue());
780 } else if (const ConstantFP *CF = dyn_cast<ConstantFP>(V)) {
781 MIB.addFPImm(CF);
782 } else if (isa<ConstantPointerNull>(V)) {
783 // Note: This assumes that all nullptr constants are zero-valued.
784 MIB.addImm(0);
785 } else {
786 // Could be an Undef. In any case insert an Undef so we can see what we
787 // dropped.
788 MIB.addReg(0U);
789 }
790 } break;
791 }
792 }
793 }
794
795 MachineInstr *
EmitDbgInstrRef(SDDbgValue * SD,DenseMap<SDValue,Register> & VRBaseMap)796 InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
797 DenseMap<SDValue, Register> &VRBaseMap) {
798 // No support for instruction references for variadic values yet.
799 if (SD->isVariadic())
800 return nullptr;
801 SDDbgOperand DbgOperand = SD->getLocationOps()[0];
802 // Instruction referencing is still in a prototype state: for now we're only
803 // going to support SDNodes within a block. Copies are not supported, they
804 // don't actually define a value.
805 if (DbgOperand.getKind() != SDDbgOperand::SDNODE)
806 return nullptr;
807
808 SDNode *Node = DbgOperand.getSDNode();
809 SDValue Op = SDValue(Node, DbgOperand.getResNo());
810 DenseMap<SDValue, Register>::iterator I = VRBaseMap.find(Op);
811 if (I==VRBaseMap.end())
812 return nullptr; // undef value: let EmitDbgValue produce a DBG_VALUE $noreg.
813
814 MDNode *Var = SD->getVariable();
815 MDNode *Expr = SD->getExpression();
816 DebugLoc DL = SD->getDebugLoc();
817
818 // Try to pick out a defining instruction at this point.
819 unsigned VReg = getVR(Op, VRBaseMap);
820 MachineInstr *ResultInstr = nullptr;
821
822 // No definition corresponds to scenarios where a vreg is live-in to a block,
823 // and doesn't have a defining instruction (yet). This can be patched up
824 // later; at this early stage of implementation, fall back to using DBG_VALUE.
825 if (!MRI->hasOneDef(VReg))
826 return nullptr;
827
828 MachineInstr &DefMI = *MRI->def_instr_begin(VReg);
829 // Some target specific opcodes can become copies. As stated above, we're
830 // ignoring those for now.
831 if (DefMI.isCopy() || DefMI.getOpcode() == TargetOpcode::SUBREG_TO_REG)
832 return nullptr;
833
834 const MCInstrDesc &RefII = TII->get(TargetOpcode::DBG_INSTR_REF);
835 auto MIB = BuildMI(*MF, DL, RefII);
836
837 // Find the operand which defines the specified VReg.
838 unsigned OperandIdx = 0;
839 for (const auto &MO : DefMI.operands()) {
840 if (MO.isReg() && MO.isDef() && MO.getReg() == VReg)
841 break;
842 ++OperandIdx;
843 }
844 assert(OperandIdx < DefMI.getNumOperands());
845
846 // Make the DBG_INSTR_REF refer to that instruction, and that operand.
847 unsigned InstrNum = DefMI.getDebugInstrNum();
848 MIB.addImm(InstrNum);
849 MIB.addImm(OperandIdx);
850 MIB.addMetadata(Var);
851 MIB.addMetadata(Expr);
852 ResultInstr = &*MIB;
853 return ResultInstr;
854 }
855
856 MachineInstr *
EmitDbgLabel(SDDbgLabel * SD)857 InstrEmitter::EmitDbgLabel(SDDbgLabel *SD) {
858 MDNode *Label = SD->getLabel();
859 DebugLoc DL = SD->getDebugLoc();
860 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
861 "Expected inlined-at fields to agree");
862
863 const MCInstrDesc &II = TII->get(TargetOpcode::DBG_LABEL);
864 MachineInstrBuilder MIB = BuildMI(*MF, DL, II);
865 MIB.addMetadata(Label);
866
867 return &*MIB;
868 }
869
870 /// EmitMachineNode - Generate machine code for a target-specific node and
871 /// needed dependencies.
872 ///
873 void InstrEmitter::
EmitMachineNode(SDNode * Node,bool IsClone,bool IsCloned,DenseMap<SDValue,Register> & VRBaseMap)874 EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
875 DenseMap<SDValue, Register> &VRBaseMap) {
876 unsigned Opc = Node->getMachineOpcode();
877
878 // Handle subreg insert/extract specially
879 if (Opc == TargetOpcode::EXTRACT_SUBREG ||
880 Opc == TargetOpcode::INSERT_SUBREG ||
881 Opc == TargetOpcode::SUBREG_TO_REG) {
882 EmitSubregNode(Node, VRBaseMap, IsClone, IsCloned);
883 return;
884 }
885
886 // Handle COPY_TO_REGCLASS specially.
887 if (Opc == TargetOpcode::COPY_TO_REGCLASS) {
888 EmitCopyToRegClassNode(Node, VRBaseMap);
889 return;
890 }
891
892 // Handle REG_SEQUENCE specially.
893 if (Opc == TargetOpcode::REG_SEQUENCE) {
894 EmitRegSequence(Node, VRBaseMap, IsClone, IsCloned);
895 return;
896 }
897
898 if (Opc == TargetOpcode::IMPLICIT_DEF)
899 // We want a unique VR for each IMPLICIT_DEF use.
900 return;
901
902 const MCInstrDesc &II = TII->get(Opc);
903 unsigned NumResults = CountResults(Node);
904 unsigned NumDefs = II.getNumDefs();
905 const MCPhysReg *ScratchRegs = nullptr;
906
907 // Handle STACKMAP and PATCHPOINT specially and then use the generic code.
908 if (Opc == TargetOpcode::STACKMAP || Opc == TargetOpcode::PATCHPOINT) {
909 // Stackmaps do not have arguments and do not preserve their calling
910 // convention. However, to simplify runtime support, they clobber the same
911 // scratch registers as AnyRegCC.
912 unsigned CC = CallingConv::AnyReg;
913 if (Opc == TargetOpcode::PATCHPOINT) {
914 CC = Node->getConstantOperandVal(PatchPointOpers::CCPos);
915 NumDefs = NumResults;
916 }
917 ScratchRegs = TLI->getScratchRegisters((CallingConv::ID) CC);
918 } else if (Opc == TargetOpcode::STATEPOINT) {
919 NumDefs = NumResults;
920 }
921
922 unsigned NumImpUses = 0;
923 unsigned NodeOperands =
924 countOperands(Node, II.getNumOperands() - NumDefs, NumImpUses);
925 bool HasVRegVariadicDefs = !MF->getTarget().usesPhysRegsForValues() &&
926 II.isVariadic() && II.variadicOpsAreDefs();
927 bool HasPhysRegOuts = NumResults > NumDefs &&
928 II.getImplicitDefs() != nullptr && !HasVRegVariadicDefs;
929 #ifndef NDEBUG
930 unsigned NumMIOperands = NodeOperands + NumResults;
931 if (II.isVariadic())
932 assert(NumMIOperands >= II.getNumOperands() &&
933 "Too few operands for a variadic node!");
934 else
935 assert(NumMIOperands >= II.getNumOperands() &&
936 NumMIOperands <= II.getNumOperands() + II.getNumImplicitDefs() +
937 NumImpUses &&
938 "#operands for dag node doesn't match .td file!");
939 #endif
940
941 // Create the new machine instruction.
942 MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II);
943
944 // Add result register values for things that are defined by this
945 // instruction.
946 if (NumResults) {
947 CreateVirtualRegisters(Node, MIB, II, IsClone, IsCloned, VRBaseMap);
948
949 // Transfer any IR flags from the SDNode to the MachineInstr
950 MachineInstr *MI = MIB.getInstr();
951 const SDNodeFlags Flags = Node->getFlags();
952 if (Flags.hasNoSignedZeros())
953 MI->setFlag(MachineInstr::MIFlag::FmNsz);
954
955 if (Flags.hasAllowReciprocal())
956 MI->setFlag(MachineInstr::MIFlag::FmArcp);
957
958 if (Flags.hasNoNaNs())
959 MI->setFlag(MachineInstr::MIFlag::FmNoNans);
960
961 if (Flags.hasNoInfs())
962 MI->setFlag(MachineInstr::MIFlag::FmNoInfs);
963
964 if (Flags.hasAllowContract())
965 MI->setFlag(MachineInstr::MIFlag::FmContract);
966
967 if (Flags.hasApproximateFuncs())
968 MI->setFlag(MachineInstr::MIFlag::FmAfn);
969
970 if (Flags.hasAllowReassociation())
971 MI->setFlag(MachineInstr::MIFlag::FmReassoc);
972
973 if (Flags.hasNoUnsignedWrap())
974 MI->setFlag(MachineInstr::MIFlag::NoUWrap);
975
976 if (Flags.hasNoSignedWrap())
977 MI->setFlag(MachineInstr::MIFlag::NoSWrap);
978
979 if (Flags.hasExact())
980 MI->setFlag(MachineInstr::MIFlag::IsExact);
981
982 if (Flags.hasNoFPExcept())
983 MI->setFlag(MachineInstr::MIFlag::NoFPExcept);
984 }
985
986 // Emit all of the actual operands of this instruction, adding them to the
987 // instruction as appropriate.
988 bool HasOptPRefs = NumDefs > NumResults;
989 assert((!HasOptPRefs || !HasPhysRegOuts) &&
990 "Unable to cope with optional defs and phys regs defs!");
991 unsigned NumSkip = HasOptPRefs ? NumDefs - NumResults : 0;
992 for (unsigned i = NumSkip; i != NodeOperands; ++i)
993 AddOperand(MIB, Node->getOperand(i), i-NumSkip+NumDefs, &II,
994 VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned);
995
996 // Add scratch registers as implicit def and early clobber
997 if (ScratchRegs)
998 for (unsigned i = 0; ScratchRegs[i]; ++i)
999 MIB.addReg(ScratchRegs[i], RegState::ImplicitDefine |
1000 RegState::EarlyClobber);
1001
1002 // Set the memory reference descriptions of this instruction now that it is
1003 // part of the function.
1004 MIB.setMemRefs(cast<MachineSDNode>(Node)->memoperands());
1005
1006 // Insert the instruction into position in the block. This needs to
1007 // happen before any custom inserter hook is called so that the
1008 // hook knows where in the block to insert the replacement code.
1009 MBB->insert(InsertPos, MIB);
1010
1011 // The MachineInstr may also define physregs instead of virtregs. These
1012 // physreg values can reach other instructions in different ways:
1013 //
1014 // 1. When there is a use of a Node value beyond the explicitly defined
1015 // virtual registers, we emit a CopyFromReg for one of the implicitly
1016 // defined physregs. This only happens when HasPhysRegOuts is true.
1017 //
1018 // 2. A CopyFromReg reading a physreg may be glued to this instruction.
1019 //
1020 // 3. A glued instruction may implicitly use a physreg.
1021 //
1022 // 4. A glued instruction may use a RegisterSDNode operand.
1023 //
1024 // Collect all the used physreg defs, and make sure that any unused physreg
1025 // defs are marked as dead.
1026 SmallVector<Register, 8> UsedRegs;
1027
1028 // Additional results must be physical register defs.
1029 if (HasPhysRegOuts) {
1030 for (unsigned i = NumDefs; i < NumResults; ++i) {
1031 Register Reg = II.getImplicitDefs()[i - NumDefs];
1032 if (!Node->hasAnyUseOfValue(i))
1033 continue;
1034 // This implicitly defined physreg has a use.
1035 UsedRegs.push_back(Reg);
1036 EmitCopyFromReg(Node, i, IsClone, IsCloned, Reg, VRBaseMap);
1037 }
1038 }
1039
1040 // Scan the glue chain for any used physregs.
1041 if (Node->getValueType(Node->getNumValues()-1) == MVT::Glue) {
1042 for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) {
1043 if (F->getOpcode() == ISD::CopyFromReg) {
1044 UsedRegs.push_back(cast<RegisterSDNode>(F->getOperand(1))->getReg());
1045 continue;
1046 } else if (F->getOpcode() == ISD::CopyToReg) {
1047 // Skip CopyToReg nodes that are internal to the glue chain.
1048 continue;
1049 }
1050 // Collect declared implicit uses.
1051 const MCInstrDesc &MCID = TII->get(F->getMachineOpcode());
1052 UsedRegs.append(MCID.getImplicitUses(),
1053 MCID.getImplicitUses() + MCID.getNumImplicitUses());
1054 // In addition to declared implicit uses, we must also check for
1055 // direct RegisterSDNode operands.
1056 for (unsigned i = 0, e = F->getNumOperands(); i != e; ++i)
1057 if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(F->getOperand(i))) {
1058 Register Reg = R->getReg();
1059 if (Reg.isPhysical())
1060 UsedRegs.push_back(Reg);
1061 }
1062 }
1063 }
1064
1065 // Finally mark unused registers as dead.
1066 if (!UsedRegs.empty() || II.getImplicitDefs() || II.hasOptionalDef())
1067 MIB->setPhysRegsDeadExcept(UsedRegs, *TRI);
1068
1069 // STATEPOINT is too 'dynamic' to have meaningful machine description.
1070 // We have to manually tie operands.
1071 if (Opc == TargetOpcode::STATEPOINT && NumDefs > 0) {
1072 assert(!HasPhysRegOuts && "STATEPOINT mishandled");
1073 MachineInstr *MI = MIB;
1074 unsigned Def = 0;
1075 int First = StatepointOpers(MI).getFirstGCPtrIdx();
1076 assert(First > 0 && "Statepoint has Defs but no GC ptr list");
1077 unsigned Use = (unsigned)First;
1078 while (Def < NumDefs) {
1079 if (MI->getOperand(Use).isReg())
1080 MI->tieOperands(Def++, Use);
1081 Use = StackMaps::getNextMetaArgIdx(MI, Use);
1082 }
1083 }
1084
1085 // Run post-isel target hook to adjust this instruction if needed.
1086 if (II.hasPostISelHook())
1087 TLI->AdjustInstrPostInstrSelection(*MIB, Node);
1088 }
1089
1090 /// EmitSpecialNode - Generate machine code for a target-independent node and
1091 /// needed dependencies.
1092 void InstrEmitter::
EmitSpecialNode(SDNode * Node,bool IsClone,bool IsCloned,DenseMap<SDValue,Register> & VRBaseMap)1093 EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
1094 DenseMap<SDValue, Register> &VRBaseMap) {
1095 switch (Node->getOpcode()) {
1096 default:
1097 #ifndef NDEBUG
1098 Node->dump();
1099 #endif
1100 llvm_unreachable("This target-independent node should have been selected!");
1101 case ISD::EntryToken:
1102 llvm_unreachable("EntryToken should have been excluded from the schedule!");
1103 case ISD::MERGE_VALUES:
1104 case ISD::TokenFactor: // fall thru
1105 break;
1106 case ISD::CopyToReg: {
1107 Register DestReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
1108 SDValue SrcVal = Node->getOperand(2);
1109 if (Register::isVirtualRegister(DestReg) && SrcVal.isMachineOpcode() &&
1110 SrcVal.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) {
1111 // Instead building a COPY to that vreg destination, build an
1112 // IMPLICIT_DEF instruction instead.
1113 BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
1114 TII->get(TargetOpcode::IMPLICIT_DEF), DestReg);
1115 break;
1116 }
1117 Register SrcReg;
1118 if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(SrcVal))
1119 SrcReg = R->getReg();
1120 else
1121 SrcReg = getVR(SrcVal, VRBaseMap);
1122
1123 if (SrcReg == DestReg) // Coalesced away the copy? Ignore.
1124 break;
1125
1126 BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TargetOpcode::COPY),
1127 DestReg).addReg(SrcReg);
1128 break;
1129 }
1130 case ISD::CopyFromReg: {
1131 unsigned SrcReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
1132 EmitCopyFromReg(Node, 0, IsClone, IsCloned, SrcReg, VRBaseMap);
1133 break;
1134 }
1135 case ISD::EH_LABEL:
1136 case ISD::ANNOTATION_LABEL: {
1137 unsigned Opc = (Node->getOpcode() == ISD::EH_LABEL)
1138 ? TargetOpcode::EH_LABEL
1139 : TargetOpcode::ANNOTATION_LABEL;
1140 MCSymbol *S = cast<LabelSDNode>(Node)->getLabel();
1141 BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
1142 TII->get(Opc)).addSym(S);
1143 break;
1144 }
1145
1146 case ISD::LIFETIME_START:
1147 case ISD::LIFETIME_END: {
1148 unsigned TarOp = (Node->getOpcode() == ISD::LIFETIME_START) ?
1149 TargetOpcode::LIFETIME_START : TargetOpcode::LIFETIME_END;
1150
1151 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Node->getOperand(1));
1152 BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TarOp))
1153 .addFrameIndex(FI->getIndex());
1154 break;
1155 }
1156
1157 case ISD::PSEUDO_PROBE: {
1158 unsigned TarOp = TargetOpcode::PSEUDO_PROBE;
1159 auto Guid = cast<PseudoProbeSDNode>(Node)->getGuid();
1160 auto Index = cast<PseudoProbeSDNode>(Node)->getIndex();
1161 auto Attr = cast<PseudoProbeSDNode>(Node)->getAttributes();
1162
1163 BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TarOp))
1164 .addImm(Guid)
1165 .addImm(Index)
1166 .addImm((uint8_t)PseudoProbeType::Block)
1167 .addImm(Attr);
1168 break;
1169 }
1170
1171 case ISD::INLINEASM:
1172 case ISD::INLINEASM_BR: {
1173 unsigned NumOps = Node->getNumOperands();
1174 if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
1175 --NumOps; // Ignore the glue operand.
1176
1177 // Create the inline asm machine instruction.
1178 unsigned TgtOpc = Node->getOpcode() == ISD::INLINEASM_BR
1179 ? TargetOpcode::INLINEASM_BR
1180 : TargetOpcode::INLINEASM;
1181 MachineInstrBuilder MIB =
1182 BuildMI(*MF, Node->getDebugLoc(), TII->get(TgtOpc));
1183
1184 // Add the asm string as an external symbol operand.
1185 SDValue AsmStrV = Node->getOperand(InlineAsm::Op_AsmString);
1186 const char *AsmStr = cast<ExternalSymbolSDNode>(AsmStrV)->getSymbol();
1187 MIB.addExternalSymbol(AsmStr);
1188
1189 // Add the HasSideEffect, isAlignStack, AsmDialect, MayLoad and MayStore
1190 // bits.
1191 int64_t ExtraInfo =
1192 cast<ConstantSDNode>(Node->getOperand(InlineAsm::Op_ExtraInfo))->
1193 getZExtValue();
1194 MIB.addImm(ExtraInfo);
1195
1196 // Remember to operand index of the group flags.
1197 SmallVector<unsigned, 8> GroupIdx;
1198
1199 // Remember registers that are part of early-clobber defs.
1200 SmallVector<unsigned, 8> ECRegs;
1201
1202 // Add all of the operand registers to the instruction.
1203 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
1204 unsigned Flags =
1205 cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
1206 const unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
1207
1208 GroupIdx.push_back(MIB->getNumOperands());
1209 MIB.addImm(Flags);
1210 ++i; // Skip the ID value.
1211
1212 switch (InlineAsm::getKind(Flags)) {
1213 default: llvm_unreachable("Bad flags!");
1214 case InlineAsm::Kind_RegDef:
1215 for (unsigned j = 0; j != NumVals; ++j, ++i) {
1216 unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
1217 // FIXME: Add dead flags for physical and virtual registers defined.
1218 // For now, mark physical register defs as implicit to help fast
1219 // regalloc. This makes inline asm look a lot like calls.
1220 MIB.addReg(Reg,
1221 RegState::Define |
1222 getImplRegState(Register::isPhysicalRegister(Reg)));
1223 }
1224 break;
1225 case InlineAsm::Kind_RegDefEarlyClobber:
1226 case InlineAsm::Kind_Clobber:
1227 for (unsigned j = 0; j != NumVals; ++j, ++i) {
1228 unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
1229 MIB.addReg(Reg,
1230 RegState::Define | RegState::EarlyClobber |
1231 getImplRegState(Register::isPhysicalRegister(Reg)));
1232 ECRegs.push_back(Reg);
1233 }
1234 break;
1235 case InlineAsm::Kind_RegUse: // Use of register.
1236 case InlineAsm::Kind_Imm: // Immediate.
1237 case InlineAsm::Kind_Mem: // Addressing mode.
1238 // The addressing mode has been selected, just add all of the
1239 // operands to the machine instruction.
1240 for (unsigned j = 0; j != NumVals; ++j, ++i)
1241 AddOperand(MIB, Node->getOperand(i), 0, nullptr, VRBaseMap,
1242 /*IsDebug=*/false, IsClone, IsCloned);
1243
1244 // Manually set isTied bits.
1245 if (InlineAsm::getKind(Flags) == InlineAsm::Kind_RegUse) {
1246 unsigned DefGroup = 0;
1247 if (InlineAsm::isUseOperandTiedToDef(Flags, DefGroup)) {
1248 unsigned DefIdx = GroupIdx[DefGroup] + 1;
1249 unsigned UseIdx = GroupIdx.back() + 1;
1250 for (unsigned j = 0; j != NumVals; ++j)
1251 MIB->tieOperands(DefIdx + j, UseIdx + j);
1252 }
1253 }
1254 break;
1255 }
1256 }
1257
1258 // GCC inline assembly allows input operands to also be early-clobber
1259 // output operands (so long as the operand is written only after it's
1260 // used), but this does not match the semantics of our early-clobber flag.
1261 // If an early-clobber operand register is also an input operand register,
1262 // then remove the early-clobber flag.
1263 for (unsigned Reg : ECRegs) {
1264 if (MIB->readsRegister(Reg, TRI)) {
1265 MachineOperand *MO =
1266 MIB->findRegisterDefOperand(Reg, false, false, TRI);
1267 assert(MO && "No def operand for clobbered register?");
1268 MO->setIsEarlyClobber(false);
1269 }
1270 }
1271
1272 // Get the mdnode from the asm if it exists and add it to the instruction.
1273 SDValue MDV = Node->getOperand(InlineAsm::Op_MDNode);
1274 const MDNode *MD = cast<MDNodeSDNode>(MDV)->getMD();
1275 if (MD)
1276 MIB.addMetadata(MD);
1277
1278 MBB->insert(InsertPos, MIB);
1279 break;
1280 }
1281 }
1282 }
1283
1284 /// InstrEmitter - Construct an InstrEmitter and set it to start inserting
1285 /// at the given position in the given block.
InstrEmitter(const TargetMachine & TM,MachineBasicBlock * mbb,MachineBasicBlock::iterator insertpos)1286 InstrEmitter::InstrEmitter(const TargetMachine &TM, MachineBasicBlock *mbb,
1287 MachineBasicBlock::iterator insertpos)
1288 : MF(mbb->getParent()), MRI(&MF->getRegInfo()),
1289 TII(MF->getSubtarget().getInstrInfo()),
1290 TRI(MF->getSubtarget().getRegisterInfo()),
1291 TLI(MF->getSubtarget().getTargetLowering()), MBB(mbb),
1292 InsertPos(insertpos) {
1293 EmitDebugInstrRefs = TM.Options.ValueTrackingVariableLocations;
1294 }
1295