1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines an instruction selector for the AMDGPU target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPUISelDAGToDAG.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUInstrInfo.h"
17 #include "AMDGPUSubtarget.h"
18 #include "AMDGPUTargetMachine.h"
19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
20 #include "MCTargetDesc/R600MCTargetDesc.h"
21 #include "R600RegisterInfo.h"
22 #include "SIMachineFunctionInfo.h"
23 #include "llvm/Analysis/UniformityAnalysis.h"
24 #include "llvm/Analysis/ValueTracking.h"
25 #include "llvm/CodeGen/FunctionLoweringInfo.h"
26 #include "llvm/CodeGen/SelectionDAG.h"
27 #include "llvm/CodeGen/SelectionDAGISel.h"
28 #include "llvm/CodeGen/SelectionDAGNodes.h"
29 #include "llvm/IR/IntrinsicsAMDGPU.h"
30 #include "llvm/InitializePasses.h"
31 #include "llvm/Support/ErrorHandling.h"
32 
33 #ifdef EXPENSIVE_CHECKS
34 #include "llvm/Analysis/LoopInfo.h"
35 #include "llvm/IR/Dominators.h"
36 #endif
37 
38 #define DEBUG_TYPE "amdgpu-isel"
39 
40 using namespace llvm;
41 
42 //===----------------------------------------------------------------------===//
43 // Instruction Selector Implementation
44 //===----------------------------------------------------------------------===//
45 
46 namespace {
47 static SDValue stripBitcast(SDValue Val) {
48   return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
49 }
50 
51 // Figure out if this is really an extract of the high 16-bits of a dword.
52 static bool isExtractHiElt(SDValue In, SDValue &Out) {
53   In = stripBitcast(In);
54 
55   if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
56     if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) {
57       if (!Idx->isOne())
58         return false;
59       Out = In.getOperand(0);
60       return true;
61     }
62   }
63 
64   if (In.getOpcode() != ISD::TRUNCATE)
65     return false;
66 
67   SDValue Srl = In.getOperand(0);
68   if (Srl.getOpcode() == ISD::SRL) {
69     if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) {
70       if (ShiftAmt->getZExtValue() == 16) {
71         Out = stripBitcast(Srl.getOperand(0));
72         return true;
73       }
74     }
75   }
76 
77   return false;
78 }
79 
80 // Look through operations that obscure just looking at the low 16-bits of the
81 // same register.
82 static SDValue stripExtractLoElt(SDValue In) {
83   if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
84     if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(In.getOperand(1))) {
85       if (Idx->isZero() && In.getValueSizeInBits() <= 32)
86         return In.getOperand(0);
87     }
88   }
89 
90   if (In.getOpcode() == ISD::TRUNCATE) {
91     SDValue Src = In.getOperand(0);
92     if (Src.getValueType().getSizeInBits() == 32)
93       return stripBitcast(Src);
94   }
95 
96   return In;
97 }
98 
99 } // end anonymous namespace
100 
101 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel",
102                       "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
103 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo)
104 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis)
105 INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
106 #ifdef EXPENSIVE_CHECKS
107 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
108 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
109 #endif
110 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel",
111                     "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
112 
113 /// This pass converts a legalized DAG into a AMDGPU-specific
114 // DAG, ready for instruction scheduling.
115 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine &TM,
116                                         CodeGenOpt::Level OptLevel) {
117   return new AMDGPUDAGToDAGISel(TM, OptLevel);
118 }
119 
120 AMDGPUDAGToDAGISel::AMDGPUDAGToDAGISel(TargetMachine &TM,
121                                        CodeGenOpt::Level OptLevel)
122     : SelectionDAGISel(ID, TM, OptLevel) {
123   EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG;
124 }
125 
126 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
127 #ifdef EXPENSIVE_CHECKS
128   DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
129   LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
130   for (auto &L : LI->getLoopsInPreorder()) {
131     assert(L->isLCSSAForm(DT));
132   }
133 #endif
134   Subtarget = &MF.getSubtarget<GCNSubtarget>();
135   Mode = SIModeRegisterDefaults(MF.getFunction());
136   return SelectionDAGISel::runOnMachineFunction(MF);
137 }
138 
139 bool AMDGPUDAGToDAGISel::fp16SrcZerosHighBits(unsigned Opc) const {
140   // XXX - only need to list legal operations.
141   switch (Opc) {
142   case ISD::FADD:
143   case ISD::FSUB:
144   case ISD::FMUL:
145   case ISD::FDIV:
146   case ISD::FREM:
147   case ISD::FCANONICALIZE:
148   case ISD::UINT_TO_FP:
149   case ISD::SINT_TO_FP:
150   case ISD::FABS:
151     // Fabs is lowered to a bit operation, but it's an and which will clear the
152     // high bits anyway.
153   case ISD::FSQRT:
154   case ISD::FSIN:
155   case ISD::FCOS:
156   case ISD::FPOWI:
157   case ISD::FPOW:
158   case ISD::FLOG:
159   case ISD::FLOG2:
160   case ISD::FLOG10:
161   case ISD::FEXP:
162   case ISD::FEXP2:
163   case ISD::FCEIL:
164   case ISD::FTRUNC:
165   case ISD::FRINT:
166   case ISD::FNEARBYINT:
167   case ISD::FROUND:
168   case ISD::FFLOOR:
169   case ISD::FMINNUM:
170   case ISD::FMAXNUM:
171   case ISD::FLDEXP:
172   case AMDGPUISD::FRACT:
173   case AMDGPUISD::CLAMP:
174   case AMDGPUISD::COS_HW:
175   case AMDGPUISD::SIN_HW:
176   case AMDGPUISD::FMIN3:
177   case AMDGPUISD::FMAX3:
178   case AMDGPUISD::FMED3:
179   case AMDGPUISD::FMAD_FTZ:
180   case AMDGPUISD::RCP:
181   case AMDGPUISD::RSQ:
182   case AMDGPUISD::RCP_IFLAG:
183     // On gfx10, all 16-bit instructions preserve the high bits.
184     return Subtarget->getGeneration() <= AMDGPUSubtarget::GFX9;
185   case ISD::FP_ROUND:
186     // We may select fptrunc (fma/mad) to mad_mixlo, which does not zero the
187     // high bits on gfx9.
188     // TODO: If we had the source node we could see if the source was fma/mad
189     return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
190   case ISD::FMA:
191   case ISD::FMAD:
192   case AMDGPUISD::DIV_FIXUP:
193     return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
194   default:
195     // fcopysign, select and others may be lowered to 32-bit bit operations
196     // which don't zero the high bits.
197     return false;
198   }
199 }
200 
201 void AMDGPUDAGToDAGISel::getAnalysisUsage(AnalysisUsage &AU) const {
202   AU.addRequired<AMDGPUArgumentUsageInfo>();
203   AU.addRequired<UniformityInfoWrapperPass>();
204 #ifdef EXPENSIVE_CHECKS
205   AU.addRequired<DominatorTreeWrapperPass>();
206   AU.addRequired<LoopInfoWrapperPass>();
207 #endif
208   SelectionDAGISel::getAnalysisUsage(AU);
209 }
210 
211 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
212   assert(Subtarget->d16PreservesUnusedBits());
213   MVT VT = N->getValueType(0).getSimpleVT();
214   if (VT != MVT::v2i16 && VT != MVT::v2f16)
215     return false;
216 
217   SDValue Lo = N->getOperand(0);
218   SDValue Hi = N->getOperand(1);
219 
220   LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi));
221 
222   // build_vector lo, (load ptr) -> load_d16_hi ptr, lo
223   // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo
224   // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo
225 
226   // Need to check for possible indirect dependencies on the other half of the
227   // vector to avoid introducing a cycle.
228   if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) {
229     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
230 
231     SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo);
232     SDValue Ops[] = {
233       LdHi->getChain(), LdHi->getBasePtr(), TiedIn
234     };
235 
236     unsigned LoadOp = AMDGPUISD::LOAD_D16_HI;
237     if (LdHi->getMemoryVT() == MVT::i8) {
238       LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ?
239         AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8;
240     } else {
241       assert(LdHi->getMemoryVT() == MVT::i16);
242     }
243 
244     SDValue NewLoadHi =
245       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList,
246                                   Ops, LdHi->getMemoryVT(),
247                                   LdHi->getMemOperand());
248 
249     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi);
250     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1));
251     return true;
252   }
253 
254   // build_vector (load ptr), hi -> load_d16_lo ptr, hi
255   // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi
256   // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi
257   LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo));
258   if (LdLo && Lo.hasOneUse()) {
259     SDValue TiedIn = getHi16Elt(Hi);
260     if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode()))
261       return false;
262 
263     SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
264     unsigned LoadOp = AMDGPUISD::LOAD_D16_LO;
265     if (LdLo->getMemoryVT() == MVT::i8) {
266       LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ?
267         AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8;
268     } else {
269       assert(LdLo->getMemoryVT() == MVT::i16);
270     }
271 
272     TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn);
273 
274     SDValue Ops[] = {
275       LdLo->getChain(), LdLo->getBasePtr(), TiedIn
276     };
277 
278     SDValue NewLoadLo =
279       CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList,
280                                   Ops, LdLo->getMemoryVT(),
281                                   LdLo->getMemOperand());
282 
283     CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo);
284     CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1));
285     return true;
286   }
287 
288   return false;
289 }
290 
291 void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
292   if (!Subtarget->d16PreservesUnusedBits())
293     return;
294 
295   SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
296 
297   bool MadeChange = false;
298   while (Position != CurDAG->allnodes_begin()) {
299     SDNode *N = &*--Position;
300     if (N->use_empty())
301       continue;
302 
303     switch (N->getOpcode()) {
304     case ISD::BUILD_VECTOR:
305       MadeChange |= matchLoadD16FromBuildVector(N);
306       break;
307     default:
308       break;
309     }
310   }
311 
312   if (MadeChange) {
313     CurDAG->RemoveDeadNodes();
314     LLVM_DEBUG(dbgs() << "After PreProcess:\n";
315                CurDAG->dump(););
316   }
317 }
318 
319 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N,
320                                            bool Negated) const {
321   if (N->isUndef())
322     return true;
323 
324   const SIInstrInfo *TII = Subtarget->getInstrInfo();
325   if (Negated) {
326     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
327       return TII->isInlineConstant(-C->getAPIntValue());
328 
329     if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
330       return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt());
331 
332   } else {
333     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
334       return TII->isInlineConstant(C->getAPIntValue());
335 
336     if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
337       return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt());
338   }
339 
340   return false;
341 }
342 
343 /// Determine the register class for \p OpNo
344 /// \returns The register class of the virtual register that will be used for
345 /// the given operand number \OpNo or NULL if the register class cannot be
346 /// determined.
347 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
348                                                           unsigned OpNo) const {
349   if (!N->isMachineOpcode()) {
350     if (N->getOpcode() == ISD::CopyToReg) {
351       Register Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
352       if (Reg.isVirtual()) {
353         MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo();
354         return MRI.getRegClass(Reg);
355       }
356 
357       const SIRegisterInfo *TRI
358         = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo();
359       return TRI->getPhysRegBaseClass(Reg);
360     }
361 
362     return nullptr;
363   }
364 
365   switch (N->getMachineOpcode()) {
366   default: {
367     const MCInstrDesc &Desc =
368         Subtarget->getInstrInfo()->get(N->getMachineOpcode());
369     unsigned OpIdx = Desc.getNumDefs() + OpNo;
370     if (OpIdx >= Desc.getNumOperands())
371       return nullptr;
372     int RegClass = Desc.operands()[OpIdx].RegClass;
373     if (RegClass == -1)
374       return nullptr;
375 
376     return Subtarget->getRegisterInfo()->getRegClass(RegClass);
377   }
378   case AMDGPU::REG_SEQUENCE: {
379     unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
380     const TargetRegisterClass *SuperRC =
381         Subtarget->getRegisterInfo()->getRegClass(RCID);
382 
383     SDValue SubRegOp = N->getOperand(OpNo + 1);
384     unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
385     return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
386                                                               SubRegIdx);
387   }
388   }
389 }
390 
391 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain,
392                                          SDValue Glue) const {
393   SmallVector <SDValue, 8> Ops;
394   Ops.push_back(NewChain); // Replace the chain.
395   for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
396     Ops.push_back(N->getOperand(i));
397 
398   Ops.push_back(Glue);
399   return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
400 }
401 
402 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
403   const SITargetLowering& Lowering =
404     *static_cast<const SITargetLowering*>(getTargetLowering());
405 
406   assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain");
407 
408   SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val);
409   return glueCopyToOp(N, M0, M0.getValue(1));
410 }
411 
412 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const {
413   unsigned AS = cast<MemSDNode>(N)->getAddressSpace();
414   if (AS == AMDGPUAS::LOCAL_ADDRESS) {
415     if (Subtarget->ldsRequiresM0Init())
416       return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
417   } else if (AS == AMDGPUAS::REGION_ADDRESS) {
418     MachineFunction &MF = CurDAG->getMachineFunction();
419     unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize();
420     return
421         glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32));
422   }
423   return N;
424 }
425 
426 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm,
427                                                   EVT VT) const {
428   SDNode *Lo = CurDAG->getMachineNode(
429       AMDGPU::S_MOV_B32, DL, MVT::i32,
430       CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32));
431   SDNode *Hi =
432       CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
433                              CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32));
434   const SDValue Ops[] = {
435       CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
436       SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
437       SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
438 
439   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops);
440 }
441 
442 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
443   EVT VT = N->getValueType(0);
444   unsigned NumVectorElts = VT.getVectorNumElements();
445   EVT EltVT = VT.getVectorElementType();
446   SDLoc DL(N);
447   SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
448 
449   if (NumVectorElts == 1) {
450     CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
451                          RegClass);
452     return;
453   }
454 
455   assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not "
456                                   "supported yet");
457   // 32 = Max Num Vector Elements
458   // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
459   // 1 = Vector Register Class
460   SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
461 
462   bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() ==
463                Triple::amdgcn;
464   RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
465   bool IsRegSeq = true;
466   unsigned NOps = N->getNumOperands();
467   for (unsigned i = 0; i < NOps; i++) {
468     // XXX: Why is this here?
469     if (isa<RegisterSDNode>(N->getOperand(i))) {
470       IsRegSeq = false;
471       break;
472     }
473     unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
474                          : R600RegisterInfo::getSubRegFromChannel(i);
475     RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
476     RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32);
477   }
478   if (NOps != NumVectorElts) {
479     // Fill in the missing undef elements if this was a scalar_to_vector.
480     assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
481     MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
482                                                    DL, EltVT);
483     for (unsigned i = NOps; i < NumVectorElts; ++i) {
484       unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i)
485                            : R600RegisterInfo::getSubRegFromChannel(i);
486       RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
487       RegSeqArgs[1 + (2 * i) + 1] =
488           CurDAG->getTargetConstant(Sub, DL, MVT::i32);
489     }
490   }
491 
492   if (!IsRegSeq)
493     SelectCode(N);
494   CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
495 }
496 
497 void AMDGPUDAGToDAGISel::Select(SDNode *N) {
498   unsigned int Opc = N->getOpcode();
499   if (N->isMachineOpcode()) {
500     N->setNodeId(-1);
501     return;   // Already selected.
502   }
503 
504   // isa<MemSDNode> almost works but is slightly too permissive for some DS
505   // intrinsics.
506   if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) ||
507       Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
508       Opc == AMDGPUISD::ATOMIC_LOAD_FMAX) {
509     N = glueCopyToM0LDSInit(N);
510     SelectCode(N);
511     return;
512   }
513 
514   switch (Opc) {
515   default:
516     break;
517   // We are selecting i64 ADD here instead of custom lower it during
518   // DAG legalization, so we can fold some i64 ADDs used for address
519   // calculation into the LOAD and STORE instructions.
520   case ISD::ADDC:
521   case ISD::ADDE:
522   case ISD::SUBC:
523   case ISD::SUBE: {
524     if (N->getValueType(0) != MVT::i64)
525       break;
526 
527     SelectADD_SUB_I64(N);
528     return;
529   }
530   case ISD::UADDO_CARRY:
531   case ISD::USUBO_CARRY:
532     if (N->getValueType(0) != MVT::i32)
533       break;
534 
535     SelectAddcSubb(N);
536     return;
537   case ISD::UADDO:
538   case ISD::USUBO: {
539     SelectUADDO_USUBO(N);
540     return;
541   }
542   case AMDGPUISD::FMUL_W_CHAIN: {
543     SelectFMUL_W_CHAIN(N);
544     return;
545   }
546   case AMDGPUISD::FMA_W_CHAIN: {
547     SelectFMA_W_CHAIN(N);
548     return;
549   }
550 
551   case ISD::SCALAR_TO_VECTOR:
552   case ISD::BUILD_VECTOR: {
553     EVT VT = N->getValueType(0);
554     unsigned NumVectorElts = VT.getVectorNumElements();
555     if (VT.getScalarSizeInBits() == 16) {
556       if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) {
557         if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) {
558           ReplaceNode(N, Packed);
559           return;
560         }
561       }
562 
563       break;
564     }
565 
566     assert(VT.getVectorElementType().bitsEq(MVT::i32));
567     unsigned RegClassID =
568         SIRegisterInfo::getSGPRClassForBitWidth(NumVectorElts * 32)->getID();
569     SelectBuildVector(N, RegClassID);
570     return;
571   }
572   case ISD::BUILD_PAIR: {
573     SDValue RC, SubReg0, SubReg1;
574     SDLoc DL(N);
575     if (N->getValueType(0) == MVT::i128) {
576       RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32);
577       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
578       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
579     } else if (N->getValueType(0) == MVT::i64) {
580       RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
581       SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
582       SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
583     } else {
584       llvm_unreachable("Unhandled value type for BUILD_PAIR");
585     }
586     const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
587                             N->getOperand(1), SubReg1 };
588     ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
589                                           N->getValueType(0), Ops));
590     return;
591   }
592 
593   case ISD::Constant:
594   case ISD::ConstantFP: {
595     if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
596       break;
597 
598     uint64_t Imm;
599     if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
600       Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
601     else {
602       ConstantSDNode *C = cast<ConstantSDNode>(N);
603       Imm = C->getZExtValue();
604     }
605 
606     SDLoc DL(N);
607     ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0)));
608     return;
609   }
610   case AMDGPUISD::BFE_I32:
611   case AMDGPUISD::BFE_U32: {
612     // There is a scalar version available, but unlike the vector version which
613     // has a separate operand for the offset and width, the scalar version packs
614     // the width and offset into a single operand. Try to move to the scalar
615     // version if the offsets are constant, so that we can try to keep extended
616     // loads of kernel arguments in SGPRs.
617 
618     // TODO: Technically we could try to pattern match scalar bitshifts of
619     // dynamic values, but it's probably not useful.
620     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
621     if (!Offset)
622       break;
623 
624     ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
625     if (!Width)
626       break;
627 
628     bool Signed = Opc == AMDGPUISD::BFE_I32;
629 
630     uint32_t OffsetVal = Offset->getZExtValue();
631     uint32_t WidthVal = Width->getZExtValue();
632 
633     ReplaceNode(N, getBFE32(Signed, SDLoc(N), N->getOperand(0), OffsetVal,
634                             WidthVal));
635     return;
636   }
637   case AMDGPUISD::DIV_SCALE: {
638     SelectDIV_SCALE(N);
639     return;
640   }
641   case AMDGPUISD::MAD_I64_I32:
642   case AMDGPUISD::MAD_U64_U32: {
643     SelectMAD_64_32(N);
644     return;
645   }
646   case ISD::SMUL_LOHI:
647   case ISD::UMUL_LOHI:
648     return SelectMUL_LOHI(N);
649   case ISD::CopyToReg: {
650     const SITargetLowering& Lowering =
651       *static_cast<const SITargetLowering*>(getTargetLowering());
652     N = Lowering.legalizeTargetIndependentNode(N, *CurDAG);
653     break;
654   }
655   case ISD::AND:
656   case ISD::SRL:
657   case ISD::SRA:
658   case ISD::SIGN_EXTEND_INREG:
659     if (N->getValueType(0) != MVT::i32)
660       break;
661 
662     SelectS_BFE(N);
663     return;
664   case ISD::BRCOND:
665     SelectBRCOND(N);
666     return;
667   case AMDGPUISD::CVT_PKRTZ_F16_F32:
668   case AMDGPUISD::CVT_PKNORM_I16_F32:
669   case AMDGPUISD::CVT_PKNORM_U16_F32:
670   case AMDGPUISD::CVT_PK_U16_U32:
671   case AMDGPUISD::CVT_PK_I16_I32: {
672     // Hack around using a legal type if f16 is illegal.
673     if (N->getValueType(0) == MVT::i32) {
674       MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16;
675       N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT),
676                               { N->getOperand(0), N->getOperand(1) });
677       SelectCode(N);
678       return;
679     }
680 
681     break;
682   }
683   case ISD::INTRINSIC_W_CHAIN: {
684     SelectINTRINSIC_W_CHAIN(N);
685     return;
686   }
687   case ISD::INTRINSIC_WO_CHAIN: {
688     SelectINTRINSIC_WO_CHAIN(N);
689     return;
690   }
691   case ISD::INTRINSIC_VOID: {
692     SelectINTRINSIC_VOID(N);
693     return;
694   }
695   }
696 
697   SelectCode(N);
698 }
699 
700 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
701   const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
702   const Instruction *Term = BB->getTerminator();
703   return Term->getMetadata("amdgpu.uniform") ||
704          Term->getMetadata("structurizecfg.uniform");
705 }
706 
707 bool AMDGPUDAGToDAGISel::isUnneededShiftMask(const SDNode *N,
708                                              unsigned ShAmtBits) const {
709   assert(N->getOpcode() == ISD::AND);
710 
711   const APInt &RHS = cast<ConstantSDNode>(N->getOperand(1))->getAPIntValue();
712   if (RHS.countr_one() >= ShAmtBits)
713     return true;
714 
715   const APInt &LHSKnownZeros = CurDAG->computeKnownBits(N->getOperand(0)).Zero;
716   return (LHSKnownZeros | RHS).countr_one() >= ShAmtBits;
717 }
718 
719 static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr,
720                                           SDValue &N0, SDValue &N1) {
721   if (Addr.getValueType() == MVT::i64 && Addr.getOpcode() == ISD::BITCAST &&
722       Addr.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
723     // As we split 64-bit `or` earlier, it's complicated pattern to match, i.e.
724     // (i64 (bitcast (v2i32 (build_vector
725     //                        (or (extract_vector_elt V, 0), OFFSET),
726     //                        (extract_vector_elt V, 1)))))
727     SDValue Lo = Addr.getOperand(0).getOperand(0);
728     if (Lo.getOpcode() == ISD::OR && DAG.isBaseWithConstantOffset(Lo)) {
729       SDValue BaseLo = Lo.getOperand(0);
730       SDValue BaseHi = Addr.getOperand(0).getOperand(1);
731       // Check that split base (Lo and Hi) are extracted from the same one.
732       if (BaseLo.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
733           BaseHi.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
734           BaseLo.getOperand(0) == BaseHi.getOperand(0) &&
735           // Lo is statically extracted from index 0.
736           isa<ConstantSDNode>(BaseLo.getOperand(1)) &&
737           BaseLo.getConstantOperandVal(1) == 0 &&
738           // Hi is statically extracted from index 0.
739           isa<ConstantSDNode>(BaseHi.getOperand(1)) &&
740           BaseHi.getConstantOperandVal(1) == 1) {
741         N0 = BaseLo.getOperand(0).getOperand(0);
742         N1 = Lo.getOperand(1);
743         return true;
744       }
745     }
746   }
747   return false;
748 }
749 
750 bool AMDGPUDAGToDAGISel::isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
751                                                     SDValue &RHS) const {
752   if (CurDAG->isBaseWithConstantOffset(Addr)) {
753     LHS = Addr.getOperand(0);
754     RHS = Addr.getOperand(1);
755     return true;
756   }
757 
758   if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, LHS, RHS)) {
759     assert(LHS && RHS && isa<ConstantSDNode>(RHS));
760     return true;
761   }
762 
763   return false;
764 }
765 
766 StringRef AMDGPUDAGToDAGISel::getPassName() const {
767   return "AMDGPU DAG->DAG Pattern Instruction Selection";
768 }
769 
770 //===----------------------------------------------------------------------===//
771 // Complex Patterns
772 //===----------------------------------------------------------------------===//
773 
774 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
775                                             SDValue &Offset) {
776   return false;
777 }
778 
779 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
780                                             SDValue &Offset) {
781   ConstantSDNode *C;
782   SDLoc DL(Addr);
783 
784   if ((C = dyn_cast<ConstantSDNode>(Addr))) {
785     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
786     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
787   } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
788              (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
789     Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
790     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
791   } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
792             (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
793     Base = Addr.getOperand(0);
794     Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
795   } else {
796     Base = Addr;
797     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
798   }
799 
800   return true;
801 }
802 
803 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
804                                                        const SDLoc &DL) const {
805   SDNode *Mov = CurDAG->getMachineNode(
806     AMDGPU::S_MOV_B32, DL, MVT::i32,
807     CurDAG->getTargetConstant(Val, DL, MVT::i32));
808   return SDValue(Mov, 0);
809 }
810 
811 // FIXME: Should only handle uaddo_carry/usubo_carry
812 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
813   SDLoc DL(N);
814   SDValue LHS = N->getOperand(0);
815   SDValue RHS = N->getOperand(1);
816 
817   unsigned Opcode = N->getOpcode();
818   bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE);
819   bool ProduceCarry =
820       ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC;
821   bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE;
822 
823   SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
824   SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
825 
826   SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
827                                        DL, MVT::i32, LHS, Sub0);
828   SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
829                                        DL, MVT::i32, LHS, Sub1);
830 
831   SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
832                                        DL, MVT::i32, RHS, Sub0);
833   SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
834                                        DL, MVT::i32, RHS, Sub1);
835 
836   SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
837 
838   static const unsigned OpcMap[2][2][2] = {
839       {{AMDGPU::S_SUB_U32, AMDGPU::S_ADD_U32},
840        {AMDGPU::V_SUB_CO_U32_e32, AMDGPU::V_ADD_CO_U32_e32}},
841       {{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32},
842        {AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}};
843 
844   unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd];
845   unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd];
846 
847   SDNode *AddLo;
848   if (!ConsumeCarry) {
849     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
850     AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args);
851   } else {
852     SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) };
853     AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
854   }
855   SDValue AddHiArgs[] = {
856     SDValue(Hi0, 0),
857     SDValue(Hi1, 0),
858     SDValue(AddLo, 1)
859   };
860   SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs);
861 
862   SDValue RegSequenceArgs[] = {
863     CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
864     SDValue(AddLo,0),
865     Sub0,
866     SDValue(AddHi,0),
867     Sub1,
868   };
869   SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
870                                                MVT::i64, RegSequenceArgs);
871 
872   if (ProduceCarry) {
873     // Replace the carry-use
874     ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1));
875   }
876 
877   // Replace the remaining uses.
878   ReplaceNode(N, RegSequence);
879 }
880 
881 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) {
882   SDLoc DL(N);
883   SDValue LHS = N->getOperand(0);
884   SDValue RHS = N->getOperand(1);
885   SDValue CI = N->getOperand(2);
886 
887   if (N->isDivergent()) {
888     unsigned Opc = N->getOpcode() == ISD::UADDO_CARRY ? AMDGPU::V_ADDC_U32_e64
889                                                       : AMDGPU::V_SUBB_U32_e64;
890     CurDAG->SelectNodeTo(
891         N, Opc, N->getVTList(),
892         {LHS, RHS, CI,
893          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
894   } else {
895     unsigned Opc = N->getOpcode() == ISD::UADDO_CARRY ? AMDGPU::S_ADD_CO_PSEUDO
896                                                       : AMDGPU::S_SUB_CO_PSEUDO;
897     CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI});
898   }
899 }
900 
901 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
902   // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
903   // carry out despite the _i32 name. These were renamed in VI to _U32.
904   // FIXME: We should probably rename the opcodes here.
905   bool IsAdd = N->getOpcode() == ISD::UADDO;
906   bool IsVALU = N->isDivergent();
907 
908   for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E;
909        ++UI)
910     if (UI.getUse().getResNo() == 1) {
911       if ((IsAdd && (UI->getOpcode() != ISD::UADDO_CARRY)) ||
912           (!IsAdd && (UI->getOpcode() != ISD::USUBO_CARRY))) {
913         IsVALU = true;
914         break;
915       }
916     }
917 
918   if (IsVALU) {
919     unsigned Opc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
920 
921     CurDAG->SelectNodeTo(
922         N, Opc, N->getVTList(),
923         {N->getOperand(0), N->getOperand(1),
924          CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
925   } else {
926     unsigned Opc = N->getOpcode() == ISD::UADDO ? AMDGPU::S_UADDO_PSEUDO
927                                                 : AMDGPU::S_USUBO_PSEUDO;
928 
929     CurDAG->SelectNodeTo(N, Opc, N->getVTList(),
930                          {N->getOperand(0), N->getOperand(1)});
931   }
932 }
933 
934 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) {
935   SDLoc SL(N);
936   //  src0_modifiers, src0,  src1_modifiers, src1, src2_modifiers, src2, clamp, omod
937   SDValue Ops[10];
938 
939   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]);
940   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
941   SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]);
942   Ops[8] = N->getOperand(0);
943   Ops[9] = N->getOperand(4);
944 
945   // If there are no source modifiers, prefer fmac over fma because it can use
946   // the smaller VOP2 encoding.
947   bool UseFMAC = Subtarget->hasDLInsts() &&
948                  cast<ConstantSDNode>(Ops[0])->isZero() &&
949                  cast<ConstantSDNode>(Ops[2])->isZero() &&
950                  cast<ConstantSDNode>(Ops[4])->isZero();
951   unsigned Opcode = UseFMAC ? AMDGPU::V_FMAC_F32_e64 : AMDGPU::V_FMA_F32_e64;
952   CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), Ops);
953 }
954 
955 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) {
956   SDLoc SL(N);
957   //    src0_modifiers, src0,  src1_modifiers, src1, clamp, omod
958   SDValue Ops[8];
959 
960   SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]);
961   SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
962   Ops[6] = N->getOperand(0);
963   Ops[7] = N->getOperand(3);
964 
965   CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops);
966 }
967 
968 // We need to handle this here because tablegen doesn't support matching
969 // instructions with multiple outputs.
970 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
971   SDLoc SL(N);
972   EVT VT = N->getValueType(0);
973 
974   assert(VT == MVT::f32 || VT == MVT::f64);
975 
976   unsigned Opc
977     = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64_e64 : AMDGPU::V_DIV_SCALE_F32_e64;
978 
979   // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp,
980   // omod
981   SDValue Ops[8];
982   SelectVOP3BMods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
983   SelectVOP3BMods(N->getOperand(1), Ops[3], Ops[2]);
984   SelectVOP3BMods(N->getOperand(2), Ops[5], Ops[4]);
985   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
986 }
987 
988 // We need to handle this here because tablegen doesn't support matching
989 // instructions with multiple outputs.
990 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
991   SDLoc SL(N);
992   bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32;
993   unsigned Opc;
994   if (Subtarget->hasMADIntraFwdBug())
995     Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
996                  : AMDGPU::V_MAD_U64_U32_gfx11_e64;
997   else
998     Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
999 
1000   SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1001   SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
1002                     Clamp };
1003   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1004 }
1005 
1006 // We need to handle this here because tablegen doesn't support matching
1007 // instructions with multiple outputs.
1008 void AMDGPUDAGToDAGISel::SelectMUL_LOHI(SDNode *N) {
1009   SDLoc SL(N);
1010   bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
1011   unsigned Opc;
1012   if (Subtarget->hasMADIntraFwdBug())
1013     Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
1014                  : AMDGPU::V_MAD_U64_U32_gfx11_e64;
1015   else
1016     Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1017 
1018   SDValue Zero = CurDAG->getTargetConstant(0, SL, MVT::i64);
1019   SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1020   SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Zero, Clamp};
1021   SDNode *Mad = CurDAG->getMachineNode(Opc, SL, N->getVTList(), Ops);
1022   if (!SDValue(N, 0).use_empty()) {
1023     SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32);
1024     SDNode *Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
1025                                         MVT::i32, SDValue(Mad, 0), Sub0);
1026     ReplaceUses(SDValue(N, 0), SDValue(Lo, 0));
1027   }
1028   if (!SDValue(N, 1).use_empty()) {
1029     SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32);
1030     SDNode *Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
1031                                         MVT::i32, SDValue(Mad, 0), Sub1);
1032     ReplaceUses(SDValue(N, 1), SDValue(Hi, 0));
1033   }
1034   CurDAG->RemoveDeadNode(N);
1035 }
1036 
1037 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset) const {
1038   if (!isUInt<16>(Offset))
1039     return false;
1040 
1041   if (!Base || Subtarget->hasUsableDSOffset() ||
1042       Subtarget->unsafeDSOffsetFoldingEnabled())
1043     return true;
1044 
1045   // On Southern Islands instruction with a negative base value and an offset
1046   // don't seem to work.
1047   return CurDAG->SignBitIsZero(Base);
1048 }
1049 
1050 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1051                                               SDValue &Offset) const {
1052   SDLoc DL(Addr);
1053   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1054     SDValue N0 = Addr.getOperand(0);
1055     SDValue N1 = Addr.getOperand(1);
1056     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1057     if (isDSOffsetLegal(N0, C1->getSExtValue())) {
1058       // (add n0, c0)
1059       Base = N0;
1060       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1061       return true;
1062     }
1063   } else if (Addr.getOpcode() == ISD::SUB) {
1064     // sub C, x -> add (sub 0, x), C
1065     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1066       int64_t ByteOffset = C->getSExtValue();
1067       if (isDSOffsetLegal(SDValue(), ByteOffset)) {
1068         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1069 
1070         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1071         // the known bits in isDSOffsetLegal. We need to emit the selected node
1072         // here, so this is thrown away.
1073         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1074                                       Zero, Addr.getOperand(1));
1075 
1076         if (isDSOffsetLegal(Sub, ByteOffset)) {
1077           SmallVector<SDValue, 3> Opnds;
1078           Opnds.push_back(Zero);
1079           Opnds.push_back(Addr.getOperand(1));
1080 
1081           // FIXME: Select to VOP3 version for with-carry.
1082           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1083           if (Subtarget->hasAddNoCarry()) {
1084             SubOp = AMDGPU::V_SUB_U32_e64;
1085             Opnds.push_back(
1086                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1087           }
1088 
1089           MachineSDNode *MachineSub =
1090               CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds);
1091 
1092           Base = SDValue(MachineSub, 0);
1093           Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
1094           return true;
1095         }
1096       }
1097     }
1098   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1099     // If we have a constant address, prefer to put the constant into the
1100     // offset. This can save moves to load the constant address since multiple
1101     // operations can share the zero base address register, and enables merging
1102     // into read2 / write2 instructions.
1103 
1104     SDLoc DL(Addr);
1105 
1106     if (isDSOffsetLegal(SDValue(), CAddr->getZExtValue())) {
1107       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1108       MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1109                                  DL, MVT::i32, Zero);
1110       Base = SDValue(MovZero, 0);
1111       Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1112       return true;
1113     }
1114   }
1115 
1116   // default case
1117   Base = Addr;
1118   Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
1119   return true;
1120 }
1121 
1122 bool AMDGPUDAGToDAGISel::isDSOffset2Legal(SDValue Base, unsigned Offset0,
1123                                           unsigned Offset1,
1124                                           unsigned Size) const {
1125   if (Offset0 % Size != 0 || Offset1 % Size != 0)
1126     return false;
1127   if (!isUInt<8>(Offset0 / Size) || !isUInt<8>(Offset1 / Size))
1128     return false;
1129 
1130   if (!Base || Subtarget->hasUsableDSOffset() ||
1131       Subtarget->unsafeDSOffsetFoldingEnabled())
1132     return true;
1133 
1134   // On Southern Islands instruction with a negative base value and an offset
1135   // don't seem to work.
1136   return CurDAG->SignBitIsZero(Base);
1137 }
1138 
1139 bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegal(SDValue Base,
1140                                                 uint64_t FlatVariant) const {
1141   if (FlatVariant != SIInstrFlags::FlatScratch)
1142     return true;
1143   // When value in 32-bit Base can be negative calculate scratch offset using
1144   // 32-bit add instruction, otherwise use Base(unsigned) + offset.
1145   return CurDAG->SignBitIsZero(Base);
1146 }
1147 
1148 // TODO: If offset is too big, put low 16-bit into offset.
1149 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1150                                                    SDValue &Offset0,
1151                                                    SDValue &Offset1) const {
1152   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 4);
1153 }
1154 
1155 bool AMDGPUDAGToDAGISel::SelectDS128Bit8ByteAligned(SDValue Addr, SDValue &Base,
1156                                                     SDValue &Offset0,
1157                                                     SDValue &Offset1) const {
1158   return SelectDSReadWrite2(Addr, Base, Offset0, Offset1, 8);
1159 }
1160 
1161 bool AMDGPUDAGToDAGISel::SelectDSReadWrite2(SDValue Addr, SDValue &Base,
1162                                             SDValue &Offset0, SDValue &Offset1,
1163                                             unsigned Size) const {
1164   SDLoc DL(Addr);
1165 
1166   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1167     SDValue N0 = Addr.getOperand(0);
1168     SDValue N1 = Addr.getOperand(1);
1169     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1170     unsigned OffsetValue0 = C1->getZExtValue();
1171     unsigned OffsetValue1 = OffsetValue0 + Size;
1172 
1173     // (add n0, c0)
1174     if (isDSOffset2Legal(N0, OffsetValue0, OffsetValue1, Size)) {
1175       Base = N0;
1176       Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1177       Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1178       return true;
1179     }
1180   } else if (Addr.getOpcode() == ISD::SUB) {
1181     // sub C, x -> add (sub 0, x), C
1182     if (const ConstantSDNode *C =
1183             dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1184       unsigned OffsetValue0 = C->getZExtValue();
1185       unsigned OffsetValue1 = OffsetValue0 + Size;
1186 
1187       if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1188         SDLoc DL(Addr);
1189         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1190 
1191         // XXX - This is kind of hacky. Create a dummy sub node so we can check
1192         // the known bits in isDSOffsetLegal. We need to emit the selected node
1193         // here, so this is thrown away.
1194         SDValue Sub =
1195             CurDAG->getNode(ISD::SUB, DL, MVT::i32, Zero, Addr.getOperand(1));
1196 
1197         if (isDSOffset2Legal(Sub, OffsetValue0, OffsetValue1, Size)) {
1198           SmallVector<SDValue, 3> Opnds;
1199           Opnds.push_back(Zero);
1200           Opnds.push_back(Addr.getOperand(1));
1201           unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1202           if (Subtarget->hasAddNoCarry()) {
1203             SubOp = AMDGPU::V_SUB_U32_e64;
1204             Opnds.push_back(
1205                 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1206           }
1207 
1208           MachineSDNode *MachineSub = CurDAG->getMachineNode(
1209               SubOp, DL, MVT::getIntegerVT(Size * 8), Opnds);
1210 
1211           Base = SDValue(MachineSub, 0);
1212           Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1213           Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1214           return true;
1215         }
1216       }
1217     }
1218   } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1219     unsigned OffsetValue0 = CAddr->getZExtValue();
1220     unsigned OffsetValue1 = OffsetValue0 + Size;
1221 
1222     if (isDSOffset2Legal(SDValue(), OffsetValue0, OffsetValue1, Size)) {
1223       SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1224       MachineSDNode *MovZero =
1225           CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32, Zero);
1226       Base = SDValue(MovZero, 0);
1227       Offset0 = CurDAG->getTargetConstant(OffsetValue0 / Size, DL, MVT::i8);
1228       Offset1 = CurDAG->getTargetConstant(OffsetValue1 / Size, DL, MVT::i8);
1229       return true;
1230     }
1231   }
1232 
1233   // default case
1234 
1235   Base = Addr;
1236   Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
1237   Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
1238   return true;
1239 }
1240 
1241 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, SDValue &VAddr,
1242                                      SDValue &SOffset, SDValue &Offset,
1243                                      SDValue &Offen, SDValue &Idxen,
1244                                      SDValue &Addr64) const {
1245   // Subtarget prefers to use flat instruction
1246   // FIXME: This should be a pattern predicate and not reach here
1247   if (Subtarget->useFlatForGlobal())
1248     return false;
1249 
1250   SDLoc DL(Addr);
1251 
1252   Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1253   Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1254   Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
1255   SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1256 
1257   ConstantSDNode *C1 = nullptr;
1258   SDValue N0 = Addr;
1259   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1260     C1 = cast<ConstantSDNode>(Addr.getOperand(1));
1261     if (isUInt<32>(C1->getZExtValue()))
1262       N0 = Addr.getOperand(0);
1263     else
1264       C1 = nullptr;
1265   }
1266 
1267   if (N0.getOpcode() == ISD::ADD) {
1268     // (add N2, N3) -> addr64, or
1269     // (add (add N2, N3), C1) -> addr64
1270     SDValue N2 = N0.getOperand(0);
1271     SDValue N3 = N0.getOperand(1);
1272     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1273 
1274     if (N2->isDivergent()) {
1275       if (N3->isDivergent()) {
1276         // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
1277         // addr64, and construct the resource from a 0 address.
1278         Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1279         VAddr = N0;
1280       } else {
1281         // N2 is divergent, N3 is not.
1282         Ptr = N3;
1283         VAddr = N2;
1284       }
1285     } else {
1286       // N2 is not divergent.
1287       Ptr = N2;
1288       VAddr = N3;
1289     }
1290     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1291   } else if (N0->isDivergent()) {
1292     // N0 is divergent. Use it as the addr64, and construct the resource from a
1293     // 0 address.
1294     Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1295     VAddr = N0;
1296     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1297   } else {
1298     // N0 -> offset, or
1299     // (N0 + C1) -> offset
1300     VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
1301     Ptr = N0;
1302   }
1303 
1304   if (!C1) {
1305     // No offset.
1306     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1307     return true;
1308   }
1309 
1310   if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) {
1311     // Legal offset for instruction.
1312     Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
1313     return true;
1314   }
1315 
1316   // Illegal offset, store it in soffset.
1317   Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1318   SOffset =
1319       SDValue(CurDAG->getMachineNode(
1320                   AMDGPU::S_MOV_B32, DL, MVT::i32,
1321                   CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
1322               0);
1323   return true;
1324 }
1325 
1326 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1327                                            SDValue &VAddr, SDValue &SOffset,
1328                                            SDValue &Offset) const {
1329   SDValue Ptr, Offen, Idxen, Addr64;
1330 
1331   // addr64 bit was removed for volcanic islands.
1332   // FIXME: This should be a pattern predicate and not reach here
1333   if (!Subtarget->hasAddr64())
1334     return false;
1335 
1336   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1337     return false;
1338 
1339   ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1340   if (C->getSExtValue()) {
1341     SDLoc DL(Addr);
1342 
1343     const SITargetLowering& Lowering =
1344       *static_cast<const SITargetLowering*>(getTargetLowering());
1345 
1346     SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
1347     return true;
1348   }
1349 
1350   return false;
1351 }
1352 
1353 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1354   SDLoc DL(N);
1355 
1356   auto *FI = dyn_cast<FrameIndexSDNode>(N);
1357   SDValue TFI =
1358       FI ? CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)) : N;
1359 
1360   // We rebase the base address into an absolute stack address and hence
1361   // use constant 0 for soffset. This value must be retained until
1362   // frame elimination and eliminateFrameIndex will choose the appropriate
1363   // frame register if need be.
1364   return std::pair(TFI, CurDAG->getTargetConstant(0, DL, MVT::i32));
1365 }
1366 
1367 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
1368                                                  SDValue Addr, SDValue &Rsrc,
1369                                                  SDValue &VAddr, SDValue &SOffset,
1370                                                  SDValue &ImmOffset) const {
1371 
1372   SDLoc DL(Addr);
1373   MachineFunction &MF = CurDAG->getMachineFunction();
1374   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1375 
1376   Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1377 
1378   if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1379     int64_t Imm = CAddr->getSExtValue();
1380     const int64_t NullPtr =
1381         AMDGPUTargetMachine::getNullPointerValue(AMDGPUAS::PRIVATE_ADDRESS);
1382     // Don't fold null pointer.
1383     if (Imm != NullPtr) {
1384       const uint32_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset();
1385       SDValue HighBits =
1386           CurDAG->getTargetConstant(Imm & ~MaxOffset, DL, MVT::i32);
1387       MachineSDNode *MovHighBits = CurDAG->getMachineNode(
1388         AMDGPU::V_MOV_B32_e32, DL, MVT::i32, HighBits);
1389       VAddr = SDValue(MovHighBits, 0);
1390 
1391       SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1392       ImmOffset = CurDAG->getTargetConstant(Imm & MaxOffset, DL, MVT::i32);
1393       return true;
1394     }
1395   }
1396 
1397   if (CurDAG->isBaseWithConstantOffset(Addr)) {
1398     // (add n0, c1)
1399 
1400     SDValue N0 = Addr.getOperand(0);
1401     SDValue N1 = Addr.getOperand(1);
1402 
1403     // Offsets in vaddr must be positive if range checking is enabled.
1404     //
1405     // The total computation of vaddr + soffset + offset must not overflow.  If
1406     // vaddr is negative, even if offset is 0 the sgpr offset add will end up
1407     // overflowing.
1408     //
1409     // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would
1410     // always perform a range check. If a negative vaddr base index was used,
1411     // this would fail the range check. The overall address computation would
1412     // compute a valid address, but this doesn't happen due to the range
1413     // check. For out-of-bounds MUBUF loads, a 0 is returned.
1414     //
1415     // Therefore it should be safe to fold any VGPR offset on gfx9 into the
1416     // MUBUF vaddr, but not on older subtargets which can only do this if the
1417     // sign bit is known 0.
1418     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1419     if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) &&
1420         (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1421          CurDAG->SignBitIsZero(N0))) {
1422       std::tie(VAddr, SOffset) = foldFrameIndex(N0);
1423       ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
1424       return true;
1425     }
1426   }
1427 
1428   // (node)
1429   std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
1430   ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1431   return true;
1432 }
1433 
1434 static bool IsCopyFromSGPR(const SIRegisterInfo &TRI, SDValue Val) {
1435   if (Val.getOpcode() != ISD::CopyFromReg)
1436     return false;
1437   auto Reg = cast<RegisterSDNode>(Val.getOperand(1))->getReg();
1438   if (!Reg.isPhysical())
1439     return false;
1440   auto RC = TRI.getPhysRegBaseClass(Reg);
1441   return RC && TRI.isSGPRClass(RC);
1442 }
1443 
1444 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
1445                                                   SDValue Addr,
1446                                                   SDValue &SRsrc,
1447                                                   SDValue &SOffset,
1448                                                   SDValue &Offset) const {
1449   const SIRegisterInfo *TRI =
1450       static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
1451   MachineFunction &MF = CurDAG->getMachineFunction();
1452   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1453   SDLoc DL(Addr);
1454 
1455   // CopyFromReg <sgpr>
1456   if (IsCopyFromSGPR(*TRI, Addr)) {
1457     SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1458     SOffset = Addr;
1459     Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1460     return true;
1461   }
1462 
1463   ConstantSDNode *CAddr;
1464   if (Addr.getOpcode() == ISD::ADD) {
1465     // Add (CopyFromReg <sgpr>) <constant>
1466     CAddr = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
1467     if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue()))
1468       return false;
1469     if (!IsCopyFromSGPR(*TRI, Addr.getOperand(0)))
1470       return false;
1471 
1472     SOffset = Addr.getOperand(0);
1473   } else if ((CAddr = dyn_cast<ConstantSDNode>(Addr)) &&
1474              SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue())) {
1475     // <constant>
1476     SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1477   } else {
1478     return false;
1479   }
1480 
1481   SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1482 
1483   Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i32);
1484   return true;
1485 }
1486 
1487 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1488                                            SDValue &SOffset, SDValue &Offset
1489                                            ) const {
1490   SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1491   const SIInstrInfo *TII =
1492     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
1493 
1494   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1495     return false;
1496 
1497   if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1498       !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1499       !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1500     uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1501                     APInt::getAllOnes(32).getZExtValue(); // Size
1502     SDLoc DL(Addr);
1503 
1504     const SITargetLowering& Lowering =
1505       *static_cast<const SITargetLowering*>(getTargetLowering());
1506 
1507     SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
1508     return true;
1509   }
1510   return false;
1511 }
1512 
1513 // Find a load or store from corresponding pattern root.
1514 // Roots may be build_vector, bitconvert or their combinations.
1515 static MemSDNode* findMemSDNode(SDNode *N) {
1516   N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode();
1517   if (MemSDNode *MN = dyn_cast<MemSDNode>(N))
1518     return MN;
1519   assert(isa<BuildVectorSDNode>(N));
1520   for (SDValue V : N->op_values())
1521     if (MemSDNode *MN =
1522           dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V)))
1523       return MN;
1524   llvm_unreachable("cannot find MemSDNode in the pattern!");
1525 }
1526 
1527 bool AMDGPUDAGToDAGISel::SelectFlatOffsetImpl(SDNode *N, SDValue Addr,
1528                                               SDValue &VAddr, SDValue &Offset,
1529                                               uint64_t FlatVariant) const {
1530   int64_t OffsetVal = 0;
1531 
1532   unsigned AS = findMemSDNode(N)->getAddressSpace();
1533 
1534   bool CanHaveFlatSegmentOffsetBug =
1535       Subtarget->hasFlatSegmentOffsetBug() &&
1536       FlatVariant == SIInstrFlags::FLAT &&
1537       (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS);
1538 
1539   if (Subtarget->hasFlatInstOffsets() && !CanHaveFlatSegmentOffsetBug) {
1540     SDValue N0, N1;
1541     if (isBaseWithConstantOffset64(Addr, N0, N1) &&
1542         isFlatScratchBaseLegal(N0, FlatVariant)) {
1543       int64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
1544 
1545       const SIInstrInfo *TII = Subtarget->getInstrInfo();
1546       if (TII->isLegalFLATOffset(COffsetVal, AS, FlatVariant)) {
1547         Addr = N0;
1548         OffsetVal = COffsetVal;
1549       } else {
1550         // If the offset doesn't fit, put the low bits into the offset field and
1551         // add the rest.
1552         //
1553         // For a FLAT instruction the hardware decides whether to access
1554         // global/scratch/shared memory based on the high bits of vaddr,
1555         // ignoring the offset field, so we have to ensure that when we add
1556         // remainder to vaddr it still points into the same underlying object.
1557         // The easiest way to do that is to make sure that we split the offset
1558         // into two pieces that are both >= 0 or both <= 0.
1559 
1560         SDLoc DL(N);
1561         uint64_t RemainderOffset;
1562 
1563         std::tie(OffsetVal, RemainderOffset) =
1564             TII->splitFlatOffset(COffsetVal, AS, FlatVariant);
1565 
1566         SDValue AddOffsetLo =
1567             getMaterializedScalarImm32(Lo_32(RemainderOffset), DL);
1568         SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
1569 
1570         if (Addr.getValueType().getSizeInBits() == 32) {
1571           SmallVector<SDValue, 3> Opnds;
1572           Opnds.push_back(N0);
1573           Opnds.push_back(AddOffsetLo);
1574           unsigned AddOp = AMDGPU::V_ADD_CO_U32_e32;
1575           if (Subtarget->hasAddNoCarry()) {
1576             AddOp = AMDGPU::V_ADD_U32_e64;
1577             Opnds.push_back(Clamp);
1578           }
1579           Addr = SDValue(CurDAG->getMachineNode(AddOp, DL, MVT::i32, Opnds), 0);
1580         } else {
1581           // TODO: Should this try to use a scalar add pseudo if the base address
1582           // is uniform and saddr is usable?
1583           SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
1584           SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
1585 
1586           SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1587                                                 DL, MVT::i32, N0, Sub0);
1588           SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
1589                                                 DL, MVT::i32, N0, Sub1);
1590 
1591           SDValue AddOffsetHi =
1592               getMaterializedScalarImm32(Hi_32(RemainderOffset), DL);
1593 
1594           SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1);
1595 
1596           SDNode *Add =
1597               CurDAG->getMachineNode(AMDGPU::V_ADD_CO_U32_e64, DL, VTs,
1598                                      {AddOffsetLo, SDValue(N0Lo, 0), Clamp});
1599 
1600           SDNode *Addc = CurDAG->getMachineNode(
1601               AMDGPU::V_ADDC_U32_e64, DL, VTs,
1602               {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp});
1603 
1604           SDValue RegSequenceArgs[] = {
1605               CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32),
1606               SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1};
1607 
1608           Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1609                                                 MVT::i64, RegSequenceArgs),
1610                          0);
1611         }
1612       }
1613     }
1614   }
1615 
1616   VAddr = Addr;
1617   Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16);
1618   return true;
1619 }
1620 
1621 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N, SDValue Addr,
1622                                           SDValue &VAddr,
1623                                           SDValue &Offset) const {
1624   return SelectFlatOffsetImpl(N, Addr, VAddr, Offset, SIInstrFlags::FLAT);
1625 }
1626 
1627 bool AMDGPUDAGToDAGISel::SelectGlobalOffset(SDNode *N, SDValue Addr,
1628                                             SDValue &VAddr,
1629                                             SDValue &Offset) const {
1630   return SelectFlatOffsetImpl(N, Addr, VAddr, Offset, SIInstrFlags::FlatGlobal);
1631 }
1632 
1633 bool AMDGPUDAGToDAGISel::SelectScratchOffset(SDNode *N, SDValue Addr,
1634                                              SDValue &VAddr,
1635                                              SDValue &Offset) const {
1636   return SelectFlatOffsetImpl(N, Addr, VAddr, Offset,
1637                               SIInstrFlags::FlatScratch);
1638 }
1639 
1640 // If this matches zero_extend i32:x, return x
1641 static SDValue matchZExtFromI32(SDValue Op) {
1642   if (Op.getOpcode() != ISD::ZERO_EXTEND)
1643     return SDValue();
1644 
1645   SDValue ExtSrc = Op.getOperand(0);
1646   return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
1647 }
1648 
1649 // Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
1650 bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N,
1651                                            SDValue Addr,
1652                                            SDValue &SAddr,
1653                                            SDValue &VOffset,
1654                                            SDValue &Offset) const {
1655   int64_t ImmOffset = 0;
1656 
1657   // Match the immediate offset first, which canonically is moved as low as
1658   // possible.
1659 
1660   SDValue LHS, RHS;
1661   if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
1662     int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
1663     const SIInstrInfo *TII = Subtarget->getInstrInfo();
1664 
1665     if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::GLOBAL_ADDRESS,
1666                                SIInstrFlags::FlatGlobal)) {
1667       Addr = LHS;
1668       ImmOffset = COffsetVal;
1669     } else if (!LHS->isDivergent()) {
1670       if (COffsetVal > 0) {
1671         SDLoc SL(N);
1672         // saddr + large_offset -> saddr +
1673         //                         (voffset = large_offset & ~MaxOffset) +
1674         //                         (large_offset & MaxOffset);
1675         int64_t SplitImmOffset, RemainderOffset;
1676         std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
1677             COffsetVal, AMDGPUAS::GLOBAL_ADDRESS, SIInstrFlags::FlatGlobal);
1678 
1679         if (isUInt<32>(RemainderOffset)) {
1680           SDNode *VMov = CurDAG->getMachineNode(
1681               AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
1682               CurDAG->getTargetConstant(RemainderOffset, SDLoc(), MVT::i32));
1683           VOffset = SDValue(VMov, 0);
1684           SAddr = LHS;
1685           Offset = CurDAG->getTargetConstant(SplitImmOffset, SDLoc(), MVT::i16);
1686           return true;
1687         }
1688       }
1689 
1690       // We are adding a 64 bit SGPR and a constant. If constant bus limit
1691       // is 1 we would need to perform 1 or 2 extra moves for each half of
1692       // the constant and it is better to do a scalar add and then issue a
1693       // single VALU instruction to materialize zero. Otherwise it is less
1694       // instructions to perform VALU adds with immediates or inline literals.
1695       unsigned NumLiterals =
1696           !TII->isInlineConstant(APInt(32, COffsetVal & 0xffffffff)) +
1697           !TII->isInlineConstant(APInt(32, COffsetVal >> 32));
1698       if (Subtarget->getConstantBusLimit(AMDGPU::V_ADD_U32_e64) > NumLiterals)
1699         return false;
1700     }
1701   }
1702 
1703   // Match the variable offset.
1704   if (Addr.getOpcode() == ISD::ADD) {
1705     LHS = Addr.getOperand(0);
1706     RHS = Addr.getOperand(1);
1707 
1708     if (!LHS->isDivergent()) {
1709       // add (i64 sgpr), (zero_extend (i32 vgpr))
1710       if (SDValue ZextRHS = matchZExtFromI32(RHS)) {
1711         SAddr = LHS;
1712         VOffset = ZextRHS;
1713       }
1714     }
1715 
1716     if (!SAddr && !RHS->isDivergent()) {
1717       // add (zero_extend (i32 vgpr)), (i64 sgpr)
1718       if (SDValue ZextLHS = matchZExtFromI32(LHS)) {
1719         SAddr = RHS;
1720         VOffset = ZextLHS;
1721       }
1722     }
1723 
1724     if (SAddr) {
1725       Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1726       return true;
1727     }
1728   }
1729 
1730   if (Addr->isDivergent() || Addr.getOpcode() == ISD::UNDEF ||
1731       isa<ConstantSDNode>(Addr))
1732     return false;
1733 
1734   // It's cheaper to materialize a single 32-bit zero for vaddr than the two
1735   // moves required to copy a 64-bit SGPR to VGPR.
1736   SAddr = Addr;
1737   SDNode *VMov =
1738       CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, SDLoc(Addr), MVT::i32,
1739                              CurDAG->getTargetConstant(0, SDLoc(), MVT::i32));
1740   VOffset = SDValue(VMov, 0);
1741   Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1742   return true;
1743 }
1744 
1745 static SDValue SelectSAddrFI(SelectionDAG *CurDAG, SDValue SAddr) {
1746   if (auto FI = dyn_cast<FrameIndexSDNode>(SAddr)) {
1747     SAddr = CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0));
1748   } else if (SAddr.getOpcode() == ISD::ADD &&
1749              isa<FrameIndexSDNode>(SAddr.getOperand(0))) {
1750     // Materialize this into a scalar move for scalar address to avoid
1751     // readfirstlane.
1752     auto FI = cast<FrameIndexSDNode>(SAddr.getOperand(0));
1753     SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
1754                                               FI->getValueType(0));
1755     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, SDLoc(SAddr),
1756                                            MVT::i32, TFI, SAddr.getOperand(1)),
1757                     0);
1758   }
1759 
1760   return SAddr;
1761 }
1762 
1763 // Match (32-bit SGPR base) + sext(imm offset)
1764 bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *Parent, SDValue Addr,
1765                                             SDValue &SAddr,
1766                                             SDValue &Offset) const {
1767   if (Addr->isDivergent())
1768     return false;
1769 
1770   SDLoc DL(Addr);
1771 
1772   int64_t COffsetVal = 0;
1773 
1774   if (CurDAG->isBaseWithConstantOffset(Addr) &&
1775       isFlatScratchBaseLegal(Addr.getOperand(0))) {
1776     COffsetVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
1777     SAddr = Addr.getOperand(0);
1778   } else {
1779     SAddr = Addr;
1780   }
1781 
1782   SAddr = SelectSAddrFI(CurDAG, SAddr);
1783 
1784   const SIInstrInfo *TII = Subtarget->getInstrInfo();
1785 
1786   if (!TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS,
1787                               SIInstrFlags::FlatScratch)) {
1788     int64_t SplitImmOffset, RemainderOffset;
1789     std::tie(SplitImmOffset, RemainderOffset) = TII->splitFlatOffset(
1790         COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, SIInstrFlags::FlatScratch);
1791 
1792     COffsetVal = SplitImmOffset;
1793 
1794     SDValue AddOffset =
1795         SAddr.getOpcode() == ISD::TargetFrameIndex
1796             ? getMaterializedScalarImm32(Lo_32(RemainderOffset), DL)
1797             : CurDAG->getTargetConstant(RemainderOffset, DL, MVT::i32);
1798     SAddr = SDValue(CurDAG->getMachineNode(AMDGPU::S_ADD_I32, DL, MVT::i32,
1799                                            SAddr, AddOffset),
1800                     0);
1801   }
1802 
1803   Offset = CurDAG->getTargetConstant(COffsetVal, DL, MVT::i16);
1804 
1805   return true;
1806 }
1807 
1808 // Check whether the flat scratch SVS swizzle bug affects this access.
1809 bool AMDGPUDAGToDAGISel::checkFlatScratchSVSSwizzleBug(
1810     SDValue VAddr, SDValue SAddr, uint64_t ImmOffset) const {
1811   if (!Subtarget->hasFlatScratchSVSSwizzleBug())
1812     return false;
1813 
1814   // The bug affects the swizzling of SVS accesses if there is any carry out
1815   // from the two low order bits (i.e. from bit 1 into bit 2) when adding
1816   // voffset to (soffset + inst_offset).
1817   KnownBits VKnown = CurDAG->computeKnownBits(VAddr);
1818   KnownBits SKnown = KnownBits::computeForAddSub(
1819       true, false, CurDAG->computeKnownBits(SAddr),
1820       KnownBits::makeConstant(APInt(32, ImmOffset)));
1821   uint64_t VMax = VKnown.getMaxValue().getZExtValue();
1822   uint64_t SMax = SKnown.getMaxValue().getZExtValue();
1823   return (VMax & 3) + (SMax & 3) >= 4;
1824 }
1825 
1826 bool AMDGPUDAGToDAGISel::SelectScratchSVAddr(SDNode *N, SDValue Addr,
1827                                              SDValue &VAddr, SDValue &SAddr,
1828                                              SDValue &Offset) const  {
1829   int64_t ImmOffset = 0;
1830 
1831   SDValue LHS, RHS;
1832   if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
1833     int64_t COffsetVal = cast<ConstantSDNode>(RHS)->getSExtValue();
1834     const SIInstrInfo *TII = Subtarget->getInstrInfo();
1835 
1836     if (TII->isLegalFLATOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true)) {
1837       Addr = LHS;
1838       ImmOffset = COffsetVal;
1839     } else if (!LHS->isDivergent() && COffsetVal > 0) {
1840       SDLoc SL(N);
1841       // saddr + large_offset -> saddr + (vaddr = large_offset & ~MaxOffset) +
1842       //                         (large_offset & MaxOffset);
1843       int64_t SplitImmOffset, RemainderOffset;
1844       std::tie(SplitImmOffset, RemainderOffset)
1845         = TII->splitFlatOffset(COffsetVal, AMDGPUAS::PRIVATE_ADDRESS, true);
1846 
1847       if (isUInt<32>(RemainderOffset)) {
1848         SDNode *VMov = CurDAG->getMachineNode(
1849           AMDGPU::V_MOV_B32_e32, SL, MVT::i32,
1850           CurDAG->getTargetConstant(RemainderOffset, SDLoc(), MVT::i32));
1851         VAddr = SDValue(VMov, 0);
1852         SAddr = LHS;
1853         if (!isFlatScratchBaseLegal(SAddr) || !isFlatScratchBaseLegal(VAddr))
1854           return false;
1855         if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, SplitImmOffset))
1856           return false;
1857         Offset = CurDAG->getTargetConstant(SplitImmOffset, SDLoc(), MVT::i16);
1858         return true;
1859       }
1860     }
1861   }
1862 
1863   if (Addr.getOpcode() != ISD::ADD)
1864     return false;
1865 
1866   LHS = Addr.getOperand(0);
1867   RHS = Addr.getOperand(1);
1868 
1869   if (!LHS->isDivergent() && RHS->isDivergent()) {
1870     SAddr = LHS;
1871     VAddr = RHS;
1872   } else if (!RHS->isDivergent() && LHS->isDivergent()) {
1873     SAddr = RHS;
1874     VAddr = LHS;
1875   } else {
1876     return false;
1877   }
1878 
1879   if (!isFlatScratchBaseLegal(SAddr) || !isFlatScratchBaseLegal(VAddr))
1880     return false;
1881 
1882   if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, ImmOffset))
1883     return false;
1884   SAddr = SelectSAddrFI(CurDAG, SAddr);
1885   Offset = CurDAG->getTargetConstant(ImmOffset, SDLoc(), MVT::i16);
1886   return true;
1887 }
1888 
1889 // Match an immediate (if Offset is not null) or an SGPR (if SOffset is
1890 // not null) offset. If Imm32Only is true, match only 32-bit immediate
1891 // offsets available on CI.
1892 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1893                                           SDValue *SOffset, SDValue *Offset,
1894                                           bool Imm32Only, bool IsBuffer) const {
1895   assert((!SOffset || !Offset) &&
1896          "Cannot match both soffset and offset at the same time!");
1897 
1898   ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
1899   if (!C) {
1900     if (!SOffset)
1901       return false;
1902     if (ByteOffsetNode.getValueType().isScalarInteger() &&
1903         ByteOffsetNode.getValueType().getSizeInBits() == 32) {
1904       *SOffset = ByteOffsetNode;
1905       return true;
1906     }
1907     if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) {
1908       if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) {
1909         *SOffset = ByteOffsetNode.getOperand(0);
1910         return true;
1911       }
1912     }
1913     return false;
1914   }
1915 
1916   SDLoc SL(ByteOffsetNode);
1917 
1918   // GFX9 and GFX10 have signed byte immediate offsets. The immediate
1919   // offset for S_BUFFER instructions is unsigned.
1920   int64_t ByteOffset = IsBuffer ? C->getZExtValue() : C->getSExtValue();
1921   std::optional<int64_t> EncodedOffset =
1922       AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset, IsBuffer);
1923   if (EncodedOffset && Offset && !Imm32Only) {
1924     *Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1925     return true;
1926   }
1927 
1928   // SGPR and literal offsets are unsigned.
1929   if (ByteOffset < 0)
1930     return false;
1931 
1932   EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset);
1933   if (EncodedOffset && Offset && Imm32Only) {
1934     *Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32);
1935     return true;
1936   }
1937 
1938   if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset))
1939     return false;
1940 
1941   if (SOffset) {
1942     SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
1943     *SOffset = SDValue(
1944         CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0);
1945     return true;
1946   }
1947 
1948   return false;
1949 }
1950 
1951 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
1952   if (Addr.getValueType() != MVT::i32)
1953     return Addr;
1954 
1955   // Zero-extend a 32-bit address.
1956   SDLoc SL(Addr);
1957 
1958   const MachineFunction &MF = CurDAG->getMachineFunction();
1959   const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1960   unsigned AddrHiVal = Info->get32BitAddressHighBits();
1961   SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
1962 
1963   const SDValue Ops[] = {
1964     CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32),
1965     Addr,
1966     CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
1967     SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi),
1968             0),
1969     CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32),
1970   };
1971 
1972   return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64,
1973                                         Ops), 0);
1974 }
1975 
1976 // Match a base and an immediate (if Offset is not null) or an SGPR (if
1977 // SOffset is not null) or an immediate+SGPR offset. If Imm32Only is
1978 // true, match only 32-bit immediate offsets available on CI.
1979 bool AMDGPUDAGToDAGISel::SelectSMRDBaseOffset(SDValue Addr, SDValue &SBase,
1980                                               SDValue *SOffset, SDValue *Offset,
1981                                               bool Imm32Only,
1982                                               bool IsBuffer) const {
1983   if (SOffset && Offset) {
1984     assert(!Imm32Only && !IsBuffer);
1985     SDValue B;
1986     return SelectSMRDBaseOffset(Addr, B, nullptr, Offset) &&
1987            SelectSMRDBaseOffset(B, SBase, SOffset, nullptr);
1988   }
1989 
1990   // A 32-bit (address + offset) should not cause unsigned 32-bit integer
1991   // wraparound, because s_load instructions perform the addition in 64 bits.
1992   if (Addr.getValueType() == MVT::i32 && Addr.getOpcode() == ISD::ADD &&
1993       !Addr->getFlags().hasNoUnsignedWrap())
1994     return false;
1995 
1996   SDValue N0, N1;
1997   // Extract the base and offset if possible.
1998   if (CurDAG->isBaseWithConstantOffset(Addr) || Addr.getOpcode() == ISD::ADD) {
1999     N0 = Addr.getOperand(0);
2000     N1 = Addr.getOperand(1);
2001   } else if (getBaseWithOffsetUsingSplitOR(*CurDAG, Addr, N0, N1)) {
2002     assert(N0 && N1 && isa<ConstantSDNode>(N1));
2003   }
2004   if (!N0 || !N1)
2005     return false;
2006   if (SelectSMRDOffset(N1, SOffset, Offset, Imm32Only, IsBuffer)) {
2007     SBase = N0;
2008     return true;
2009   }
2010   if (SelectSMRDOffset(N0, SOffset, Offset, Imm32Only, IsBuffer)) {
2011     SBase = N1;
2012     return true;
2013   }
2014   return false;
2015 }
2016 
2017 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
2018                                     SDValue *SOffset, SDValue *Offset,
2019                                     bool Imm32Only) const {
2020   if (SelectSMRDBaseOffset(Addr, SBase, SOffset, Offset, Imm32Only)) {
2021     SBase = Expand32BitAddress(SBase);
2022     return true;
2023   }
2024 
2025   if (Addr.getValueType() == MVT::i32 && Offset && !SOffset) {
2026     SBase = Expand32BitAddress(Addr);
2027     *Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
2028     return true;
2029   }
2030 
2031   return false;
2032 }
2033 
2034 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
2035                                        SDValue &Offset) const {
2036   return SelectSMRD(Addr, SBase, /* SOffset */ nullptr, &Offset);
2037 }
2038 
2039 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
2040                                          SDValue &Offset) const {
2041   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2042   return SelectSMRD(Addr, SBase, /* SOffset */ nullptr, &Offset,
2043                     /* Imm32Only */ true);
2044 }
2045 
2046 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
2047                                         SDValue &SOffset) const {
2048   return SelectSMRD(Addr, SBase, &SOffset, /* Offset */ nullptr);
2049 }
2050 
2051 bool AMDGPUDAGToDAGISel::SelectSMRDSgprImm(SDValue Addr, SDValue &SBase,
2052                                            SDValue &SOffset,
2053                                            SDValue &Offset) const {
2054   return SelectSMRD(Addr, SBase, &SOffset, &Offset);
2055 }
2056 
2057 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue N, SDValue &Offset) const {
2058   return SelectSMRDOffset(N, /* SOffset */ nullptr, &Offset,
2059                           /* Imm32Only */ false, /* IsBuffer */ true);
2060 }
2061 
2062 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue N,
2063                                                SDValue &Offset) const {
2064   assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2065   return SelectSMRDOffset(N, /* SOffset */ nullptr, &Offset,
2066                           /* Imm32Only */ true, /* IsBuffer */ true);
2067 }
2068 
2069 bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,
2070                                                  SDValue &Offset) const {
2071   // Match the (soffset + offset) pair as a 32-bit register base and
2072   // an immediate offset.
2073   return N.getValueType() == MVT::i32 &&
2074          SelectSMRDBaseOffset(N, /* SBase */ SOffset, /* SOffset*/ nullptr,
2075                               &Offset, /* Imm32Only */ false,
2076                               /* IsBuffer */ true);
2077 }
2078 
2079 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
2080                                             SDValue &Base,
2081                                             SDValue &Offset) const {
2082   SDLoc DL(Index);
2083 
2084   if (CurDAG->isBaseWithConstantOffset(Index)) {
2085     SDValue N0 = Index.getOperand(0);
2086     SDValue N1 = Index.getOperand(1);
2087     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
2088 
2089     // (add n0, c0)
2090     // Don't peel off the offset (c0) if doing so could possibly lead
2091     // the base (n0) to be negative.
2092     // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset.
2093     if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0) ||
2094         (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) {
2095       Base = N0;
2096       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
2097       return true;
2098     }
2099   }
2100 
2101   if (isa<ConstantSDNode>(Index))
2102     return false;
2103 
2104   Base = Index;
2105   Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
2106   return true;
2107 }
2108 
2109 SDNode *AMDGPUDAGToDAGISel::getBFE32(bool IsSigned, const SDLoc &DL,
2110                                      SDValue Val, uint32_t Offset,
2111                                      uint32_t Width) {
2112   if (Val->isDivergent()) {
2113     unsigned Opcode = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2114     SDValue Off = CurDAG->getTargetConstant(Offset, DL, MVT::i32);
2115     SDValue W = CurDAG->getTargetConstant(Width, DL, MVT::i32);
2116 
2117     return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, Off, W);
2118   }
2119   unsigned Opcode = IsSigned ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2120   // Transformation function, pack the offset and width of a BFE into
2121   // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
2122   // source, bits [5:0] contain the offset and bits [22:16] the width.
2123   uint32_t PackedVal = Offset | (Width << 16);
2124   SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
2125 
2126   return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
2127 }
2128 
2129 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
2130   // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
2131   // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
2132   // Predicate: 0 < b <= c < 32
2133 
2134   const SDValue &Shl = N->getOperand(0);
2135   ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
2136   ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2137 
2138   if (B && C) {
2139     uint32_t BVal = B->getZExtValue();
2140     uint32_t CVal = C->getZExtValue();
2141 
2142     if (0 < BVal && BVal <= CVal && CVal < 32) {
2143       bool Signed = N->getOpcode() == ISD::SRA;
2144       ReplaceNode(N, getBFE32(Signed, SDLoc(N), Shl.getOperand(0), CVal - BVal,
2145                   32 - CVal));
2146       return;
2147     }
2148   }
2149   SelectCode(N);
2150 }
2151 
2152 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
2153   switch (N->getOpcode()) {
2154   case ISD::AND:
2155     if (N->getOperand(0).getOpcode() == ISD::SRL) {
2156       // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
2157       // Predicate: isMask(mask)
2158       const SDValue &Srl = N->getOperand(0);
2159       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
2160       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
2161 
2162       if (Shift && Mask) {
2163         uint32_t ShiftVal = Shift->getZExtValue();
2164         uint32_t MaskVal = Mask->getZExtValue();
2165 
2166         if (isMask_32(MaskVal)) {
2167           uint32_t WidthVal = llvm::popcount(MaskVal);
2168           ReplaceNode(N, getBFE32(false, SDLoc(N), Srl.getOperand(0), ShiftVal,
2169                                   WidthVal));
2170           return;
2171         }
2172       }
2173     }
2174     break;
2175   case ISD::SRL:
2176     if (N->getOperand(0).getOpcode() == ISD::AND) {
2177       // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
2178       // Predicate: isMask(mask >> b)
2179       const SDValue &And = N->getOperand(0);
2180       ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
2181       ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
2182 
2183       if (Shift && Mask) {
2184         uint32_t ShiftVal = Shift->getZExtValue();
2185         uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
2186 
2187         if (isMask_32(MaskVal)) {
2188           uint32_t WidthVal = llvm::popcount(MaskVal);
2189           ReplaceNode(N, getBFE32(false, SDLoc(N), And.getOperand(0), ShiftVal,
2190                       WidthVal));
2191           return;
2192         }
2193       }
2194     } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
2195       SelectS_BFEFromShifts(N);
2196       return;
2197     }
2198     break;
2199   case ISD::SRA:
2200     if (N->getOperand(0).getOpcode() == ISD::SHL) {
2201       SelectS_BFEFromShifts(N);
2202       return;
2203     }
2204     break;
2205 
2206   case ISD::SIGN_EXTEND_INREG: {
2207     // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
2208     SDValue Src = N->getOperand(0);
2209     if (Src.getOpcode() != ISD::SRL)
2210       break;
2211 
2212     const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
2213     if (!Amt)
2214       break;
2215 
2216     unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2217     ReplaceNode(N, getBFE32(true, SDLoc(N), Src.getOperand(0),
2218                             Amt->getZExtValue(), Width));
2219     return;
2220   }
2221   }
2222 
2223   SelectCode(N);
2224 }
2225 
2226 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
2227   assert(N->getOpcode() == ISD::BRCOND);
2228   if (!N->hasOneUse())
2229     return false;
2230 
2231   SDValue Cond = N->getOperand(1);
2232   if (Cond.getOpcode() == ISD::CopyToReg)
2233     Cond = Cond.getOperand(2);
2234 
2235   if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
2236     return false;
2237 
2238   MVT VT = Cond.getOperand(0).getSimpleValueType();
2239   if (VT == MVT::i32)
2240     return true;
2241 
2242   if (VT == MVT::i64) {
2243     auto ST = static_cast<const GCNSubtarget *>(Subtarget);
2244 
2245     ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
2246     return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64();
2247   }
2248 
2249   return false;
2250 }
2251 
2252 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
2253   SDValue Cond = N->getOperand(1);
2254 
2255   if (Cond.isUndef()) {
2256     CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other,
2257                          N->getOperand(2), N->getOperand(0));
2258     return;
2259   }
2260 
2261   const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget);
2262   const SIRegisterInfo *TRI = ST->getRegisterInfo();
2263 
2264   bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
2265   unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ;
2266   Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC();
2267   SDLoc SL(N);
2268 
2269   if (!UseSCCBr) {
2270     // This is the case that we are selecting to S_CBRANCH_VCCNZ.  We have not
2271     // analyzed what generates the vcc value, so we do not know whether vcc
2272     // bits for disabled lanes are 0.  Thus we need to mask out bits for
2273     // disabled lanes.
2274     //
2275     // For the case that we select S_CBRANCH_SCC1 and it gets
2276     // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls
2277     // SIInstrInfo::moveToVALU which inserts the S_AND).
2278     //
2279     // We could add an analysis of what generates the vcc value here and omit
2280     // the S_AND when is unnecessary. But it would be better to add a separate
2281     // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it
2282     // catches both cases.
2283     Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32
2284                                                          : AMDGPU::S_AND_B64,
2285                      SL, MVT::i1,
2286                      CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO
2287                                                         : AMDGPU::EXEC,
2288                                          MVT::i1),
2289                     Cond),
2290                    0);
2291   }
2292 
2293   SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond);
2294   CurDAG->SelectNodeTo(N, BrOp, MVT::Other,
2295                        N->getOperand(2), // Basic Block
2296                        VCC.getValue(0));
2297 }
2298 
2299 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
2300   // The address is assumed to be uniform, so if it ends up in a VGPR, it will
2301   // be copied to an SGPR with readfirstlane.
2302   unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
2303     AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2304 
2305   SDValue Chain = N->getOperand(0);
2306   SDValue Ptr = N->getOperand(2);
2307   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2308   MachineMemOperand *MMO = M->getMemOperand();
2309   bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2310 
2311   SDValue Offset;
2312   if (CurDAG->isBaseWithConstantOffset(Ptr)) {
2313     SDValue PtrBase = Ptr.getOperand(0);
2314     SDValue PtrOffset = Ptr.getOperand(1);
2315 
2316     const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue();
2317     if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue())) {
2318       N = glueCopyToM0(N, PtrBase);
2319       Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
2320     }
2321   }
2322 
2323   if (!Offset) {
2324     N = glueCopyToM0(N, Ptr);
2325     Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2326   }
2327 
2328   SDValue Ops[] = {
2329     Offset,
2330     CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
2331     Chain,
2332     N->getOperand(N->getNumOperands() - 1) // New glue
2333   };
2334 
2335   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2336   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2337 }
2338 
2339 // We need to handle this here because tablegen doesn't support matching
2340 // instructions with multiple outputs.
2341 void AMDGPUDAGToDAGISel::SelectDSBvhStackIntrinsic(SDNode *N) {
2342   unsigned Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
2343   SDValue Ops[] = {N->getOperand(2), N->getOperand(3), N->getOperand(4),
2344                    N->getOperand(5), N->getOperand(0)};
2345 
2346   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2347   MachineMemOperand *MMO = M->getMemOperand();
2348   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2349   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2350 }
2351 
2352 static unsigned gwsIntrinToOpcode(unsigned IntrID) {
2353   switch (IntrID) {
2354   case Intrinsic::amdgcn_ds_gws_init:
2355     return AMDGPU::DS_GWS_INIT;
2356   case Intrinsic::amdgcn_ds_gws_barrier:
2357     return AMDGPU::DS_GWS_BARRIER;
2358   case Intrinsic::amdgcn_ds_gws_sema_v:
2359     return AMDGPU::DS_GWS_SEMA_V;
2360   case Intrinsic::amdgcn_ds_gws_sema_br:
2361     return AMDGPU::DS_GWS_SEMA_BR;
2362   case Intrinsic::amdgcn_ds_gws_sema_p:
2363     return AMDGPU::DS_GWS_SEMA_P;
2364   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2365     return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
2366   default:
2367     llvm_unreachable("not a gws intrinsic");
2368   }
2369 }
2370 
2371 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) {
2372   if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
2373       !Subtarget->hasGWSSemaReleaseAll()) {
2374     // Let this error.
2375     SelectCode(N);
2376     return;
2377   }
2378 
2379   // Chain, intrinsic ID, vsrc, offset
2380   const bool HasVSrc = N->getNumOperands() == 4;
2381   assert(HasVSrc || N->getNumOperands() == 3);
2382 
2383   SDLoc SL(N);
2384   SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2);
2385   int ImmOffset = 0;
2386   MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2387   MachineMemOperand *MMO = M->getMemOperand();
2388 
2389   // Don't worry if the offset ends up in a VGPR. Only one lane will have
2390   // effect, so SIFixSGPRCopies will validly insert readfirstlane.
2391 
2392   // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
2393   // offset field) % 64. Some versions of the programming guide omit the m0
2394   // part, or claim it's from offset 0.
2395   if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) {
2396     // If we have a constant offset, try to use the 0 in m0 as the base.
2397     // TODO: Look into changing the default m0 initialization value. If the
2398     // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
2399     // the immediate offset.
2400     glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32));
2401     ImmOffset = ConstOffset->getZExtValue();
2402   } else {
2403     if (CurDAG->isBaseWithConstantOffset(BaseOffset)) {
2404       ImmOffset = BaseOffset.getConstantOperandVal(1);
2405       BaseOffset = BaseOffset.getOperand(0);
2406     }
2407 
2408     // Prefer to do the shift in an SGPR since it should be possible to use m0
2409     // as the result directly. If it's already an SGPR, it will be eliminated
2410     // later.
2411     SDNode *SGPROffset
2412       = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
2413                                BaseOffset);
2414     // Shift to offset in m0
2415     SDNode *M0Base
2416       = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
2417                                SDValue(SGPROffset, 0),
2418                                CurDAG->getTargetConstant(16, SL, MVT::i32));
2419     glueCopyToM0(N, SDValue(M0Base, 0));
2420   }
2421 
2422   SDValue Chain = N->getOperand(0);
2423   SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
2424 
2425   const unsigned Opc = gwsIntrinToOpcode(IntrID);
2426   SmallVector<SDValue, 5> Ops;
2427   if (HasVSrc)
2428     Ops.push_back(N->getOperand(2));
2429   Ops.push_back(OffsetField);
2430   Ops.push_back(Chain);
2431 
2432   SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2433   CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2434 }
2435 
2436 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) {
2437   if (Subtarget->getLDSBankCount() != 16) {
2438     // This is a single instruction with a pattern.
2439     SelectCode(N);
2440     return;
2441   }
2442 
2443   SDLoc DL(N);
2444 
2445   // This requires 2 instructions. It is possible to write a pattern to support
2446   // this, but the generated isel emitter doesn't correctly deal with multiple
2447   // output instructions using the same physical register input. The copy to m0
2448   // is incorrectly placed before the second instruction.
2449   //
2450   // TODO: Match source modifiers.
2451   //
2452   // def : Pat <
2453   //   (int_amdgcn_interp_p1_f16
2454   //    (VOP3Mods f32:$src0, i32:$src0_modifiers),
2455   //                             (i32 timm:$attrchan), (i32 timm:$attr),
2456   //                             (i1 timm:$high), M0),
2457   //   (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr,
2458   //       timm:$attrchan, 0,
2459   //       (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> {
2460   //   let Predicates = [has16BankLDS];
2461   // }
2462 
2463   // 16 bank LDS
2464   SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0,
2465                                       N->getOperand(5), SDValue());
2466 
2467   SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other);
2468 
2469   SDNode *InterpMov =
2470     CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, {
2471         CurDAG->getTargetConstant(2, DL, MVT::i32), // P0
2472         N->getOperand(3),  // Attr
2473         N->getOperand(2),  // Attrchan
2474         ToM0.getValue(1) // In glue
2475   });
2476 
2477   SDNode *InterpP1LV =
2478     CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, {
2479         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers
2480         N->getOperand(1), // Src0
2481         N->getOperand(3), // Attr
2482         N->getOperand(2), // Attrchan
2483         CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers
2484         SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high
2485         N->getOperand(4), // high
2486         CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp
2487         CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod
2488         SDValue(InterpMov, 1)
2489   });
2490 
2491   CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0));
2492 }
2493 
2494 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
2495   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2496   switch (IntrID) {
2497   case Intrinsic::amdgcn_ds_append:
2498   case Intrinsic::amdgcn_ds_consume: {
2499     if (N->getValueType(0) != MVT::i32)
2500       break;
2501     SelectDSAppendConsume(N, IntrID);
2502     return;
2503   }
2504   case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2505     SelectDSBvhStackIntrinsic(N);
2506     return;
2507   }
2508 
2509   SelectCode(N);
2510 }
2511 
2512 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) {
2513   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2514   unsigned Opcode;
2515   switch (IntrID) {
2516   case Intrinsic::amdgcn_wqm:
2517     Opcode = AMDGPU::WQM;
2518     break;
2519   case Intrinsic::amdgcn_softwqm:
2520     Opcode = AMDGPU::SOFT_WQM;
2521     break;
2522   case Intrinsic::amdgcn_wwm:
2523   case Intrinsic::amdgcn_strict_wwm:
2524     Opcode = AMDGPU::STRICT_WWM;
2525     break;
2526   case Intrinsic::amdgcn_strict_wqm:
2527     Opcode = AMDGPU::STRICT_WQM;
2528     break;
2529   case Intrinsic::amdgcn_interp_p1_f16:
2530     SelectInterpP1F16(N);
2531     return;
2532   case Intrinsic::amdgcn_inverse_ballot:
2533     switch (N->getOperand(1).getValueSizeInBits()) {
2534     case 32:
2535       Opcode = AMDGPU::S_INVERSE_BALLOT_U32;
2536       break;
2537     case 64:
2538       Opcode = AMDGPU::S_INVERSE_BALLOT_U64;
2539       break;
2540     default:
2541       llvm_unreachable("Unsupported size for inverse ballot mask.");
2542     }
2543     break;
2544   default:
2545     SelectCode(N);
2546     return;
2547   }
2548 
2549   SDValue Src = N->getOperand(1);
2550   CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src});
2551 }
2552 
2553 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) {
2554   unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2555   switch (IntrID) {
2556   case Intrinsic::amdgcn_ds_gws_init:
2557   case Intrinsic::amdgcn_ds_gws_barrier:
2558   case Intrinsic::amdgcn_ds_gws_sema_v:
2559   case Intrinsic::amdgcn_ds_gws_sema_br:
2560   case Intrinsic::amdgcn_ds_gws_sema_p:
2561   case Intrinsic::amdgcn_ds_gws_sema_release_all:
2562     SelectDS_GWS(N, IntrID);
2563     return;
2564   default:
2565     break;
2566   }
2567 
2568   SelectCode(N);
2569 }
2570 
2571 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
2572                                             unsigned &Mods,
2573                                             bool IsCanonicalizing,
2574                                             bool AllowAbs) const {
2575   Mods = SISrcMods::NONE;
2576   Src = In;
2577 
2578   if (Src.getOpcode() == ISD::FNEG) {
2579     Mods |= SISrcMods::NEG;
2580     Src = Src.getOperand(0);
2581   } else if (Src.getOpcode() == ISD::FSUB && IsCanonicalizing) {
2582     // Fold fsub [+-]0 into fneg. This may not have folded depending on the
2583     // denormal mode, but we're implicitly canonicalizing in a source operand.
2584     auto *LHS = dyn_cast<ConstantFPSDNode>(Src.getOperand(0));
2585     if (LHS && LHS->isZero()) {
2586       Mods |= SISrcMods::NEG;
2587       Src = Src.getOperand(1);
2588     }
2589   }
2590 
2591   if (AllowAbs && Src.getOpcode() == ISD::FABS) {
2592     Mods |= SISrcMods::ABS;
2593     Src = Src.getOperand(0);
2594   }
2595 
2596   return true;
2597 }
2598 
2599 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
2600                                         SDValue &SrcMods) const {
2601   unsigned Mods;
2602   if (SelectVOP3ModsImpl(In, Src, Mods, /*IsCanonicalizing=*/true,
2603                          /*AllowAbs=*/true)) {
2604     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2605     return true;
2606   }
2607 
2608   return false;
2609 }
2610 
2611 bool AMDGPUDAGToDAGISel::SelectVOP3ModsNonCanonicalizing(
2612     SDValue In, SDValue &Src, SDValue &SrcMods) const {
2613   unsigned Mods;
2614   if (SelectVOP3ModsImpl(In, Src, Mods, /*IsCanonicalizing=*/false,
2615                          /*AllowAbs=*/true)) {
2616     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2617     return true;
2618   }
2619 
2620   return false;
2621 }
2622 
2623 bool AMDGPUDAGToDAGISel::SelectVOP3BMods(SDValue In, SDValue &Src,
2624                                          SDValue &SrcMods) const {
2625   unsigned Mods;
2626   if (SelectVOP3ModsImpl(In, Src, Mods,
2627                          /*IsCanonicalizing=*/true,
2628                          /*AllowAbs=*/false)) {
2629     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2630     return true;
2631   }
2632 
2633   return false;
2634 }
2635 
2636 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
2637   if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
2638     return false;
2639 
2640   Src = In;
2641   return true;
2642 }
2643 
2644 bool AMDGPUDAGToDAGISel::SelectVINTERPModsImpl(SDValue In, SDValue &Src,
2645                                                SDValue &SrcMods,
2646                                                bool OpSel) const {
2647   unsigned Mods;
2648   if (SelectVOP3ModsImpl(In, Src, Mods,
2649                          /*IsCanonicalizing=*/true,
2650                          /*AllowAbs=*/false)) {
2651     if (OpSel)
2652       Mods |= SISrcMods::OP_SEL_0;
2653     SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2654     return true;
2655   }
2656 
2657   return false;
2658 }
2659 
2660 bool AMDGPUDAGToDAGISel::SelectVINTERPMods(SDValue In, SDValue &Src,
2661                                            SDValue &SrcMods) const {
2662   return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ false);
2663 }
2664 
2665 bool AMDGPUDAGToDAGISel::SelectVINTERPModsHi(SDValue In, SDValue &Src,
2666                                              SDValue &SrcMods) const {
2667   return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ true);
2668 }
2669 
2670 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
2671                                          SDValue &SrcMods, SDValue &Clamp,
2672                                          SDValue &Omod) const {
2673   SDLoc DL(In);
2674   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2675   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2676 
2677   return SelectVOP3Mods(In, Src, SrcMods);
2678 }
2679 
2680 bool AMDGPUDAGToDAGISel::SelectVOP3BMods0(SDValue In, SDValue &Src,
2681                                           SDValue &SrcMods, SDValue &Clamp,
2682                                           SDValue &Omod) const {
2683   SDLoc DL(In);
2684   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2685   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2686 
2687   return SelectVOP3BMods(In, Src, SrcMods);
2688 }
2689 
2690 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
2691                                          SDValue &Clamp, SDValue &Omod) const {
2692   Src = In;
2693 
2694   SDLoc DL(In);
2695   Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2696   Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2697 
2698   return true;
2699 }
2700 
2701 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
2702                                          SDValue &SrcMods, bool IsDOT) const {
2703   unsigned Mods = SISrcMods::NONE;
2704   Src = In;
2705 
2706   // TODO: Handle G_FSUB 0 as fneg
2707   if (Src.getOpcode() == ISD::FNEG) {
2708     Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
2709     Src = Src.getOperand(0);
2710   }
2711 
2712   if (Src.getOpcode() == ISD::BUILD_VECTOR && Src.getNumOperands() == 2 &&
2713       (!IsDOT || !Subtarget->hasDOTOpSelHazard())) {
2714     unsigned VecMods = Mods;
2715 
2716     SDValue Lo = stripBitcast(Src.getOperand(0));
2717     SDValue Hi = stripBitcast(Src.getOperand(1));
2718 
2719     if (Lo.getOpcode() == ISD::FNEG) {
2720       Lo = stripBitcast(Lo.getOperand(0));
2721       Mods ^= SISrcMods::NEG;
2722     }
2723 
2724     if (Hi.getOpcode() == ISD::FNEG) {
2725       Hi = stripBitcast(Hi.getOperand(0));
2726       Mods ^= SISrcMods::NEG_HI;
2727     }
2728 
2729     if (isExtractHiElt(Lo, Lo))
2730       Mods |= SISrcMods::OP_SEL_0;
2731 
2732     if (isExtractHiElt(Hi, Hi))
2733       Mods |= SISrcMods::OP_SEL_1;
2734 
2735     unsigned VecSize = Src.getValueSizeInBits();
2736     Lo = stripExtractLoElt(Lo);
2737     Hi = stripExtractLoElt(Hi);
2738 
2739     if (Lo.getValueSizeInBits() > VecSize) {
2740       Lo = CurDAG->getTargetExtractSubreg(
2741         (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
2742         MVT::getIntegerVT(VecSize), Lo);
2743     }
2744 
2745     if (Hi.getValueSizeInBits() > VecSize) {
2746       Hi = CurDAG->getTargetExtractSubreg(
2747         (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, SDLoc(In),
2748         MVT::getIntegerVT(VecSize), Hi);
2749     }
2750 
2751     assert(Lo.getValueSizeInBits() <= VecSize &&
2752            Hi.getValueSizeInBits() <= VecSize);
2753 
2754     if (Lo == Hi && !isInlineImmediate(Lo.getNode())) {
2755       // Really a scalar input. Just select from the low half of the register to
2756       // avoid packing.
2757 
2758       if (VecSize == 32 || VecSize == Lo.getValueSizeInBits()) {
2759         Src = Lo;
2760       } else {
2761         assert(Lo.getValueSizeInBits() == 32 && VecSize == 64);
2762 
2763         SDLoc SL(In);
2764         SDValue Undef = SDValue(
2765           CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, SL,
2766                                  Lo.getValueType()), 0);
2767         auto RC = Lo->isDivergent() ? AMDGPU::VReg_64RegClassID
2768                                     : AMDGPU::SReg_64RegClassID;
2769         const SDValue Ops[] = {
2770           CurDAG->getTargetConstant(RC, SL, MVT::i32),
2771           Lo, CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
2772           Undef, CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32) };
2773 
2774         Src = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SL,
2775                                              Src.getValueType(), Ops), 0);
2776       }
2777       SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2778       return true;
2779     }
2780 
2781     if (VecSize == 64 && Lo == Hi && isa<ConstantFPSDNode>(Lo)) {
2782       uint64_t Lit = cast<ConstantFPSDNode>(Lo)->getValueAPF()
2783                       .bitcastToAPInt().getZExtValue();
2784       if (AMDGPU::isInlinableLiteral32(Lit, Subtarget->hasInv2PiInlineImm())) {
2785         Src = CurDAG->getTargetConstant(Lit, SDLoc(In), MVT::i64);
2786         SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2787         return true;
2788       }
2789     }
2790 
2791     Mods = VecMods;
2792   }
2793 
2794   // Packed instructions do not have abs modifiers.
2795   Mods |= SISrcMods::OP_SEL_1;
2796 
2797   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2798   return true;
2799 }
2800 
2801 bool AMDGPUDAGToDAGISel::SelectVOP3PModsDOT(SDValue In, SDValue &Src,
2802                                             SDValue &SrcMods) const {
2803   return SelectVOP3PMods(In, Src, SrcMods, true);
2804 }
2805 
2806 bool AMDGPUDAGToDAGISel::SelectDotIUVOP3PMods(SDValue In, SDValue &Src) const {
2807   const ConstantSDNode *C = cast<ConstantSDNode>(In);
2808   // Literal i1 value set in intrinsic, represents SrcMods for the next operand.
2809   // 1 promotes packed values to signed, 0 treats them as unsigned.
2810   assert(C->getAPIntValue().getBitWidth() == 1 && "expected i1 value");
2811 
2812   unsigned Mods = SISrcMods::OP_SEL_1;
2813   unsigned SrcSign = C->getZExtValue();
2814   if (SrcSign == 1)
2815     Mods ^= SISrcMods::NEG;
2816 
2817   Src = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2818   return true;
2819 }
2820 
2821 bool AMDGPUDAGToDAGISel::SelectWMMAOpSelVOP3PMods(SDValue In,
2822                                                   SDValue &Src) const {
2823   const ConstantSDNode *C = cast<ConstantSDNode>(In);
2824   assert(C->getAPIntValue().getBitWidth() == 1 && "expected i1 value");
2825 
2826   unsigned Mods = SISrcMods::OP_SEL_1;
2827   unsigned SrcVal = C->getZExtValue();
2828   if (SrcVal == 1)
2829     Mods |= SISrcMods::OP_SEL_0;
2830 
2831   Src = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2832   return true;
2833 }
2834 
2835 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
2836                                          SDValue &SrcMods) const {
2837   Src = In;
2838   // FIXME: Handle op_sel
2839   SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
2840   return true;
2841 }
2842 
2843 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
2844                                              SDValue &SrcMods) const {
2845   // FIXME: Handle op_sel
2846   return SelectVOP3Mods(In, Src, SrcMods);
2847 }
2848 
2849 // The return value is not whether the match is possible (which it always is),
2850 // but whether or not it a conversion is really used.
2851 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
2852                                                    unsigned &Mods) const {
2853   Mods = 0;
2854   SelectVOP3ModsImpl(In, Src, Mods);
2855 
2856   if (Src.getOpcode() == ISD::FP_EXTEND) {
2857     Src = Src.getOperand(0);
2858     assert(Src.getValueType() == MVT::f16);
2859     Src = stripBitcast(Src);
2860 
2861     // Be careful about folding modifiers if we already have an abs. fneg is
2862     // applied last, so we don't want to apply an earlier fneg.
2863     if ((Mods & SISrcMods::ABS) == 0) {
2864       unsigned ModsTmp;
2865       SelectVOP3ModsImpl(Src, Src, ModsTmp);
2866 
2867       if ((ModsTmp & SISrcMods::NEG) != 0)
2868         Mods ^= SISrcMods::NEG;
2869 
2870       if ((ModsTmp & SISrcMods::ABS) != 0)
2871         Mods |= SISrcMods::ABS;
2872     }
2873 
2874     // op_sel/op_sel_hi decide the source type and source.
2875     // If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
2876     // If the sources's op_sel is set, it picks the high half of the source
2877     // register.
2878 
2879     Mods |= SISrcMods::OP_SEL_1;
2880     if (isExtractHiElt(Src, Src)) {
2881       Mods |= SISrcMods::OP_SEL_0;
2882 
2883       // TODO: Should we try to look for neg/abs here?
2884     }
2885 
2886     return true;
2887   }
2888 
2889   return false;
2890 }
2891 
2892 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
2893                                                   SDValue &SrcMods) const {
2894   unsigned Mods = 0;
2895   if (!SelectVOP3PMadMixModsImpl(In, Src, Mods))
2896     return false;
2897   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2898   return true;
2899 }
2900 
2901 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
2902                                                SDValue &SrcMods) const {
2903   unsigned Mods = 0;
2904   SelectVOP3PMadMixModsImpl(In, Src, Mods);
2905   SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2906   return true;
2907 }
2908 
2909 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
2910   if (In.isUndef())
2911     return CurDAG->getUNDEF(MVT::i32);
2912 
2913   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) {
2914     SDLoc SL(In);
2915     return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32);
2916   }
2917 
2918   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) {
2919     SDLoc SL(In);
2920     return CurDAG->getConstant(
2921       C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32);
2922   }
2923 
2924   SDValue Src;
2925   if (isExtractHiElt(In, Src))
2926     return Src;
2927 
2928   return SDValue();
2929 }
2930 
2931 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
2932   assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn);
2933 
2934   const SIRegisterInfo *SIRI =
2935     static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
2936   const SIInstrInfo * SII =
2937     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
2938 
2939   unsigned Limit = 0;
2940   bool AllUsesAcceptSReg = true;
2941   for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
2942     Limit < 10 && U != E; ++U, ++Limit) {
2943     const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo());
2944 
2945     // If the register class is unknown, it could be an unknown
2946     // register class that needs to be an SGPR, e.g. an inline asm
2947     // constraint
2948     if (!RC || SIRI->isSGPRClass(RC))
2949       return false;
2950 
2951     if (RC != &AMDGPU::VS_32RegClass) {
2952       AllUsesAcceptSReg = false;
2953       SDNode * User = *U;
2954       if (User->isMachineOpcode()) {
2955         unsigned Opc = User->getMachineOpcode();
2956         const MCInstrDesc &Desc = SII->get(Opc);
2957         if (Desc.isCommutable()) {
2958           unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo();
2959           unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
2960           if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) {
2961             unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs();
2962             const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo);
2963             if (CommutedRC == &AMDGPU::VS_32RegClass)
2964               AllUsesAcceptSReg = true;
2965           }
2966         }
2967       }
2968       // If "AllUsesAcceptSReg == false" so far we haven't succeeded
2969       // commuting current user. This means have at least one use
2970       // that strictly require VGPR. Thus, we will not attempt to commute
2971       // other user instructions.
2972       if (!AllUsesAcceptSReg)
2973         break;
2974     }
2975   }
2976   return !AllUsesAcceptSReg && (Limit < 10);
2977 }
2978 
2979 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const {
2980   auto Ld = cast<LoadSDNode>(N);
2981 
2982   if (N->isDivergent() && !AMDGPUInstrInfo::isUniformMMO(Ld->getMemOperand()))
2983     return false;
2984 
2985   return Ld->getAlign() >= Align(4) &&
2986          ((Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
2987            Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
2988           (Subtarget->getScalarizeGlobalBehavior() &&
2989            Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
2990            Ld->isSimple() &&
2991            static_cast<const SITargetLowering *>(getTargetLowering())
2992                ->isMemOpHasNoClobberedMemOperand(N)));
2993 }
2994 
2995 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
2996   const AMDGPUTargetLowering& Lowering =
2997     *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
2998   bool IsModified = false;
2999   do {
3000     IsModified = false;
3001 
3002     // Go over all selected nodes and try to fold them a bit more
3003     SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin();
3004     while (Position != CurDAG->allnodes_end()) {
3005       SDNode *Node = &*Position++;
3006       MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node);
3007       if (!MachineNode)
3008         continue;
3009 
3010       SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
3011       if (ResNode != Node) {
3012         if (ResNode)
3013           ReplaceUses(Node, ResNode);
3014         IsModified = true;
3015       }
3016     }
3017     CurDAG->RemoveDeadNodes();
3018   } while (IsModified);
3019 }
3020 
3021 char AMDGPUDAGToDAGISel::ID = 0;
3022