1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the targeting of the InstructionSelector class for
10 // SPIRV.
11 // TODO: This should be generated by TableGen.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "MCTargetDesc/SPIRVBaseInfo.h"
16 #include "MCTargetDesc/SPIRVMCTargetDesc.h"
17 #include "SPIRV.h"
18 #include "SPIRVGlobalRegistry.h"
19 #include "SPIRVInstrInfo.h"
20 #include "SPIRVRegisterBankInfo.h"
21 #include "SPIRVRegisterInfo.h"
22 #include "SPIRVTargetMachine.h"
23 #include "SPIRVUtils.h"
24 #include "llvm/ADT/APFloat.h"
25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
28 #include "llvm/CodeGen/MachineInstrBuilder.h"
29 #include "llvm/CodeGen/MachineRegisterInfo.h"
30 #include "llvm/IR/IntrinsicsSPIRV.h"
31 #include "llvm/Support/Debug.h"
32 
33 #define DEBUG_TYPE "spirv-isel"
34 
35 using namespace llvm;
36 namespace CL = SPIRV::OpenCLExtInst;
37 namespace GL = SPIRV::GLSLExtInst;
38 
39 using ExtInstList =
40     std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
41 
42 namespace {
43 
44 #define GET_GLOBALISEL_PREDICATE_BITSET
45 #include "SPIRVGenGlobalISel.inc"
46 #undef GET_GLOBALISEL_PREDICATE_BITSET
47 
48 class SPIRVInstructionSelector : public InstructionSelector {
49   const SPIRVSubtarget &STI;
50   const SPIRVInstrInfo &TII;
51   const SPIRVRegisterInfo &TRI;
52   const RegisterBankInfo &RBI;
53   SPIRVGlobalRegistry &GR;
54   MachineRegisterInfo *MRI;
55 
56 public:
57   SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
58                            const SPIRVSubtarget &ST,
59                            const RegisterBankInfo &RBI);
60   void setupMF(MachineFunction &MF, GISelKnownBits *KB,
61                CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
62                BlockFrequencyInfo *BFI) override;
63   // Common selection code. Instruction-specific selection occurs in spvSelect.
64   bool select(MachineInstr &I) override;
65   static const char *getName() { return DEBUG_TYPE; }
66 
67 #define GET_GLOBALISEL_PREDICATES_DECL
68 #include "SPIRVGenGlobalISel.inc"
69 #undef GET_GLOBALISEL_PREDICATES_DECL
70 
71 #define GET_GLOBALISEL_TEMPORARIES_DECL
72 #include "SPIRVGenGlobalISel.inc"
73 #undef GET_GLOBALISEL_TEMPORARIES_DECL
74 
75 private:
76   // tblgen-erated 'select' implementation, used as the initial selector for
77   // the patterns that don't require complex C++.
78   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
79 
80   // All instruction-specific selection that didn't happen in "select()".
81   // Is basically a large Switch/Case delegating to all other select method.
82   bool spvSelect(Register ResVReg, const SPIRVType *ResType,
83                  MachineInstr &I) const;
84 
85   bool selectGlobalValue(Register ResVReg, MachineInstr &I,
86                          const MachineInstr *Init = nullptr) const;
87 
88   bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType,
89                          MachineInstr &I, Register SrcReg,
90                          unsigned Opcode) const;
91   bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
92                   unsigned Opcode) const;
93 
94   bool selectLoad(Register ResVReg, const SPIRVType *ResType,
95                   MachineInstr &I) const;
96   bool selectStore(MachineInstr &I) const;
97 
98   bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
99 
100   bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType,
101                        MachineInstr &I, unsigned NewOpcode) const;
102 
103   bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType,
104                            MachineInstr &I) const;
105 
106   bool selectFence(MachineInstr &I) const;
107 
108   bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType,
109                            MachineInstr &I) const;
110 
111   bool selectBitreverse(Register ResVReg, const SPIRVType *ResType,
112                         MachineInstr &I) const;
113 
114   bool selectConstVector(Register ResVReg, const SPIRVType *ResType,
115                          MachineInstr &I) const;
116 
117   bool selectCmp(Register ResVReg, const SPIRVType *ResType,
118                  unsigned comparisonOpcode, MachineInstr &I) const;
119 
120   bool selectICmp(Register ResVReg, const SPIRVType *ResType,
121                   MachineInstr &I) const;
122   bool selectFCmp(Register ResVReg, const SPIRVType *ResType,
123                   MachineInstr &I) const;
124 
125   void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
126                    int OpIdx) const;
127   void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
128                     int OpIdx) const;
129 
130   bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm,
131                    MachineInstr &I) const;
132 
133   bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
134                     bool IsSigned) const;
135   bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
136                   bool IsSigned, unsigned Opcode) const;
137   bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
138                  bool IsSigned) const;
139 
140   bool selectTrunc(Register ResVReg, const SPIRVType *ResType,
141                    MachineInstr &I) const;
142 
143   bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
144                        const SPIRVType *intTy, const SPIRVType *boolTy) const;
145 
146   bool selectOpUndef(Register ResVReg, const SPIRVType *ResType,
147                      MachineInstr &I) const;
148   bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType,
149                        MachineInstr &I) const;
150   bool selectExtractVal(Register ResVReg, const SPIRVType *ResType,
151                         MachineInstr &I) const;
152   bool selectInsertVal(Register ResVReg, const SPIRVType *ResType,
153                        MachineInstr &I) const;
154   bool selectExtractElt(Register ResVReg, const SPIRVType *ResType,
155                         MachineInstr &I) const;
156   bool selectInsertElt(Register ResVReg, const SPIRVType *ResType,
157                        MachineInstr &I) const;
158   bool selectGEP(Register ResVReg, const SPIRVType *ResType,
159                  MachineInstr &I) const;
160 
161   bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType,
162                         MachineInstr &I) const;
163 
164   bool selectBranch(MachineInstr &I) const;
165   bool selectBranchCond(MachineInstr &I) const;
166 
167   bool selectPhi(Register ResVReg, const SPIRVType *ResType,
168                  MachineInstr &I) const;
169 
170   bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
171                      MachineInstr &I, CL::OpenCLExtInst CLInst) const;
172   bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
173                      MachineInstr &I, CL::OpenCLExtInst CLInst,
174                      GL::GLSLExtInst GLInst) const;
175   bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
176                      MachineInstr &I, const ExtInstList &ExtInsts) const;
177 
178   bool selectLog10(Register ResVReg, const SPIRVType *ResType,
179                    MachineInstr &I) const;
180 
181   Register buildI32Constant(uint32_t Val, MachineInstr &I,
182                             const SPIRVType *ResType = nullptr) const;
183 
184   Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const;
185   Register buildOnesVal(bool AllOnes, const SPIRVType *ResType,
186                         MachineInstr &I) const;
187 };
188 
189 } // end anonymous namespace
190 
191 #define GET_GLOBALISEL_IMPL
192 #include "SPIRVGenGlobalISel.inc"
193 #undef GET_GLOBALISEL_IMPL
194 
195 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
196                                                    const SPIRVSubtarget &ST,
197                                                    const RegisterBankInfo &RBI)
198     : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
199       TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
200 #define GET_GLOBALISEL_PREDICATES_INIT
201 #include "SPIRVGenGlobalISel.inc"
202 #undef GET_GLOBALISEL_PREDICATES_INIT
203 #define GET_GLOBALISEL_TEMPORARIES_INIT
204 #include "SPIRVGenGlobalISel.inc"
205 #undef GET_GLOBALISEL_TEMPORARIES_INIT
206 {
207 }
208 
209 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB,
210                                        CodeGenCoverage *CoverageInfo,
211                                        ProfileSummaryInfo *PSI,
212                                        BlockFrequencyInfo *BFI) {
213   MRI = &MF.getRegInfo();
214   GR.setCurrentFunc(MF);
215   InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
216 }
217 
218 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI);
219 
220 // Defined in SPIRVLegalizerInfo.cpp.
221 extern bool isTypeFoldingSupported(unsigned Opcode);
222 
223 bool SPIRVInstructionSelector::select(MachineInstr &I) {
224   assert(I.getParent() && "Instruction should be in a basic block!");
225   assert(I.getParent()->getParent() && "Instruction should be in a function!");
226 
227   Register Opcode = I.getOpcode();
228   // If it's not a GMIR instruction, we've selected it already.
229   if (!isPreISelGenericOpcode(Opcode)) {
230     if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
231       auto *Def = MRI->getVRegDef(I.getOperand(1).getReg());
232       if (isTypeFoldingSupported(Def->getOpcode())) {
233         auto Res = selectImpl(I, *CoverageInfo);
234         assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
235         if (Res)
236           return Res;
237       }
238       MRI->replaceRegWith(I.getOperand(1).getReg(), I.getOperand(0).getReg());
239       I.removeFromParent();
240       return true;
241     } else if (I.getNumDefs() == 1) {
242       // Make all vregs 32 bits (for SPIR-V IDs).
243       MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32));
244     }
245     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
246   }
247 
248   if (I.getNumOperands() != I.getNumExplicitOperands()) {
249     LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
250     return false;
251   }
252 
253   // Common code for getting return reg+type, and removing selected instr
254   // from parent occurs here. Instr-specific selection happens in spvSelect().
255   bool HasDefs = I.getNumDefs() > 0;
256   Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0);
257   SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr;
258   assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
259   if (spvSelect(ResVReg, ResType, I)) {
260     if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs).
261       MRI->setType(ResVReg, LLT::scalar(32));
262     I.removeFromParent();
263     return true;
264   }
265   return false;
266 }
267 
268 bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
269                                          const SPIRVType *ResType,
270                                          MachineInstr &I) const {
271   assert(!isTypeFoldingSupported(I.getOpcode()) ||
272          I.getOpcode() == TargetOpcode::G_CONSTANT);
273   const unsigned Opcode = I.getOpcode();
274   switch (Opcode) {
275   case TargetOpcode::G_CONSTANT:
276     return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(),
277                        I);
278   case TargetOpcode::G_GLOBAL_VALUE:
279     return selectGlobalValue(ResVReg, I);
280   case TargetOpcode::G_IMPLICIT_DEF:
281     return selectOpUndef(ResVReg, ResType, I);
282 
283   case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
284   case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
285     return selectIntrinsic(ResVReg, ResType, I);
286   case TargetOpcode::G_BITREVERSE:
287     return selectBitreverse(ResVReg, ResType, I);
288 
289   case TargetOpcode::G_BUILD_VECTOR:
290     return selectConstVector(ResVReg, ResType, I);
291 
292   case TargetOpcode::G_SHUFFLE_VECTOR: {
293     MachineBasicBlock &BB = *I.getParent();
294     auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
295                    .addDef(ResVReg)
296                    .addUse(GR.getSPIRVTypeID(ResType))
297                    .addUse(I.getOperand(1).getReg())
298                    .addUse(I.getOperand(2).getReg());
299     for (auto V : I.getOperand(3).getShuffleMask())
300       MIB.addImm(V);
301     return MIB.constrainAllUses(TII, TRI, RBI);
302   }
303   case TargetOpcode::G_MEMMOVE:
304   case TargetOpcode::G_MEMCPY:
305   case TargetOpcode::G_MEMSET:
306     return selectMemOperation(ResVReg, I);
307 
308   case TargetOpcode::G_ICMP:
309     return selectICmp(ResVReg, ResType, I);
310   case TargetOpcode::G_FCMP:
311     return selectFCmp(ResVReg, ResType, I);
312 
313   case TargetOpcode::G_FRAME_INDEX:
314     return selectFrameIndex(ResVReg, ResType, I);
315 
316   case TargetOpcode::G_LOAD:
317     return selectLoad(ResVReg, ResType, I);
318   case TargetOpcode::G_STORE:
319     return selectStore(I);
320 
321   case TargetOpcode::G_BR:
322     return selectBranch(I);
323   case TargetOpcode::G_BRCOND:
324     return selectBranchCond(I);
325 
326   case TargetOpcode::G_PHI:
327     return selectPhi(ResVReg, ResType, I);
328 
329   case TargetOpcode::G_FPTOSI:
330     return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
331   case TargetOpcode::G_FPTOUI:
332     return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
333 
334   case TargetOpcode::G_SITOFP:
335     return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF);
336   case TargetOpcode::G_UITOFP:
337     return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF);
338 
339   case TargetOpcode::G_CTPOP:
340     return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount);
341   case TargetOpcode::G_SMIN:
342     return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin);
343   case TargetOpcode::G_UMIN:
344     return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin);
345 
346   case TargetOpcode::G_SMAX:
347     return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax);
348   case TargetOpcode::G_UMAX:
349     return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax);
350 
351   case TargetOpcode::G_FMA:
352     return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma);
353 
354   case TargetOpcode::G_FPOW:
355     return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow);
356   case TargetOpcode::G_FPOWI:
357     return selectExtInst(ResVReg, ResType, I, CL::pown);
358 
359   case TargetOpcode::G_FEXP:
360     return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp);
361   case TargetOpcode::G_FEXP2:
362     return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2);
363 
364   case TargetOpcode::G_FLOG:
365     return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log);
366   case TargetOpcode::G_FLOG2:
367     return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2);
368   case TargetOpcode::G_FLOG10:
369     return selectLog10(ResVReg, ResType, I);
370 
371   case TargetOpcode::G_FABS:
372     return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs);
373   case TargetOpcode::G_ABS:
374     return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs);
375 
376   case TargetOpcode::G_FMINNUM:
377   case TargetOpcode::G_FMINIMUM:
378     return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::FMin);
379   case TargetOpcode::G_FMAXNUM:
380   case TargetOpcode::G_FMAXIMUM:
381     return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::FMax);
382 
383   case TargetOpcode::G_FCOPYSIGN:
384     return selectExtInst(ResVReg, ResType, I, CL::copysign);
385 
386   case TargetOpcode::G_FCEIL:
387     return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil);
388   case TargetOpcode::G_FFLOOR:
389     return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor);
390 
391   case TargetOpcode::G_FCOS:
392     return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos);
393   case TargetOpcode::G_FSIN:
394     return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin);
395 
396   case TargetOpcode::G_FSQRT:
397     return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt);
398 
399   case TargetOpcode::G_CTTZ:
400   case TargetOpcode::G_CTTZ_ZERO_UNDEF:
401     return selectExtInst(ResVReg, ResType, I, CL::ctz);
402   case TargetOpcode::G_CTLZ:
403   case TargetOpcode::G_CTLZ_ZERO_UNDEF:
404     return selectExtInst(ResVReg, ResType, I, CL::clz);
405 
406   case TargetOpcode::G_INTRINSIC_ROUND:
407     return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round);
408   case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
409     return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
410   case TargetOpcode::G_INTRINSIC_TRUNC:
411     return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc);
412   case TargetOpcode::G_FRINT:
413   case TargetOpcode::G_FNEARBYINT:
414     return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
415 
416   case TargetOpcode::G_SMULH:
417     return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi);
418   case TargetOpcode::G_UMULH:
419     return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi);
420 
421   case TargetOpcode::G_SEXT:
422     return selectExt(ResVReg, ResType, I, true);
423   case TargetOpcode::G_ANYEXT:
424   case TargetOpcode::G_ZEXT:
425     return selectExt(ResVReg, ResType, I, false);
426   case TargetOpcode::G_TRUNC:
427     return selectTrunc(ResVReg, ResType, I);
428   case TargetOpcode::G_FPTRUNC:
429   case TargetOpcode::G_FPEXT:
430     return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert);
431 
432   case TargetOpcode::G_PTRTOINT:
433     return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU);
434   case TargetOpcode::G_INTTOPTR:
435     return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr);
436   case TargetOpcode::G_BITCAST:
437     return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
438   case TargetOpcode::G_ADDRSPACE_CAST:
439     return selectAddrSpaceCast(ResVReg, ResType, I);
440   case TargetOpcode::G_PTR_ADD: {
441     // Currently, we get G_PTR_ADD only as a result of translating
442     // global variables, initialized with constant expressions like GV + Const
443     // (see test opencl/basic/progvar_prog_scope_init.ll).
444     // TODO: extend the handler once we have other cases.
445     assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
446     Register GV = I.getOperand(1).getReg();
447     MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV);
448     assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
449             (*II).getOpcode() == TargetOpcode::COPY ||
450             (*II).getOpcode() == SPIRV::OpVariable) &&
451            isImm(I.getOperand(2), MRI));
452     Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I);
453     MachineBasicBlock &BB = *I.getParent();
454     auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
455                    .addDef(ResVReg)
456                    .addUse(GR.getSPIRVTypeID(ResType))
457                    .addImm(static_cast<uint32_t>(
458                        SPIRV::Opcode::InBoundsPtrAccessChain))
459                    .addUse(GV)
460                    .addUse(Idx)
461                    .addUse(I.getOperand(2).getReg());
462     return MIB.constrainAllUses(TII, TRI, RBI);
463   }
464 
465   case TargetOpcode::G_ATOMICRMW_OR:
466     return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr);
467   case TargetOpcode::G_ATOMICRMW_ADD:
468     return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd);
469   case TargetOpcode::G_ATOMICRMW_AND:
470     return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd);
471   case TargetOpcode::G_ATOMICRMW_MAX:
472     return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax);
473   case TargetOpcode::G_ATOMICRMW_MIN:
474     return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin);
475   case TargetOpcode::G_ATOMICRMW_SUB:
476     return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub);
477   case TargetOpcode::G_ATOMICRMW_XOR:
478     return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor);
479   case TargetOpcode::G_ATOMICRMW_UMAX:
480     return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax);
481   case TargetOpcode::G_ATOMICRMW_UMIN:
482     return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin);
483   case TargetOpcode::G_ATOMICRMW_XCHG:
484     return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange);
485   case TargetOpcode::G_ATOMIC_CMPXCHG:
486     return selectAtomicCmpXchg(ResVReg, ResType, I);
487 
488   case TargetOpcode::G_FENCE:
489     return selectFence(I);
490 
491   default:
492     return false;
493   }
494 }
495 
496 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
497                                              const SPIRVType *ResType,
498                                              MachineInstr &I,
499                                              CL::OpenCLExtInst CLInst) const {
500   return selectExtInst(ResVReg, ResType, I,
501                        {{SPIRV::InstructionSet::OpenCL_std, CLInst}});
502 }
503 
504 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
505                                              const SPIRVType *ResType,
506                                              MachineInstr &I,
507                                              CL::OpenCLExtInst CLInst,
508                                              GL::GLSLExtInst GLInst) const {
509   ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
510                           {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
511   return selectExtInst(ResVReg, ResType, I, ExtInsts);
512 }
513 
514 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
515                                              const SPIRVType *ResType,
516                                              MachineInstr &I,
517                                              const ExtInstList &Insts) const {
518 
519   for (const auto &Ex : Insts) {
520     SPIRV::InstructionSet::InstructionSet Set = Ex.first;
521     uint32_t Opcode = Ex.second;
522     if (STI.canUseExtInstSet(Set)) {
523       MachineBasicBlock &BB = *I.getParent();
524       auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
525                      .addDef(ResVReg)
526                      .addUse(GR.getSPIRVTypeID(ResType))
527                      .addImm(static_cast<uint32_t>(Set))
528                      .addImm(Opcode);
529       const unsigned NumOps = I.getNumOperands();
530       for (unsigned i = 1; i < NumOps; ++i)
531         MIB.add(I.getOperand(i));
532       return MIB.constrainAllUses(TII, TRI, RBI);
533     }
534   }
535   return false;
536 }
537 
538 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg,
539                                                  const SPIRVType *ResType,
540                                                  MachineInstr &I,
541                                                  Register SrcReg,
542                                                  unsigned Opcode) const {
543   return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
544       .addDef(ResVReg)
545       .addUse(GR.getSPIRVTypeID(ResType))
546       .addUse(SrcReg)
547       .constrainAllUses(TII, TRI, RBI);
548 }
549 
550 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
551                                           const SPIRVType *ResType,
552                                           MachineInstr &I,
553                                           unsigned Opcode) const {
554   return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(),
555                            Opcode);
556 }
557 
558 static SPIRV::Scope::Scope getScope(SyncScope::ID Ord) {
559   switch (Ord) {
560   case SyncScope::SingleThread:
561     return SPIRV::Scope::Invocation;
562   case SyncScope::System:
563     return SPIRV::Scope::Device;
564   default:
565     llvm_unreachable("Unsupported synchronization Scope ID.");
566   }
567 }
568 
569 static void addMemoryOperands(MachineMemOperand *MemOp,
570                               MachineInstrBuilder &MIB) {
571   uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
572   if (MemOp->isVolatile())
573     SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
574   if (MemOp->isNonTemporal())
575     SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
576   if (MemOp->getAlign().value())
577     SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
578 
579   if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
580     MIB.addImm(SpvMemOp);
581     if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
582       MIB.addImm(MemOp->getAlign().value());
583   }
584 }
585 
586 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) {
587   uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
588   if (Flags & MachineMemOperand::Flags::MOVolatile)
589     SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
590   if (Flags & MachineMemOperand::Flags::MONonTemporal)
591     SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
592 
593   if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
594     MIB.addImm(SpvMemOp);
595 }
596 
597 bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
598                                           const SPIRVType *ResType,
599                                           MachineInstr &I) const {
600   unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
601   Register Ptr = I.getOperand(1 + OpOffset).getReg();
602   auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
603                  .addDef(ResVReg)
604                  .addUse(GR.getSPIRVTypeID(ResType))
605                  .addUse(Ptr);
606   if (!I.getNumMemOperands()) {
607     assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
608            I.getOpcode() ==
609                TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
610     addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
611   } else {
612     addMemoryOperands(*I.memoperands_begin(), MIB);
613   }
614   return MIB.constrainAllUses(TII, TRI, RBI);
615 }
616 
617 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
618   unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
619   Register StoreVal = I.getOperand(0 + OpOffset).getReg();
620   Register Ptr = I.getOperand(1 + OpOffset).getReg();
621   MachineBasicBlock &BB = *I.getParent();
622   auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore))
623                  .addUse(Ptr)
624                  .addUse(StoreVal);
625   if (!I.getNumMemOperands()) {
626     assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
627            I.getOpcode() ==
628                TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
629     addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
630   } else {
631     addMemoryOperands(*I.memoperands_begin(), MIB);
632   }
633   return MIB.constrainAllUses(TII, TRI, RBI);
634 }
635 
636 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
637                                                   MachineInstr &I) const {
638   MachineBasicBlock &BB = *I.getParent();
639   Register SrcReg = I.getOperand(1).getReg();
640   if (I.getOpcode() == TargetOpcode::G_MEMSET) {
641     assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
642     unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI);
643     unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI);
644     SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
645     SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII);
646     Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII);
647     SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType(
648         ArrTy, I, TII, SPIRV::StorageClass::UniformConstant);
649     // TODO: check if we have such GV, add init, use buildGlobalVariable.
650     Type *LLVMArrTy = ArrayType::get(
651         IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num);
652     GlobalVariable *GV =
653         new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage);
654     Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
655     GR.add(GV, GR.CurMF, VarReg);
656 
657     buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {});
658     BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable))
659         .addDef(VarReg)
660         .addUse(GR.getSPIRVTypeID(VarTy))
661         .addImm(SPIRV::StorageClass::UniformConstant)
662         .addUse(Const)
663         .constrainAllUses(TII, TRI, RBI);
664     SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType(
665         ValTy, I, TII, SPIRV::StorageClass::UniformConstant);
666     SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
667     selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast);
668   }
669   auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized))
670                  .addUse(I.getOperand(0).getReg())
671                  .addUse(SrcReg)
672                  .addUse(I.getOperand(2).getReg());
673   if (I.getNumMemOperands())
674     addMemoryOperands(*I.memoperands_begin(), MIB);
675   bool Result = MIB.constrainAllUses(TII, TRI, RBI);
676   if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg())
677     BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg)
678         .addUse(MIB->getOperand(0).getReg());
679   return Result;
680 }
681 
682 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
683                                                const SPIRVType *ResType,
684                                                MachineInstr &I,
685                                                unsigned NewOpcode) const {
686   assert(I.hasOneMemOperand());
687   const MachineMemOperand *MemOp = *I.memoperands_begin();
688   uint32_t Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID()));
689   Register ScopeReg = buildI32Constant(Scope, I);
690 
691   Register Ptr = I.getOperand(1).getReg();
692   // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll
693   // auto ScSem =
694   // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr));
695   AtomicOrdering AO = MemOp->getSuccessOrdering();
696   uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
697   Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I);
698 
699   return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode))
700       .addDef(ResVReg)
701       .addUse(GR.getSPIRVTypeID(ResType))
702       .addUse(Ptr)
703       .addUse(ScopeReg)
704       .addUse(MemSemReg)
705       .addUse(I.getOperand(2).getReg())
706       .constrainAllUses(TII, TRI, RBI);
707 }
708 
709 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
710   AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm());
711   uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
712   Register MemSemReg = buildI32Constant(MemSem, I);
713   SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm());
714   uint32_t Scope = static_cast<uint32_t>(getScope(Ord));
715   Register ScopeReg = buildI32Constant(Scope, I);
716   MachineBasicBlock &BB = *I.getParent();
717   return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier))
718       .addUse(ScopeReg)
719       .addUse(MemSemReg)
720       .constrainAllUses(TII, TRI, RBI);
721 }
722 
723 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
724                                                    const SPIRVType *ResType,
725                                                    MachineInstr &I) const {
726   Register ScopeReg;
727   Register MemSemEqReg;
728   Register MemSemNeqReg;
729   Register Ptr = I.getOperand(2).getReg();
730   if (!isa<GIntrinsic>(I)) {
731     assert(I.hasOneMemOperand());
732     const MachineMemOperand *MemOp = *I.memoperands_begin();
733     unsigned Scope = static_cast<uint32_t>(getScope(MemOp->getSyncScopeID()));
734     ScopeReg = buildI32Constant(Scope, I);
735 
736     unsigned ScSem = static_cast<uint32_t>(
737         getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)));
738     AtomicOrdering AO = MemOp->getSuccessOrdering();
739     unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem;
740     MemSemEqReg = buildI32Constant(MemSemEq, I);
741     AtomicOrdering FO = MemOp->getFailureOrdering();
742     unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem;
743     MemSemNeqReg =
744         MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I);
745   } else {
746     ScopeReg = I.getOperand(5).getReg();
747     MemSemEqReg = I.getOperand(6).getReg();
748     MemSemNeqReg = I.getOperand(7).getReg();
749   }
750 
751   Register Cmp = I.getOperand(3).getReg();
752   Register Val = I.getOperand(4).getReg();
753   SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val);
754   Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass);
755   const DebugLoc &DL = I.getDebugLoc();
756   bool Result =
757       BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange))
758           .addDef(ACmpRes)
759           .addUse(GR.getSPIRVTypeID(SpvValTy))
760           .addUse(Ptr)
761           .addUse(ScopeReg)
762           .addUse(MemSemEqReg)
763           .addUse(MemSemNeqReg)
764           .addUse(Val)
765           .addUse(Cmp)
766           .constrainAllUses(TII, TRI, RBI);
767   Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
768   SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
769   Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual))
770                 .addDef(CmpSuccReg)
771                 .addUse(GR.getSPIRVTypeID(BoolTy))
772                 .addUse(ACmpRes)
773                 .addUse(Cmp)
774                 .constrainAllUses(TII, TRI, RBI);
775   Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
776   Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
777                 .addDef(TmpReg)
778                 .addUse(GR.getSPIRVTypeID(ResType))
779                 .addUse(ACmpRes)
780                 .addUse(GR.getOrCreateUndef(I, ResType, TII))
781                 .addImm(0)
782                 .constrainAllUses(TII, TRI, RBI);
783   Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
784                 .addDef(ResVReg)
785                 .addUse(GR.getSPIRVTypeID(ResType))
786                 .addUse(CmpSuccReg)
787                 .addUse(TmpReg)
788                 .addImm(1)
789                 .constrainAllUses(TII, TRI, RBI);
790   return Result;
791 }
792 
793 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) {
794   switch (SC) {
795   case SPIRV::StorageClass::Workgroup:
796   case SPIRV::StorageClass::CrossWorkgroup:
797   case SPIRV::StorageClass::Function:
798     return true;
799   default:
800     return false;
801   }
802 }
803 
804 // In SPIR-V address space casting can only happen to and from the Generic
805 // storage class. We can also only case Workgroup, CrossWorkgroup, or Function
806 // pointers to and from Generic pointers. As such, we can convert e.g. from
807 // Workgroup to Function by going via a Generic pointer as an intermediary. All
808 // other combinations can only be done by a bitcast, and are probably not safe.
809 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
810                                                    const SPIRVType *ResType,
811                                                    MachineInstr &I) const {
812   // If the AddrSpaceCast user is single and in OpConstantComposite or
813   // OpVariable, we should select OpSpecConstantOp.
814   auto UIs = MRI->use_instructions(ResVReg);
815   if (!UIs.empty() && ++UIs.begin() == UIs.end() &&
816       (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite ||
817        UIs.begin()->getOpcode() == SPIRV::OpVariable ||
818        isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) {
819     Register NewReg = I.getOperand(1).getReg();
820     MachineBasicBlock &BB = *I.getParent();
821     SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
822     ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII,
823                                              SPIRV::StorageClass::Generic);
824     bool Result =
825         BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
826             .addDef(ResVReg)
827             .addUse(GR.getSPIRVTypeID(ResType))
828             .addImm(static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric))
829             .addUse(NewReg)
830             .constrainAllUses(TII, TRI, RBI);
831     return Result;
832   }
833   Register SrcPtr = I.getOperand(1).getReg();
834   SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr);
835   SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr);
836   SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg);
837 
838   // Casting from an eligable pointer to Generic.
839   if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
840     return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
841   // Casting from Generic to an eligable pointer.
842   if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC))
843     return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
844   // Casting between 2 eligable pointers using Generic as an intermediary.
845   if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) {
846     Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass);
847     SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType(
848         SrcPtrTy, I, TII, SPIRV::StorageClass::Generic);
849     MachineBasicBlock &BB = *I.getParent();
850     const DebugLoc &DL = I.getDebugLoc();
851     bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric))
852                        .addDef(Tmp)
853                        .addUse(GR.getSPIRVTypeID(GenericPtrTy))
854                        .addUse(SrcPtr)
855                        .constrainAllUses(TII, TRI, RBI);
856     return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr))
857                           .addDef(ResVReg)
858                           .addUse(GR.getSPIRVTypeID(ResType))
859                           .addUse(Tmp)
860                           .constrainAllUses(TII, TRI, RBI);
861   }
862   // TODO Should this case just be disallowed completely?
863   // We're casting 2 other arbitrary address spaces, so have to bitcast.
864   return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
865 }
866 
867 static unsigned getFCmpOpcode(unsigned PredNum) {
868   auto Pred = static_cast<CmpInst::Predicate>(PredNum);
869   switch (Pred) {
870   case CmpInst::FCMP_OEQ:
871     return SPIRV::OpFOrdEqual;
872   case CmpInst::FCMP_OGE:
873     return SPIRV::OpFOrdGreaterThanEqual;
874   case CmpInst::FCMP_OGT:
875     return SPIRV::OpFOrdGreaterThan;
876   case CmpInst::FCMP_OLE:
877     return SPIRV::OpFOrdLessThanEqual;
878   case CmpInst::FCMP_OLT:
879     return SPIRV::OpFOrdLessThan;
880   case CmpInst::FCMP_ONE:
881     return SPIRV::OpFOrdNotEqual;
882   case CmpInst::FCMP_ORD:
883     return SPIRV::OpOrdered;
884   case CmpInst::FCMP_UEQ:
885     return SPIRV::OpFUnordEqual;
886   case CmpInst::FCMP_UGE:
887     return SPIRV::OpFUnordGreaterThanEqual;
888   case CmpInst::FCMP_UGT:
889     return SPIRV::OpFUnordGreaterThan;
890   case CmpInst::FCMP_ULE:
891     return SPIRV::OpFUnordLessThanEqual;
892   case CmpInst::FCMP_ULT:
893     return SPIRV::OpFUnordLessThan;
894   case CmpInst::FCMP_UNE:
895     return SPIRV::OpFUnordNotEqual;
896   case CmpInst::FCMP_UNO:
897     return SPIRV::OpUnordered;
898   default:
899     llvm_unreachable("Unknown predicate type for FCmp");
900   }
901 }
902 
903 static unsigned getICmpOpcode(unsigned PredNum) {
904   auto Pred = static_cast<CmpInst::Predicate>(PredNum);
905   switch (Pred) {
906   case CmpInst::ICMP_EQ:
907     return SPIRV::OpIEqual;
908   case CmpInst::ICMP_NE:
909     return SPIRV::OpINotEqual;
910   case CmpInst::ICMP_SGE:
911     return SPIRV::OpSGreaterThanEqual;
912   case CmpInst::ICMP_SGT:
913     return SPIRV::OpSGreaterThan;
914   case CmpInst::ICMP_SLE:
915     return SPIRV::OpSLessThanEqual;
916   case CmpInst::ICMP_SLT:
917     return SPIRV::OpSLessThan;
918   case CmpInst::ICMP_UGE:
919     return SPIRV::OpUGreaterThanEqual;
920   case CmpInst::ICMP_UGT:
921     return SPIRV::OpUGreaterThan;
922   case CmpInst::ICMP_ULE:
923     return SPIRV::OpULessThanEqual;
924   case CmpInst::ICMP_ULT:
925     return SPIRV::OpULessThan;
926   default:
927     llvm_unreachable("Unknown predicate type for ICmp");
928   }
929 }
930 
931 static unsigned getPtrCmpOpcode(unsigned Pred) {
932   switch (static_cast<CmpInst::Predicate>(Pred)) {
933   case CmpInst::ICMP_EQ:
934     return SPIRV::OpPtrEqual;
935   case CmpInst::ICMP_NE:
936     return SPIRV::OpPtrNotEqual;
937   default:
938     llvm_unreachable("Unknown predicate type for pointer comparison");
939   }
940 }
941 
942 // Return the logical operation, or abort if none exists.
943 static unsigned getBoolCmpOpcode(unsigned PredNum) {
944   auto Pred = static_cast<CmpInst::Predicate>(PredNum);
945   switch (Pred) {
946   case CmpInst::ICMP_EQ:
947     return SPIRV::OpLogicalEqual;
948   case CmpInst::ICMP_NE:
949     return SPIRV::OpLogicalNotEqual;
950   default:
951     llvm_unreachable("Unknown predicate type for Bool comparison");
952   }
953 }
954 
955 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
956                                                 const SPIRVType *ResType,
957                                                 MachineInstr &I) const {
958   MachineBasicBlock &BB = *I.getParent();
959   return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse))
960       .addDef(ResVReg)
961       .addUse(GR.getSPIRVTypeID(ResType))
962       .addUse(I.getOperand(1).getReg())
963       .constrainAllUses(TII, TRI, RBI);
964 }
965 
966 bool SPIRVInstructionSelector::selectConstVector(Register ResVReg,
967                                                  const SPIRVType *ResType,
968                                                  MachineInstr &I) const {
969   // TODO: only const case is supported for now.
970   assert(std::all_of(
971       I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) {
972         if (MO.isDef())
973           return true;
974         if (!MO.isReg())
975           return false;
976         SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg());
977         assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE &&
978                ConstTy->getOperand(1).isReg());
979         Register ConstReg = ConstTy->getOperand(1).getReg();
980         const MachineInstr *Const = this->MRI->getVRegDef(ConstReg);
981         assert(Const);
982         return (Const->getOpcode() == TargetOpcode::G_CONSTANT ||
983                 Const->getOpcode() == TargetOpcode::G_FCONSTANT);
984       }));
985 
986   auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
987                      TII.get(SPIRV::OpConstantComposite))
988                  .addDef(ResVReg)
989                  .addUse(GR.getSPIRVTypeID(ResType));
990   for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
991     MIB.addUse(I.getOperand(i).getReg());
992   return MIB.constrainAllUses(TII, TRI, RBI);
993 }
994 
995 bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
996                                          const SPIRVType *ResType,
997                                          unsigned CmpOpc,
998                                          MachineInstr &I) const {
999   Register Cmp0 = I.getOperand(2).getReg();
1000   Register Cmp1 = I.getOperand(3).getReg();
1001   assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
1002              GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
1003          "CMP operands should have the same type");
1004   return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc))
1005       .addDef(ResVReg)
1006       .addUse(GR.getSPIRVTypeID(ResType))
1007       .addUse(Cmp0)
1008       .addUse(Cmp1)
1009       .constrainAllUses(TII, TRI, RBI);
1010 }
1011 
1012 bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
1013                                           const SPIRVType *ResType,
1014                                           MachineInstr &I) const {
1015   auto Pred = I.getOperand(1).getPredicate();
1016   unsigned CmpOpc;
1017 
1018   Register CmpOperand = I.getOperand(2).getReg();
1019   if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer))
1020     CmpOpc = getPtrCmpOpcode(Pred);
1021   else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool))
1022     CmpOpc = getBoolCmpOpcode(Pred);
1023   else
1024     CmpOpc = getICmpOpcode(Pred);
1025   return selectCmp(ResVReg, ResType, CmpOpc, I);
1026 }
1027 
1028 void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB,
1029                                             const MachineInstr &I,
1030                                             int OpIdx) const {
1031   assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
1032          "Expected G_FCONSTANT");
1033   const ConstantFP *FPImm = I.getOperand(1).getFPImm();
1034   addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB);
1035 }
1036 
1037 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB,
1038                                            const MachineInstr &I,
1039                                            int OpIdx) const {
1040   assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1041          "Expected G_CONSTANT");
1042   addNumImm(I.getOperand(1).getCImm()->getValue(), MIB);
1043 }
1044 
1045 Register
1046 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
1047                                            const SPIRVType *ResType) const {
1048   Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32);
1049   const SPIRVType *SpvI32Ty =
1050       ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII);
1051   // Find a constant in DT or build a new one.
1052   auto ConstInt = ConstantInt::get(LLVMTy, Val);
1053   Register NewReg = GR.find(ConstInt, GR.CurMF);
1054   if (!NewReg.isValid()) {
1055     NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
1056     GR.add(ConstInt, GR.CurMF, NewReg);
1057     MachineInstr *MI;
1058     MachineBasicBlock &BB = *I.getParent();
1059     if (Val == 0) {
1060       MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1061                .addDef(NewReg)
1062                .addUse(GR.getSPIRVTypeID(SpvI32Ty));
1063     } else {
1064       MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
1065                .addDef(NewReg)
1066                .addUse(GR.getSPIRVTypeID(SpvI32Ty))
1067                .addImm(APInt(32, Val).getZExtValue());
1068     }
1069     constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
1070   }
1071   return NewReg;
1072 }
1073 
1074 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
1075                                           const SPIRVType *ResType,
1076                                           MachineInstr &I) const {
1077   unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate());
1078   return selectCmp(ResVReg, ResType, CmpOp, I);
1079 }
1080 
1081 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType,
1082                                                  MachineInstr &I) const {
1083   if (ResType->getOpcode() == SPIRV::OpTypeVector)
1084     return GR.getOrCreateConsIntVector(0, I, ResType, TII);
1085   return GR.getOrCreateConstInt(0, I, ResType, TII);
1086 }
1087 
1088 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
1089                                                 const SPIRVType *ResType,
1090                                                 MachineInstr &I) const {
1091   unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
1092   APInt One =
1093       AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0);
1094   if (ResType->getOpcode() == SPIRV::OpTypeVector)
1095     return GR.getOrCreateConsIntVector(One.getZExtValue(), I, ResType, TII);
1096   return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII);
1097 }
1098 
1099 bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
1100                                             const SPIRVType *ResType,
1101                                             MachineInstr &I,
1102                                             bool IsSigned) const {
1103   // To extend a bool, we need to use OpSelect between constants.
1104   Register ZeroReg = buildZerosVal(ResType, I);
1105   Register OneReg = buildOnesVal(IsSigned, ResType, I);
1106   bool IsScalarBool =
1107       GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool);
1108   unsigned Opcode =
1109       IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond;
1110   return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1111       .addDef(ResVReg)
1112       .addUse(GR.getSPIRVTypeID(ResType))
1113       .addUse(I.getOperand(1).getReg())
1114       .addUse(OneReg)
1115       .addUse(ZeroReg)
1116       .constrainAllUses(TII, TRI, RBI);
1117 }
1118 
1119 bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
1120                                           const SPIRVType *ResType,
1121                                           MachineInstr &I, bool IsSigned,
1122                                           unsigned Opcode) const {
1123   Register SrcReg = I.getOperand(1).getReg();
1124   // We can convert bool value directly to float type without OpConvert*ToF,
1125   // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
1126   if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) {
1127     unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
1128     SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
1129     if (ResType->getOpcode() == SPIRV::OpTypeVector) {
1130       const unsigned NumElts = ResType->getOperand(2).getImm();
1131       TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII);
1132     }
1133     SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1134     selectSelect(SrcReg, TmpType, I, false);
1135   }
1136   return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode);
1137 }
1138 
1139 bool SPIRVInstructionSelector::selectExt(Register ResVReg,
1140                                          const SPIRVType *ResType,
1141                                          MachineInstr &I, bool IsSigned) const {
1142   if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool))
1143     return selectSelect(ResVReg, ResType, I, IsSigned);
1144   unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1145   return selectUnOp(ResVReg, ResType, I, Opcode);
1146 }
1147 
1148 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
1149                                                Register ResVReg,
1150                                                MachineInstr &I,
1151                                                const SPIRVType *IntTy,
1152                                                const SPIRVType *BoolTy) const {
1153   // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
1154   Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1155   bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector;
1156   unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
1157   Register Zero = buildZerosVal(IntTy, I);
1158   Register One = buildOnesVal(false, IntTy, I);
1159   MachineBasicBlock &BB = *I.getParent();
1160   BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1161       .addDef(BitIntReg)
1162       .addUse(GR.getSPIRVTypeID(IntTy))
1163       .addUse(IntReg)
1164       .addUse(One)
1165       .constrainAllUses(TII, TRI, RBI);
1166   return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
1167       .addDef(ResVReg)
1168       .addUse(GR.getSPIRVTypeID(BoolTy))
1169       .addUse(BitIntReg)
1170       .addUse(Zero)
1171       .constrainAllUses(TII, TRI, RBI);
1172 }
1173 
1174 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
1175                                            const SPIRVType *ResType,
1176                                            MachineInstr &I) const {
1177   if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) {
1178     Register IntReg = I.getOperand(1).getReg();
1179     const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg);
1180     return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType);
1181   }
1182   bool IsSigned = GR.isScalarOrVectorSigned(ResType);
1183   unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1184   return selectUnOp(ResVReg, ResType, I, Opcode);
1185 }
1186 
1187 bool SPIRVInstructionSelector::selectConst(Register ResVReg,
1188                                            const SPIRVType *ResType,
1189                                            const APInt &Imm,
1190                                            MachineInstr &I) const {
1191   unsigned TyOpcode = ResType->getOpcode();
1192   assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero());
1193   MachineBasicBlock &BB = *I.getParent();
1194   if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) &&
1195       Imm.isZero())
1196     return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1197         .addDef(ResVReg)
1198         .addUse(GR.getSPIRVTypeID(ResType))
1199         .constrainAllUses(TII, TRI, RBI);
1200   if (TyOpcode == SPIRV::OpTypeInt) {
1201     assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!");
1202     Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII);
1203     if (Reg == ResVReg)
1204       return true;
1205     return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
1206         .addDef(ResVReg)
1207         .addUse(Reg)
1208         .constrainAllUses(TII, TRI, RBI);
1209   }
1210   auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
1211                  .addDef(ResVReg)
1212                  .addUse(GR.getSPIRVTypeID(ResType));
1213   // <=32-bit integers should be caught by the sdag pattern.
1214   assert(Imm.getBitWidth() > 32);
1215   addNumImm(Imm, MIB);
1216   return MIB.constrainAllUses(TII, TRI, RBI);
1217 }
1218 
1219 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
1220                                              const SPIRVType *ResType,
1221                                              MachineInstr &I) const {
1222   return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
1223       .addDef(ResVReg)
1224       .addUse(GR.getSPIRVTypeID(ResType))
1225       .constrainAllUses(TII, TRI, RBI);
1226 }
1227 
1228 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) {
1229   assert(MO.isReg());
1230   const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg());
1231   if (TypeInst->getOpcode() != SPIRV::ASSIGN_TYPE)
1232     return false;
1233   assert(TypeInst->getOperand(1).isReg());
1234   MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg());
1235   return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT;
1236 }
1237 
1238 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) {
1239   const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg());
1240   MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg());
1241   assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT);
1242   return ImmInst->getOperand(1).getCImm()->getZExtValue();
1243 }
1244 
1245 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
1246                                                const SPIRVType *ResType,
1247                                                MachineInstr &I) const {
1248   MachineBasicBlock &BB = *I.getParent();
1249   auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert))
1250                  .addDef(ResVReg)
1251                  .addUse(GR.getSPIRVTypeID(ResType))
1252                  // object to insert
1253                  .addUse(I.getOperand(3).getReg())
1254                  // composite to insert into
1255                  .addUse(I.getOperand(2).getReg());
1256   for (unsigned i = 4; i < I.getNumOperands(); i++)
1257     MIB.addImm(foldImm(I.getOperand(i), MRI));
1258   return MIB.constrainAllUses(TII, TRI, RBI);
1259 }
1260 
1261 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
1262                                                 const SPIRVType *ResType,
1263                                                 MachineInstr &I) const {
1264   MachineBasicBlock &BB = *I.getParent();
1265   auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
1266                  .addDef(ResVReg)
1267                  .addUse(GR.getSPIRVTypeID(ResType))
1268                  .addUse(I.getOperand(2).getReg());
1269   for (unsigned i = 3; i < I.getNumOperands(); i++)
1270     MIB.addImm(foldImm(I.getOperand(i), MRI));
1271   return MIB.constrainAllUses(TII, TRI, RBI);
1272 }
1273 
1274 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
1275                                                const SPIRVType *ResType,
1276                                                MachineInstr &I) const {
1277   if (isImm(I.getOperand(4), MRI))
1278     return selectInsertVal(ResVReg, ResType, I);
1279   MachineBasicBlock &BB = *I.getParent();
1280   return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic))
1281       .addDef(ResVReg)
1282       .addUse(GR.getSPIRVTypeID(ResType))
1283       .addUse(I.getOperand(2).getReg())
1284       .addUse(I.getOperand(3).getReg())
1285       .addUse(I.getOperand(4).getReg())
1286       .constrainAllUses(TII, TRI, RBI);
1287 }
1288 
1289 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
1290                                                 const SPIRVType *ResType,
1291                                                 MachineInstr &I) const {
1292   if (isImm(I.getOperand(3), MRI))
1293     return selectExtractVal(ResVReg, ResType, I);
1294   MachineBasicBlock &BB = *I.getParent();
1295   return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic))
1296       .addDef(ResVReg)
1297       .addUse(GR.getSPIRVTypeID(ResType))
1298       .addUse(I.getOperand(2).getReg())
1299       .addUse(I.getOperand(3).getReg())
1300       .constrainAllUses(TII, TRI, RBI);
1301 }
1302 
1303 bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
1304                                          const SPIRVType *ResType,
1305                                          MachineInstr &I) const {
1306   const bool IsGEPInBounds = I.getOperand(2).getImm();
1307 
1308   // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only
1309   // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however,
1310   // we have to use Op[InBounds]AccessChain.
1311   const unsigned Opcode = STI.isVulkanEnv()
1312                               ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain
1313                                                : SPIRV::OpAccessChain)
1314                               : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
1315                                                : SPIRV::OpPtrAccessChain);
1316 
1317   auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1318                  .addDef(ResVReg)
1319                  .addUse(GR.getSPIRVTypeID(ResType))
1320                  // Object to get a pointer to.
1321                  .addUse(I.getOperand(3).getReg());
1322   // Adding indices.
1323   const unsigned StartingIndex =
1324       (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain)
1325           ? 5
1326           : 4;
1327   for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
1328     Res.addUse(I.getOperand(i).getReg());
1329   return Res.constrainAllUses(TII, TRI, RBI);
1330 }
1331 
1332 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
1333                                                const SPIRVType *ResType,
1334                                                MachineInstr &I) const {
1335   MachineBasicBlock &BB = *I.getParent();
1336   switch (cast<GIntrinsic>(I).getIntrinsicID()) {
1337   case Intrinsic::spv_load:
1338     return selectLoad(ResVReg, ResType, I);
1339   case Intrinsic::spv_store:
1340     return selectStore(I);
1341   case Intrinsic::spv_extractv:
1342     return selectExtractVal(ResVReg, ResType, I);
1343   case Intrinsic::spv_insertv:
1344     return selectInsertVal(ResVReg, ResType, I);
1345   case Intrinsic::spv_extractelt:
1346     return selectExtractElt(ResVReg, ResType, I);
1347   case Intrinsic::spv_insertelt:
1348     return selectInsertElt(ResVReg, ResType, I);
1349   case Intrinsic::spv_gep:
1350     return selectGEP(ResVReg, ResType, I);
1351   case Intrinsic::spv_unref_global:
1352   case Intrinsic::spv_init_global: {
1353     MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
1354     MachineInstr *Init = I.getNumExplicitOperands() > 2
1355                              ? MRI->getVRegDef(I.getOperand(2).getReg())
1356                              : nullptr;
1357     assert(MI);
1358     return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init);
1359   }
1360   case Intrinsic::spv_undef: {
1361     auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
1362                    .addDef(ResVReg)
1363                    .addUse(GR.getSPIRVTypeID(ResType));
1364     return MIB.constrainAllUses(TII, TRI, RBI);
1365   }
1366   case Intrinsic::spv_const_composite: {
1367     // If no values are attached, the composite is null constant.
1368     bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
1369     unsigned Opcode =
1370         IsNull ? SPIRV::OpConstantNull : SPIRV::OpConstantComposite;
1371     auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1372                    .addDef(ResVReg)
1373                    .addUse(GR.getSPIRVTypeID(ResType));
1374     // skip type MD node we already used when generated assign.type for this
1375     if (!IsNull) {
1376       for (unsigned i = I.getNumExplicitDefs() + 1;
1377            i < I.getNumExplicitOperands(); ++i) {
1378         MIB.addUse(I.getOperand(i).getReg());
1379       }
1380     }
1381     return MIB.constrainAllUses(TII, TRI, RBI);
1382   }
1383   case Intrinsic::spv_assign_name: {
1384     auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName));
1385     MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg());
1386     for (unsigned i = I.getNumExplicitDefs() + 2;
1387          i < I.getNumExplicitOperands(); ++i) {
1388       MIB.addImm(I.getOperand(i).getImm());
1389     }
1390     return MIB.constrainAllUses(TII, TRI, RBI);
1391   }
1392   case Intrinsic::spv_switch: {
1393     auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch));
1394     for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
1395       if (I.getOperand(i).isReg())
1396         MIB.addReg(I.getOperand(i).getReg());
1397       else if (I.getOperand(i).isCImm())
1398         addNumImm(I.getOperand(i).getCImm()->getValue(), MIB);
1399       else if (I.getOperand(i).isMBB())
1400         MIB.addMBB(I.getOperand(i).getMBB());
1401       else
1402         llvm_unreachable("Unexpected OpSwitch operand");
1403     }
1404     return MIB.constrainAllUses(TII, TRI, RBI);
1405   }
1406   case Intrinsic::spv_cmpxchg:
1407     return selectAtomicCmpXchg(ResVReg, ResType, I);
1408   case Intrinsic::spv_unreachable:
1409     BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable));
1410     break;
1411   case Intrinsic::spv_alloca:
1412     return selectFrameIndex(ResVReg, ResType, I);
1413   case Intrinsic::spv_assume:
1414     if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume))
1415       BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR))
1416           .addUse(I.getOperand(1).getReg());
1417     break;
1418   case Intrinsic::spv_expect:
1419     if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume))
1420       BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR))
1421           .addDef(ResVReg)
1422           .addUse(GR.getSPIRVTypeID(ResType))
1423           .addUse(I.getOperand(2).getReg())
1424           .addUse(I.getOperand(3).getReg());
1425     break;
1426   default:
1427     llvm_unreachable("Intrinsic selection not implemented");
1428   }
1429   return true;
1430 }
1431 
1432 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
1433                                                 const SPIRVType *ResType,
1434                                                 MachineInstr &I) const {
1435   return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable))
1436       .addDef(ResVReg)
1437       .addUse(GR.getSPIRVTypeID(ResType))
1438       .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
1439       .constrainAllUses(TII, TRI, RBI);
1440 }
1441 
1442 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
1443   // InstructionSelector walks backwards through the instructions. We can use
1444   // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
1445   // first, so can generate an OpBranchConditional here. If there is no
1446   // G_BRCOND, we just use OpBranch for a regular unconditional branch.
1447   const MachineInstr *PrevI = I.getPrevNode();
1448   MachineBasicBlock &MBB = *I.getParent();
1449   if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
1450     return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
1451         .addUse(PrevI->getOperand(0).getReg())
1452         .addMBB(PrevI->getOperand(1).getMBB())
1453         .addMBB(I.getOperand(0).getMBB())
1454         .constrainAllUses(TII, TRI, RBI);
1455   }
1456   return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch))
1457       .addMBB(I.getOperand(0).getMBB())
1458       .constrainAllUses(TII, TRI, RBI);
1459 }
1460 
1461 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
1462   // InstructionSelector walks backwards through the instructions. For an
1463   // explicit conditional branch with no fallthrough, we use both a G_BR and a
1464   // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
1465   // generate the OpBranchConditional in selectBranch above.
1466   //
1467   // If an OpBranchConditional has been generated, we simply return, as the work
1468   // is alread done. If there is no OpBranchConditional, LLVM must be relying on
1469   // implicit fallthrough to the next basic block, so we need to create an
1470   // OpBranchConditional with an explicit "false" argument pointing to the next
1471   // basic block that LLVM would fall through to.
1472   const MachineInstr *NextI = I.getNextNode();
1473   // Check if this has already been successfully selected.
1474   if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
1475     return true;
1476   // Must be relying on implicit block fallthrough, so generate an
1477   // OpBranchConditional with the "next" basic block as the "false" target.
1478   MachineBasicBlock &MBB = *I.getParent();
1479   unsigned NextMBBNum = MBB.getNextNode()->getNumber();
1480   MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum);
1481   return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
1482       .addUse(I.getOperand(0).getReg())
1483       .addMBB(I.getOperand(1).getMBB())
1484       .addMBB(NextMBB)
1485       .constrainAllUses(TII, TRI, RBI);
1486 }
1487 
1488 bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
1489                                          const SPIRVType *ResType,
1490                                          MachineInstr &I) const {
1491   auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi))
1492                  .addDef(ResVReg)
1493                  .addUse(GR.getSPIRVTypeID(ResType));
1494   const unsigned NumOps = I.getNumOperands();
1495   for (unsigned i = 1; i < NumOps; i += 2) {
1496     MIB.addUse(I.getOperand(i + 0).getReg());
1497     MIB.addMBB(I.getOperand(i + 1).getMBB());
1498   }
1499   return MIB.constrainAllUses(TII, TRI, RBI);
1500 }
1501 
1502 bool SPIRVInstructionSelector::selectGlobalValue(
1503     Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
1504   // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
1505   MachineIRBuilder MIRBuilder(I);
1506   const GlobalValue *GV = I.getOperand(1).getGlobal();
1507   Type *GVType = GV->getValueType();
1508   SPIRVType *PointerBaseType;
1509   if (GVType->isArrayTy()) {
1510     SPIRVType *ArrayElementType =
1511         GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder,
1512                                 SPIRV::AccessQualifier::ReadWrite, false);
1513     PointerBaseType = GR.getOrCreateSPIRVArrayType(
1514         ArrayElementType, GVType->getArrayNumElements(), I, TII);
1515   } else {
1516     PointerBaseType = GR.getOrCreateSPIRVType(
1517         GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
1518   }
1519   SPIRVType *ResType = GR.getOrCreateSPIRVPointerType(
1520       PointerBaseType, I, TII,
1521       addressSpaceToStorageClass(GV->getAddressSpace()));
1522   std::string GlobalIdent = GV->getGlobalIdentifier();
1523   // We have functions as operands in tests with blocks of instruction e.g. in
1524   // transcoding/global_block.ll. These operands are not used and should be
1525   // substituted by zero constants. Their type is expected to be always
1526   // OpTypePointer Function %uchar.
1527   if (isa<Function>(GV)) {
1528     const Constant *ConstVal = GV;
1529     MachineBasicBlock &BB = *I.getParent();
1530     Register NewReg = GR.find(ConstVal, GR.CurMF);
1531     if (!NewReg.isValid()) {
1532       Register NewReg = ResVReg;
1533       GR.add(ConstVal, GR.CurMF, NewReg);
1534       return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1535           .addDef(NewReg)
1536           .addUse(GR.getSPIRVTypeID(ResType))
1537           .constrainAllUses(TII, TRI, RBI);
1538     }
1539     assert(NewReg != ResVReg);
1540     return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
1541         .addDef(ResVReg)
1542         .addUse(NewReg)
1543         .constrainAllUses(TII, TRI, RBI);
1544   }
1545   auto GlobalVar = cast<GlobalVariable>(GV);
1546   assert(GlobalVar->getName() != "llvm.global.annotations");
1547 
1548   bool HasInit = GlobalVar->hasInitializer() &&
1549                  !isa<UndefValue>(GlobalVar->getInitializer());
1550   // Skip empty declaration for GVs with initilaizers till we get the decl with
1551   // passed initializer.
1552   if (HasInit && !Init)
1553     return true;
1554 
1555   unsigned AddrSpace = GV->getAddressSpace();
1556   SPIRV::StorageClass::StorageClass Storage =
1557       addressSpaceToStorageClass(AddrSpace);
1558   bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage &&
1559                   Storage != SPIRV::StorageClass::Function;
1560   SPIRV::LinkageType::LinkageType LnkType =
1561       (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
1562           ? SPIRV::LinkageType::Import
1563           : SPIRV::LinkageType::Export;
1564 
1565   Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV,
1566                                         Storage, Init, GlobalVar->isConstant(),
1567                                         HasLnkTy, LnkType, MIRBuilder, true);
1568   return Reg.isValid();
1569 }
1570 
1571 bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
1572                                            const SPIRVType *ResType,
1573                                            MachineInstr &I) const {
1574   if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
1575     return selectExtInst(ResVReg, ResType, I, CL::log10);
1576   }
1577 
1578   // There is no log10 instruction in the GLSL Extended Instruction set, so it
1579   // is implemented as:
1580   // log10(x) = log2(x) * (1 / log2(10))
1581   //          = log2(x) * 0.30103
1582 
1583   MachineIRBuilder MIRBuilder(I);
1584   MachineBasicBlock &BB = *I.getParent();
1585 
1586   // Build log2(x).
1587   Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1588   bool Result =
1589       BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1590           .addDef(VarReg)
1591           .addUse(GR.getSPIRVTypeID(ResType))
1592           .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1593           .addImm(GL::Log2)
1594           .add(I.getOperand(1))
1595           .constrainAllUses(TII, TRI, RBI);
1596 
1597   // Build 0.30103.
1598   assert(ResType->getOpcode() == SPIRV::OpTypeVector ||
1599          ResType->getOpcode() == SPIRV::OpTypeFloat);
1600   // TODO: Add matrix implementation once supported by the HLSL frontend.
1601   const SPIRVType *SpirvScalarType =
1602       ResType->getOpcode() == SPIRV::OpTypeVector
1603           ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg())
1604           : ResType;
1605   Register ScaleReg =
1606       GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType);
1607 
1608   // Multiply log2(x) by 0.30103 to get log10(x) result.
1609   auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
1610                     ? SPIRV::OpVectorTimesScalar
1611                     : SPIRV::OpFMulS;
1612   Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1613                 .addDef(ResVReg)
1614                 .addUse(GR.getSPIRVTypeID(ResType))
1615                 .addUse(VarReg)
1616                 .addUse(ScaleReg)
1617                 .constrainAllUses(TII, TRI, RBI);
1618 
1619   return Result;
1620 }
1621 
1622 namespace llvm {
1623 InstructionSelector *
1624 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
1625                                const SPIRVSubtarget &Subtarget,
1626                                const RegisterBankInfo &RBI) {
1627   return new SPIRVInstructionSelector(TM, Subtarget, RBI);
1628 }
1629 } // namespace llvm
1630