1 //===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // The pass emits SPIRV intrinsics keeping essential high-level information for
10 // the translation of LLVM IR to SPIR-V.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SPIRV.h"
15 #include "SPIRVTargetMachine.h"
16 #include "SPIRVUtils.h"
17 #include "llvm/IR/IRBuilder.h"
18 #include "llvm/IR/InstIterator.h"
19 #include "llvm/IR/InstVisitor.h"
20 #include "llvm/IR/IntrinsicsSPIRV.h"
21 
22 #include <queue>
23 
24 // This pass performs the following transformation on LLVM IR level required
25 // for the following translation to SPIR-V:
26 // - replaces direct usages of aggregate constants with target-specific
27 //   intrinsics;
28 // - replaces aggregates-related instructions (extract/insert, ld/st, etc)
29 //   with a target-specific intrinsics;
30 // - emits intrinsics for the global variable initializers since IRTranslator
31 //   doesn't handle them and it's not very convenient to translate them
32 //   ourselves;
33 // - emits intrinsics to keep track of the string names assigned to the values;
34 // - emits intrinsics to keep track of constants (this is necessary to have an
35 //   LLVM IR constant after the IRTranslation is completed) for their further
36 //   deduplication;
37 // - emits intrinsics to keep track of original LLVM types of the values
38 //   to be able to emit proper SPIR-V types eventually.
39 //
40 // TODO: consider removing spv.track.constant in favor of spv.assign.type.
41 
42 using namespace llvm;
43 
44 namespace llvm {
45 void initializeSPIRVEmitIntrinsicsPass(PassRegistry &);
46 } // namespace llvm
47 
48 namespace {
49 class SPIRVEmitIntrinsics
50     : public FunctionPass,
51       public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
52   SPIRVTargetMachine *TM = nullptr;
53   IRBuilder<> *IRB = nullptr;
54   Function *F = nullptr;
55   bool TrackConstants = true;
56   DenseMap<Instruction *, Constant *> AggrConsts;
57   DenseSet<Instruction *> AggrStores;
58   void preprocessCompositeConstants();
59   CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
60                             Value *Arg, Value *Arg2) {
61     ConstantAsMetadata *CM = ValueAsMetadata::getConstant(Arg);
62     MDTuple *TyMD = MDNode::get(F->getContext(), CM);
63     MetadataAsValue *VMD = MetadataAsValue::get(F->getContext(), TyMD);
64     return IRB->CreateIntrinsic(IntrID, {Types}, {Arg2, VMD});
65   }
66   void replaceMemInstrUses(Instruction *Old, Instruction *New);
67   void processInstrAfterVisit(Instruction *I);
68   void insertAssignTypeIntrs(Instruction *I);
69   void processGlobalValue(GlobalVariable &GV);
70 
71 public:
72   static char ID;
73   SPIRVEmitIntrinsics() : FunctionPass(ID) {
74     initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry());
75   }
76   SPIRVEmitIntrinsics(SPIRVTargetMachine *_TM) : FunctionPass(ID), TM(_TM) {
77     initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry());
78   }
79   Instruction *visitInstruction(Instruction &I) { return &I; }
80   Instruction *visitSwitchInst(SwitchInst &I);
81   Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
82   Instruction *visitBitCastInst(BitCastInst &I);
83   Instruction *visitInsertElementInst(InsertElementInst &I);
84   Instruction *visitExtractElementInst(ExtractElementInst &I);
85   Instruction *visitInsertValueInst(InsertValueInst &I);
86   Instruction *visitExtractValueInst(ExtractValueInst &I);
87   Instruction *visitLoadInst(LoadInst &I);
88   Instruction *visitStoreInst(StoreInst &I);
89   Instruction *visitAllocaInst(AllocaInst &I);
90   Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
91   Instruction *visitUnreachableInst(UnreachableInst &I);
92   bool runOnFunction(Function &F) override;
93 };
94 } // namespace
95 
96 char SPIRVEmitIntrinsics::ID = 0;
97 
98 INITIALIZE_PASS(SPIRVEmitIntrinsics, "emit-intrinsics", "SPIRV emit intrinsics",
99                 false, false)
100 
101 static inline bool isAssignTypeInstr(const Instruction *I) {
102   return isa<IntrinsicInst>(I) &&
103          cast<IntrinsicInst>(I)->getIntrinsicID() == Intrinsic::spv_assign_type;
104 }
105 
106 static bool isMemInstrToReplace(Instruction *I) {
107   return isa<StoreInst>(I) || isa<LoadInst>(I) || isa<InsertValueInst>(I) ||
108          isa<ExtractValueInst>(I) || isa<AtomicCmpXchgInst>(I);
109 }
110 
111 static bool isAggrToReplace(const Value *V) {
112   return isa<ConstantAggregate>(V) || isa<ConstantDataArray>(V) ||
113          (isa<ConstantAggregateZero>(V) && !V->getType()->isVectorTy());
114 }
115 
116 static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
117   if (isa<PHINode>(I))
118     B.SetInsertPoint(I->getParent(), I->getParent()->getFirstInsertionPt());
119   else
120     B.SetInsertPoint(I);
121 }
122 
123 static bool requireAssignType(Instruction *I) {
124   IntrinsicInst *Intr = dyn_cast<IntrinsicInst>(I);
125   if (Intr) {
126     switch (Intr->getIntrinsicID()) {
127     case Intrinsic::invariant_start:
128     case Intrinsic::invariant_end:
129       return false;
130     }
131   }
132   return true;
133 }
134 
135 void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
136                                               Instruction *New) {
137   while (!Old->user_empty()) {
138     auto *U = Old->user_back();
139     if (isAssignTypeInstr(U)) {
140       IRB->SetInsertPoint(U);
141       SmallVector<Value *, 2> Args = {New, U->getOperand(1)};
142       IRB->CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args);
143       U->eraseFromParent();
144     } else if (isMemInstrToReplace(U) || isa<ReturnInst>(U) ||
145                isa<CallInst>(U)) {
146       U->replaceUsesOfWith(Old, New);
147     } else {
148       llvm_unreachable("illegal aggregate intrinsic user");
149     }
150   }
151   Old->eraseFromParent();
152 }
153 
154 void SPIRVEmitIntrinsics::preprocessCompositeConstants() {
155   std::queue<Instruction *> Worklist;
156   for (auto &I : instructions(F))
157     Worklist.push(&I);
158 
159   while (!Worklist.empty()) {
160     auto *I = Worklist.front();
161     assert(I);
162     bool KeepInst = false;
163     for (const auto &Op : I->operands()) {
164       auto BuildCompositeIntrinsic = [&KeepInst, &Worklist, &I, &Op,
165                                       this](Constant *AggrC,
166                                             ArrayRef<Value *> Args) {
167         IRB->SetInsertPoint(I);
168         auto *CCI =
169             IRB->CreateIntrinsic(Intrinsic::spv_const_composite, {}, {Args});
170         Worklist.push(CCI);
171         I->replaceUsesOfWith(Op, CCI);
172         KeepInst = true;
173         AggrConsts[CCI] = AggrC;
174       };
175 
176       if (auto *AggrC = dyn_cast<ConstantAggregate>(Op)) {
177         SmallVector<Value *> Args(AggrC->op_begin(), AggrC->op_end());
178         BuildCompositeIntrinsic(AggrC, Args);
179       } else if (auto *AggrC = dyn_cast<ConstantDataArray>(Op)) {
180         SmallVector<Value *> Args;
181         for (unsigned i = 0; i < AggrC->getNumElements(); ++i)
182           Args.push_back(AggrC->getElementAsConstant(i));
183         BuildCompositeIntrinsic(AggrC, Args);
184       } else if (isa<ConstantAggregateZero>(Op) &&
185                  !Op->getType()->isVectorTy()) {
186         auto *AggrC = cast<ConstantAggregateZero>(Op);
187         SmallVector<Value *> Args(AggrC->op_begin(), AggrC->op_end());
188         BuildCompositeIntrinsic(AggrC, Args);
189       }
190     }
191     if (!KeepInst)
192       Worklist.pop();
193   }
194 }
195 
196 Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
197   SmallVector<Value *, 4> Args;
198   for (auto &Op : I.operands())
199     if (Op.get()->getType()->isSized())
200       Args.push_back(Op);
201   IRB->SetInsertPoint(&I);
202   IRB->CreateIntrinsic(Intrinsic::spv_switch, {I.getOperand(0)->getType()},
203                        {Args});
204   return &I;
205 }
206 
207 Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
208   SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(0)->getType()};
209   SmallVector<Value *, 4> Args;
210   Args.push_back(IRB->getInt1(I.isInBounds()));
211   for (auto &Op : I.operands())
212     Args.push_back(Op);
213   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
214   I.replaceAllUsesWith(NewI);
215   I.eraseFromParent();
216   return NewI;
217 }
218 
219 Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
220   SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(0)->getType()};
221   SmallVector<Value *> Args(I.op_begin(), I.op_end());
222   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args});
223   std::string InstName = I.hasName() ? I.getName().str() : "";
224   I.replaceAllUsesWith(NewI);
225   I.eraseFromParent();
226   NewI->setName(InstName);
227   return NewI;
228 }
229 
230 Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
231   SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(0)->getType(),
232                                   I.getOperand(1)->getType(),
233                                   I.getOperand(2)->getType()};
234   SmallVector<Value *> Args(I.op_begin(), I.op_end());
235   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
236   std::string InstName = I.hasName() ? I.getName().str() : "";
237   I.replaceAllUsesWith(NewI);
238   I.eraseFromParent();
239   NewI->setName(InstName);
240   return NewI;
241 }
242 
243 Instruction *
244 SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
245   SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
246                                   I.getIndexOperand()->getType()};
247   SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
248   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
249   std::string InstName = I.hasName() ? I.getName().str() : "";
250   I.replaceAllUsesWith(NewI);
251   I.eraseFromParent();
252   NewI->setName(InstName);
253   return NewI;
254 }
255 
256 Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
257   SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
258   SmallVector<Value *> Args;
259   for (auto &Op : I.operands())
260     if (isa<UndefValue>(Op))
261       Args.push_back(UndefValue::get(IRB->getInt32Ty()));
262     else
263       Args.push_back(Op);
264   for (auto &Op : I.indices())
265     Args.push_back(IRB->getInt32(Op));
266   Instruction *NewI =
267       IRB->CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args});
268   replaceMemInstrUses(&I, NewI);
269   return NewI;
270 }
271 
272 Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
273   SmallVector<Value *> Args;
274   for (auto &Op : I.operands())
275     Args.push_back(Op);
276   for (auto &Op : I.indices())
277     Args.push_back(IRB->getInt32(Op));
278   auto *NewI =
279       IRB->CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args});
280   I.replaceAllUsesWith(NewI);
281   I.eraseFromParent();
282   return NewI;
283 }
284 
285 Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
286   if (!I.getType()->isAggregateType())
287     return &I;
288   TrackConstants = false;
289   const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
290   MachineMemOperand::Flags Flags =
291       TLI->getLoadMemOperandFlags(I, F->getParent()->getDataLayout());
292   auto *NewI =
293       IRB->CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()},
294                            {I.getPointerOperand(), IRB->getInt16(Flags),
295                             IRB->getInt8(I.getAlign().value())});
296   replaceMemInstrUses(&I, NewI);
297   return NewI;
298 }
299 
300 Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
301   if (!AggrStores.contains(&I))
302     return &I;
303   TrackConstants = false;
304   const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
305   MachineMemOperand::Flags Flags =
306       TLI->getStoreMemOperandFlags(I, F->getParent()->getDataLayout());
307   auto *PtrOp = I.getPointerOperand();
308   auto *NewI = IRB->CreateIntrinsic(
309       Intrinsic::spv_store, {I.getValueOperand()->getType(), PtrOp->getType()},
310       {I.getValueOperand(), PtrOp, IRB->getInt16(Flags),
311        IRB->getInt8(I.getAlign().value())});
312   I.eraseFromParent();
313   return NewI;
314 }
315 
316 Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
317   TrackConstants = false;
318   Type *PtrTy = I.getType();
319   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
320   std::string InstName = I.hasName() ? I.getName().str() : "";
321   I.replaceAllUsesWith(NewI);
322   I.eraseFromParent();
323   NewI->setName(InstName);
324   return NewI;
325 }
326 
327 Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
328   assert(I.getType()->isAggregateType() && "Aggregate result is expected");
329   SmallVector<Value *> Args;
330   for (auto &Op : I.operands())
331     Args.push_back(Op);
332   Args.push_back(IRB->getInt32(I.getSyncScopeID()));
333   Args.push_back(IRB->getInt32(
334       static_cast<uint32_t>(getMemSemantics(I.getSuccessOrdering()))));
335   Args.push_back(IRB->getInt32(
336       static_cast<uint32_t>(getMemSemantics(I.getFailureOrdering()))));
337   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_cmpxchg,
338                                     {I.getPointerOperand()->getType()}, {Args});
339   replaceMemInstrUses(&I, NewI);
340   return NewI;
341 }
342 
343 Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
344   IRB->SetInsertPoint(&I);
345   IRB->CreateIntrinsic(Intrinsic::spv_unreachable, {}, {});
346   return &I;
347 }
348 
349 void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV) {
350   // Skip special artifical variable llvm.global.annotations.
351   if (GV.getName() == "llvm.global.annotations")
352     return;
353   if (GV.hasInitializer() && !isa<UndefValue>(GV.getInitializer())) {
354     Constant *Init = GV.getInitializer();
355     Type *Ty = isAggrToReplace(Init) ? IRB->getInt32Ty() : Init->getType();
356     Constant *Const = isAggrToReplace(Init) ? IRB->getInt32(1) : Init;
357     auto *InitInst = IRB->CreateIntrinsic(Intrinsic::spv_init_global,
358                                           {GV.getType(), Ty}, {&GV, Const});
359     InitInst->setArgOperand(1, Init);
360   }
361   if ((!GV.hasInitializer() || isa<UndefValue>(GV.getInitializer())) &&
362       GV.getNumUses() == 0)
363     IRB->CreateIntrinsic(Intrinsic::spv_unref_global, GV.getType(), &GV);
364 }
365 
366 void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I) {
367   Type *Ty = I->getType();
368   if (!Ty->isVoidTy() && requireAssignType(I)) {
369     setInsertPointSkippingPhis(*IRB, I->getNextNode());
370     Type *TypeToAssign = Ty;
371     if (auto *II = dyn_cast<IntrinsicInst>(I)) {
372       if (II->getIntrinsicID() == Intrinsic::spv_const_composite) {
373         auto t = AggrConsts.find(II);
374         assert(t != AggrConsts.end());
375         TypeToAssign = t->second->getType();
376       }
377     }
378     Constant *Const = Constant::getNullValue(TypeToAssign);
379     buildIntrWithMD(Intrinsic::spv_assign_type, {Ty}, Const, I);
380   }
381   for (const auto &Op : I->operands()) {
382     if (isa<ConstantPointerNull>(Op) || isa<UndefValue>(Op) ||
383         // Check GetElementPtrConstantExpr case.
384         (isa<ConstantExpr>(Op) && isa<GEPOperator>(Op))) {
385       setInsertPointSkippingPhis(*IRB, I);
386       if (isa<UndefValue>(Op) && Op->getType()->isAggregateType())
387         buildIntrWithMD(Intrinsic::spv_assign_type, {IRB->getInt32Ty()}, Op,
388                         UndefValue::get(IRB->getInt32Ty()));
389       else
390         buildIntrWithMD(Intrinsic::spv_assign_type, {Op->getType()}, Op, Op);
391     }
392   }
393 }
394 
395 void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I) {
396   auto *II = dyn_cast<IntrinsicInst>(I);
397   if (II && II->getIntrinsicID() == Intrinsic::spv_const_composite &&
398       TrackConstants) {
399     IRB->SetInsertPoint(I->getNextNode());
400     Type *Ty = IRB->getInt32Ty();
401     auto t = AggrConsts.find(I);
402     assert(t != AggrConsts.end());
403     auto *NewOp =
404         buildIntrWithMD(Intrinsic::spv_track_constant, {Ty, Ty}, t->second, I);
405     I->replaceAllUsesWith(NewOp);
406     NewOp->setArgOperand(0, I);
407   }
408   for (const auto &Op : I->operands()) {
409     if ((isa<ConstantAggregateZero>(Op) && Op->getType()->isVectorTy()) ||
410         isa<PHINode>(I) || isa<SwitchInst>(I))
411       TrackConstants = false;
412     if ((isa<ConstantData>(Op) || isa<ConstantExpr>(Op)) && TrackConstants) {
413       unsigned OpNo = Op.getOperandNo();
414       if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
415                  (II->paramHasAttr(OpNo, Attribute::ImmArg))))
416         continue;
417       IRB->SetInsertPoint(I);
418       auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant,
419                                     {Op->getType(), Op->getType()}, Op, Op);
420       I->setOperand(OpNo, NewOp);
421     }
422   }
423   if (I->hasName()) {
424     setInsertPointSkippingPhis(*IRB, I->getNextNode());
425     std::vector<Value *> Args = {I};
426     addStringImm(I->getName(), *IRB, Args);
427     IRB->CreateIntrinsic(Intrinsic::spv_assign_name, {I->getType()}, Args);
428   }
429 }
430 
431 bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
432   if (Func.isDeclaration())
433     return false;
434   F = &Func;
435   IRB = new IRBuilder<>(Func.getContext());
436   AggrConsts.clear();
437   AggrStores.clear();
438 
439   // StoreInst's operand type can be changed during the next transformations,
440   // so we need to store it in the set. Also store already transformed types.
441   for (auto &I : instructions(Func)) {
442     StoreInst *SI = dyn_cast<StoreInst>(&I);
443     if (!SI)
444       continue;
445     Type *ElTy = SI->getValueOperand()->getType();
446     PointerType *PTy = cast<PointerType>(SI->getOperand(1)->getType());
447     if (ElTy->isAggregateType() || ElTy->isVectorTy() ||
448         !PTy->isOpaqueOrPointeeTypeMatches(ElTy))
449       AggrStores.insert(&I);
450   }
451 
452   IRB->SetInsertPoint(&Func.getEntryBlock().front());
453   for (auto &GV : Func.getParent()->globals())
454     processGlobalValue(GV);
455 
456   preprocessCompositeConstants();
457   SmallVector<Instruction *> Worklist;
458   for (auto &I : instructions(Func))
459     Worklist.push_back(&I);
460 
461   for (auto &I : Worklist)
462     insertAssignTypeIntrs(I);
463 
464   for (auto *I : Worklist) {
465     TrackConstants = true;
466     if (!I->getType()->isVoidTy() || isa<StoreInst>(I))
467       IRB->SetInsertPoint(I->getNextNode());
468     I = visit(*I);
469     processInstrAfterVisit(I);
470   }
471   return true;
472 }
473 
474 FunctionPass *llvm::createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM) {
475   return new SPIRVEmitIntrinsics(TM);
476 }
477