1 //===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // The pass emits SPIRV intrinsics keeping essential high-level information for
10 // the translation of LLVM IR to SPIR-V.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SPIRV.h"
15 #include "SPIRVTargetMachine.h"
16 #include "SPIRVUtils.h"
17 #include "llvm/IR/IRBuilder.h"
18 #include "llvm/IR/InstIterator.h"
19 #include "llvm/IR/InstVisitor.h"
20 #include "llvm/IR/IntrinsicsSPIRV.h"
21 
22 #include <queue>
23 
24 // This pass performs the following transformation on LLVM IR level required
25 // for the following translation to SPIR-V:
26 // - replaces direct usages of aggregate constants with target-specific
27 //   intrinsics;
28 // - replaces aggregates-related instructions (extract/insert, ld/st, etc)
29 //   with a target-specific intrinsics;
30 // - emits intrinsics for the global variable initializers since IRTranslator
31 //   doesn't handle them and it's not very convenient to translate them
32 //   ourselves;
33 // - emits intrinsics to keep track of the string names assigned to the values;
34 // - emits intrinsics to keep track of constants (this is necessary to have an
35 //   LLVM IR constant after the IRTranslation is completed) for their further
36 //   deduplication;
37 // - emits intrinsics to keep track of original LLVM types of the values
38 //   to be able to emit proper SPIR-V types eventually.
39 //
40 // TODO: consider removing spv.track.constant in favor of spv.assign.type.
41 
42 using namespace llvm;
43 
44 namespace llvm {
45 void initializeSPIRVEmitIntrinsicsPass(PassRegistry &);
46 } // namespace llvm
47 
48 namespace {
49 class SPIRVEmitIntrinsics
50     : public FunctionPass,
51       public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
52   SPIRVTargetMachine *TM = nullptr;
53   IRBuilder<> *IRB = nullptr;
54   Function *F = nullptr;
55   bool TrackConstants = true;
56   DenseMap<Instruction *, Constant *> AggrConsts;
57   DenseSet<Instruction *> AggrStores;
58   void preprocessCompositeConstants();
59   void preprocessUndefs();
60   CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types,
61                             Value *Arg, Value *Arg2) {
62     ConstantAsMetadata *CM = ValueAsMetadata::getConstant(Arg);
63     MDTuple *TyMD = MDNode::get(F->getContext(), CM);
64     MetadataAsValue *VMD = MetadataAsValue::get(F->getContext(), TyMD);
65     return IRB->CreateIntrinsic(IntrID, {Types}, {Arg2, VMD});
66   }
67   void replaceMemInstrUses(Instruction *Old, Instruction *New);
68   void processInstrAfterVisit(Instruction *I);
69   void insertAssignTypeIntrs(Instruction *I);
70   void processGlobalValue(GlobalVariable &GV);
71 
72 public:
73   static char ID;
74   SPIRVEmitIntrinsics() : FunctionPass(ID) {
75     initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry());
76   }
77   SPIRVEmitIntrinsics(SPIRVTargetMachine *_TM) : FunctionPass(ID), TM(_TM) {
78     initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry());
79   }
80   Instruction *visitInstruction(Instruction &I) { return &I; }
81   Instruction *visitSwitchInst(SwitchInst &I);
82   Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
83   Instruction *visitBitCastInst(BitCastInst &I);
84   Instruction *visitInsertElementInst(InsertElementInst &I);
85   Instruction *visitExtractElementInst(ExtractElementInst &I);
86   Instruction *visitInsertValueInst(InsertValueInst &I);
87   Instruction *visitExtractValueInst(ExtractValueInst &I);
88   Instruction *visitLoadInst(LoadInst &I);
89   Instruction *visitStoreInst(StoreInst &I);
90   Instruction *visitAllocaInst(AllocaInst &I);
91   Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
92   Instruction *visitUnreachableInst(UnreachableInst &I);
93   bool runOnFunction(Function &F) override;
94 };
95 } // namespace
96 
97 char SPIRVEmitIntrinsics::ID = 0;
98 
99 INITIALIZE_PASS(SPIRVEmitIntrinsics, "emit-intrinsics", "SPIRV emit intrinsics",
100                 false, false)
101 
102 static inline bool isAssignTypeInstr(const Instruction *I) {
103   return isa<IntrinsicInst>(I) &&
104          cast<IntrinsicInst>(I)->getIntrinsicID() == Intrinsic::spv_assign_type;
105 }
106 
107 static bool isMemInstrToReplace(Instruction *I) {
108   return isa<StoreInst>(I) || isa<LoadInst>(I) || isa<InsertValueInst>(I) ||
109          isa<ExtractValueInst>(I) || isa<AtomicCmpXchgInst>(I);
110 }
111 
112 static bool isAggrToReplace(const Value *V) {
113   return isa<ConstantAggregate>(V) || isa<ConstantDataArray>(V) ||
114          (isa<ConstantAggregateZero>(V) && !V->getType()->isVectorTy());
115 }
116 
117 static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
118   if (isa<PHINode>(I))
119     B.SetInsertPoint(I->getParent(), I->getParent()->getFirstInsertionPt());
120   else
121     B.SetInsertPoint(I);
122 }
123 
124 static bool requireAssignType(Instruction *I) {
125   IntrinsicInst *Intr = dyn_cast<IntrinsicInst>(I);
126   if (Intr) {
127     switch (Intr->getIntrinsicID()) {
128     case Intrinsic::invariant_start:
129     case Intrinsic::invariant_end:
130       return false;
131     }
132   }
133   return true;
134 }
135 
136 void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
137                                               Instruction *New) {
138   while (!Old->user_empty()) {
139     auto *U = Old->user_back();
140     if (isAssignTypeInstr(U)) {
141       IRB->SetInsertPoint(U);
142       SmallVector<Value *, 2> Args = {New, U->getOperand(1)};
143       IRB->CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args);
144       U->eraseFromParent();
145     } else if (isMemInstrToReplace(U) || isa<ReturnInst>(U) ||
146                isa<CallInst>(U)) {
147       U->replaceUsesOfWith(Old, New);
148     } else {
149       llvm_unreachable("illegal aggregate intrinsic user");
150     }
151   }
152   Old->eraseFromParent();
153 }
154 
155 void SPIRVEmitIntrinsics::preprocessUndefs() {
156   std::queue<Instruction *> Worklist;
157   for (auto &I : instructions(F))
158     Worklist.push(&I);
159 
160   while (!Worklist.empty()) {
161     Instruction *I = Worklist.front();
162     Worklist.pop();
163 
164     for (auto &Op : I->operands()) {
165       auto *AggrUndef = dyn_cast<UndefValue>(Op);
166       if (!AggrUndef || !Op->getType()->isAggregateType())
167         continue;
168 
169       IRB->SetInsertPoint(I);
170       auto *IntrUndef = IRB->CreateIntrinsic(Intrinsic::spv_undef, {}, {});
171       Worklist.push(IntrUndef);
172       I->replaceUsesOfWith(Op, IntrUndef);
173       AggrConsts[IntrUndef] = AggrUndef;
174     }
175   }
176 }
177 
178 void SPIRVEmitIntrinsics::preprocessCompositeConstants() {
179   std::queue<Instruction *> Worklist;
180   for (auto &I : instructions(F))
181     Worklist.push(&I);
182 
183   while (!Worklist.empty()) {
184     auto *I = Worklist.front();
185     assert(I);
186     bool KeepInst = false;
187     for (const auto &Op : I->operands()) {
188       auto BuildCompositeIntrinsic = [&KeepInst, &Worklist, &I, &Op,
189                                       this](Constant *AggrC,
190                                             ArrayRef<Value *> Args) {
191         IRB->SetInsertPoint(I);
192         auto *CCI =
193             IRB->CreateIntrinsic(Intrinsic::spv_const_composite, {}, {Args});
194         Worklist.push(CCI);
195         I->replaceUsesOfWith(Op, CCI);
196         KeepInst = true;
197         AggrConsts[CCI] = AggrC;
198       };
199 
200       if (auto *AggrC = dyn_cast<ConstantAggregate>(Op)) {
201         SmallVector<Value *> Args(AggrC->op_begin(), AggrC->op_end());
202         BuildCompositeIntrinsic(AggrC, Args);
203       } else if (auto *AggrC = dyn_cast<ConstantDataArray>(Op)) {
204         SmallVector<Value *> Args;
205         for (unsigned i = 0; i < AggrC->getNumElements(); ++i)
206           Args.push_back(AggrC->getElementAsConstant(i));
207         BuildCompositeIntrinsic(AggrC, Args);
208       } else if (isa<ConstantAggregateZero>(Op) &&
209                  !Op->getType()->isVectorTy()) {
210         auto *AggrC = cast<ConstantAggregateZero>(Op);
211         SmallVector<Value *> Args(AggrC->op_begin(), AggrC->op_end());
212         BuildCompositeIntrinsic(AggrC, Args);
213       }
214     }
215     if (!KeepInst)
216       Worklist.pop();
217   }
218 }
219 
220 Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
221   SmallVector<Value *, 4> Args;
222   for (auto &Op : I.operands())
223     if (Op.get()->getType()->isSized())
224       Args.push_back(Op);
225   IRB->SetInsertPoint(&I);
226   IRB->CreateIntrinsic(Intrinsic::spv_switch, {I.getOperand(0)->getType()},
227                        {Args});
228   return &I;
229 }
230 
231 Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
232   SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(0)->getType()};
233   SmallVector<Value *, 4> Args;
234   Args.push_back(IRB->getInt1(I.isInBounds()));
235   for (auto &Op : I.operands())
236     Args.push_back(Op);
237   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
238   I.replaceAllUsesWith(NewI);
239   I.eraseFromParent();
240   return NewI;
241 }
242 
243 Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
244   SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(0)->getType()};
245   SmallVector<Value *> Args(I.op_begin(), I.op_end());
246   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args});
247   std::string InstName = I.hasName() ? I.getName().str() : "";
248   I.replaceAllUsesWith(NewI);
249   I.eraseFromParent();
250   NewI->setName(InstName);
251   return NewI;
252 }
253 
254 Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
255   SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(0)->getType(),
256                                   I.getOperand(1)->getType(),
257                                   I.getOperand(2)->getType()};
258   SmallVector<Value *> Args(I.op_begin(), I.op_end());
259   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
260   std::string InstName = I.hasName() ? I.getName().str() : "";
261   I.replaceAllUsesWith(NewI);
262   I.eraseFromParent();
263   NewI->setName(InstName);
264   return NewI;
265 }
266 
267 Instruction *
268 SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
269   SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
270                                   I.getIndexOperand()->getType()};
271   SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
272   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
273   std::string InstName = I.hasName() ? I.getName().str() : "";
274   I.replaceAllUsesWith(NewI);
275   I.eraseFromParent();
276   NewI->setName(InstName);
277   return NewI;
278 }
279 
280 Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
281   SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
282   SmallVector<Value *> Args;
283   for (auto &Op : I.operands())
284     if (isa<UndefValue>(Op))
285       Args.push_back(UndefValue::get(IRB->getInt32Ty()));
286     else
287       Args.push_back(Op);
288   for (auto &Op : I.indices())
289     Args.push_back(IRB->getInt32(Op));
290   Instruction *NewI =
291       IRB->CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args});
292   replaceMemInstrUses(&I, NewI);
293   return NewI;
294 }
295 
296 Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
297   SmallVector<Value *> Args;
298   for (auto &Op : I.operands())
299     Args.push_back(Op);
300   for (auto &Op : I.indices())
301     Args.push_back(IRB->getInt32(Op));
302   auto *NewI =
303       IRB->CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args});
304   I.replaceAllUsesWith(NewI);
305   I.eraseFromParent();
306   return NewI;
307 }
308 
309 Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
310   if (!I.getType()->isAggregateType())
311     return &I;
312   TrackConstants = false;
313   const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
314   MachineMemOperand::Flags Flags =
315       TLI->getLoadMemOperandFlags(I, F->getParent()->getDataLayout());
316   auto *NewI =
317       IRB->CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()},
318                            {I.getPointerOperand(), IRB->getInt16(Flags),
319                             IRB->getInt8(I.getAlign().value())});
320   replaceMemInstrUses(&I, NewI);
321   return NewI;
322 }
323 
324 Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
325   if (!AggrStores.contains(&I))
326     return &I;
327   TrackConstants = false;
328   const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
329   MachineMemOperand::Flags Flags =
330       TLI->getStoreMemOperandFlags(I, F->getParent()->getDataLayout());
331   auto *PtrOp = I.getPointerOperand();
332   auto *NewI = IRB->CreateIntrinsic(
333       Intrinsic::spv_store, {I.getValueOperand()->getType(), PtrOp->getType()},
334       {I.getValueOperand(), PtrOp, IRB->getInt16(Flags),
335        IRB->getInt8(I.getAlign().value())});
336   I.eraseFromParent();
337   return NewI;
338 }
339 
340 Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
341   TrackConstants = false;
342   Type *PtrTy = I.getType();
343   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
344   std::string InstName = I.hasName() ? I.getName().str() : "";
345   I.replaceAllUsesWith(NewI);
346   I.eraseFromParent();
347   NewI->setName(InstName);
348   return NewI;
349 }
350 
351 Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
352   assert(I.getType()->isAggregateType() && "Aggregate result is expected");
353   SmallVector<Value *> Args;
354   for (auto &Op : I.operands())
355     Args.push_back(Op);
356   Args.push_back(IRB->getInt32(I.getSyncScopeID()));
357   Args.push_back(IRB->getInt32(
358       static_cast<uint32_t>(getMemSemantics(I.getSuccessOrdering()))));
359   Args.push_back(IRB->getInt32(
360       static_cast<uint32_t>(getMemSemantics(I.getFailureOrdering()))));
361   auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_cmpxchg,
362                                     {I.getPointerOperand()->getType()}, {Args});
363   replaceMemInstrUses(&I, NewI);
364   return NewI;
365 }
366 
367 Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
368   IRB->SetInsertPoint(&I);
369   IRB->CreateIntrinsic(Intrinsic::spv_unreachable, {}, {});
370   return &I;
371 }
372 
373 void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV) {
374   // Skip special artifical variable llvm.global.annotations.
375   if (GV.getName() == "llvm.global.annotations")
376     return;
377   if (GV.hasInitializer() && !isa<UndefValue>(GV.getInitializer())) {
378     Constant *Init = GV.getInitializer();
379     Type *Ty = isAggrToReplace(Init) ? IRB->getInt32Ty() : Init->getType();
380     Constant *Const = isAggrToReplace(Init) ? IRB->getInt32(1) : Init;
381     auto *InitInst = IRB->CreateIntrinsic(Intrinsic::spv_init_global,
382                                           {GV.getType(), Ty}, {&GV, Const});
383     InitInst->setArgOperand(1, Init);
384   }
385   if ((!GV.hasInitializer() || isa<UndefValue>(GV.getInitializer())) &&
386       GV.getNumUses() == 0)
387     IRB->CreateIntrinsic(Intrinsic::spv_unref_global, GV.getType(), &GV);
388 }
389 
390 void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I) {
391   Type *Ty = I->getType();
392   if (!Ty->isVoidTy() && requireAssignType(I)) {
393     setInsertPointSkippingPhis(*IRB, I->getNextNode());
394     Type *TypeToAssign = Ty;
395     if (auto *II = dyn_cast<IntrinsicInst>(I)) {
396       if (II->getIntrinsicID() == Intrinsic::spv_const_composite ||
397           II->getIntrinsicID() == Intrinsic::spv_undef) {
398         auto t = AggrConsts.find(II);
399         assert(t != AggrConsts.end());
400         TypeToAssign = t->second->getType();
401       }
402     }
403     Constant *Const = Constant::getNullValue(TypeToAssign);
404     buildIntrWithMD(Intrinsic::spv_assign_type, {Ty}, Const, I);
405   }
406   for (const auto &Op : I->operands()) {
407     if (isa<ConstantPointerNull>(Op) || isa<UndefValue>(Op) ||
408         // Check GetElementPtrConstantExpr case.
409         (isa<ConstantExpr>(Op) && isa<GEPOperator>(Op))) {
410       setInsertPointSkippingPhis(*IRB, I);
411       if (isa<UndefValue>(Op) && Op->getType()->isAggregateType())
412         buildIntrWithMD(Intrinsic::spv_assign_type, {IRB->getInt32Ty()}, Op,
413                         UndefValue::get(IRB->getInt32Ty()));
414       else
415         buildIntrWithMD(Intrinsic::spv_assign_type, {Op->getType()}, Op, Op);
416     }
417   }
418 }
419 
420 void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I) {
421   auto *II = dyn_cast<IntrinsicInst>(I);
422   if (II && II->getIntrinsicID() == Intrinsic::spv_const_composite &&
423       TrackConstants) {
424     IRB->SetInsertPoint(I->getNextNode());
425     Type *Ty = IRB->getInt32Ty();
426     auto t = AggrConsts.find(I);
427     assert(t != AggrConsts.end());
428     auto *NewOp =
429         buildIntrWithMD(Intrinsic::spv_track_constant, {Ty, Ty}, t->second, I);
430     I->replaceAllUsesWith(NewOp);
431     NewOp->setArgOperand(0, I);
432   }
433   for (const auto &Op : I->operands()) {
434     if ((isa<ConstantAggregateZero>(Op) && Op->getType()->isVectorTy()) ||
435         isa<PHINode>(I) || isa<SwitchInst>(I))
436       TrackConstants = false;
437     if ((isa<ConstantData>(Op) || isa<ConstantExpr>(Op)) && TrackConstants) {
438       unsigned OpNo = Op.getOperandNo();
439       if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
440                  (II->paramHasAttr(OpNo, Attribute::ImmArg))))
441         continue;
442       IRB->SetInsertPoint(I);
443       auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant,
444                                     {Op->getType(), Op->getType()}, Op, Op);
445       I->setOperand(OpNo, NewOp);
446     }
447   }
448   if (I->hasName()) {
449     setInsertPointSkippingPhis(*IRB, I->getNextNode());
450     std::vector<Value *> Args = {I};
451     addStringImm(I->getName(), *IRB, Args);
452     IRB->CreateIntrinsic(Intrinsic::spv_assign_name, {I->getType()}, Args);
453   }
454 }
455 
456 bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
457   if (Func.isDeclaration())
458     return false;
459   F = &Func;
460   IRB = new IRBuilder<>(Func.getContext());
461   AggrConsts.clear();
462   AggrStores.clear();
463 
464   // StoreInst's operand type can be changed during the next transformations,
465   // so we need to store it in the set. Also store already transformed types.
466   for (auto &I : instructions(Func)) {
467     StoreInst *SI = dyn_cast<StoreInst>(&I);
468     if (!SI)
469       continue;
470     Type *ElTy = SI->getValueOperand()->getType();
471     PointerType *PTy = cast<PointerType>(SI->getOperand(1)->getType());
472     if (ElTy->isAggregateType() || ElTy->isVectorTy() ||
473         !PTy->isOpaqueOrPointeeTypeMatches(ElTy))
474       AggrStores.insert(&I);
475   }
476 
477   IRB->SetInsertPoint(&Func.getEntryBlock().front());
478   for (auto &GV : Func.getParent()->globals())
479     processGlobalValue(GV);
480 
481   preprocessUndefs();
482   preprocessCompositeConstants();
483   SmallVector<Instruction *> Worklist;
484   for (auto &I : instructions(Func))
485     Worklist.push_back(&I);
486 
487   for (auto &I : Worklist)
488     insertAssignTypeIntrs(I);
489 
490   for (auto *I : Worklist) {
491     TrackConstants = true;
492     if (!I->getType()->isVoidTy() || isa<StoreInst>(I))
493       IRB->SetInsertPoint(I->getNextNode());
494     I = visit(*I);
495     processInstrAfterVisit(I);
496   }
497   return true;
498 }
499 
500 FunctionPass *llvm::createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM) {
501   return new SPIRVEmitIntrinsics(TM);
502 }
503