1 //===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the IntrinsicLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/CodeGen/IntrinsicLowering.h"
14 #include "llvm/ADT/SmallVector.h"
15 #include "llvm/IR/CallSite.h"
16 #include "llvm/IR/Constants.h"
17 #include "llvm/IR/DataLayout.h"
18 #include "llvm/IR/DerivedTypes.h"
19 #include "llvm/IR/IRBuilder.h"
20 #include "llvm/IR/Module.h"
21 #include "llvm/IR/Type.h"
22 #include "llvm/Support/ErrorHandling.h"
23 #include "llvm/Support/raw_ostream.h"
24 using namespace llvm;
25 
26 /// This function is used when we want to lower an intrinsic call to a call of
27 /// an external function. This handles hard cases such as when there was already
28 /// a prototype for the external function, but that prototype doesn't match the
29 /// arguments we expect to pass in.
30 template <class ArgIt>
ReplaceCallWith(const char * NewFn,CallInst * CI,ArgIt ArgBegin,ArgIt ArgEnd,Type * RetTy)31 static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI,
32                                  ArgIt ArgBegin, ArgIt ArgEnd,
33                                  Type *RetTy) {
34   // If we haven't already looked up this function, check to see if the
35   // program already contains a function with this name.
36   Module *M = CI->getModule();
37   // Get or insert the definition now.
38   std::vector<Type *> ParamTys;
39   for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
40     ParamTys.push_back((*I)->getType());
41   FunctionCallee FCache =
42       M->getOrInsertFunction(NewFn, FunctionType::get(RetTy, ParamTys, false));
43 
44   IRBuilder<> Builder(CI->getParent(), CI->getIterator());
45   SmallVector<Value *, 8> Args(ArgBegin, ArgEnd);
46   CallInst *NewCI = Builder.CreateCall(FCache, Args);
47   NewCI->setName(CI->getName());
48   if (!CI->use_empty())
49     CI->replaceAllUsesWith(NewCI);
50   return NewCI;
51 }
52 
53 /// Emit the code to lower bswap of V before the specified instruction IP.
LowerBSWAP(LLVMContext & Context,Value * V,Instruction * IP)54 static Value *LowerBSWAP(LLVMContext &Context, Value *V, Instruction *IP) {
55   assert(V->getType()->isIntOrIntVectorTy() && "Can't bswap a non-integer type!");
56 
57   unsigned BitSize = V->getType()->getScalarSizeInBits();
58 
59   IRBuilder<> Builder(IP);
60 
61   switch(BitSize) {
62   default: llvm_unreachable("Unhandled type size of value to byteswap!");
63   case 16: {
64     Value *Tmp1 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8),
65                                     "bswap.2");
66     Value *Tmp2 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8),
67                                      "bswap.1");
68     V = Builder.CreateOr(Tmp1, Tmp2, "bswap.i16");
69     break;
70   }
71   case 32: {
72     Value *Tmp4 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 24),
73                                     "bswap.4");
74     Value *Tmp3 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8),
75                                     "bswap.3");
76     Value *Tmp2 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8),
77                                      "bswap.2");
78     Value *Tmp1 = Builder.CreateLShr(V,ConstantInt::get(V->getType(), 24),
79                                      "bswap.1");
80     Tmp3 = Builder.CreateAnd(Tmp3,
81                          ConstantInt::get(V->getType(), 0xFF0000),
82                              "bswap.and3");
83     Tmp2 = Builder.CreateAnd(Tmp2,
84                            ConstantInt::get(V->getType(), 0xFF00),
85                              "bswap.and2");
86     Tmp4 = Builder.CreateOr(Tmp4, Tmp3, "bswap.or1");
87     Tmp2 = Builder.CreateOr(Tmp2, Tmp1, "bswap.or2");
88     V = Builder.CreateOr(Tmp4, Tmp2, "bswap.i32");
89     break;
90   }
91   case 64: {
92     Value *Tmp8 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 56),
93                                     "bswap.8");
94     Value *Tmp7 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 40),
95                                     "bswap.7");
96     Value *Tmp6 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 24),
97                                     "bswap.6");
98     Value *Tmp5 = Builder.CreateShl(V, ConstantInt::get(V->getType(), 8),
99                                     "bswap.5");
100     Value* Tmp4 = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 8),
101                                      "bswap.4");
102     Value* Tmp3 = Builder.CreateLShr(V,
103                                      ConstantInt::get(V->getType(), 24),
104                                      "bswap.3");
105     Value* Tmp2 = Builder.CreateLShr(V,
106                                      ConstantInt::get(V->getType(), 40),
107                                      "bswap.2");
108     Value* Tmp1 = Builder.CreateLShr(V,
109                                      ConstantInt::get(V->getType(), 56),
110                                      "bswap.1");
111     Tmp7 = Builder.CreateAnd(Tmp7,
112                              ConstantInt::get(V->getType(),
113                                               0xFF000000000000ULL),
114                              "bswap.and7");
115     Tmp6 = Builder.CreateAnd(Tmp6,
116                              ConstantInt::get(V->getType(),
117                                               0xFF0000000000ULL),
118                              "bswap.and6");
119     Tmp5 = Builder.CreateAnd(Tmp5,
120                         ConstantInt::get(V->getType(),
121                              0xFF00000000ULL),
122                              "bswap.and5");
123     Tmp4 = Builder.CreateAnd(Tmp4,
124                         ConstantInt::get(V->getType(),
125                              0xFF000000ULL),
126                              "bswap.and4");
127     Tmp3 = Builder.CreateAnd(Tmp3,
128                              ConstantInt::get(V->getType(),
129                              0xFF0000ULL),
130                              "bswap.and3");
131     Tmp2 = Builder.CreateAnd(Tmp2,
132                              ConstantInt::get(V->getType(),
133                              0xFF00ULL),
134                              "bswap.and2");
135     Tmp8 = Builder.CreateOr(Tmp8, Tmp7, "bswap.or1");
136     Tmp6 = Builder.CreateOr(Tmp6, Tmp5, "bswap.or2");
137     Tmp4 = Builder.CreateOr(Tmp4, Tmp3, "bswap.or3");
138     Tmp2 = Builder.CreateOr(Tmp2, Tmp1, "bswap.or4");
139     Tmp8 = Builder.CreateOr(Tmp8, Tmp6, "bswap.or5");
140     Tmp4 = Builder.CreateOr(Tmp4, Tmp2, "bswap.or6");
141     V = Builder.CreateOr(Tmp8, Tmp4, "bswap.i64");
142     break;
143   }
144   }
145   return V;
146 }
147 
148 /// Emit the code to lower ctpop of V before the specified instruction IP.
LowerCTPOP(LLVMContext & Context,Value * V,Instruction * IP)149 static Value *LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP) {
150   assert(V->getType()->isIntegerTy() && "Can't ctpop a non-integer type!");
151 
152   static const uint64_t MaskValues[6] = {
153     0x5555555555555555ULL, 0x3333333333333333ULL,
154     0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
155     0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL
156   };
157 
158   IRBuilder<> Builder(IP);
159 
160   unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
161   unsigned WordSize = (BitSize + 63) / 64;
162   Value *Count = ConstantInt::get(V->getType(), 0);
163 
164   for (unsigned n = 0; n < WordSize; ++n) {
165     Value *PartValue = V;
166     for (unsigned i = 1, ct = 0; i < (BitSize>64 ? 64 : BitSize);
167          i <<= 1, ++ct) {
168       Value *MaskCst = ConstantInt::get(V->getType(), MaskValues[ct]);
169       Value *LHS = Builder.CreateAnd(PartValue, MaskCst, "cppop.and1");
170       Value *VShift = Builder.CreateLShr(PartValue,
171                                         ConstantInt::get(V->getType(), i),
172                                          "ctpop.sh");
173       Value *RHS = Builder.CreateAnd(VShift, MaskCst, "cppop.and2");
174       PartValue = Builder.CreateAdd(LHS, RHS, "ctpop.step");
175     }
176     Count = Builder.CreateAdd(PartValue, Count, "ctpop.part");
177     if (BitSize > 64) {
178       V = Builder.CreateLShr(V, ConstantInt::get(V->getType(), 64),
179                              "ctpop.part.sh");
180       BitSize -= 64;
181     }
182   }
183 
184   return Count;
185 }
186 
187 /// Emit the code to lower ctlz of V before the specified instruction IP.
LowerCTLZ(LLVMContext & Context,Value * V,Instruction * IP)188 static Value *LowerCTLZ(LLVMContext &Context, Value *V, Instruction *IP) {
189 
190   IRBuilder<> Builder(IP);
191 
192   unsigned BitSize = V->getType()->getPrimitiveSizeInBits();
193   for (unsigned i = 1; i < BitSize; i <<= 1) {
194     Value *ShVal = ConstantInt::get(V->getType(), i);
195     ShVal = Builder.CreateLShr(V, ShVal, "ctlz.sh");
196     V = Builder.CreateOr(V, ShVal, "ctlz.step");
197   }
198 
199   V = Builder.CreateNot(V);
200   return LowerCTPOP(Context, V, IP);
201 }
202 
ReplaceFPIntrinsicWithCall(CallInst * CI,const char * Fname,const char * Dname,const char * LDname)203 static void ReplaceFPIntrinsicWithCall(CallInst *CI, const char *Fname,
204                                        const char *Dname,
205                                        const char *LDname) {
206   CallSite CS(CI);
207   switch (CI->getArgOperand(0)->getType()->getTypeID()) {
208   default: llvm_unreachable("Invalid type in intrinsic");
209   case Type::FloatTyID:
210     ReplaceCallWith(Fname, CI, CS.arg_begin(), CS.arg_end(),
211                   Type::getFloatTy(CI->getContext()));
212     break;
213   case Type::DoubleTyID:
214     ReplaceCallWith(Dname, CI, CS.arg_begin(), CS.arg_end(),
215                   Type::getDoubleTy(CI->getContext()));
216     break;
217   case Type::X86_FP80TyID:
218   case Type::FP128TyID:
219   case Type::PPC_FP128TyID:
220     ReplaceCallWith(LDname, CI, CS.arg_begin(), CS.arg_end(),
221                   CI->getArgOperand(0)->getType());
222     break;
223   }
224 }
225 
LowerIntrinsicCall(CallInst * CI)226 void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
227   IRBuilder<> Builder(CI);
228   LLVMContext &Context = CI->getContext();
229 
230   const Function *Callee = CI->getCalledFunction();
231   assert(Callee && "Cannot lower an indirect call!");
232 
233   CallSite CS(CI);
234   switch (Callee->getIntrinsicID()) {
235   case Intrinsic::not_intrinsic:
236     report_fatal_error("Cannot lower a call to a non-intrinsic function '"+
237                       Callee->getName() + "'!");
238   default:
239     report_fatal_error("Code generator does not support intrinsic function '"+
240                       Callee->getName()+"'!");
241 
242   case Intrinsic::expect: {
243     // Just replace __builtin_expect(exp, c) with EXP.
244     Value *V = CI->getArgOperand(0);
245     CI->replaceAllUsesWith(V);
246     break;
247   }
248 
249   case Intrinsic::ctpop:
250     CI->replaceAllUsesWith(LowerCTPOP(Context, CI->getArgOperand(0), CI));
251     break;
252 
253   case Intrinsic::bswap:
254     CI->replaceAllUsesWith(LowerBSWAP(Context, CI->getArgOperand(0), CI));
255     break;
256 
257   case Intrinsic::ctlz:
258     CI->replaceAllUsesWith(LowerCTLZ(Context, CI->getArgOperand(0), CI));
259     break;
260 
261   case Intrinsic::cttz: {
262     // cttz(x) -> ctpop(~X & (X-1))
263     Value *Src = CI->getArgOperand(0);
264     Value *NotSrc = Builder.CreateNot(Src);
265     NotSrc->setName(Src->getName() + ".not");
266     Value *SrcM1 = ConstantInt::get(Src->getType(), 1);
267     SrcM1 = Builder.CreateSub(Src, SrcM1);
268     Src = LowerCTPOP(Context, Builder.CreateAnd(NotSrc, SrcM1), CI);
269     CI->replaceAllUsesWith(Src);
270     break;
271   }
272 
273   case Intrinsic::stacksave:
274   case Intrinsic::stackrestore: {
275     if (!Warned)
276       errs() << "WARNING: this target does not support the llvm.stack"
277              << (Callee->getIntrinsicID() == Intrinsic::stacksave ?
278                "save" : "restore") << " intrinsic.\n";
279     Warned = true;
280     if (Callee->getIntrinsicID() == Intrinsic::stacksave)
281       CI->replaceAllUsesWith(Constant::getNullValue(CI->getType()));
282     break;
283   }
284 
285   case Intrinsic::get_dynamic_area_offset:
286     errs() << "WARNING: this target does not support the custom llvm.get."
287               "dynamic.area.offset.  It is being lowered to a constant 0\n";
288     // Just lower it to a constant 0 because for most targets
289     // @llvm.get.dynamic.area.offset is lowered to zero.
290     CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 0));
291     break;
292   case Intrinsic::returnaddress:
293   case Intrinsic::frameaddress:
294     errs() << "WARNING: this target does not support the llvm."
295            << (Callee->getIntrinsicID() == Intrinsic::returnaddress ?
296              "return" : "frame") << "address intrinsic.\n";
297     CI->replaceAllUsesWith(
298         ConstantPointerNull::get(cast<PointerType>(CI->getType())));
299     break;
300   case Intrinsic::addressofreturnaddress:
301     errs() << "WARNING: this target does not support the "
302               "llvm.addressofreturnaddress intrinsic.\n";
303     CI->replaceAllUsesWith(
304         ConstantPointerNull::get(cast<PointerType>(CI->getType())));
305     break;
306 
307   case Intrinsic::prefetch:
308     break;    // Simply strip out prefetches on unsupported architectures
309 
310   case Intrinsic::pcmarker:
311     break;    // Simply strip out pcmarker on unsupported architectures
312   case Intrinsic::readcyclecounter: {
313     errs() << "WARNING: this target does not support the llvm.readcyclecoun"
314            << "ter intrinsic.  It is being lowered to a constant 0\n";
315     CI->replaceAllUsesWith(ConstantInt::get(Type::getInt64Ty(Context), 0));
316     break;
317   }
318 
319   case Intrinsic::dbg_declare:
320   case Intrinsic::dbg_label:
321     break;    // Simply strip out debugging intrinsics
322 
323   case Intrinsic::eh_typeid_for:
324     // Return something different to eh_selector.
325     CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 1));
326     break;
327 
328   case Intrinsic::annotation:
329   case Intrinsic::ptr_annotation:
330     // Just drop the annotation, but forward the value
331     CI->replaceAllUsesWith(CI->getOperand(0));
332     break;
333 
334   case Intrinsic::assume:
335   case Intrinsic::var_annotation:
336     break;   // Strip out these intrinsics
337 
338   case Intrinsic::memcpy: {
339     Type *IntPtr = DL.getIntPtrType(Context);
340     Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
341                                         /* isSigned */ false);
342     Value *Ops[3];
343     Ops[0] = CI->getArgOperand(0);
344     Ops[1] = CI->getArgOperand(1);
345     Ops[2] = Size;
346     ReplaceCallWith("memcpy", CI, Ops, Ops+3, CI->getArgOperand(0)->getType());
347     break;
348   }
349   case Intrinsic::memmove: {
350     Type *IntPtr = DL.getIntPtrType(Context);
351     Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
352                                         /* isSigned */ false);
353     Value *Ops[3];
354     Ops[0] = CI->getArgOperand(0);
355     Ops[1] = CI->getArgOperand(1);
356     Ops[2] = Size;
357     ReplaceCallWith("memmove", CI, Ops, Ops+3, CI->getArgOperand(0)->getType());
358     break;
359   }
360   case Intrinsic::memset: {
361     Value *Op0 = CI->getArgOperand(0);
362     Type *IntPtr = DL.getIntPtrType(Op0->getType());
363     Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
364                                         /* isSigned */ false);
365     Value *Ops[3];
366     Ops[0] = Op0;
367     // Extend the amount to i32.
368     Ops[1] = Builder.CreateIntCast(CI->getArgOperand(1),
369                                    Type::getInt32Ty(Context),
370                                    /* isSigned */ false);
371     Ops[2] = Size;
372     ReplaceCallWith("memset", CI, Ops, Ops+3, CI->getArgOperand(0)->getType());
373     break;
374   }
375   case Intrinsic::sqrt: {
376     ReplaceFPIntrinsicWithCall(CI, "sqrtf", "sqrt", "sqrtl");
377     break;
378   }
379   case Intrinsic::log: {
380     ReplaceFPIntrinsicWithCall(CI, "logf", "log", "logl");
381     break;
382   }
383   case Intrinsic::log2: {
384     ReplaceFPIntrinsicWithCall(CI, "log2f", "log2", "log2l");
385     break;
386   }
387   case Intrinsic::log10: {
388     ReplaceFPIntrinsicWithCall(CI, "log10f", "log10", "log10l");
389     break;
390   }
391   case Intrinsic::exp: {
392     ReplaceFPIntrinsicWithCall(CI, "expf", "exp", "expl");
393     break;
394   }
395   case Intrinsic::exp2: {
396     ReplaceFPIntrinsicWithCall(CI, "exp2f", "exp2", "exp2l");
397     break;
398   }
399   case Intrinsic::pow: {
400     ReplaceFPIntrinsicWithCall(CI, "powf", "pow", "powl");
401     break;
402   }
403   case Intrinsic::sin: {
404     ReplaceFPIntrinsicWithCall(CI, "sinf", "sin", "sinl");
405     break;
406   }
407   case Intrinsic::cos: {
408     ReplaceFPIntrinsicWithCall(CI, "cosf", "cos", "cosl");
409     break;
410   }
411   case Intrinsic::floor: {
412     ReplaceFPIntrinsicWithCall(CI, "floorf", "floor", "floorl");
413     break;
414   }
415   case Intrinsic::ceil: {
416     ReplaceFPIntrinsicWithCall(CI, "ceilf", "ceil", "ceill");
417     break;
418   }
419   case Intrinsic::trunc: {
420     ReplaceFPIntrinsicWithCall(CI, "truncf", "trunc", "truncl");
421     break;
422   }
423   case Intrinsic::round: {
424     ReplaceFPIntrinsicWithCall(CI, "roundf", "round", "roundl");
425     break;
426   }
427   case Intrinsic::copysign: {
428     ReplaceFPIntrinsicWithCall(CI, "copysignf", "copysign", "copysignl");
429     break;
430   }
431   case Intrinsic::flt_rounds:
432      // Lower to "round to the nearest"
433      if (!CI->getType()->isVoidTy())
434        CI->replaceAllUsesWith(ConstantInt::get(CI->getType(), 1));
435      break;
436   case Intrinsic::invariant_start:
437   case Intrinsic::lifetime_start:
438     // Discard region information.
439     CI->replaceAllUsesWith(UndefValue::get(CI->getType()));
440     break;
441   case Intrinsic::invariant_end:
442   case Intrinsic::lifetime_end:
443     // Discard region information.
444     break;
445   }
446 
447   assert(CI->use_empty() &&
448          "Lowering should have eliminated any uses of the intrinsic call!");
449   CI->eraseFromParent();
450 }
451 
LowerToByteSwap(CallInst * CI)452 bool IntrinsicLowering::LowerToByteSwap(CallInst *CI) {
453   // Verify this is a simple bswap.
454   if (CI->getNumArgOperands() != 1 ||
455       CI->getType() != CI->getArgOperand(0)->getType() ||
456       !CI->getType()->isIntegerTy())
457     return false;
458 
459   IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
460   if (!Ty)
461     return false;
462 
463   // Okay, we can do this xform, do so now.
464   Module *M = CI->getModule();
465   Function *Int = Intrinsic::getDeclaration(M, Intrinsic::bswap, Ty);
466 
467   Value *Op = CI->getArgOperand(0);
468   Op = CallInst::Create(Int, Op, CI->getName(), CI);
469 
470   CI->replaceAllUsesWith(Op);
471   CI->eraseFromParent();
472   return true;
473 }
474