1 //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10 // llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
15 #include "llvm/Analysis/ObjCARCInstKind.h"
16 #include "llvm/Analysis/ObjCARCUtil.h"
17 #include "llvm/Analysis/TargetTransformInfo.h"
18 #include "llvm/CodeGen/Passes.h"
19 #include "llvm/CodeGen/TargetLowering.h"
20 #include "llvm/CodeGen/TargetPassConfig.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/IRBuilder.h"
23 #include "llvm/IR/Instructions.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/Module.h"
26 #include "llvm/IR/Type.h"
27 #include "llvm/InitializePasses.h"
28 #include "llvm/Pass.h"
29 #include "llvm/Support/Casting.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
32 
33 using namespace llvm;
34 
35 /// Threshold to leave statically sized memory intrinsic calls. Calls of known
36 /// size larger than this will be expanded by the pass. Calls of unknown or
37 /// lower size will be left for expansion in codegen.
38 static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
39     "mem-intrinsic-expand-size",
40     cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1),
41     cl::Hidden);
42 
43 namespace {
44 
45 struct PreISelIntrinsicLowering {
46   const TargetMachine &TM;
47   const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
48 
49   /// If this is true, assume it's preferably to leave memory intrinsic calls
50   /// for replacement with a library call later. Otherwise this depends on
51   /// TargetLoweringInfo availability of the corresponding function.
52   const bool UseMemIntrinsicLibFunc;
53 
54   explicit PreISelIntrinsicLowering(
55       const TargetMachine &TM_,
56       function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
57       bool UseMemIntrinsicLibFunc_ = true)
58       : TM(TM_), LookupTTI(LookupTTI_),
59         UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
60 
61   static bool shouldExpandMemIntrinsicWithSize(Value *Size,
62                                                const TargetTransformInfo &TTI);
63   bool expandMemIntrinsicUses(Function &F) const;
64   bool lowerIntrinsics(Module &M) const;
65 };
66 
67 } // namespace
68 
69 static bool lowerLoadRelative(Function &F) {
70   if (F.use_empty())
71     return false;
72 
73   bool Changed = false;
74   Type *Int32Ty = Type::getInt32Ty(F.getContext());
75   Type *Int32PtrTy = Int32Ty->getPointerTo();
76   Type *Int8Ty = Type::getInt8Ty(F.getContext());
77 
78   for (Use &U : llvm::make_early_inc_range(F.uses())) {
79     auto CI = dyn_cast<CallInst>(U.getUser());
80     if (!CI || CI->getCalledOperand() != &F)
81       continue;
82 
83     IRBuilder<> B(CI);
84     Value *OffsetPtr =
85         B.CreateGEP(Int8Ty, CI->getArgOperand(0), CI->getArgOperand(1));
86     Value *OffsetPtrI32 = B.CreateBitCast(OffsetPtr, Int32PtrTy);
87     Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtrI32, Align(4));
88 
89     Value *ResultPtr = B.CreateGEP(Int8Ty, CI->getArgOperand(0), OffsetI32);
90 
91     CI->replaceAllUsesWith(ResultPtr);
92     CI->eraseFromParent();
93     Changed = true;
94   }
95 
96   return Changed;
97 }
98 
99 // ObjCARC has knowledge about whether an obj-c runtime function needs to be
100 // always tail-called or never tail-called.
101 static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
102   objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(&F);
103   if (objcarc::IsAlwaysTail(Kind))
104     return CallInst::TCK_Tail;
105   else if (objcarc::IsNeverTail(Kind))
106     return CallInst::TCK_NoTail;
107   return CallInst::TCK_None;
108 }
109 
110 static bool lowerObjCCall(Function &F, const char *NewFn,
111                           bool setNonLazyBind = false) {
112   assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
113          "Pre-ISel intrinsics do lower into regular function calls");
114   if (F.use_empty())
115     return false;
116 
117   // If we haven't already looked up this function, check to see if the
118   // program already contains a function with this name.
119   Module *M = F.getParent();
120   FunctionCallee FCache = M->getOrInsertFunction(NewFn, F.getFunctionType());
121 
122   if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) {
123     Fn->setLinkage(F.getLinkage());
124     if (setNonLazyBind && !Fn->isWeakForLinker()) {
125       // If we have Native ARC, set nonlazybind attribute for these APIs for
126       // performance.
127       Fn->addFnAttr(Attribute::NonLazyBind);
128     }
129   }
130 
131   CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
132 
133   for (Use &U : llvm::make_early_inc_range(F.uses())) {
134     auto *CB = cast<CallBase>(U.getUser());
135 
136     if (CB->getCalledFunction() != &F) {
137       objcarc::ARCInstKind Kind = objcarc::getAttachedARCFunctionKind(CB);
138       (void)Kind;
139       assert((Kind == objcarc::ARCInstKind::RetainRV ||
140               Kind == objcarc::ARCInstKind::UnsafeClaimRV) &&
141              "use expected to be the argument of operand bundle "
142              "\"clang.arc.attachedcall\"");
143       U.set(FCache.getCallee());
144       continue;
145     }
146 
147     auto *CI = cast<CallInst>(CB);
148     assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
149 
150     IRBuilder<> Builder(CI->getParent(), CI->getIterator());
151     SmallVector<Value *, 8> Args(CI->args());
152     SmallVector<llvm::OperandBundleDef, 1> BundleList;
153     CI->getOperandBundlesAsDefs(BundleList);
154     CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList);
155     NewCI->setName(CI->getName());
156 
157     // Try to set the most appropriate TailCallKind based on both the current
158     // attributes and the ones that we could get from ObjCARC's special
159     // knowledge of the runtime functions.
160     //
161     // std::max respects both requirements of notail and tail here:
162     // * notail on either the call or from ObjCARC becomes notail
163     // * tail on either side is stronger than none, but not notail
164     CallInst::TailCallKind TCK = CI->getTailCallKind();
165     NewCI->setTailCallKind(std::max(TCK, OverridingTCK));
166 
167     if (!CI->use_empty())
168       CI->replaceAllUsesWith(NewCI);
169     CI->eraseFromParent();
170   }
171 
172   return true;
173 }
174 
175 // TODO: Should refine based on estimated number of accesses (e.g. does it
176 // require splitting based on alignment)
177 bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
178     Value *Size, const TargetTransformInfo &TTI) {
179   ConstantInt *CI = dyn_cast<ConstantInt>(Size);
180   if (!CI)
181     return true;
182   uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
183                            ? MemIntrinsicExpandSizeThresholdOpt
184                            : TTI.getMaxMemIntrinsicInlineSizeThreshold();
185   uint64_t SizeVal = CI->getZExtValue();
186 
187   // Treat a threshold of 0 as a special case to force expansion of all
188   // intrinsics, including size 0.
189   return SizeVal > Threshold || Threshold == 0;
190 }
191 
192 static bool canEmitLibcall(const TargetMachine &TM, Function *F,
193                            RTLIB::Libcall LC) {
194   // TODO: Should this consider the address space of the memcpy?
195   const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering();
196   return TLI->getLibcallName(LC) != nullptr;
197 }
198 
199 // TODO: Handle atomic memcpy and memcpy.inline
200 // TODO: Pass ScalarEvolution
201 bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
202   Intrinsic::ID ID = F.getIntrinsicID();
203   bool Changed = false;
204 
205   for (User *U : llvm::make_early_inc_range(F.users())) {
206     Instruction *Inst = cast<Instruction>(U);
207 
208     switch (ID) {
209     case Intrinsic::memcpy: {
210       auto *Memcpy = cast<MemCpyInst>(Inst);
211       Function *ParentFunc = Memcpy->getFunction();
212       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
213       if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {
214         if (UseMemIntrinsicLibFunc &&
215             canEmitLibcall(TM, ParentFunc, RTLIB::MEMCPY))
216           break;
217 
218         // TODO: For optsize, emit the loop into a separate function
219         expandMemCpyAsLoop(Memcpy, TTI);
220         Changed = true;
221         Memcpy->eraseFromParent();
222       }
223 
224       break;
225     }
226     case Intrinsic::memmove: {
227       auto *Memmove = cast<MemMoveInst>(Inst);
228       Function *ParentFunc = Memmove->getFunction();
229       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
230       if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {
231         if (UseMemIntrinsicLibFunc &&
232             canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE))
233           break;
234 
235         if (expandMemMoveAsLoop(Memmove, TTI)) {
236           Changed = true;
237           Memmove->eraseFromParent();
238         }
239       }
240 
241       break;
242     }
243     case Intrinsic::memset: {
244       auto *Memset = cast<MemSetInst>(Inst);
245       Function *ParentFunc = Memset->getFunction();
246       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
247       if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) {
248         if (UseMemIntrinsicLibFunc &&
249             canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET))
250           break;
251 
252         expandMemSetAsLoop(Memset);
253         Changed = true;
254         Memset->eraseFromParent();
255       }
256 
257       break;
258     }
259     default:
260       llvm_unreachable("unhandled intrinsic");
261     }
262   }
263 
264   return Changed;
265 }
266 
267 bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
268   bool Changed = false;
269   for (Function &F : M) {
270     switch (F.getIntrinsicID()) {
271     default:
272       break;
273     case Intrinsic::memcpy:
274     case Intrinsic::memmove:
275     case Intrinsic::memset:
276       Changed |= expandMemIntrinsicUses(F);
277       break;
278     case Intrinsic::load_relative:
279       Changed |= lowerLoadRelative(F);
280       break;
281     case Intrinsic::objc_autorelease:
282       Changed |= lowerObjCCall(F, "objc_autorelease");
283       break;
284     case Intrinsic::objc_autoreleasePoolPop:
285       Changed |= lowerObjCCall(F, "objc_autoreleasePoolPop");
286       break;
287     case Intrinsic::objc_autoreleasePoolPush:
288       Changed |= lowerObjCCall(F, "objc_autoreleasePoolPush");
289       break;
290     case Intrinsic::objc_autoreleaseReturnValue:
291       Changed |= lowerObjCCall(F, "objc_autoreleaseReturnValue");
292       break;
293     case Intrinsic::objc_copyWeak:
294       Changed |= lowerObjCCall(F, "objc_copyWeak");
295       break;
296     case Intrinsic::objc_destroyWeak:
297       Changed |= lowerObjCCall(F, "objc_destroyWeak");
298       break;
299     case Intrinsic::objc_initWeak:
300       Changed |= lowerObjCCall(F, "objc_initWeak");
301       break;
302     case Intrinsic::objc_loadWeak:
303       Changed |= lowerObjCCall(F, "objc_loadWeak");
304       break;
305     case Intrinsic::objc_loadWeakRetained:
306       Changed |= lowerObjCCall(F, "objc_loadWeakRetained");
307       break;
308     case Intrinsic::objc_moveWeak:
309       Changed |= lowerObjCCall(F, "objc_moveWeak");
310       break;
311     case Intrinsic::objc_release:
312       Changed |= lowerObjCCall(F, "objc_release", true);
313       break;
314     case Intrinsic::objc_retain:
315       Changed |= lowerObjCCall(F, "objc_retain", true);
316       break;
317     case Intrinsic::objc_retainAutorelease:
318       Changed |= lowerObjCCall(F, "objc_retainAutorelease");
319       break;
320     case Intrinsic::objc_retainAutoreleaseReturnValue:
321       Changed |= lowerObjCCall(F, "objc_retainAutoreleaseReturnValue");
322       break;
323     case Intrinsic::objc_retainAutoreleasedReturnValue:
324       Changed |= lowerObjCCall(F, "objc_retainAutoreleasedReturnValue");
325       break;
326     case Intrinsic::objc_retainBlock:
327       Changed |= lowerObjCCall(F, "objc_retainBlock");
328       break;
329     case Intrinsic::objc_storeStrong:
330       Changed |= lowerObjCCall(F, "objc_storeStrong");
331       break;
332     case Intrinsic::objc_storeWeak:
333       Changed |= lowerObjCCall(F, "objc_storeWeak");
334       break;
335     case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
336       Changed |= lowerObjCCall(F, "objc_unsafeClaimAutoreleasedReturnValue");
337       break;
338     case Intrinsic::objc_retainedObject:
339       Changed |= lowerObjCCall(F, "objc_retainedObject");
340       break;
341     case Intrinsic::objc_unretainedObject:
342       Changed |= lowerObjCCall(F, "objc_unretainedObject");
343       break;
344     case Intrinsic::objc_unretainedPointer:
345       Changed |= lowerObjCCall(F, "objc_unretainedPointer");
346       break;
347     case Intrinsic::objc_retain_autorelease:
348       Changed |= lowerObjCCall(F, "objc_retain_autorelease");
349       break;
350     case Intrinsic::objc_sync_enter:
351       Changed |= lowerObjCCall(F, "objc_sync_enter");
352       break;
353     case Intrinsic::objc_sync_exit:
354       Changed |= lowerObjCCall(F, "objc_sync_exit");
355       break;
356     }
357   }
358   return Changed;
359 }
360 
361 namespace {
362 
363 class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
364 public:
365   static char ID;
366 
367   PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
368 
369   void getAnalysisUsage(AnalysisUsage &AU) const override {
370     AU.addRequired<TargetTransformInfoWrapperPass>();
371     AU.addRequired<TargetPassConfig>();
372   }
373 
374   bool runOnModule(Module &M) override {
375     auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
376       return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
377     };
378 
379     const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
380     PreISelIntrinsicLowering Lowering(TM, LookupTTI);
381     return Lowering.lowerIntrinsics(M);
382   }
383 };
384 
385 } // end anonymous namespace
386 
387 char PreISelIntrinsicLoweringLegacyPass::ID;
388 
389 INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
390                       "pre-isel-intrinsic-lowering",
391                       "Pre-ISel Intrinsic Lowering", false, false)
392 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
393 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
394 INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
395                     "pre-isel-intrinsic-lowering",
396                     "Pre-ISel Intrinsic Lowering", false, false)
397 
398 ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
399   return new PreISelIntrinsicLoweringLegacyPass();
400 }
401 
402 PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M,
403                                                     ModuleAnalysisManager &AM) {
404   auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
405 
406   auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
407     return FAM.getResult<TargetIRAnalysis>(F);
408   };
409 
410   PreISelIntrinsicLowering Lowering(TM, LookupTTI);
411   if (!Lowering.lowerIntrinsics(M))
412     return PreservedAnalyses::all();
413   else
414     return PreservedAnalyses::none();
415 }
416