1 //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
10 // llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
15 #include "llvm/Analysis/ObjCARCInstKind.h"
16 #include "llvm/Analysis/ObjCARCUtil.h"
17 #include "llvm/Analysis/TargetTransformInfo.h"
18 #include "llvm/CodeGen/Passes.h"
19 #include "llvm/CodeGen/TargetLowering.h"
20 #include "llvm/CodeGen/TargetPassConfig.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/IRBuilder.h"
23 #include "llvm/IR/Instructions.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/Module.h"
26 #include "llvm/IR/Type.h"
27 #include "llvm/InitializePasses.h"
28 #include "llvm/Pass.h"
29 #include "llvm/Support/Casting.h"
30 #include "llvm/Target/TargetMachine.h"
31 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
32 
33 using namespace llvm;
34 
35 /// Threshold to leave statically sized memory intrinsic calls. Calls of known
36 /// size larger than this will be expanded by the pass. Calls of unknown or
37 /// lower size will be left for expansion in codegen.
38 static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
39     "mem-intrinsic-expand-size",
40     cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1),
41     cl::Hidden);
42 
43 namespace {
44 
45 struct PreISelIntrinsicLowering {
46   const TargetMachine &TM;
47   const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
48 
49   /// If this is true, assume it's preferably to leave memory intrinsic calls
50   /// for replacement with a library call later. Otherwise this depends on
51   /// TargetLoweringInfo availability of the corresponding function.
52   const bool UseMemIntrinsicLibFunc;
53 
54   explicit PreISelIntrinsicLowering(
55       const TargetMachine &TM_,
56       function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
57       bool UseMemIntrinsicLibFunc_ = true)
58       : TM(TM_), LookupTTI(LookupTTI_),
59         UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
60 
61   static bool shouldExpandMemIntrinsicWithSize(Value *Size,
62                                                const TargetTransformInfo &TTI);
63   bool expandMemIntrinsicUses(Function &F) const;
64   bool lowerIntrinsics(Module &M) const;
65 };
66 
67 } // namespace
68 
69 static bool lowerLoadRelative(Function &F) {
70   if (F.use_empty())
71     return false;
72 
73   bool Changed = false;
74   Type *Int32Ty = Type::getInt32Ty(F.getContext());
75 
76   for (Use &U : llvm::make_early_inc_range(F.uses())) {
77     auto CI = dyn_cast<CallInst>(U.getUser());
78     if (!CI || CI->getCalledOperand() != &F)
79       continue;
80 
81     IRBuilder<> B(CI);
82     Value *OffsetPtr =
83         B.CreatePtrAdd(CI->getArgOperand(0), CI->getArgOperand(1));
84     Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtr, Align(4));
85 
86     Value *ResultPtr = B.CreatePtrAdd(CI->getArgOperand(0), OffsetI32);
87 
88     CI->replaceAllUsesWith(ResultPtr);
89     CI->eraseFromParent();
90     Changed = true;
91   }
92 
93   return Changed;
94 }
95 
96 // ObjCARC has knowledge about whether an obj-c runtime function needs to be
97 // always tail-called or never tail-called.
98 static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
99   objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(&F);
100   if (objcarc::IsAlwaysTail(Kind))
101     return CallInst::TCK_Tail;
102   else if (objcarc::IsNeverTail(Kind))
103     return CallInst::TCK_NoTail;
104   return CallInst::TCK_None;
105 }
106 
107 static bool lowerObjCCall(Function &F, const char *NewFn,
108                           bool setNonLazyBind = false) {
109   assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
110          "Pre-ISel intrinsics do lower into regular function calls");
111   if (F.use_empty())
112     return false;
113 
114   // If we haven't already looked up this function, check to see if the
115   // program already contains a function with this name.
116   Module *M = F.getParent();
117   FunctionCallee FCache = M->getOrInsertFunction(NewFn, F.getFunctionType());
118 
119   if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) {
120     Fn->setLinkage(F.getLinkage());
121     if (setNonLazyBind && !Fn->isWeakForLinker()) {
122       // If we have Native ARC, set nonlazybind attribute for these APIs for
123       // performance.
124       Fn->addFnAttr(Attribute::NonLazyBind);
125     }
126   }
127 
128   CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
129 
130   for (Use &U : llvm::make_early_inc_range(F.uses())) {
131     auto *CB = cast<CallBase>(U.getUser());
132 
133     if (CB->getCalledFunction() != &F) {
134       objcarc::ARCInstKind Kind = objcarc::getAttachedARCFunctionKind(CB);
135       (void)Kind;
136       assert((Kind == objcarc::ARCInstKind::RetainRV ||
137               Kind == objcarc::ARCInstKind::UnsafeClaimRV) &&
138              "use expected to be the argument of operand bundle "
139              "\"clang.arc.attachedcall\"");
140       U.set(FCache.getCallee());
141       continue;
142     }
143 
144     auto *CI = cast<CallInst>(CB);
145     assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
146 
147     IRBuilder<> Builder(CI->getParent(), CI->getIterator());
148     SmallVector<Value *, 8> Args(CI->args());
149     SmallVector<llvm::OperandBundleDef, 1> BundleList;
150     CI->getOperandBundlesAsDefs(BundleList);
151     CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList);
152     NewCI->setName(CI->getName());
153 
154     // Try to set the most appropriate TailCallKind based on both the current
155     // attributes and the ones that we could get from ObjCARC's special
156     // knowledge of the runtime functions.
157     //
158     // std::max respects both requirements of notail and tail here:
159     // * notail on either the call or from ObjCARC becomes notail
160     // * tail on either side is stronger than none, but not notail
161     CallInst::TailCallKind TCK = CI->getTailCallKind();
162     NewCI->setTailCallKind(std::max(TCK, OverridingTCK));
163 
164     // Transfer the 'returned' attribute from the intrinsic to the call site.
165     // By applying this only to intrinsic call sites, we avoid applying it to
166     // non-ARC explicit calls to things like objc_retain which have not been
167     // auto-upgraded to use the intrinsics.
168     unsigned Index;
169     if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) &&
170         Index)
171       NewCI->addParamAttr(Index - AttributeList::FirstArgIndex,
172                           Attribute::Returned);
173 
174     if (!CI->use_empty())
175       CI->replaceAllUsesWith(NewCI);
176     CI->eraseFromParent();
177   }
178 
179   return true;
180 }
181 
182 // TODO: Should refine based on estimated number of accesses (e.g. does it
183 // require splitting based on alignment)
184 bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
185     Value *Size, const TargetTransformInfo &TTI) {
186   ConstantInt *CI = dyn_cast<ConstantInt>(Size);
187   if (!CI)
188     return true;
189   uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
190                            ? MemIntrinsicExpandSizeThresholdOpt
191                            : TTI.getMaxMemIntrinsicInlineSizeThreshold();
192   uint64_t SizeVal = CI->getZExtValue();
193 
194   // Treat a threshold of 0 as a special case to force expansion of all
195   // intrinsics, including size 0.
196   return SizeVal > Threshold || Threshold == 0;
197 }
198 
199 static bool canEmitLibcall(const TargetMachine &TM, Function *F,
200                            RTLIB::Libcall LC) {
201   // TODO: Should this consider the address space of the memcpy?
202   const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering();
203   return TLI->getLibcallName(LC) != nullptr;
204 }
205 
206 // TODO: Handle atomic memcpy and memcpy.inline
207 // TODO: Pass ScalarEvolution
208 bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
209   Intrinsic::ID ID = F.getIntrinsicID();
210   bool Changed = false;
211 
212   for (User *U : llvm::make_early_inc_range(F.users())) {
213     Instruction *Inst = cast<Instruction>(U);
214 
215     switch (ID) {
216     case Intrinsic::memcpy: {
217       auto *Memcpy = cast<MemCpyInst>(Inst);
218       Function *ParentFunc = Memcpy->getFunction();
219       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
220       if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {
221         if (UseMemIntrinsicLibFunc &&
222             canEmitLibcall(TM, ParentFunc, RTLIB::MEMCPY))
223           break;
224 
225         // TODO: For optsize, emit the loop into a separate function
226         expandMemCpyAsLoop(Memcpy, TTI);
227         Changed = true;
228         Memcpy->eraseFromParent();
229       }
230 
231       break;
232     }
233     case Intrinsic::memmove: {
234       auto *Memmove = cast<MemMoveInst>(Inst);
235       Function *ParentFunc = Memmove->getFunction();
236       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
237       if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {
238         if (UseMemIntrinsicLibFunc &&
239             canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE))
240           break;
241 
242         if (expandMemMoveAsLoop(Memmove, TTI)) {
243           Changed = true;
244           Memmove->eraseFromParent();
245         }
246       }
247 
248       break;
249     }
250     case Intrinsic::memset: {
251       auto *Memset = cast<MemSetInst>(Inst);
252       Function *ParentFunc = Memset->getFunction();
253       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
254       if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) {
255         if (UseMemIntrinsicLibFunc &&
256             canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET))
257           break;
258 
259         expandMemSetAsLoop(Memset);
260         Changed = true;
261         Memset->eraseFromParent();
262       }
263 
264       break;
265     }
266     default:
267       llvm_unreachable("unhandled intrinsic");
268     }
269   }
270 
271   return Changed;
272 }
273 
274 bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
275   bool Changed = false;
276   for (Function &F : M) {
277     switch (F.getIntrinsicID()) {
278     default:
279       break;
280     case Intrinsic::memcpy:
281     case Intrinsic::memmove:
282     case Intrinsic::memset:
283       Changed |= expandMemIntrinsicUses(F);
284       break;
285     case Intrinsic::load_relative:
286       Changed |= lowerLoadRelative(F);
287       break;
288     case Intrinsic::objc_autorelease:
289       Changed |= lowerObjCCall(F, "objc_autorelease");
290       break;
291     case Intrinsic::objc_autoreleasePoolPop:
292       Changed |= lowerObjCCall(F, "objc_autoreleasePoolPop");
293       break;
294     case Intrinsic::objc_autoreleasePoolPush:
295       Changed |= lowerObjCCall(F, "objc_autoreleasePoolPush");
296       break;
297     case Intrinsic::objc_autoreleaseReturnValue:
298       Changed |= lowerObjCCall(F, "objc_autoreleaseReturnValue");
299       break;
300     case Intrinsic::objc_copyWeak:
301       Changed |= lowerObjCCall(F, "objc_copyWeak");
302       break;
303     case Intrinsic::objc_destroyWeak:
304       Changed |= lowerObjCCall(F, "objc_destroyWeak");
305       break;
306     case Intrinsic::objc_initWeak:
307       Changed |= lowerObjCCall(F, "objc_initWeak");
308       break;
309     case Intrinsic::objc_loadWeak:
310       Changed |= lowerObjCCall(F, "objc_loadWeak");
311       break;
312     case Intrinsic::objc_loadWeakRetained:
313       Changed |= lowerObjCCall(F, "objc_loadWeakRetained");
314       break;
315     case Intrinsic::objc_moveWeak:
316       Changed |= lowerObjCCall(F, "objc_moveWeak");
317       break;
318     case Intrinsic::objc_release:
319       Changed |= lowerObjCCall(F, "objc_release", true);
320       break;
321     case Intrinsic::objc_retain:
322       Changed |= lowerObjCCall(F, "objc_retain", true);
323       break;
324     case Intrinsic::objc_retainAutorelease:
325       Changed |= lowerObjCCall(F, "objc_retainAutorelease");
326       break;
327     case Intrinsic::objc_retainAutoreleaseReturnValue:
328       Changed |= lowerObjCCall(F, "objc_retainAutoreleaseReturnValue");
329       break;
330     case Intrinsic::objc_retainAutoreleasedReturnValue:
331       Changed |= lowerObjCCall(F, "objc_retainAutoreleasedReturnValue");
332       break;
333     case Intrinsic::objc_retainBlock:
334       Changed |= lowerObjCCall(F, "objc_retainBlock");
335       break;
336     case Intrinsic::objc_storeStrong:
337       Changed |= lowerObjCCall(F, "objc_storeStrong");
338       break;
339     case Intrinsic::objc_storeWeak:
340       Changed |= lowerObjCCall(F, "objc_storeWeak");
341       break;
342     case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
343       Changed |= lowerObjCCall(F, "objc_unsafeClaimAutoreleasedReturnValue");
344       break;
345     case Intrinsic::objc_retainedObject:
346       Changed |= lowerObjCCall(F, "objc_retainedObject");
347       break;
348     case Intrinsic::objc_unretainedObject:
349       Changed |= lowerObjCCall(F, "objc_unretainedObject");
350       break;
351     case Intrinsic::objc_unretainedPointer:
352       Changed |= lowerObjCCall(F, "objc_unretainedPointer");
353       break;
354     case Intrinsic::objc_retain_autorelease:
355       Changed |= lowerObjCCall(F, "objc_retain_autorelease");
356       break;
357     case Intrinsic::objc_sync_enter:
358       Changed |= lowerObjCCall(F, "objc_sync_enter");
359       break;
360     case Intrinsic::objc_sync_exit:
361       Changed |= lowerObjCCall(F, "objc_sync_exit");
362       break;
363     }
364   }
365   return Changed;
366 }
367 
368 namespace {
369 
370 class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
371 public:
372   static char ID;
373 
374   PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
375 
376   void getAnalysisUsage(AnalysisUsage &AU) const override {
377     AU.addRequired<TargetTransformInfoWrapperPass>();
378     AU.addRequired<TargetPassConfig>();
379   }
380 
381   bool runOnModule(Module &M) override {
382     auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
383       return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
384     };
385 
386     const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
387     PreISelIntrinsicLowering Lowering(TM, LookupTTI);
388     return Lowering.lowerIntrinsics(M);
389   }
390 };
391 
392 } // end anonymous namespace
393 
394 char PreISelIntrinsicLoweringLegacyPass::ID;
395 
396 INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
397                       "pre-isel-intrinsic-lowering",
398                       "Pre-ISel Intrinsic Lowering", false, false)
399 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
400 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
401 INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
402                     "pre-isel-intrinsic-lowering",
403                     "Pre-ISel Intrinsic Lowering", false, false)
404 
405 ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
406   return new PreISelIntrinsicLoweringLegacyPass();
407 }
408 
409 PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M,
410                                                     ModuleAnalysisManager &AM) {
411   auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
412 
413   auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
414     return FAM.getResult<TargetIRAnalysis>(F);
415   };
416 
417   PreISelIntrinsicLowering Lowering(TM, LookupTTI);
418   if (!Lowering.lowerIntrinsics(M))
419     return PreservedAnalyses::all();
420   else
421     return PreservedAnalyses::none();
422 }
423