1 //===-- AMDGPUAlwaysInlinePass.cpp - Promote Allocas ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This pass marks all internal functions as always_inline and creates
11 /// duplicates of all other functions and marks the duplicates as always_inline.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "AMDGPU.h"
16 #include "AMDGPUTargetMachine.h"
17 #include "Utils/AMDGPUBaseInfo.h"
18 #include "llvm/ADT/SmallPtrSet.h"
19 #include "llvm/IR/Module.h"
20 #include "llvm/Transforms/Utils/Cloning.h"
21
22 using namespace llvm;
23
24 namespace {
25
26 static cl::opt<bool> StressCalls(
27 "amdgpu-stress-function-calls",
28 cl::Hidden,
29 cl::desc("Force all functions to be noinline"),
30 cl::init(false));
31
32 class AMDGPUAlwaysInline : public ModulePass {
33 bool GlobalOpt;
34
35 void recursivelyVisitUsers(GlobalValue &GV,
36 SmallPtrSetImpl<Function *> &FuncsToAlwaysInline);
37 public:
38 static char ID;
39
AMDGPUAlwaysInline(bool GlobalOpt=false)40 AMDGPUAlwaysInline(bool GlobalOpt = false) :
41 ModulePass(ID), GlobalOpt(GlobalOpt) { }
42 bool runOnModule(Module &M) override;
43
getAnalysisUsage(AnalysisUsage & AU) const44 void getAnalysisUsage(AnalysisUsage &AU) const override {
45 AU.setPreservesAll();
46 }
47 };
48
49 } // End anonymous namespace
50
51 INITIALIZE_PASS(AMDGPUAlwaysInline, "amdgpu-always-inline",
52 "AMDGPU Inline All Functions", false, false)
53
54 char AMDGPUAlwaysInline::ID = 0;
55
recursivelyVisitUsers(GlobalValue & GV,SmallPtrSetImpl<Function * > & FuncsToAlwaysInline)56 void AMDGPUAlwaysInline::recursivelyVisitUsers(
57 GlobalValue &GV,
58 SmallPtrSetImpl<Function *> &FuncsToAlwaysInline) {
59 SmallVector<User *, 16> Stack;
60
61 SmallPtrSet<const Value *, 8> Visited;
62
63 for (User *U : GV.users())
64 Stack.push_back(U);
65
66 while (!Stack.empty()) {
67 User *U = Stack.pop_back_val();
68 if (!Visited.insert(U).second)
69 continue;
70
71 if (Instruction *I = dyn_cast<Instruction>(U)) {
72 Function *F = I->getParent()->getParent();
73 if (!AMDGPU::isEntryFunctionCC(F->getCallingConv())) {
74 FuncsToAlwaysInline.insert(F);
75 Stack.push_back(F);
76 }
77
78 // No need to look at further users, but we do need to inline any callers.
79 continue;
80 }
81
82 for (User *UU : U->users())
83 Stack.push_back(UU);
84 }
85 }
86
runOnModule(Module & M)87 bool AMDGPUAlwaysInline::runOnModule(Module &M) {
88 std::vector<GlobalAlias*> AliasesToRemove;
89
90 SmallPtrSet<Function *, 8> FuncsToAlwaysInline;
91 SmallPtrSet<Function *, 8> FuncsToNoInline;
92
93 for (GlobalAlias &A : M.aliases()) {
94 if (Function* F = dyn_cast<Function>(A.getAliasee())) {
95 A.replaceAllUsesWith(F);
96 AliasesToRemove.push_back(&A);
97 }
98
99 // FIXME: If the aliasee isn't a function, it's some kind of constant expr
100 // cast that won't be inlined through.
101 }
102
103 if (GlobalOpt) {
104 for (GlobalAlias* A : AliasesToRemove) {
105 A->eraseFromParent();
106 }
107 }
108
109 // Always force inlining of any function that uses an LDS global address. This
110 // is something of a workaround because we don't have a way of supporting LDS
111 // objects defined in functions. LDS is always allocated by a kernel, and it
112 // is difficult to manage LDS usage if a function may be used by multiple
113 // kernels.
114 //
115 // OpenCL doesn't allow declaring LDS in non-kernels, so in practice this
116 // should only appear when IPO passes manages to move LDs defined in a kernel
117 // into a single user function.
118
119 for (GlobalVariable &GV : M.globals()) {
120 // TODO: Region address
121 unsigned AS = GV.getAddressSpace();
122 if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS)
123 continue;
124
125 recursivelyVisitUsers(GV, FuncsToAlwaysInline);
126 }
127
128 if (!AMDGPUTargetMachine::EnableFunctionCalls || StressCalls) {
129 auto IncompatAttr
130 = StressCalls ? Attribute::AlwaysInline : Attribute::NoInline;
131
132 for (Function &F : M) {
133 if (!F.isDeclaration() && !F.use_empty() &&
134 !F.hasFnAttribute(IncompatAttr)) {
135 if (StressCalls) {
136 if (!FuncsToAlwaysInline.count(&F))
137 FuncsToNoInline.insert(&F);
138 } else
139 FuncsToAlwaysInline.insert(&F);
140 }
141 }
142 }
143
144 for (Function *F : FuncsToAlwaysInline)
145 F->addFnAttr(Attribute::AlwaysInline);
146
147 for (Function *F : FuncsToNoInline)
148 F->addFnAttr(Attribute::NoInline);
149
150 return !FuncsToAlwaysInline.empty() || !FuncsToNoInline.empty();
151 }
152
createAMDGPUAlwaysInlinePass(bool GlobalOpt)153 ModulePass *llvm::createAMDGPUAlwaysInlinePass(bool GlobalOpt) {
154 return new AMDGPUAlwaysInline(GlobalOpt);
155 }
156
157