1 //===-- AMDGPULowerIntrinsics.cpp -----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "AMDGPU.h"
10 #include "AMDGPUSubtarget.h"
11 #include "llvm/Analysis/TargetTransformInfo.h"
12 #include "llvm/CodeGen/TargetPassConfig.h"
13 #include "llvm/IR/Constants.h"
14 #include "llvm/IR/Instructions.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/IntrinsicsAMDGPU.h"
17 #include "llvm/IR/IntrinsicsR600.h"
18 #include "llvm/IR/Module.h"
19 #include "llvm/Support/CommandLine.h"
20 #include "llvm/Target/TargetMachine.h"
21 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
22
23 #define DEBUG_TYPE "amdgpu-lower-intrinsics"
24
25 using namespace llvm;
26
27 namespace {
28
29 static int MaxStaticSize;
30
31 static cl::opt<int, true> MemIntrinsicExpandSizeThresholdOpt(
32 "amdgpu-mem-intrinsic-expand-size",
33 cl::desc("Set minimum mem intrinsic size to expand in IR"),
34 cl::location(MaxStaticSize),
35 cl::init(1024),
36 cl::Hidden);
37
38
39 class AMDGPULowerIntrinsics : public ModulePass {
40 private:
41 bool makeLIDRangeMetadata(Function &F) const;
42
43 public:
44 static char ID;
45
AMDGPULowerIntrinsics()46 AMDGPULowerIntrinsics() : ModulePass(ID) {}
47
48 bool runOnModule(Module &M) override;
49 bool expandMemIntrinsicUses(Function &F);
getPassName() const50 StringRef getPassName() const override {
51 return "AMDGPU Lower Intrinsics";
52 }
53
getAnalysisUsage(AnalysisUsage & AU) const54 void getAnalysisUsage(AnalysisUsage &AU) const override {
55 AU.addRequired<TargetTransformInfoWrapperPass>();
56 }
57 };
58
59 }
60
61 char AMDGPULowerIntrinsics::ID = 0;
62
63 char &llvm::AMDGPULowerIntrinsicsID = AMDGPULowerIntrinsics::ID;
64
65 INITIALIZE_PASS(AMDGPULowerIntrinsics, DEBUG_TYPE, "Lower intrinsics", false,
66 false)
67
68 // TODO: Should refine based on estimated number of accesses (e.g. does it
69 // require splitting based on alignment)
shouldExpandOperationWithSize(Value * Size)70 static bool shouldExpandOperationWithSize(Value *Size) {
71 ConstantInt *CI = dyn_cast<ConstantInt>(Size);
72 return !CI || (CI->getSExtValue() > MaxStaticSize);
73 }
74
expandMemIntrinsicUses(Function & F)75 bool AMDGPULowerIntrinsics::expandMemIntrinsicUses(Function &F) {
76 Intrinsic::ID ID = F.getIntrinsicID();
77 bool Changed = false;
78
79 for (auto I = F.user_begin(), E = F.user_end(); I != E;) {
80 Instruction *Inst = cast<Instruction>(*I);
81 ++I;
82
83 switch (ID) {
84 case Intrinsic::memcpy: {
85 auto *Memcpy = cast<MemCpyInst>(Inst);
86 if (shouldExpandOperationWithSize(Memcpy->getLength())) {
87 Function *ParentFunc = Memcpy->getParent()->getParent();
88 const TargetTransformInfo &TTI =
89 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*ParentFunc);
90 expandMemCpyAsLoop(Memcpy, TTI);
91 Changed = true;
92 Memcpy->eraseFromParent();
93 }
94
95 break;
96 }
97 case Intrinsic::memmove: {
98 auto *Memmove = cast<MemMoveInst>(Inst);
99 if (shouldExpandOperationWithSize(Memmove->getLength())) {
100 expandMemMoveAsLoop(Memmove);
101 Changed = true;
102 Memmove->eraseFromParent();
103 }
104
105 break;
106 }
107 case Intrinsic::memset: {
108 auto *Memset = cast<MemSetInst>(Inst);
109 if (shouldExpandOperationWithSize(Memset->getLength())) {
110 expandMemSetAsLoop(Memset);
111 Changed = true;
112 Memset->eraseFromParent();
113 }
114
115 break;
116 }
117 default:
118 break;
119 }
120 }
121
122 return Changed;
123 }
124
makeLIDRangeMetadata(Function & F) const125 bool AMDGPULowerIntrinsics::makeLIDRangeMetadata(Function &F) const {
126 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
127 if (!TPC)
128 return false;
129
130 const TargetMachine &TM = TPC->getTM<TargetMachine>();
131 bool Changed = false;
132
133 for (auto *U : F.users()) {
134 auto *CI = dyn_cast<CallInst>(U);
135 if (!CI)
136 continue;
137
138 Function *Caller = CI->getParent()->getParent();
139 const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, *Caller);
140 Changed |= ST.makeLIDRangeMetadata(CI);
141 }
142 return Changed;
143 }
144
runOnModule(Module & M)145 bool AMDGPULowerIntrinsics::runOnModule(Module &M) {
146 bool Changed = false;
147
148 for (Function &F : M) {
149 if (!F.isDeclaration())
150 continue;
151
152 switch (F.getIntrinsicID()) {
153 case Intrinsic::memcpy:
154 case Intrinsic::memmove:
155 case Intrinsic::memset:
156 if (expandMemIntrinsicUses(F))
157 Changed = true;
158 break;
159
160 case Intrinsic::amdgcn_workitem_id_x:
161 case Intrinsic::r600_read_tidig_x:
162 case Intrinsic::amdgcn_workitem_id_y:
163 case Intrinsic::r600_read_tidig_y:
164 case Intrinsic::amdgcn_workitem_id_z:
165 case Intrinsic::r600_read_tidig_z:
166 case Intrinsic::r600_read_local_size_x:
167 case Intrinsic::r600_read_local_size_y:
168 case Intrinsic::r600_read_local_size_z:
169 Changed |= makeLIDRangeMetadata(F);
170 break;
171
172 default:
173 break;
174 }
175 }
176
177 return Changed;
178 }
179
createAMDGPULowerIntrinsicsPass()180 ModulePass *llvm::createAMDGPULowerIntrinsicsPass() {
181 return new AMDGPULowerIntrinsics();
182 }
183