1 /*========================== begin_copyright_notice ============================
2
3 Copyright (C) 2017-2021 Intel Corporation
4
5 SPDX-License-Identifier: MIT
6
7 ============================= end_copyright_notice ===========================*/
8
9 #include "GenISAIntrinsics/GenIntrinsics.h"
10 #include "Compiler/CodeGenPublicEnums.h"
11 #include "Compiler/CodeGenPublic.h"
12 #include "Compiler/MetaDataUtilsWrapper.h"
13 #include "LLVM3DBuilder/MetadataBuilder.h"
14 #include "common/LLVMWarningsPush.hpp"
15 #include <llvm/IR/Module.h>
16 #include <llvm/Pass.h>
17 #include <llvmWrapper/IR/IRBuilder.h>
18 #include "common/LLVMWarningsPop.hpp"
19 #include "LLVM3DBuilder/BuiltinsFrontend.hpp"
20
21 #include "common/FunctionUpgrader.h"
22
23 using namespace IGC;
24 using namespace IGC::IGCMD;
25 using namespace llvm;
26
27 class LinkMultiRateShader : public ModulePass
28 {
29 public:
LinkMultiRateShader()30 LinkMultiRateShader() : ModulePass(ID)
31 {
32
33 }
34 static char ID;
35
36 bool runOnModule(llvm::Module& M);
37
getAnalysisUsage(llvm::AnalysisUsage & AU) const38 void getAnalysisUsage(llvm::AnalysisUsage& AU) const
39 {
40 AU.addRequired<MetaDataUtilsWrapper>();
41 AU.addRequired<CodeGenContextWrapper>();
42 }
43
getPassName() const44 virtual llvm::StringRef getPassName() const
45 {
46 return "LinkMultirateShader";
47 }
48 private:
49 void Link(Function* pixelPhase, Function* samplePhase, Module& M);
50 Function* PatchSamplePhaseSignature(
51 Function* samplePhase,
52 SmallDenseMap<unsigned int, unsigned int, 16> & linkSignature);
53 void GetPixelPhaseOutput(
54 Function* pixelPhase,
55 SmallVector<Value*, 10> & outputs,
56 const SmallDenseMap<unsigned int, unsigned int, 16> & linkSignature);
57 };
58
59 char LinkMultiRateShader::ID = 0;
60
CreateLinkMultiRateShaderPass()61 Pass* CreateLinkMultiRateShaderPass()
62 {
63 return new LinkMultiRateShader();
64 }
65
runOnModule(llvm::Module & M)66 bool LinkMultiRateShader::runOnModule(llvm::Module& M)
67 {
68 Function* pixelPhase = nullptr;
69 Function* samplePhase = nullptr;
70 NamedMDNode* pixelNode = M.getNamedMetadata("pixel_phase");
71 NamedMDNode* sampleNode = M.getNamedMetadata("sample_phase");
72 if (sampleNode == nullptr)
73 {
74 // if there is no sample phase no need to link
75 return false;
76 }
77 if (pixelNode)
78 {
79 pixelPhase = mdconst::dyn_extract<Function>(pixelNode->getOperand(0)->getOperand(0));
80 }
81 if (sampleNode)
82 {
83 samplePhase = mdconst::dyn_extract<Function>(sampleNode->getOperand(0)->getOperand(0));
84 }
85 if (pixelPhase == nullptr)
86 {
87 IRBuilder<> builder(M.getContext());
88 pixelPhase = Function::Create(FunctionType::get(builder.getVoidTy(), false),
89 GlobalValue::ExternalLinkage,
90 "multiRatePS",
91 &M);
92
93 BasicBlock* bb = BasicBlock::Create(M.getContext(), "dummyBB", pixelPhase);
94 builder.SetInsertPoint(bb);
95 builder.CreateRetVoid();
96 MetadataBuilder metadatBuilder(&M);
97 metadatBuilder.SetShadingRate(pixelPhase, PSPHASE_PIXEL);
98 }
99 Link(pixelPhase, samplePhase, M);
100 sampleNode->eraseFromParent();
101
102 MetaDataUtils* pMdUtils = getAnalysis<MetaDataUtilsWrapper>().getMetaDataUtils();
103 pMdUtils->clearFunctionsInfo();
104 IGCMetaDataHelper::addFunction(*pMdUtils, pixelPhase);
105 NamedMDNode* coarseNode = M.getNamedMetadata(NAMED_METADATA_COARSE_PHASE);
106 if (coarseNode != nullptr)
107 {
108 Function* coarsePhase =
109 llvm::mdconst::dyn_extract<Function>(coarseNode->getOperand(0)->getOperand(0));
110 IGCMetaDataHelper::addFunction(*pMdUtils, coarsePhase);
111 }
112
113 return true;
114 }
115
GetPixelPhaseOutput(Function * pixelPhase,SmallVector<Value *,10> & outputs,const SmallDenseMap<unsigned int,unsigned int,16> & linkSignature)116 void LinkMultiRateShader::GetPixelPhaseOutput(
117 Function* pixelPhase,
118 SmallVector<Value*, 10> & outputs,
119 const SmallDenseMap<unsigned int, unsigned int, 16> & linkSignature)
120 {
121 Function* phaseInput = GenISAIntrinsic::getDeclaration(
122 pixelPhase->getParent(),
123 GenISAIntrinsic::GenISA_PHASE_OUTPUT,
124 Type::getFloatTy(pixelPhase->getContext()));
125 Function* phaseHalfInput = GenISAIntrinsic::getDeclaration(
126 pixelPhase->getParent(),
127 GenISAIntrinsic::GenISA_PHASE_OUTPUT,
128 Type::getHalfTy(pixelPhase->getContext()));
129 SmallVector<Instruction*, 10> phaseIntrinsics;
130 auto getPhaseInputs = [&phaseIntrinsics, &outputs, &linkSignature, pixelPhase](User* user)
131 {
132 if (Instruction * inst = dyn_cast<Instruction>(user))
133 {
134 if (inst->getParent()->getParent() == pixelPhase)
135 {
136 unsigned int index = (unsigned int)cast<ConstantInt>(inst->getOperand(1))->getZExtValue();
137 auto it = linkSignature.find(index);
138 if (it != linkSignature.end())
139 {
140 unsigned int location = it->second;
141 outputs[location + 1] = inst->getOperand(0);
142 }
143 phaseIntrinsics.push_back(inst);
144 }
145 }
146 };
147 for (auto* user : phaseInput->users())
148 {
149 getPhaseInputs(user);
150 }
151 for (auto* user : phaseHalfInput->users())
152 {
153 getPhaseInputs(user);
154 }
155 for (auto phaseIntrinsic : phaseIntrinsics)
156 {
157 phaseIntrinsic->eraseFromParent();
158 }
159 }
160
Link(Function * pixelPhase,Function * samplePhase,Module & M)161 void LinkMultiRateShader::Link(Function* pixelPhase, Function* samplePhase, Module& M)
162 {
163 IGCLLVM::IRBuilder<> builder(M.getContext());
164 SmallDenseMap<unsigned int, unsigned int, 16> linkSignature;
165 Function* samplePhasePatched = PatchSamplePhaseSignature(samplePhase, linkSignature);
166
167 BasicBlock* returnBB = &pixelPhase->getBasicBlockList().back();
168 BasicBlock* loopHeader = BasicBlock::Create(M.getContext(), "SampleLoopHeader", pixelPhase);
169 BasicBlock* loopBlock = BasicBlock::Create(M.getContext(), "LoopBlock", pixelPhase);
170 BasicBlock* newRet = BasicBlock::Create(M.getContext(), "ret", pixelPhase);
171 returnBB->getTerminator()->eraseFromParent();
172 builder.SetInsertPoint(returnBB);
173 builder.CreateBr(loopHeader);
174 builder.SetInsertPoint(&(*pixelPhase->getEntryBlock().begin()));
175 Value* loopCounterPtr = builder.CreateAlloca(builder.getInt32Ty());
176
177 // Initialize the loop counter in the loop header
178 builder.SetInsertPoint(loopHeader);
179 unsigned int addrSpaceRsc = IGC::EncodeAS4GFXResource(*builder.getInt32(0), RENDER_TARGET);
180 PointerType* ptrTy = llvm::PointerType::get(builder.getInt32Ty(), addrSpaceRsc);
181 Value* pRsrc = ConstantPointerNull::get(ptrTy);
182 Function* sampleInfoptr = GenISAIntrinsic::getDeclaration(&M, GenISAIntrinsic::GenISA_sampleinfoptr, pRsrc->getType());
183 Value* sampleInfoValue = builder.CreateCall(sampleInfoptr, pRsrc);
184
185 Value* msaaRate = builder.CreateExtractElement(sampleInfoValue, builder.getInt32(0));
186 builder.CreateStore(msaaRate, loopCounterPtr);
187 builder.CreateBr(loopBlock);
188
189 //Loop block decrement the counter and call the sample phase
190 builder.SetInsertPoint(loopBlock);
191 Value* counter = builder.CreateLoad(loopCounterPtr);
192 counter = builder.CreateAdd(counter, builder.getInt32(-1));
193 SmallVector<Value*, 10> inputs(linkSignature.size() + 1, UndefValue::get(builder.getFloatTy()));
194 inputs[0] = counter;
195 GetPixelPhaseOutput(pixelPhase, inputs, linkSignature);
196 builder.CreateStore(counter, loopCounterPtr);
197
198 builder.CreateCall(samplePhasePatched, inputs);
199 Value* cond = builder.CreateICmp(CmpInst::Predicate::ICMP_EQ, counter, builder.getInt32(0));
200 builder.CreateCondBr(cond, newRet, loopBlock);
201
202 // The new return block is empty
203 builder.SetInsertPoint(newRet);
204 builder.CreateRetVoid();
205 }
206
PatchSamplePhaseSignature(Function * samplePhase,SmallDenseMap<unsigned int,unsigned int,16> & linkSignature)207 Function* LinkMultiRateShader::PatchSamplePhaseSignature(
208 Function* samplePhase,
209 SmallDenseMap<unsigned int, unsigned int, 16> & linkSignature)
210 {
211 SmallDenseMap<unsigned int, Value*, 16> linkArguments;
212 Module* M = samplePhase->getParent();
213 CodeGenContext* ctx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
214 LLVM3DBuilder<> builder(samplePhase->getContext(), ctx->platform.getPlatformInfo());
215
216 FunctionUpgrader FuncUpgrader;
217
218 FuncUpgrader.SetFunctionToUpgrade(samplePhase);
219
220 // find all the phase inputs
221 Function* phaseInput = GenISAIntrinsic::getDeclaration(
222 M,
223 GenISAIntrinsic::GenISA_PHASE_INPUT,
224 builder.getFloatTy());
225 Function* phaseHalfInput = GenISAIntrinsic::getDeclaration(
226 M,
227 GenISAIntrinsic::GenISA_PHASE_INPUT,
228 builder.getHalfTy());
229 Value* sampleIndex = FuncUpgrader.AddArgument("", builder.getInt32Ty());
230
231 SmallVector<Type*, 10> funcSignature;
232 funcSignature.push_back(builder.getInt32Ty());
233 unsigned int inputLocation = 0;
234 auto getLinkInfo = [samplePhase, &linkArguments, &linkSignature, &FuncUpgrader, &funcSignature, &inputLocation](User* user)
235 {
236 if (Instruction * inst = dyn_cast<Instruction>(user))
237 {
238 if (inst->getParent()->getParent() == samplePhase)
239 {
240 int index = (int)cast<ConstantInt>(inst->getOperand(0))->getZExtValue();
241 Value* arg = nullptr;
242 auto it = linkArguments.find(index);
243 if (it != linkArguments.end())
244 {
245 arg = it->second;
246 }
247 else
248 {
249 arg = FuncUpgrader.AddArgument("", inst->getType());
250 funcSignature.push_back(inst->getType());
251 linkArguments[index] = arg;
252 linkSignature[index] = inputLocation++;
253 }
254 inst->replaceAllUsesWith(arg);
255 }
256 }
257 };
258 for (auto* user : phaseInput->users())
259 {
260 getLinkInfo(user);
261 }
262 for (auto* user : phaseHalfInput->users())
263 {
264 getLinkInfo(user);
265 }
266
267 Function* newSamplePhase = FuncUpgrader.RebuildFunction();
268
269 samplePhase->eraseFromParent();
270 samplePhase = newSamplePhase;
271 samplePhase->addFnAttr(llvm::Attribute::AlwaysInline);
272 sampleIndex = FuncUpgrader.GetArgumentFromRebuild(sampleIndex);
273
274 FuncUpgrader.Clean();
275
276 // Replace sample index intrinsic
277 Function* SGV = GenISAIntrinsic::getDeclaration(
278 M,
279 GenISAIntrinsic::GenISA_DCL_SystemValue,
280 builder.getFloatTy());
281 for (auto I = SGV->user_begin(), E = SGV->user_end(); I != E; ++I)
282 {
283 if (Instruction * inst = dyn_cast<Instruction>(*I))
284 {
285 if (inst->getParent()->getParent() == samplePhase)
286 {
287 builder.SetInsertPoint(inst->getNextNode());
288 SGVUsage usage = (SGVUsage)cast<ConstantInt>(inst->getOperand(0))->getZExtValue();
289 if (usage == SAMPLEINDEX)
290 {
291 Value* sampleIndexCast = builder.CreateBitCast(sampleIndex, builder.getFloatTy());
292 inst->replaceAllUsesWith(sampleIndexCast);
293 }
294 }
295 }
296 }
297
298 Function* input = GenISAIntrinsic::getDeclaration(
299 M,
300 GenISAIntrinsic::GenISA_DCL_inputVec,
301 builder.getFloatTy());
302 for (auto I = input->user_begin(), E = input->user_end(); I != E; ++I)
303 {
304 if (Instruction * inst = dyn_cast<Instruction>(*I))
305 {
306 if (inst->getParent()->getParent() == samplePhase)
307 {
308 e_interpolation interpolationMode =
309 (e_interpolation)llvm::cast<llvm::ConstantInt>(inst->getOperand(1))->getZExtValue();
310 bool perspective = true;
311 if (interpolationMode == EINTERPOLATION_LINEARNOPERSPECTIVE ||
312 interpolationMode == EINTERPOLATION_LINEARNOPERSPECTIVECENTROID ||
313 interpolationMode == EINTERPOLATION_LINEARNOPERSPECTIVESAMPLE)
314 {
315 perspective = false;
316 }
317
318 builder.SetInsertPoint(inst);
319 Value* InputAtSamplePos = builder.CreateEvalSampleIndex(
320 inst->getOperand(0), sampleIndex, builder.getInt1(perspective));
321 inst->replaceAllUsesWith(InputAtSamplePos);
322 }
323 }
324 }
325
326 Function* rtRead = GenISAIntrinsic::getDeclaration(
327 M,
328 GenISAIntrinsic::GenISA_RenderTargetRead);
329 for (auto I = rtRead->user_begin(), E = rtRead->user_end(); I != E; ++I)
330 {
331 if (Instruction * inst = dyn_cast<Instruction>(*I))
332 {
333 if (inst->getParent()->getParent() == samplePhase)
334 {
335 builder.SetInsertPoint(inst->getNextNode());
336
337 Value* rtReadSampleFreqargs[] = {
338 inst->getOperand(0),
339 sampleIndex
340 };
341 Function* pRTSampleFreqIntrinsic = llvm::GenISAIntrinsic::getDeclaration(
342 M,
343 llvm::GenISAIntrinsic::GenISA_RenderTargetReadSampleFreq
344 );
345 CallInst* rtReadSampleFreqInst = builder.CreateCall(
346 pRTSampleFreqIntrinsic, rtReadSampleFreqargs);
347
348 inst->replaceAllUsesWith(rtReadSampleFreqInst);
349 }
350 }
351 }
352
353 Function* rtWrite = GenISAIntrinsic::getDeclaration(
354 M,
355 GenISAIntrinsic::GenISA_RTWrite,
356 builder.getFloatTy());
357 for (auto I = rtWrite->user_begin(), E = rtWrite->user_end(); I != E; ++I)
358 {
359 if (RTWritIntrinsic * inst = dyn_cast<RTWritIntrinsic>(*I))
360 {
361 if (inst->getParent()->getParent() == samplePhase)
362 {
363 inst->setPerSample();
364 inst->setSampleIndex(sampleIndex);
365 }
366 }
367 }
368
369 Function* dualBlend = GenISAIntrinsic::getDeclaration(
370 M,
371 GenISAIntrinsic::GenISA_RTDualBlendSource,
372 builder.getFloatTy());
373 for (auto I = dualBlend->user_begin(), E = dualBlend->user_end(); I != E; ++I)
374 {
375 if (RTDualBlendSourceIntrinsic * inst = dyn_cast<RTDualBlendSourceIntrinsic>(*I))
376 {
377 if (inst->getParent()->getParent() == samplePhase)
378 {
379 inst->setPerSample();
380 inst->setSampleIndex(sampleIndex);
381 }
382 }
383 }
384
385 return samplePhase;
386 }
387