1 /*========================== begin_copyright_notice ============================
2 
3 Copyright (C) 2017-2021 Intel Corporation
4 
5 SPDX-License-Identifier: MIT
6 
7 ============================= end_copyright_notice ===========================*/
8 
9 #include "GenISAIntrinsics/GenIntrinsics.h"
10 #include "Compiler/CodeGenPublicEnums.h"
11 #include "Compiler/CodeGenPublic.h"
12 #include "Compiler/MetaDataUtilsWrapper.h"
13 #include "LLVM3DBuilder/MetadataBuilder.h"
14 #include "common/LLVMWarningsPush.hpp"
15 #include <llvm/IR/Module.h>
16 #include <llvm/Pass.h>
17 #include <llvmWrapper/IR/IRBuilder.h>
18 #include "common/LLVMWarningsPop.hpp"
19 #include "LLVM3DBuilder/BuiltinsFrontend.hpp"
20 
21 #include "common/FunctionUpgrader.h"
22 
23 using namespace IGC;
24 using namespace IGC::IGCMD;
25 using namespace llvm;
26 
27 class LinkMultiRateShader : public ModulePass
28 {
29 public:
LinkMultiRateShader()30     LinkMultiRateShader() : ModulePass(ID)
31     {
32 
33     }
34     static char ID;
35 
36     bool runOnModule(llvm::Module& M);
37 
getAnalysisUsage(llvm::AnalysisUsage & AU) const38     void getAnalysisUsage(llvm::AnalysisUsage& AU) const
39     {
40         AU.addRequired<MetaDataUtilsWrapper>();
41         AU.addRequired<CodeGenContextWrapper>();
42     }
43 
getPassName() const44     virtual llvm::StringRef getPassName() const
45     {
46         return "LinkMultirateShader";
47     }
48 private:
49     void Link(Function* pixelPhase, Function* samplePhase, Module& M);
50     Function* PatchSamplePhaseSignature(
51         Function* samplePhase,
52         SmallDenseMap<unsigned int, unsigned int, 16> & linkSignature);
53     void GetPixelPhaseOutput(
54         Function* pixelPhase,
55         SmallVector<Value*, 10> & outputs,
56         const SmallDenseMap<unsigned int, unsigned int, 16> & linkSignature);
57 };
58 
59 char LinkMultiRateShader::ID = 0;
60 
CreateLinkMultiRateShaderPass()61 Pass* CreateLinkMultiRateShaderPass()
62 {
63     return new LinkMultiRateShader();
64 }
65 
runOnModule(llvm::Module & M)66 bool LinkMultiRateShader::runOnModule(llvm::Module& M)
67 {
68     Function* pixelPhase = nullptr;
69     Function* samplePhase = nullptr;
70     NamedMDNode* pixelNode = M.getNamedMetadata("pixel_phase");
71     NamedMDNode* sampleNode = M.getNamedMetadata("sample_phase");
72     if (sampleNode == nullptr)
73     {
74         // if there is no sample phase no need to link
75         return false;
76     }
77     if (pixelNode)
78     {
79         pixelPhase = mdconst::dyn_extract<Function>(pixelNode->getOperand(0)->getOperand(0));
80     }
81     if (sampleNode)
82     {
83         samplePhase = mdconst::dyn_extract<Function>(sampleNode->getOperand(0)->getOperand(0));
84     }
85     if (pixelPhase == nullptr)
86     {
87         IRBuilder<> builder(M.getContext());
88         pixelPhase = Function::Create(FunctionType::get(builder.getVoidTy(), false),
89             GlobalValue::ExternalLinkage,
90             "multiRatePS",
91             &M);
92 
93         BasicBlock* bb = BasicBlock::Create(M.getContext(), "dummyBB", pixelPhase);
94         builder.SetInsertPoint(bb);
95         builder.CreateRetVoid();
96         MetadataBuilder metadatBuilder(&M);
97         metadatBuilder.SetShadingRate(pixelPhase, PSPHASE_PIXEL);
98     }
99     Link(pixelPhase, samplePhase, M);
100     sampleNode->eraseFromParent();
101 
102     MetaDataUtils* pMdUtils = getAnalysis<MetaDataUtilsWrapper>().getMetaDataUtils();
103     pMdUtils->clearFunctionsInfo();
104     IGCMetaDataHelper::addFunction(*pMdUtils, pixelPhase);
105     NamedMDNode* coarseNode = M.getNamedMetadata(NAMED_METADATA_COARSE_PHASE);
106     if (coarseNode != nullptr)
107     {
108         Function* coarsePhase =
109             llvm::mdconst::dyn_extract<Function>(coarseNode->getOperand(0)->getOperand(0));
110         IGCMetaDataHelper::addFunction(*pMdUtils, coarsePhase);
111     }
112 
113     return true;
114 }
115 
GetPixelPhaseOutput(Function * pixelPhase,SmallVector<Value *,10> & outputs,const SmallDenseMap<unsigned int,unsigned int,16> & linkSignature)116 void LinkMultiRateShader::GetPixelPhaseOutput(
117     Function* pixelPhase,
118     SmallVector<Value*, 10> & outputs,
119     const SmallDenseMap<unsigned int, unsigned int, 16> & linkSignature)
120 {
121     Function* phaseInput = GenISAIntrinsic::getDeclaration(
122         pixelPhase->getParent(),
123         GenISAIntrinsic::GenISA_PHASE_OUTPUT,
124         Type::getFloatTy(pixelPhase->getContext()));
125     Function* phaseHalfInput = GenISAIntrinsic::getDeclaration(
126         pixelPhase->getParent(),
127         GenISAIntrinsic::GenISA_PHASE_OUTPUT,
128         Type::getHalfTy(pixelPhase->getContext()));
129     SmallVector<Instruction*, 10> phaseIntrinsics;
130     auto getPhaseInputs = [&phaseIntrinsics, &outputs, &linkSignature, pixelPhase](User* user)
131     {
132         if (Instruction * inst = dyn_cast<Instruction>(user))
133         {
134             if (inst->getParent()->getParent() == pixelPhase)
135             {
136                 unsigned int index = (unsigned int)cast<ConstantInt>(inst->getOperand(1))->getZExtValue();
137                 auto it = linkSignature.find(index);
138                 if (it != linkSignature.end())
139                 {
140                     unsigned int location = it->second;
141                     outputs[location + 1] = inst->getOperand(0);
142                 }
143                 phaseIntrinsics.push_back(inst);
144             }
145         }
146     };
147     for (auto* user : phaseInput->users())
148     {
149         getPhaseInputs(user);
150     }
151     for (auto* user : phaseHalfInput->users())
152     {
153         getPhaseInputs(user);
154     }
155     for (auto phaseIntrinsic : phaseIntrinsics)
156     {
157         phaseIntrinsic->eraseFromParent();
158     }
159 }
160 
Link(Function * pixelPhase,Function * samplePhase,Module & M)161 void LinkMultiRateShader::Link(Function* pixelPhase, Function* samplePhase, Module& M)
162 {
163     IGCLLVM::IRBuilder<> builder(M.getContext());
164     SmallDenseMap<unsigned int, unsigned int, 16> linkSignature;
165     Function* samplePhasePatched = PatchSamplePhaseSignature(samplePhase, linkSignature);
166 
167     BasicBlock* returnBB = &pixelPhase->getBasicBlockList().back();
168     BasicBlock* loopHeader = BasicBlock::Create(M.getContext(), "SampleLoopHeader", pixelPhase);
169     BasicBlock* loopBlock = BasicBlock::Create(M.getContext(), "LoopBlock", pixelPhase);
170     BasicBlock* newRet = BasicBlock::Create(M.getContext(), "ret", pixelPhase);
171     returnBB->getTerminator()->eraseFromParent();
172     builder.SetInsertPoint(returnBB);
173     builder.CreateBr(loopHeader);
174     builder.SetInsertPoint(&(*pixelPhase->getEntryBlock().begin()));
175     Value* loopCounterPtr = builder.CreateAlloca(builder.getInt32Ty());
176 
177     // Initialize the loop counter in the loop header
178     builder.SetInsertPoint(loopHeader);
179     unsigned int addrSpaceRsc = IGC::EncodeAS4GFXResource(*builder.getInt32(0), RENDER_TARGET);
180     PointerType* ptrTy = llvm::PointerType::get(builder.getInt32Ty(), addrSpaceRsc);
181     Value* pRsrc = ConstantPointerNull::get(ptrTy);
182     Function* sampleInfoptr = GenISAIntrinsic::getDeclaration(&M, GenISAIntrinsic::GenISA_sampleinfoptr, pRsrc->getType());
183     Value* sampleInfoValue = builder.CreateCall(sampleInfoptr, pRsrc);
184 
185     Value* msaaRate = builder.CreateExtractElement(sampleInfoValue, builder.getInt32(0));
186     builder.CreateStore(msaaRate, loopCounterPtr);
187     builder.CreateBr(loopBlock);
188 
189     //Loop block decrement the counter and call the sample phase
190     builder.SetInsertPoint(loopBlock);
191     Value* counter = builder.CreateLoad(loopCounterPtr);
192     counter = builder.CreateAdd(counter, builder.getInt32(-1));
193     SmallVector<Value*, 10> inputs(linkSignature.size() + 1, UndefValue::get(builder.getFloatTy()));
194     inputs[0] = counter;
195     GetPixelPhaseOutput(pixelPhase, inputs, linkSignature);
196     builder.CreateStore(counter, loopCounterPtr);
197 
198     builder.CreateCall(samplePhasePatched, inputs);
199     Value* cond = builder.CreateICmp(CmpInst::Predicate::ICMP_EQ, counter, builder.getInt32(0));
200     builder.CreateCondBr(cond, newRet, loopBlock);
201 
202     // The new return block is empty
203     builder.SetInsertPoint(newRet);
204     builder.CreateRetVoid();
205 }
206 
PatchSamplePhaseSignature(Function * samplePhase,SmallDenseMap<unsigned int,unsigned int,16> & linkSignature)207 Function* LinkMultiRateShader::PatchSamplePhaseSignature(
208     Function* samplePhase,
209     SmallDenseMap<unsigned int, unsigned int, 16> & linkSignature)
210 {
211     SmallDenseMap<unsigned int, Value*, 16> linkArguments;
212     Module* M = samplePhase->getParent();
213     CodeGenContext* ctx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
214     LLVM3DBuilder<> builder(samplePhase->getContext(), ctx->platform.getPlatformInfo());
215 
216     FunctionUpgrader FuncUpgrader;
217 
218     FuncUpgrader.SetFunctionToUpgrade(samplePhase);
219 
220     // find all the phase inputs
221     Function* phaseInput = GenISAIntrinsic::getDeclaration(
222         M,
223         GenISAIntrinsic::GenISA_PHASE_INPUT,
224         builder.getFloatTy());
225     Function* phaseHalfInput = GenISAIntrinsic::getDeclaration(
226         M,
227         GenISAIntrinsic::GenISA_PHASE_INPUT,
228         builder.getHalfTy());
229     Value* sampleIndex = FuncUpgrader.AddArgument("", builder.getInt32Ty());
230 
231     SmallVector<Type*, 10> funcSignature;
232     funcSignature.push_back(builder.getInt32Ty());
233     unsigned int inputLocation = 0;
234     auto getLinkInfo = [samplePhase, &linkArguments, &linkSignature, &FuncUpgrader, &funcSignature, &inputLocation](User* user)
235     {
236         if (Instruction * inst = dyn_cast<Instruction>(user))
237         {
238             if (inst->getParent()->getParent() == samplePhase)
239             {
240                 int index = (int)cast<ConstantInt>(inst->getOperand(0))->getZExtValue();
241                 Value* arg = nullptr;
242                 auto it = linkArguments.find(index);
243                 if (it != linkArguments.end())
244                 {
245                     arg = it->second;
246                 }
247                 else
248                 {
249                     arg = FuncUpgrader.AddArgument("", inst->getType());
250                     funcSignature.push_back(inst->getType());
251                     linkArguments[index] = arg;
252                     linkSignature[index] = inputLocation++;
253                 }
254                 inst->replaceAllUsesWith(arg);
255             }
256         }
257     };
258     for (auto* user : phaseInput->users())
259     {
260         getLinkInfo(user);
261     }
262     for (auto* user : phaseHalfInput->users())
263     {
264         getLinkInfo(user);
265     }
266 
267     Function* newSamplePhase = FuncUpgrader.RebuildFunction();
268 
269     samplePhase->eraseFromParent();
270     samplePhase = newSamplePhase;
271     samplePhase->addFnAttr(llvm::Attribute::AlwaysInline);
272     sampleIndex = FuncUpgrader.GetArgumentFromRebuild(sampleIndex);
273 
274     FuncUpgrader.Clean();
275 
276     // Replace sample index intrinsic
277     Function* SGV = GenISAIntrinsic::getDeclaration(
278         M,
279         GenISAIntrinsic::GenISA_DCL_SystemValue,
280         builder.getFloatTy());
281     for (auto I = SGV->user_begin(), E = SGV->user_end(); I != E; ++I)
282     {
283         if (Instruction * inst = dyn_cast<Instruction>(*I))
284         {
285             if (inst->getParent()->getParent() == samplePhase)
286             {
287                 builder.SetInsertPoint(inst->getNextNode());
288                 SGVUsage usage = (SGVUsage)cast<ConstantInt>(inst->getOperand(0))->getZExtValue();
289                 if (usage == SAMPLEINDEX)
290                 {
291                     Value* sampleIndexCast = builder.CreateBitCast(sampleIndex, builder.getFloatTy());
292                     inst->replaceAllUsesWith(sampleIndexCast);
293                 }
294             }
295         }
296     }
297 
298     Function* input = GenISAIntrinsic::getDeclaration(
299         M,
300         GenISAIntrinsic::GenISA_DCL_inputVec,
301         builder.getFloatTy());
302     for (auto I = input->user_begin(), E = input->user_end(); I != E; ++I)
303     {
304         if (Instruction * inst = dyn_cast<Instruction>(*I))
305         {
306             if (inst->getParent()->getParent() == samplePhase)
307             {
308                 e_interpolation interpolationMode =
309                     (e_interpolation)llvm::cast<llvm::ConstantInt>(inst->getOperand(1))->getZExtValue();
310                 bool perspective = true;
311                 if (interpolationMode == EINTERPOLATION_LINEARNOPERSPECTIVE ||
312                     interpolationMode == EINTERPOLATION_LINEARNOPERSPECTIVECENTROID ||
313                     interpolationMode == EINTERPOLATION_LINEARNOPERSPECTIVESAMPLE)
314                 {
315                     perspective = false;
316                 }
317 
318                 builder.SetInsertPoint(inst);
319                 Value* InputAtSamplePos = builder.CreateEvalSampleIndex(
320                     inst->getOperand(0), sampleIndex, builder.getInt1(perspective));
321                 inst->replaceAllUsesWith(InputAtSamplePos);
322             }
323         }
324     }
325 
326     Function* rtRead = GenISAIntrinsic::getDeclaration(
327         M,
328         GenISAIntrinsic::GenISA_RenderTargetRead);
329     for (auto I = rtRead->user_begin(), E = rtRead->user_end(); I != E; ++I)
330     {
331         if (Instruction * inst = dyn_cast<Instruction>(*I))
332         {
333             if (inst->getParent()->getParent() == samplePhase)
334             {
335                 builder.SetInsertPoint(inst->getNextNode());
336 
337                 Value* rtReadSampleFreqargs[] = {
338                     inst->getOperand(0),
339                     sampleIndex
340                 };
341                 Function* pRTSampleFreqIntrinsic = llvm::GenISAIntrinsic::getDeclaration(
342                     M,
343                     llvm::GenISAIntrinsic::GenISA_RenderTargetReadSampleFreq
344                 );
345                 CallInst* rtReadSampleFreqInst = builder.CreateCall(
346                     pRTSampleFreqIntrinsic, rtReadSampleFreqargs);
347 
348                 inst->replaceAllUsesWith(rtReadSampleFreqInst);
349             }
350         }
351     }
352 
353     Function* rtWrite = GenISAIntrinsic::getDeclaration(
354         M,
355         GenISAIntrinsic::GenISA_RTWrite,
356         builder.getFloatTy());
357     for (auto I = rtWrite->user_begin(), E = rtWrite->user_end(); I != E; ++I)
358     {
359         if (RTWritIntrinsic * inst = dyn_cast<RTWritIntrinsic>(*I))
360         {
361             if (inst->getParent()->getParent() == samplePhase)
362             {
363                 inst->setPerSample();
364                 inst->setSampleIndex(sampleIndex);
365             }
366         }
367     }
368 
369     Function* dualBlend = GenISAIntrinsic::getDeclaration(
370         M,
371         GenISAIntrinsic::GenISA_RTDualBlendSource,
372         builder.getFloatTy());
373     for (auto I = dualBlend->user_begin(), E = dualBlend->user_end(); I != E; ++I)
374     {
375         if (RTDualBlendSourceIntrinsic * inst = dyn_cast<RTDualBlendSourceIntrinsic>(*I))
376         {
377             if (inst->getParent()->getParent() == samplePhase)
378             {
379                 inst->setPerSample();
380                 inst->setSampleIndex(sampleIndex);
381             }
382         }
383     }
384 
385     return samplePhase;
386 }
387