1 /*========================== begin_copyright_notice ============================
2 
3 Copyright (C) 2017-2021 Intel Corporation
4 
5 SPDX-License-Identifier: MIT
6 
7 ============================= end_copyright_notice ===========================*/
8 
9 #pragma once
10 #include "Compiler/CISACodeGen/CISACodeGen.h"
11 #include "Compiler/CISACodeGen/helper.h"
12 #include "Compiler/CISACodeGen/LiveVars.hpp"
13 #include "Compiler/MetaDataUtilsWrapper.h"
14 #include "Compiler/CISACodeGen/WIAnalysis.hpp"
15 #include "Compiler/CISACodeGen/PositionDepAnalysis.hpp"
16 
17 #include "common/LLVMWarningsPush.hpp"
18 #include <llvm/IR/InstVisitor.h>
19 #include <llvm/ADT/DenseSet.h>
20 #include <llvm/ADT/MapVector.h>
21 #include <llvm/Analysis/LoopInfo.h>
22 #include <llvm/IR/DataLayout.h>
23 #include "common/LLVMWarningsPop.hpp"
24 
25 namespace llvm
26 {
27     class Value;
28     class PHINode;
29     class Function;
30     class BasicBlock;
31     class GenIntrinsicInst;
32 }
33 
34 namespace IGC
35 {
36 
37     struct SSource
38     {
39         llvm::Value* value;
40         VISA_Type type;      // Set if source needs to be bit-casted to different type.
41         e_modifier mod;
42         int elementOffset;
43         int SIMDOffset;
44         unsigned char region[3];
45         bool region_set;
46         e_instance instance;
47         bool fromConstantPool;
48 
SSourceIGC::SSource49         SSource() : value(nullptr), type(VISA_Type::ISA_TYPE_NUM), mod(EMOD_NONE), elementOffset(0), SIMDOffset(0), region_set(false), instance(EINSTANCE_UNSPECIFIED), fromConstantPool(false)
50         {
51         }
52     };
53 
54     struct DstModifier
55     {
56         SSource* flag;
57         bool invertFlag;
58         bool sat;
59         e_predMode predMode;
DstModifierIGC::DstModifier60         DstModifier() :
61             flag(nullptr), invertFlag(false)
62             , sat(false), predMode(EPRED_NORMAL)
63         {
64         }
65     };
66     class EmitPass;
67     class CShader;
68 
69     struct Pattern
70     {
71         virtual void Emit(EmitPass* pass, const DstModifier& modifier) = 0;
~PatternIGC::Pattern72         virtual ~Pattern() {}
73 
74         // Does the pattern allow destination saturation?
supportsSaturateIGC::Pattern75         virtual bool supportsSaturate() { return true; }
76     };
77 
78     struct SDAG
79     {
SDAGIGC::SDAG80         SDAG(Pattern* pattern, llvm::Instruction* root) : m_pattern(pattern), m_root(root)
81         {
82 
83         }
84         Pattern* m_pattern;
85         llvm::Instruction* m_root;
86     };
87 
88     struct SDestination
89     {
90         llvm::Instruction* inst;
91         llvm::Value* predicate;
92         e_modifier mod;
93         bool inversePredicate;
94         bool uniform;
95     };
96 
97     struct SBasicBlock
98     {
99         using reverse_iterator = std::vector<SDAG>::reverse_iterator;
100 
~SBasicBlockIGC::SBasicBlock101         ~SBasicBlock() {}
102         uint id;
103         llvm::BasicBlock* bb;
104         std::vector<SDAG> m_dags;
105         // caches the active lane mask (a flag variable) for this BB
106         // this is currently set only when we enable the A64 WA
107         CVariable* m_activeMask = nullptr;
108     };
109 
110     class CodeGenPatternMatch : public llvm::FunctionPass, public llvm::InstVisitor<CodeGenPatternMatch>
111     {
112     private:
113         CodeGenPatternMatch(CodeGenPatternMatch&) = delete;
114         CodeGenPatternMatch& operator =(CodeGenPatternMatch&) = delete;
115 
116     public:
117 
118         CodeGenPatternMatch();
119 
120         ~CodeGenPatternMatch();
121 
getAnalysisUsage(llvm::AnalysisUsage & AU) const122         virtual void getAnalysisUsage(llvm::AnalysisUsage& AU) const override
123         {
124             AU.addRequired<llvm::DominatorTreeWrapperPass>();
125             AU.addRequired<llvm::LoopInfoWrapperPass>();
126             AU.addRequired<WIAnalysis>();
127             AU.addRequired<LiveVarsAnalysis>();
128             AU.addRequired<MetaDataUtilsWrapper>();
129             AU.addRequired<PositionDepAnalysis>();
130             AU.addRequired<CodeGenContextWrapper>();
131             AU.setPreservesAll();
132         }
133 
134         virtual bool runOnFunction(llvm::Function& F) override;
135 
getPassName() const136         virtual llvm::StringRef getPassName() const override {
137             return "CodeGenPatternMatchPass";
138         }
139 
140         void visitCastInst(llvm::CastInst& I);
141         void visitBinaryOperator(llvm::BinaryOperator& I);
142         void visitCmpInst(llvm::CmpInst& I);
143         void visitPHINode(llvm::PHINode& I);
144         void visitUnaryInstruction(llvm::UnaryInstruction& I);
145         void visitCallInst(llvm::CallInst& I);
146         void visitIntrinsicInst(llvm::IntrinsicInst& I);
147         void visitSelectInst(llvm::SelectInst& I);
148         void visitBitCastInst(llvm::BitCastInst& I);
149         void visitIntToPtrInst(llvm::IntToPtrInst& I);
150         void visitPtrToIntInst(llvm::PtrToIntInst& I);
151         void visitAddrSpaceCast(llvm::AddrSpaceCastInst& I);
152         void visitInstruction(llvm::Instruction& I);
153         void visitExtractElementInst(llvm::ExtractElementInst& I);
154         void visitLoadInst(llvm::LoadInst& I);
155         void visitStoreInst(llvm::StoreInst& I);
156         void visitFPToSIInst(llvm::FPToSIInst& I);
157         void visitFPToUIInst(llvm::FPToUIInst& I);
158         void visitDbgInfoIntrinsic(llvm::DbgInfoIntrinsic& I);
159         void visitExtractValueInst(llvm::ExtractValueInst& I);
160         void visitInsertValueInst(llvm::InsertValueInst& I);
161         void visitBranchInst(llvm::BranchInst& I);
162 
163     public:
164         void CodeGenBlock(llvm::BasicBlock* bb);
165         void CodeGenNode(llvm::DomTreeNode* node);
166 
ForceIsolate(llvm::Value * val)167         void ForceIsolate(llvm::Value* val) { m_forceIsolates.insert(val); }
IsForceIsolated(llvm::Value * val)168         bool IsForceIsolated(llvm::Value* val) { return (m_forceIsolates.count(val) > 0) ? true : false; }
169 
170         SSource GetSource(llvm::Value* v, bool modifier, bool regioning);
171         SSource GetSource(llvm::Value* value, e_modifier mod, bool regioning);
172         bool GetRegionModifier(SSource& mod, llvm::Value*& source, bool regioning);
173         bool BitcastSearch(SSource& mod, llvm::Value*& source, bool broadcast);
174 
175         void MarkAsSource(llvm::Value* v);
176 
177         bool MatchFMA(llvm::IntrinsicInst& I);
178         bool MatchFrc(llvm::BinaryOperator& I);
179         bool MatchFloor(llvm::BinaryOperator& I);
180         bool MatchPredAdd(llvm::BinaryOperator& I);
181         bool MatchSimpleAdd(llvm::BinaryOperator& I);
182         bool MatchMad(llvm::BinaryOperator& I);
183         bool MatchLrp(llvm::BinaryOperator& I);
184         bool MatchCmpSext(llvm::Instruction& I);
185         bool MatchModifier(llvm::Instruction& I, bool SupportSrc0Mod = true);
186         bool MatchSingleInstruction(llvm::Instruction& I);
187         bool MatchCanonicalizeInstruction(llvm::Instruction& I);
188         bool MatchBranch(llvm::BranchInst& I);
189         bool MatchShuffleBroadCast(llvm::GenIntrinsicInst& I);
190         bool MatchWaveShuffleIndex(llvm::GenIntrinsicInst& I);
191         bool MatchRegisterRegion(llvm::GenIntrinsicInst& I);
192 
193         Pattern* Match(llvm::Instruction& inst);
194         bool MatchAbsNeg(llvm::Instruction& I);
195         bool MatchFloatingPointSatModifier(llvm::Instruction& I);
196         bool MatchIntegerSatModifier(llvm::Instruction& I);
197         bool MatchPredicate(llvm::SelectInst& I);
198         bool MatchSelectModifier(llvm::SelectInst& I);
199         bool MatchPow(llvm::IntrinsicInst& I);
200         bool MatchCondModifier(llvm::CmpInst& I);
201         bool MatchBoolOp(llvm::BinaryOperator& I);
202         bool MatchFunnelShiftRotate(llvm::IntrinsicInst& I);
203         bool MatchBfn(llvm::Instruction& I);
204         bool MatchCmpSelect(llvm::SelectInst& I);
205         bool MatchAdd3(llvm::Instruction& I);
206         bool MatchDpas(llvm::GenIntrinsicInst& I);
207         bool MatchDp4a(llvm::GenIntrinsicInst& I);
208         bool MatchLogicAlu(llvm::BinaryOperator& I);
209         bool MatchRsqrt(llvm::BinaryOperator& I);
210         bool MatchLoadStorePointer(llvm::Instruction& I, llvm::Value& ptrVal);
211         bool MatchBlockReadWritePointer(llvm::GenIntrinsicInst& I);
212         bool MatchURBRead(llvm::GenIntrinsicInst& I);
213         bool MatchGradient(llvm::GenIntrinsicInst& I);
214         bool MatchSampleDerivative(llvm::GenIntrinsicInst& I);
215         bool MatchDbgInstruction(llvm::DbgInfoIntrinsic& I);
216         bool MatchAvg(llvm::Instruction& I);
217 
218         bool MatchMinMax(llvm::SelectInst& I);
219         bool MatchFullMul32(llvm::Instruction& I);
220         bool MatchMulAdd16(llvm::Instruction& I);
221         bool MatchFPToIntegerWithSaturation(llvm::Instruction& I);
222         std::tuple<llvm::Value*, unsigned, VISA_Type> isFPToIntegerSatWithExactConstant(llvm::CastInst* I);
223         std::tuple<llvm::Value*, unsigned, VISA_Type> isFPToSignedIntSatWithInexactConstant(llvm::SelectInst* I);
224         std::tuple<llvm::Value*, unsigned, VISA_Type> isFPToUnsignedIntSatWithInexactConstant(llvm::SelectInst* I);
225         bool MatchIntegerTruncSatModifier(llvm::SelectInst& I);
226         std::tuple<llvm::Value*, bool, bool> isIntegerSatTrunc(llvm::SelectInst*);
227 
228         bool MatchSIToFPZExt(llvm::SIToFPInst* S2FI);
229 
230         bool matchAddPair(llvm::ExtractValueInst*);
231         bool matchSubPair(llvm::ExtractValueInst*);
232         bool matchMulPair(llvm::ExtractValueInst*);
233         bool matchPtrToPair(llvm::ExtractValueInst*);
234 
235         bool MatchUnmaskedRegionBoundary(llvm::Instruction& I, bool start);
236 
237         bool MatchInsertToStruct(llvm::InsertValueInst*);
238         bool MatchExtractFromStruct(llvm::ExtractValueInst*);
239 
AddPattern(Pattern * P)240         void AddPattern(Pattern* P)
241         {
242             m_currentPattern = P;
243         }
244 
245         void SetSrcModifier(unsigned int sourceIndex, e_modifier mod);
246         void SetPatternRoot(llvm::Instruction& inst);
247         void CreateBasicBlocks(llvm::Function* pLLVMFunc);
248         uint GetBlockId(llvm::BasicBlock* bb);
249         void HandleSubspanUse(llvm::Value* v);
250         void HandleSampleDerivative(llvm::GenIntrinsicInst& I);
251         bool IsSubspanUse(llvm::Value* v);
252         bool HasUseOutsideLoop(llvm::Value* v);
253         bool NeedVMask();
254 
255         //helper function
256         bool NeedInstruction(llvm::Instruction& I);
257         bool SIMDConstExpr(llvm::Instruction* v);
258         bool IsConstOrSimdConstExpr(llvm::Value* C);
259 
260         // Place a constant Val into the constant pool. This constant should be
261         // available in basic block UseBlock.
262         void AddToConstantPool(llvm::BasicBlock* UseBlock, llvm::Value* Val);
263 
canEmitAsUniformBool(const llvm::Value * Val) const264         bool canEmitAsUniformBool(const llvm::Value* Val) const
265         {
266             return UniformBools.count(Val) > 0;
267         }
268 
isUniform(const llvm::Value * V) const269         bool isUniform(const llvm::Value* V) const
270         {
271             return m_WI && (m_WI->isUniform(V));
272         };
273 
274     public:
275         llvm::DenseSet<llvm::Instruction*>       m_usedInstructions;
276         bool                  m_rootIsSubspanUse;
277         llvm::DenseSet<llvm::Value*>             m_subSpanUse;
278         llvm::SmallPtrSet<llvm::Value*, 8>       m_forceIsolates;
279         std::map<llvm::BasicBlock*, SBasicBlock*> m_blockMap;
280         SBasicBlock* m_blocks;
281         uint                  m_numBlocks;
282         llvm::DenseMap<llvm::Instruction*, bool> m_IsSIMDConstExpr;
283 
284         // Where we put the constant initialization.
285         llvm::MapVector<llvm::Constant*, llvm::BasicBlock*> ConstantPlacement;
286 
287         typedef std::pair<llvm::ExtractValueInst*, llvm::ExtractValueInst*> PairOutputTy;
288         typedef llvm::DenseMap<llvm::Value*, PairOutputTy> PairOutputMapTy;
289         PairOutputMapTy PairOutputMap;
290 
291         llvm::Instruction* m_root;
292         Pattern* m_currentPattern;
293 
294         CPlatform             m_Platform;
295         bool                  m_AllowContractions;
296         bool                  m_NeedVMask;
297         bool                  m_samplertoRenderTargetEnable;
298     public:
299         static char ID;
300 
301     private:
302         CodeGenContext* m_ctx; // Caching codegen context, valid only within runOnFunction().
303         llvm::DominatorTree* DT;
304         llvm::LoopInfo* LI;
305         const llvm::DataLayout* m_DL;
306         WIAnalysis* m_WI;
307         LiveVars* m_LivenessInfo;
308         PositionDepAnalysis* m_PosDep;
309         llvm::BumpPtrAllocator m_allocator;
310         // The set of boolean values stored as predicates of a single element.
311         // Otherwise, they are expanded to the SIMD size.
312         llvm::DenseSet<const llvm::Value*> UniformBools;
313 
314         // Find bool values that will be emitted as uniform variables.
315         // Otherwise they will be expanded to the SIMD size, by default.
316         void gatherUniformBools(llvm::Value* Val);
317 
318         // Return true if it is dbg related instruction
319         bool IsDbgInst(llvm::Instruction& inst) const;
320     };
321 
322     //helper
323     bool GetModifier(llvm::Instruction& modifier, e_modifier& mod, llvm::Value*& source);
324     bool GetModifier(llvm::Value& modifier, e_modifier& mod, llvm::Value*& source);
325     bool IsNegate(llvm::Instruction* modifier, llvm::Value*& negateSource);
326     bool IsZero(llvm::Value* v);
327     bool isAbs(llvm::Value* abs, e_modifier& mod, llvm::Value*& source);
328     bool isSat(llvm::Instruction* sat, llvm::Value*& source, bool& isUnsigned);
329     bool isOne(llvm::Value* zero);
330     bool isMinOrMax(llvm::Value* inst, llvm::Value*& source0, llvm::Value*& source1, bool& isMin, bool& isUnsigned);
331     bool isCandidateForConstantPool(llvm::Value * val);
332     e_modifier CombineModifier(e_modifier mod1, e_modifier mod2);
333 
GetNbSources(llvm::Instruction & v)334     static uint GetNbSources(llvm::Instruction& v)
335     {
336         if (llvm::CallInst * intrin = llvm::dyn_cast<llvm::CallInst>(&v))
337         {
338             return intrin->getNumArgOperands();
339         }
340         return v.getNumOperands();
341     }
342 
343 } // namespace IGC
344