1 //===- AMDGPUInstructionSelector --------------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file declares the targeting of the InstructionSelector class for
10 /// AMDGPU.
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
14 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
15 
16 #include "SIDefines.h"
17 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
18 #include "llvm/IR/InstrTypes.h"
19 
20 namespace {
21 #define GET_GLOBALISEL_PREDICATE_BITSET
22 #define AMDGPUSubtarget GCNSubtarget
23 #include "AMDGPUGenGlobalISel.inc"
24 #undef GET_GLOBALISEL_PREDICATE_BITSET
25 #undef AMDGPUSubtarget
26 }
27 
28 namespace llvm {
29 
30 namespace AMDGPU {
31 struct ImageDimIntrinsicInfo;
32 }
33 
34 class AMDGPURegisterBankInfo;
35 class AMDGPUTargetMachine;
36 class BlockFrequencyInfo;
37 class ProfileSummaryInfo;
38 class GCNSubtarget;
39 class MachineInstr;
40 class MachineIRBuilder;
41 class MachineOperand;
42 class MachineRegisterInfo;
43 class RegisterBank;
44 class SIInstrInfo;
45 class SIRegisterInfo;
46 class TargetRegisterClass;
47 
48 class AMDGPUInstructionSelector final : public InstructionSelector {
49 private:
50   MachineRegisterInfo *MRI;
51   const GCNSubtarget *Subtarget;
52 
53 public:
54   AMDGPUInstructionSelector(const GCNSubtarget &STI,
55                             const AMDGPURegisterBankInfo &RBI,
56                             const AMDGPUTargetMachine &TM);
57 
58   bool select(MachineInstr &I) override;
59   static const char *getName();
60 
61   void setupMF(MachineFunction &MF, GISelKnownBits *KB,
62                CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
63                BlockFrequencyInfo *BFI) override;
64 
65 private:
66   struct GEPInfo {
67     SmallVector<unsigned, 2> SgprParts;
68     SmallVector<unsigned, 2> VgprParts;
69     int64_t Imm = 0;
70   };
71 
72   bool isSGPR(Register Reg) const;
73 
74   bool isInstrUniform(const MachineInstr &MI) const;
75   bool isVCC(Register Reg, const MachineRegisterInfo &MRI) const;
76 
77   const RegisterBank *getArtifactRegBank(
78     Register Reg, const MachineRegisterInfo &MRI,
79     const TargetRegisterInfo &TRI) const;
80 
81   /// tblgen-erated 'select' implementation.
82   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
83 
84   MachineOperand getSubOperand64(MachineOperand &MO,
85                                  const TargetRegisterClass &SubRC,
86                                  unsigned SubIdx) const;
87 
88   bool constrainCopyLikeIntrin(MachineInstr &MI, unsigned NewOpc) const;
89   bool selectCOPY(MachineInstr &I) const;
90   bool selectPHI(MachineInstr &I) const;
91   bool selectG_TRUNC(MachineInstr &I) const;
92   bool selectG_SZA_EXT(MachineInstr &I) const;
93   bool selectG_FPEXT(MachineInstr &I) const;
94   bool selectG_CONSTANT(MachineInstr &I) const;
95   bool selectG_FNEG(MachineInstr &I) const;
96   bool selectG_FABS(MachineInstr &I) const;
97   bool selectG_AND_OR_XOR(MachineInstr &I) const;
98   bool selectG_ADD_SUB(MachineInstr &I) const;
99   bool selectG_UADDO_USUBO_UADDE_USUBE(MachineInstr &I) const;
100   bool selectG_AMDGPU_MAD_64_32(MachineInstr &I) const;
101   bool selectG_EXTRACT(MachineInstr &I) const;
102   bool selectG_FMA_FMAD(MachineInstr &I) const;
103   bool selectG_MERGE_VALUES(MachineInstr &I) const;
104   bool selectG_UNMERGE_VALUES(MachineInstr &I) const;
105   bool selectG_BUILD_VECTOR(MachineInstr &I) const;
106   bool selectG_PTR_ADD(MachineInstr &I) const;
107   bool selectG_IMPLICIT_DEF(MachineInstr &I) const;
108   bool selectG_INSERT(MachineInstr &I) const;
109   bool selectG_SBFX_UBFX(MachineInstr &I) const;
110 
111   bool selectInterpP1F16(MachineInstr &MI) const;
112   bool selectWritelane(MachineInstr &MI) const;
113   bool selectDivScale(MachineInstr &MI) const;
114   bool selectIntrinsicCmp(MachineInstr &MI) const;
115   bool selectBallot(MachineInstr &I) const;
116   bool selectInverseBallot(MachineInstr &I) const;
117   bool selectRelocConstant(MachineInstr &I) const;
118   bool selectGroupStaticSize(MachineInstr &I) const;
119   bool selectReturnAddress(MachineInstr &I) const;
120   bool selectG_INTRINSIC(MachineInstr &I) const;
121 
122   bool selectEndCfIntrinsic(MachineInstr &MI) const;
123   bool selectDSOrderedIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;
124   bool selectDSGWSIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;
125   bool selectDSAppendConsume(MachineInstr &MI, bool IsAppend) const;
126   bool selectSBarrier(MachineInstr &MI) const;
127   bool selectDSBvhStackIntrinsic(MachineInstr &MI) const;
128 
129   bool selectImageIntrinsic(MachineInstr &MI,
130                             const AMDGPU::ImageDimIntrinsicInfo *Intr) const;
131   bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I) const;
132   int getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) const;
133   bool selectG_ICMP_or_FCMP(MachineInstr &I) const;
134   bool hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const;
135   void getAddrModeInfo(const MachineInstr &Load, const MachineRegisterInfo &MRI,
136                        SmallVectorImpl<GEPInfo> &AddrInfo) const;
137 
138   void initM0(MachineInstr &I) const;
139   bool selectG_LOAD_STORE_ATOMICRMW(MachineInstr &I) const;
140   bool selectG_SELECT(MachineInstr &I) const;
141   bool selectG_BRCOND(MachineInstr &I) const;
142   bool selectG_GLOBAL_VALUE(MachineInstr &I) const;
143   bool selectG_PTRMASK(MachineInstr &I) const;
144   bool selectG_EXTRACT_VECTOR_ELT(MachineInstr &I) const;
145   bool selectG_INSERT_VECTOR_ELT(MachineInstr &I) const;
146   bool selectBufferLoadLds(MachineInstr &MI) const;
147   bool selectGlobalLoadLds(MachineInstr &MI) const;
148   bool selectBVHIntrinsic(MachineInstr &I) const;
149   bool selectSMFMACIntrin(MachineInstr &I) const;
150   bool selectWaveAddress(MachineInstr &I) const;
151   bool selectStackRestore(MachineInstr &MI) const;
152   bool selectNamedBarrierInst(MachineInstr &I, Intrinsic::ID IID) const;
153   bool selectSBarrierSignalIsfirst(MachineInstr &I, Intrinsic::ID IID) const;
154   bool selectSBarrierLeave(MachineInstr &I) const;
155 
156   std::pair<Register, unsigned> selectVOP3ModsImpl(MachineOperand &Root,
157                                                    bool IsCanonicalizing = true,
158                                                    bool AllowAbs = true,
159                                                    bool OpSel = false) const;
160 
161   Register copyToVGPRIfSrcFolded(Register Src, unsigned Mods,
162                                  MachineOperand Root, MachineInstr *InsertPt,
163                                  bool ForceVGPR = false) const;
164 
165   InstructionSelector::ComplexRendererFns
166   selectVCSRC(MachineOperand &Root) const;
167 
168   InstructionSelector::ComplexRendererFns
169   selectVSRC0(MachineOperand &Root) const;
170 
171   InstructionSelector::ComplexRendererFns
172   selectVOP3Mods0(MachineOperand &Root) const;
173   InstructionSelector::ComplexRendererFns
174   selectVOP3BMods0(MachineOperand &Root) const;
175   InstructionSelector::ComplexRendererFns
176   selectVOP3OMods(MachineOperand &Root) const;
177   InstructionSelector::ComplexRendererFns
178   selectVOP3Mods(MachineOperand &Root) const;
179   InstructionSelector::ComplexRendererFns
180   selectVOP3ModsNonCanonicalizing(MachineOperand &Root) const;
181   InstructionSelector::ComplexRendererFns
182   selectVOP3BMods(MachineOperand &Root) const;
183 
184   ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
185 
186   std::pair<Register, unsigned>
187   selectVOP3PModsImpl(Register Src, const MachineRegisterInfo &MRI,
188                       bool IsDOT = false) const;
189 
190   InstructionSelector::ComplexRendererFns
191   selectVOP3PMods(MachineOperand &Root) const;
192 
193   InstructionSelector::ComplexRendererFns
194   selectVOP3PModsDOT(MachineOperand &Root) const;
195 
196   InstructionSelector::ComplexRendererFns
197   selectDotIUVOP3PMods(MachineOperand &Root) const;
198 
199   InstructionSelector::ComplexRendererFns
200   selectWMMAOpSelVOP3PMods(MachineOperand &Root) const;
201 
202   InstructionSelector::ComplexRendererFns
203   selectVOP3OpSelMods(MachineOperand &Root) const;
204 
205   InstructionSelector::ComplexRendererFns
206   selectVINTERPMods(MachineOperand &Root) const;
207   InstructionSelector::ComplexRendererFns
208   selectVINTERPModsHi(MachineOperand &Root) const;
209 
210   bool selectSmrdOffset(MachineOperand &Root, Register &Base, Register *SOffset,
211                         int64_t *Offset) const;
212   InstructionSelector::ComplexRendererFns
213   selectSmrdImm(MachineOperand &Root) const;
214   InstructionSelector::ComplexRendererFns
215   selectSmrdImm32(MachineOperand &Root) const;
216   InstructionSelector::ComplexRendererFns
217   selectSmrdSgpr(MachineOperand &Root) const;
218   InstructionSelector::ComplexRendererFns
219   selectSmrdSgprImm(MachineOperand &Root) const;
220 
221   std::pair<Register, int> selectFlatOffsetImpl(MachineOperand &Root,
222                                                 uint64_t FlatVariant) const;
223 
224   InstructionSelector::ComplexRendererFns
225   selectFlatOffset(MachineOperand &Root) const;
226   InstructionSelector::ComplexRendererFns
227   selectGlobalOffset(MachineOperand &Root) const;
228   InstructionSelector::ComplexRendererFns
229   selectScratchOffset(MachineOperand &Root) const;
230 
231   InstructionSelector::ComplexRendererFns
232   selectGlobalSAddr(MachineOperand &Root) const;
233 
234   InstructionSelector::ComplexRendererFns
235   selectScratchSAddr(MachineOperand &Root) const;
236   bool checkFlatScratchSVSSwizzleBug(Register VAddr, Register SAddr,
237                                      uint64_t ImmOffset) const;
238   InstructionSelector::ComplexRendererFns
239   selectScratchSVAddr(MachineOperand &Root) const;
240 
241   InstructionSelector::ComplexRendererFns
242   selectMUBUFScratchOffen(MachineOperand &Root) const;
243   InstructionSelector::ComplexRendererFns
244   selectMUBUFScratchOffset(MachineOperand &Root) const;
245 
246   bool isDSOffsetLegal(Register Base, int64_t Offset) const;
247   bool isDSOffset2Legal(Register Base, int64_t Offset0, int64_t Offset1,
248                         unsigned Size) const;
249   bool isFlatScratchBaseLegal(Register Addr) const;
250   bool isFlatScratchBaseLegalSV(Register Addr) const;
251   bool isFlatScratchBaseLegalSVImm(Register Addr) const;
252 
253   std::pair<Register, unsigned>
254   selectDS1Addr1OffsetImpl(MachineOperand &Root) const;
255   InstructionSelector::ComplexRendererFns
256   selectDS1Addr1Offset(MachineOperand &Root) const;
257 
258   InstructionSelector::ComplexRendererFns
259   selectDS64Bit4ByteAligned(MachineOperand &Root) const;
260 
261   InstructionSelector::ComplexRendererFns
262   selectDS128Bit8ByteAligned(MachineOperand &Root) const;
263 
264   std::pair<Register, unsigned> selectDSReadWrite2Impl(MachineOperand &Root,
265                                                        unsigned size) const;
266   InstructionSelector::ComplexRendererFns
267   selectDSReadWrite2(MachineOperand &Root, unsigned size) const;
268 
269   std::pair<Register, int64_t>
270   getPtrBaseWithConstantOffset(Register Root,
271                                const MachineRegisterInfo &MRI) const;
272 
273   // Parse out a chain of up to two g_ptr_add instructions.
274   // g_ptr_add (n0, _)
275   // g_ptr_add (n0, (n1 = g_ptr_add n2, n3))
276   struct MUBUFAddressData {
277     Register N0, N2, N3;
278     int64_t Offset = 0;
279   };
280 
281   bool shouldUseAddr64(MUBUFAddressData AddrData) const;
282 
283   void splitIllegalMUBUFOffset(MachineIRBuilder &B,
284                                Register &SOffset, int64_t &ImmOffset) const;
285 
286   MUBUFAddressData parseMUBUFAddress(Register Src) const;
287 
288   bool selectMUBUFAddr64Impl(MachineOperand &Root, Register &VAddr,
289                              Register &RSrcReg, Register &SOffset,
290                              int64_t &Offset) const;
291 
292   bool selectMUBUFOffsetImpl(MachineOperand &Root, Register &RSrcReg,
293                              Register &SOffset, int64_t &Offset) const;
294 
295   InstructionSelector::ComplexRendererFns
296   selectBUFSOffset(MachineOperand &Root) const;
297 
298   InstructionSelector::ComplexRendererFns
299   selectMUBUFAddr64(MachineOperand &Root) const;
300 
301   InstructionSelector::ComplexRendererFns
302   selectMUBUFOffset(MachineOperand &Root) const;
303 
304   ComplexRendererFns selectSMRDBufferImm(MachineOperand &Root) const;
305   ComplexRendererFns selectSMRDBufferImm32(MachineOperand &Root) const;
306   ComplexRendererFns selectSMRDBufferSgprImm(MachineOperand &Root) const;
307 
308   std::pair<Register, unsigned> selectVOP3PMadMixModsImpl(MachineOperand &Root,
309                                                           bool &Matched) const;
310   ComplexRendererFns selectVOP3PMadMixModsExt(MachineOperand &Root) const;
311   ComplexRendererFns selectVOP3PMadMixMods(MachineOperand &Root) const;
312 
313   void renderTruncImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
314                         int OpIdx = -1) const;
315 
316   void renderTruncTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
317                        int OpIdx) const;
318 
319   void renderOpSelTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
320                        int OpIdx) const;
321 
322   void renderNegateImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
323                        int OpIdx) const;
324 
325   void renderBitcastImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
326                         int OpIdx) const;
327 
328   void renderPopcntImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
329                        int OpIdx) const;
330   void renderExtractCPol(MachineInstrBuilder &MIB, const MachineInstr &MI,
331                          int OpIdx) const;
332   void renderExtractSWZ(MachineInstrBuilder &MIB, const MachineInstr &MI,
333                         int OpIdx) const;
334   void renderSetGLC(MachineInstrBuilder &MIB, const MachineInstr &MI,
335                     int OpIdx) const;
336 
337   void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,
338                         int OpIdx) const;
339 
340   void renderFPPow2ToExponent(MachineInstrBuilder &MIB, const MachineInstr &MI,
341                               int OpIdx) const;
342 
343   bool isInlineImmediate16(int64_t Imm) const;
344   bool isInlineImmediate32(int64_t Imm) const;
345   bool isInlineImmediate64(int64_t Imm) const;
346   bool isInlineImmediate(const APFloat &Imm) const;
347 
348   // Returns true if TargetOpcode::G_AND MachineInstr `MI`'s masking of the
349   // shift amount operand's `ShAmtBits` bits is unneeded.
350   bool isUnneededShiftMask(const MachineInstr &MI, unsigned ShAmtBits) const;
351 
352   const SIInstrInfo &TII;
353   const SIRegisterInfo &TRI;
354   const AMDGPURegisterBankInfo &RBI;
355   const AMDGPUTargetMachine &TM;
356   const GCNSubtarget &STI;
357   bool EnableLateStructurizeCFG;
358 #define GET_GLOBALISEL_PREDICATES_DECL
359 #define AMDGPUSubtarget GCNSubtarget
360 #include "AMDGPUGenGlobalISel.inc"
361 #undef GET_GLOBALISEL_PREDICATES_DECL
362 #undef AMDGPUSubtarget
363 
364 #define GET_GLOBALISEL_TEMPORARIES_DECL
365 #include "AMDGPUGenGlobalISel.inc"
366 #undef GET_GLOBALISEL_TEMPORARIES_DECL
367 };
368 
369 } // End llvm namespace.
370 #endif
371