1 //===-- AMDGPUSubtarget.cpp - AMDGPU Subtarget Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Implements the AMDGPU specific subclass of TargetSubtarget.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPUSubtarget.h"
15 #include "AMDGPUCallLowering.h"
16 #include "AMDGPUInstructionSelector.h"
17 #include "AMDGPULegalizerInfo.h"
18 #include "AMDGPURegisterBankInfo.h"
19 #include "AMDGPUTargetMachine.h"
20 #include "R600Subtarget.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
25 #include "llvm/CodeGen/MachineScheduler.h"
26 #include "llvm/CodeGen/TargetFrameLowering.h"
27 #include "llvm/IR/IntrinsicsAMDGPU.h"
28 #include "llvm/IR/IntrinsicsR600.h"
29 #include "llvm/IR/MDBuilder.h"
30 #include "llvm/MC/MCSubtargetInfo.h"
31 #include <algorithm>
32 
33 using namespace llvm;
34 
35 #define DEBUG_TYPE "amdgpu-subtarget"
36 
37 #define GET_SUBTARGETINFO_TARGET_DESC
38 #define GET_SUBTARGETINFO_CTOR
39 #define AMDGPUSubtarget GCNSubtarget
40 #include "AMDGPUGenSubtargetInfo.inc"
41 #undef AMDGPUSubtarget
42 
43 static cl::opt<bool> EnablePowerSched(
44   "amdgpu-enable-power-sched",
45   cl::desc("Enable scheduling to minimize mAI power bursts"),
46   cl::init(false));
47 
48 static cl::opt<bool> EnableVGPRIndexMode(
49   "amdgpu-vgpr-index-mode",
50   cl::desc("Use GPR indexing mode instead of movrel for vector indexing"),
51   cl::init(false));
52 
53 static cl::opt<bool> UseAA("amdgpu-use-aa-in-codegen",
54                            cl::desc("Enable the use of AA during codegen."),
55                            cl::init(true));
56 
57 static cl::opt<unsigned> NSAThreshold("amdgpu-nsa-threshold",
58                                       cl::desc("Number of addresses from which to enable MIMG NSA."),
59                                       cl::init(3), cl::Hidden);
60 
61 GCNSubtarget::~GCNSubtarget() = default;
62 
63 GCNSubtarget &
initializeSubtargetDependencies(const Triple & TT,StringRef GPU,StringRef FS)64 GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
65                                               StringRef GPU, StringRef FS) {
66   // Determine default and user-specified characteristics
67   //
68   // We want to be able to turn these off, but making this a subtarget feature
69   // for SI has the unhelpful behavior that it unsets everything else if you
70   // disable it.
71   //
72   // Similarly we want enable-prt-strict-null to be on by default and not to
73   // unset everything else if it is disabled
74 
75   SmallString<256> FullFS("+promote-alloca,+load-store-opt,+enable-ds128,");
76 
77   // Turn on features that HSA ABI requires. Also turn on FlatForGlobal by default
78   if (isAmdHsaOS())
79     FullFS += "+flat-for-global,+unaligned-access-mode,+trap-handler,";
80 
81   FullFS += "+enable-prt-strict-null,"; // This is overridden by a disable in FS
82 
83   // Disable mutually exclusive bits.
84   if (FS.contains_insensitive("+wavefrontsize")) {
85     if (!FS.contains_insensitive("wavefrontsize16"))
86       FullFS += "-wavefrontsize16,";
87     if (!FS.contains_insensitive("wavefrontsize32"))
88       FullFS += "-wavefrontsize32,";
89     if (!FS.contains_insensitive("wavefrontsize64"))
90       FullFS += "-wavefrontsize64,";
91   }
92 
93   FullFS += FS;
94 
95   ParseSubtargetFeatures(GPU, /*TuneCPU*/ GPU, FullFS);
96 
97   // Implement the "generic" processors, which acts as the default when no
98   // generation features are enabled (e.g for -mcpu=''). HSA OS defaults to
99   // the first amdgcn target that supports flat addressing. Other OSes defaults
100   // to the first amdgcn target.
101   if (Gen == AMDGPUSubtarget::INVALID) {
102      Gen = TT.getOS() == Triple::AMDHSA ? AMDGPUSubtarget::SEA_ISLANDS
103                                         : AMDGPUSubtarget::SOUTHERN_ISLANDS;
104   }
105 
106   // We don't support FP64 for EG/NI atm.
107   assert(!hasFP64() || (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS));
108 
109   // Targets must either support 64-bit offsets for MUBUF instructions, and/or
110   // support flat operations, otherwise they cannot access a 64-bit global
111   // address space
112   assert(hasAddr64() || hasFlat());
113   // Unless +-flat-for-global is specified, turn on FlatForGlobal for targets
114   // that do not support ADDR64 variants of MUBUF instructions. Such targets
115   // cannot use a 64 bit offset with a MUBUF instruction to access the global
116   // address space
117   if (!hasAddr64() && !FS.contains("flat-for-global") && !FlatForGlobal) {
118     ToggleFeature(AMDGPU::FeatureFlatForGlobal);
119     FlatForGlobal = true;
120   }
121   // Unless +-flat-for-global is specified, use MUBUF instructions for global
122   // address space access if flat operations are not available.
123   if (!hasFlat() && !FS.contains("flat-for-global") && FlatForGlobal) {
124     ToggleFeature(AMDGPU::FeatureFlatForGlobal);
125     FlatForGlobal = false;
126   }
127 
128   // Set defaults if needed.
129   if (MaxPrivateElementSize == 0)
130     MaxPrivateElementSize = 4;
131 
132   if (LDSBankCount == 0)
133     LDSBankCount = 32;
134 
135   if (TT.getArch() == Triple::amdgcn) {
136     if (LocalMemorySize == 0)
137       LocalMemorySize = 32768;
138 
139     // Do something sensible for unspecified target.
140     if (!HasMovrel && !HasVGPRIndexMode)
141       HasMovrel = true;
142   }
143 
144   AddressableLocalMemorySize = LocalMemorySize;
145 
146   if (AMDGPU::isGFX10Plus(*this) &&
147       !getFeatureBits().test(AMDGPU::FeatureCuMode))
148     LocalMemorySize *= 2;
149 
150   // Don't crash on invalid devices.
151   if (WavefrontSizeLog2 == 0)
152     WavefrontSizeLog2 = 5;
153 
154   HasFminFmaxLegacy = getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS;
155   HasSMulHi = getGeneration() >= AMDGPUSubtarget::GFX9;
156 
157   TargetID.setTargetIDFromFeaturesString(FS);
158 
159   LLVM_DEBUG(dbgs() << "xnack setting for subtarget: "
160                     << TargetID.getXnackSetting() << '\n');
161   LLVM_DEBUG(dbgs() << "sramecc setting for subtarget: "
162                     << TargetID.getSramEccSetting() << '\n');
163 
164   return *this;
165 }
166 
AMDGPUSubtarget(const Triple & TT)167 AMDGPUSubtarget::AMDGPUSubtarget(const Triple &TT) : TargetTriple(TT) {}
168 
GCNSubtarget(const Triple & TT,StringRef GPU,StringRef FS,const GCNTargetMachine & TM)169 GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
170                            const GCNTargetMachine &TM)
171     : // clang-format off
172     AMDGPUGenSubtargetInfo(TT, GPU, /*TuneCPU*/ GPU, FS),
173     AMDGPUSubtarget(TT),
174     TargetTriple(TT),
175     TargetID(*this),
176     InstrItins(getInstrItineraryForCPU(GPU)),
177     InstrInfo(initializeSubtargetDependencies(TT, GPU, FS)),
178     TLInfo(TM, *this),
179     FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0) {
180   // clang-format on
181   MaxWavesPerEU = AMDGPU::IsaInfo::getMaxWavesPerEU(this);
182   EUsPerCU = AMDGPU::IsaInfo::getEUsPerCU(this);
183   CallLoweringInfo.reset(new AMDGPUCallLowering(*getTargetLowering()));
184   InlineAsmLoweringInfo.reset(new InlineAsmLowering(getTargetLowering()));
185   Legalizer.reset(new AMDGPULegalizerInfo(*this, TM));
186   RegBankInfo.reset(new AMDGPURegisterBankInfo(*this));
187   InstSelector.reset(new AMDGPUInstructionSelector(
188   *this, *static_cast<AMDGPURegisterBankInfo *>(RegBankInfo.get()), TM));
189 }
190 
getConstantBusLimit(unsigned Opcode) const191 unsigned GCNSubtarget::getConstantBusLimit(unsigned Opcode) const {
192   if (getGeneration() < GFX10)
193     return 1;
194 
195   switch (Opcode) {
196   case AMDGPU::V_LSHLREV_B64_e64:
197   case AMDGPU::V_LSHLREV_B64_gfx10:
198   case AMDGPU::V_LSHLREV_B64_e64_gfx11:
199   case AMDGPU::V_LSHL_B64_e64:
200   case AMDGPU::V_LSHRREV_B64_e64:
201   case AMDGPU::V_LSHRREV_B64_gfx10:
202   case AMDGPU::V_LSHRREV_B64_e64_gfx11:
203   case AMDGPU::V_LSHR_B64_e64:
204   case AMDGPU::V_ASHRREV_I64_e64:
205   case AMDGPU::V_ASHRREV_I64_gfx10:
206   case AMDGPU::V_ASHRREV_I64_e64_gfx11:
207   case AMDGPU::V_ASHR_I64_e64:
208     return 1;
209   }
210 
211   return 2;
212 }
213 
214 /// This list was mostly derived from experimentation.
zeroesHigh16BitsOfDest(unsigned Opcode) const215 bool GCNSubtarget::zeroesHigh16BitsOfDest(unsigned Opcode) const {
216   switch (Opcode) {
217   case AMDGPU::V_CVT_F16_F32_e32:
218   case AMDGPU::V_CVT_F16_F32_e64:
219   case AMDGPU::V_CVT_F16_U16_e32:
220   case AMDGPU::V_CVT_F16_U16_e64:
221   case AMDGPU::V_CVT_F16_I16_e32:
222   case AMDGPU::V_CVT_F16_I16_e64:
223   case AMDGPU::V_RCP_F16_e64:
224   case AMDGPU::V_RCP_F16_e32:
225   case AMDGPU::V_RSQ_F16_e64:
226   case AMDGPU::V_RSQ_F16_e32:
227   case AMDGPU::V_SQRT_F16_e64:
228   case AMDGPU::V_SQRT_F16_e32:
229   case AMDGPU::V_LOG_F16_e64:
230   case AMDGPU::V_LOG_F16_e32:
231   case AMDGPU::V_EXP_F16_e64:
232   case AMDGPU::V_EXP_F16_e32:
233   case AMDGPU::V_SIN_F16_e64:
234   case AMDGPU::V_SIN_F16_e32:
235   case AMDGPU::V_COS_F16_e64:
236   case AMDGPU::V_COS_F16_e32:
237   case AMDGPU::V_FLOOR_F16_e64:
238   case AMDGPU::V_FLOOR_F16_e32:
239   case AMDGPU::V_CEIL_F16_e64:
240   case AMDGPU::V_CEIL_F16_e32:
241   case AMDGPU::V_TRUNC_F16_e64:
242   case AMDGPU::V_TRUNC_F16_e32:
243   case AMDGPU::V_RNDNE_F16_e64:
244   case AMDGPU::V_RNDNE_F16_e32:
245   case AMDGPU::V_FRACT_F16_e64:
246   case AMDGPU::V_FRACT_F16_e32:
247   case AMDGPU::V_FREXP_MANT_F16_e64:
248   case AMDGPU::V_FREXP_MANT_F16_e32:
249   case AMDGPU::V_FREXP_EXP_I16_F16_e64:
250   case AMDGPU::V_FREXP_EXP_I16_F16_e32:
251   case AMDGPU::V_LDEXP_F16_e64:
252   case AMDGPU::V_LDEXP_F16_e32:
253   case AMDGPU::V_LSHLREV_B16_e64:
254   case AMDGPU::V_LSHLREV_B16_e32:
255   case AMDGPU::V_LSHRREV_B16_e64:
256   case AMDGPU::V_LSHRREV_B16_e32:
257   case AMDGPU::V_ASHRREV_I16_e64:
258   case AMDGPU::V_ASHRREV_I16_e32:
259   case AMDGPU::V_ADD_U16_e64:
260   case AMDGPU::V_ADD_U16_e32:
261   case AMDGPU::V_SUB_U16_e64:
262   case AMDGPU::V_SUB_U16_e32:
263   case AMDGPU::V_SUBREV_U16_e64:
264   case AMDGPU::V_SUBREV_U16_e32:
265   case AMDGPU::V_MUL_LO_U16_e64:
266   case AMDGPU::V_MUL_LO_U16_e32:
267   case AMDGPU::V_ADD_F16_e64:
268   case AMDGPU::V_ADD_F16_e32:
269   case AMDGPU::V_SUB_F16_e64:
270   case AMDGPU::V_SUB_F16_e32:
271   case AMDGPU::V_SUBREV_F16_e64:
272   case AMDGPU::V_SUBREV_F16_e32:
273   case AMDGPU::V_MUL_F16_e64:
274   case AMDGPU::V_MUL_F16_e32:
275   case AMDGPU::V_MAX_F16_e64:
276   case AMDGPU::V_MAX_F16_e32:
277   case AMDGPU::V_MIN_F16_e64:
278   case AMDGPU::V_MIN_F16_e32:
279   case AMDGPU::V_MAX_U16_e64:
280   case AMDGPU::V_MAX_U16_e32:
281   case AMDGPU::V_MIN_U16_e64:
282   case AMDGPU::V_MIN_U16_e32:
283   case AMDGPU::V_MAX_I16_e64:
284   case AMDGPU::V_MAX_I16_e32:
285   case AMDGPU::V_MIN_I16_e64:
286   case AMDGPU::V_MIN_I16_e32:
287   case AMDGPU::V_MAD_F16_e64:
288   case AMDGPU::V_MAD_U16_e64:
289   case AMDGPU::V_MAD_I16_e64:
290   case AMDGPU::V_FMA_F16_e64:
291   case AMDGPU::V_DIV_FIXUP_F16_e64:
292     // On gfx10, all 16-bit instructions preserve the high bits.
293     return getGeneration() <= AMDGPUSubtarget::GFX9;
294   case AMDGPU::V_MADAK_F16:
295   case AMDGPU::V_MADMK_F16:
296   case AMDGPU::V_MAC_F16_e64:
297   case AMDGPU::V_MAC_F16_e32:
298   case AMDGPU::V_FMAMK_F16:
299   case AMDGPU::V_FMAAK_F16:
300   case AMDGPU::V_FMAC_F16_e64:
301   case AMDGPU::V_FMAC_F16_e32:
302     // In gfx9, the preferred handling of the unused high 16-bits changed. Most
303     // instructions maintain the legacy behavior of 0ing. Some instructions
304     // changed to preserving the high bits.
305     return getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
306   case AMDGPU::V_MAD_MIXLO_F16:
307   case AMDGPU::V_MAD_MIXHI_F16:
308   default:
309     return false;
310   }
311 }
312 
313 // Returns the maximum per-workgroup LDS allocation size (in bytes) that still
314 // allows the given function to achieve an occupancy of NWaves waves per
315 // SIMD / EU, taking into account only the function's *maximum* workgroup size.
316 unsigned
getMaxLocalMemSizeWithWaveCount(unsigned NWaves,const Function & F) const317 AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,
318                                                  const Function &F) const {
319   const unsigned WaveSize = getWavefrontSize();
320   const unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
321   const unsigned WavesPerWorkgroup =
322       std::max(1u, (WorkGroupSize + WaveSize - 1) / WaveSize);
323 
324   const unsigned WorkGroupsPerCU =
325       std::max(1u, (NWaves * getEUsPerCU()) / WavesPerWorkgroup);
326 
327   return getLocalMemorySize() / WorkGroupsPerCU;
328 }
329 
330 // FIXME: Should return min,max range.
331 //
332 // Returns the maximum occupancy, in number of waves per SIMD / EU, that can
333 // be achieved when only the given function is running on the machine; and
334 // taking into account the overall number of wave slots, the (maximum) workgroup
335 // size, and the per-workgroup LDS allocation size.
getOccupancyWithLocalMemSize(uint32_t Bytes,const Function & F) const336 unsigned AMDGPUSubtarget::getOccupancyWithLocalMemSize(uint32_t Bytes,
337   const Function &F) const {
338   const unsigned MaxWorkGroupSize = getFlatWorkGroupSizes(F).second;
339   const unsigned MaxWorkGroupsPerCu = getMaxWorkGroupsPerCU(MaxWorkGroupSize);
340   if (!MaxWorkGroupsPerCu)
341     return 0;
342 
343   const unsigned WaveSize = getWavefrontSize();
344 
345   // FIXME: Do we need to account for alignment requirement of LDS rounding the
346   // size up?
347   // Compute restriction based on LDS usage
348   unsigned NumGroups = getLocalMemorySize() / (Bytes ? Bytes : 1u);
349 
350   // This can be queried with more LDS than is possible, so just assume the
351   // worst.
352   if (NumGroups == 0)
353     return 1;
354 
355   NumGroups = std::min(MaxWorkGroupsPerCu, NumGroups);
356 
357   // Round to the number of waves per CU.
358   const unsigned MaxGroupNumWaves = divideCeil(MaxWorkGroupSize, WaveSize);
359   unsigned MaxWaves = NumGroups * MaxGroupNumWaves;
360 
361   // Number of waves per EU (SIMD).
362   MaxWaves = divideCeil(MaxWaves, getEUsPerCU());
363 
364   // Clamp to the maximum possible number of waves.
365   MaxWaves = std::min(MaxWaves, getMaxWavesPerEU());
366 
367   // FIXME: Needs to be a multiple of the group size?
368   //MaxWaves = MaxGroupNumWaves * (MaxWaves / MaxGroupNumWaves);
369 
370   assert(MaxWaves > 0 && MaxWaves <= getMaxWavesPerEU() &&
371          "computed invalid occupancy");
372   return MaxWaves;
373 }
374 
375 unsigned
getOccupancyWithLocalMemSize(const MachineFunction & MF) const376 AMDGPUSubtarget::getOccupancyWithLocalMemSize(const MachineFunction &MF) const {
377   const auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
378   return getOccupancyWithLocalMemSize(MFI->getLDSSize(), MF.getFunction());
379 }
380 
381 std::pair<unsigned, unsigned>
getDefaultFlatWorkGroupSize(CallingConv::ID CC) const382 AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const {
383   switch (CC) {
384   case CallingConv::AMDGPU_VS:
385   case CallingConv::AMDGPU_LS:
386   case CallingConv::AMDGPU_HS:
387   case CallingConv::AMDGPU_ES:
388   case CallingConv::AMDGPU_GS:
389   case CallingConv::AMDGPU_PS:
390     return std::pair(1, getWavefrontSize());
391   default:
392     return std::pair(1u, getMaxFlatWorkGroupSize());
393   }
394 }
395 
getFlatWorkGroupSizes(const Function & F) const396 std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
397   const Function &F) const {
398   // Default minimum/maximum flat work group sizes.
399   std::pair<unsigned, unsigned> Default =
400     getDefaultFlatWorkGroupSize(F.getCallingConv());
401 
402   // Requested minimum/maximum flat work group sizes.
403   std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
404     F, "amdgpu-flat-work-group-size", Default);
405 
406   // Make sure requested minimum is less than requested maximum.
407   if (Requested.first > Requested.second)
408     return Default;
409 
410   // Make sure requested values do not violate subtarget's specifications.
411   if (Requested.first < getMinFlatWorkGroupSize())
412     return Default;
413   if (Requested.second > getMaxFlatWorkGroupSize())
414     return Default;
415 
416   return Requested;
417 }
418 
getWavesPerEU(const Function & F,std::pair<unsigned,unsigned> FlatWorkGroupSizes) const419 std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
420     const Function &F, std::pair<unsigned, unsigned> FlatWorkGroupSizes) const {
421   // Default minimum/maximum number of waves per execution unit.
422   std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU());
423 
424   // If minimum/maximum flat work group sizes were explicitly requested using
425   // "amdgpu-flat-work-group-size" attribute, then set default minimum/maximum
426   // number of waves per execution unit to values implied by requested
427   // minimum/maximum flat work group sizes.
428   unsigned MinImpliedByFlatWorkGroupSize =
429     getWavesPerEUForWorkGroup(FlatWorkGroupSizes.second);
430   Default.first = MinImpliedByFlatWorkGroupSize;
431 
432   // Requested minimum/maximum number of waves per execution unit.
433   std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
434     F, "amdgpu-waves-per-eu", Default, true);
435 
436   // Make sure requested minimum is less than requested maximum.
437   if (Requested.second && Requested.first > Requested.second)
438     return Default;
439 
440   // Make sure requested values do not violate subtarget's specifications.
441   if (Requested.first < getMinWavesPerEU() ||
442       Requested.second > getMaxWavesPerEU())
443     return Default;
444 
445   // Make sure requested values are compatible with values implied by requested
446   // minimum/maximum flat work group sizes.
447   if (Requested.first < MinImpliedByFlatWorkGroupSize)
448     return Default;
449 
450   return Requested;
451 }
452 
getReqdWorkGroupSize(const Function & Kernel,unsigned Dim)453 static unsigned getReqdWorkGroupSize(const Function &Kernel, unsigned Dim) {
454   auto Node = Kernel.getMetadata("reqd_work_group_size");
455   if (Node && Node->getNumOperands() == 3)
456     return mdconst::extract<ConstantInt>(Node->getOperand(Dim))->getZExtValue();
457   return std::numeric_limits<unsigned>::max();
458 }
459 
isMesaKernel(const Function & F) const460 bool AMDGPUSubtarget::isMesaKernel(const Function &F) const {
461   return isMesa3DOS() && !AMDGPU::isShader(F.getCallingConv());
462 }
463 
getMaxWorkitemID(const Function & Kernel,unsigned Dimension) const464 unsigned AMDGPUSubtarget::getMaxWorkitemID(const Function &Kernel,
465                                            unsigned Dimension) const {
466   unsigned ReqdSize = getReqdWorkGroupSize(Kernel, Dimension);
467   if (ReqdSize != std::numeric_limits<unsigned>::max())
468     return ReqdSize - 1;
469   return getFlatWorkGroupSizes(Kernel).second - 1;
470 }
471 
makeLIDRangeMetadata(Instruction * I) const472 bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
473   Function *Kernel = I->getParent()->getParent();
474   unsigned MinSize = 0;
475   unsigned MaxSize = getFlatWorkGroupSizes(*Kernel).second;
476   bool IdQuery = false;
477 
478   // If reqd_work_group_size is present it narrows value down.
479   if (auto *CI = dyn_cast<CallInst>(I)) {
480     const Function *F = CI->getCalledFunction();
481     if (F) {
482       unsigned Dim = UINT_MAX;
483       switch (F->getIntrinsicID()) {
484       case Intrinsic::amdgcn_workitem_id_x:
485       case Intrinsic::r600_read_tidig_x:
486         IdQuery = true;
487         [[fallthrough]];
488       case Intrinsic::r600_read_local_size_x:
489         Dim = 0;
490         break;
491       case Intrinsic::amdgcn_workitem_id_y:
492       case Intrinsic::r600_read_tidig_y:
493         IdQuery = true;
494         [[fallthrough]];
495       case Intrinsic::r600_read_local_size_y:
496         Dim = 1;
497         break;
498       case Intrinsic::amdgcn_workitem_id_z:
499       case Intrinsic::r600_read_tidig_z:
500         IdQuery = true;
501         [[fallthrough]];
502       case Intrinsic::r600_read_local_size_z:
503         Dim = 2;
504         break;
505       default:
506         break;
507       }
508 
509       if (Dim <= 3) {
510         unsigned ReqdSize = getReqdWorkGroupSize(*Kernel, Dim);
511         if (ReqdSize != std::numeric_limits<unsigned>::max())
512           MinSize = MaxSize = ReqdSize;
513       }
514     }
515   }
516 
517   if (!MaxSize)
518     return false;
519 
520   // Range metadata is [Lo, Hi). For ID query we need to pass max size
521   // as Hi. For size query we need to pass Hi + 1.
522   if (IdQuery)
523     MinSize = 0;
524   else
525     ++MaxSize;
526 
527   MDBuilder MDB(I->getContext());
528   MDNode *MaxWorkGroupSizeRange = MDB.createRange(APInt(32, MinSize),
529                                                   APInt(32, MaxSize));
530   I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);
531   return true;
532 }
533 
getImplicitArgNumBytes(const Function & F) const534 unsigned AMDGPUSubtarget::getImplicitArgNumBytes(const Function &F) const {
535   assert(AMDGPU::isKernel(F.getCallingConv()));
536 
537   // We don't allocate the segment if we know the implicit arguments weren't
538   // used, even if the ABI implies we need them.
539   if (F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))
540     return 0;
541 
542   if (isMesaKernel(F))
543     return 16;
544 
545   // Assume all implicit inputs are used by default
546   unsigned NBytes = (AMDGPU::getAmdhsaCodeObjectVersion() >= 5) ? 256 : 56;
547   return F.getFnAttributeAsParsedInteger("amdgpu-implicitarg-num-bytes",
548                                          NBytes);
549 }
550 
getExplicitKernArgSize(const Function & F,Align & MaxAlign) const551 uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F,
552                                                  Align &MaxAlign) const {
553   assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
554          F.getCallingConv() == CallingConv::SPIR_KERNEL);
555 
556   const DataLayout &DL = F.getParent()->getDataLayout();
557   uint64_t ExplicitArgBytes = 0;
558   MaxAlign = Align(1);
559 
560   for (const Argument &Arg : F.args()) {
561     const bool IsByRef = Arg.hasByRefAttr();
562     Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
563     Align Alignment = DL.getValueOrABITypeAlignment(
564         IsByRef ? Arg.getParamAlign() : std::nullopt, ArgTy);
565     uint64_t AllocSize = DL.getTypeAllocSize(ArgTy);
566     ExplicitArgBytes = alignTo(ExplicitArgBytes, Alignment) + AllocSize;
567     MaxAlign = std::max(MaxAlign, Alignment);
568   }
569 
570   return ExplicitArgBytes;
571 }
572 
getKernArgSegmentSize(const Function & F,Align & MaxAlign) const573 unsigned AMDGPUSubtarget::getKernArgSegmentSize(const Function &F,
574                                                 Align &MaxAlign) const {
575   uint64_t ExplicitArgBytes = getExplicitKernArgSize(F, MaxAlign);
576 
577   unsigned ExplicitOffset = getExplicitKernelArgOffset(F);
578 
579   uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes;
580   unsigned ImplicitBytes = getImplicitArgNumBytes(F);
581   if (ImplicitBytes != 0) {
582     const Align Alignment = getAlignmentForImplicitArgPtr();
583     TotalSize = alignTo(ExplicitArgBytes, Alignment) + ImplicitBytes;
584     MaxAlign = std::max(MaxAlign, Alignment);
585   }
586 
587   // Being able to dereference past the end is useful for emitting scalar loads.
588   return alignTo(TotalSize, 4);
589 }
590 
getAMDGPUDwarfFlavour() const591 AMDGPUDwarfFlavour AMDGPUSubtarget::getAMDGPUDwarfFlavour() const {
592   return getWavefrontSize() == 32 ? AMDGPUDwarfFlavour::Wave32
593                                   : AMDGPUDwarfFlavour::Wave64;
594 }
595 
overrideSchedPolicy(MachineSchedPolicy & Policy,unsigned NumRegionInstrs) const596 void GCNSubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
597                                       unsigned NumRegionInstrs) const {
598   // Track register pressure so the scheduler can try to decrease
599   // pressure once register usage is above the threshold defined by
600   // SIRegisterInfo::getRegPressureSetLimit()
601   Policy.ShouldTrackPressure = true;
602 
603   // Enabling both top down and bottom up scheduling seems to give us less
604   // register spills than just using one of these approaches on its own.
605   Policy.OnlyTopDown = false;
606   Policy.OnlyBottomUp = false;
607 
608   // Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler.
609   if (!enableSIScheduler())
610     Policy.ShouldTrackLaneMasks = true;
611 }
612 
hasMadF16() const613 bool GCNSubtarget::hasMadF16() const {
614   return InstrInfo.pseudoToMCOpcode(AMDGPU::V_MAD_F16_e64) != -1;
615 }
616 
useVGPRIndexMode() const617 bool GCNSubtarget::useVGPRIndexMode() const {
618   return !hasMovrel() || (EnableVGPRIndexMode && hasVGPRIndexMode());
619 }
620 
useAA() const621 bool GCNSubtarget::useAA() const { return UseAA; }
622 
getOccupancyWithNumSGPRs(unsigned SGPRs) const623 unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
624   if (getGeneration() >= AMDGPUSubtarget::GFX10)
625     return getMaxWavesPerEU();
626 
627   if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
628     if (SGPRs <= 80)
629       return 10;
630     if (SGPRs <= 88)
631       return 9;
632     if (SGPRs <= 100)
633       return 8;
634     return 7;
635   }
636   if (SGPRs <= 48)
637     return 10;
638   if (SGPRs <= 56)
639     return 9;
640   if (SGPRs <= 64)
641     return 8;
642   if (SGPRs <= 72)
643     return 7;
644   if (SGPRs <= 80)
645     return 6;
646   return 5;
647 }
648 
getOccupancyWithNumVGPRs(unsigned NumVGPRs) const649 unsigned GCNSubtarget::getOccupancyWithNumVGPRs(unsigned NumVGPRs) const {
650   return AMDGPU::IsaInfo::getNumWavesPerEUWithNumVGPRs(this, NumVGPRs);
651 }
652 
653 unsigned
getBaseReservedNumSGPRs(const bool HasFlatScratch) const654 GCNSubtarget::getBaseReservedNumSGPRs(const bool HasFlatScratch) const {
655   if (getGeneration() >= AMDGPUSubtarget::GFX10)
656     return 2; // VCC. FLAT_SCRATCH and XNACK are no longer in SGPRs.
657 
658   if (HasFlatScratch || HasArchitectedFlatScratch) {
659     if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
660       return 6; // FLAT_SCRATCH, XNACK, VCC (in that order).
661     if (getGeneration() == AMDGPUSubtarget::SEA_ISLANDS)
662       return 4; // FLAT_SCRATCH, VCC (in that order).
663   }
664 
665   if (isXNACKEnabled())
666     return 4; // XNACK, VCC (in that order).
667   return 2; // VCC.
668 }
669 
getReservedNumSGPRs(const MachineFunction & MF) const670 unsigned GCNSubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
671   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
672   return getBaseReservedNumSGPRs(MFI.hasFlatScratchInit());
673 }
674 
getReservedNumSGPRs(const Function & F) const675 unsigned GCNSubtarget::getReservedNumSGPRs(const Function &F) const {
676   // In principle we do not need to reserve SGPR pair used for flat_scratch if
677   // we know flat instructions do not access the stack anywhere in the
678   // program. For now assume it's needed if we have flat instructions.
679   const bool KernelUsesFlatScratch = hasFlatAddressSpace();
680   return getBaseReservedNumSGPRs(KernelUsesFlatScratch);
681 }
682 
computeOccupancy(const Function & F,unsigned LDSSize,unsigned NumSGPRs,unsigned NumVGPRs) const683 unsigned GCNSubtarget::computeOccupancy(const Function &F, unsigned LDSSize,
684                                         unsigned NumSGPRs,
685                                         unsigned NumVGPRs) const {
686   unsigned Occupancy =
687     std::min(getMaxWavesPerEU(),
688              getOccupancyWithLocalMemSize(LDSSize, F));
689   if (NumSGPRs)
690     Occupancy = std::min(Occupancy, getOccupancyWithNumSGPRs(NumSGPRs));
691   if (NumVGPRs)
692     Occupancy = std::min(Occupancy, getOccupancyWithNumVGPRs(NumVGPRs));
693   return Occupancy;
694 }
695 
getBaseMaxNumSGPRs(const Function & F,std::pair<unsigned,unsigned> WavesPerEU,unsigned PreloadedSGPRs,unsigned ReservedNumSGPRs) const696 unsigned GCNSubtarget::getBaseMaxNumSGPRs(
697     const Function &F, std::pair<unsigned, unsigned> WavesPerEU,
698     unsigned PreloadedSGPRs, unsigned ReservedNumSGPRs) const {
699   // Compute maximum number of SGPRs function can use using default/requested
700   // minimum number of waves per execution unit.
701   unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, false);
702   unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU.first, true);
703 
704   // Check if maximum number of SGPRs was explicitly requested using
705   // "amdgpu-num-sgpr" attribute.
706   if (F.hasFnAttribute("amdgpu-num-sgpr")) {
707     unsigned Requested =
708         F.getFnAttributeAsParsedInteger("amdgpu-num-sgpr", MaxNumSGPRs);
709 
710     // Make sure requested value does not violate subtarget's specifications.
711     if (Requested && (Requested <= ReservedNumSGPRs))
712       Requested = 0;
713 
714     // If more SGPRs are required to support the input user/system SGPRs,
715     // increase to accommodate them.
716     //
717     // FIXME: This really ends up using the requested number of SGPRs + number
718     // of reserved special registers in total. Theoretically you could re-use
719     // the last input registers for these special registers, but this would
720     // require a lot of complexity to deal with the weird aliasing.
721     unsigned InputNumSGPRs = PreloadedSGPRs;
722     if (Requested && Requested < InputNumSGPRs)
723       Requested = InputNumSGPRs;
724 
725     // Make sure requested value is compatible with values implied by
726     // default/requested minimum/maximum number of waves per execution unit.
727     if (Requested && Requested > getMaxNumSGPRs(WavesPerEU.first, false))
728       Requested = 0;
729     if (WavesPerEU.second &&
730         Requested && Requested < getMinNumSGPRs(WavesPerEU.second))
731       Requested = 0;
732 
733     if (Requested)
734       MaxNumSGPRs = Requested;
735   }
736 
737   if (hasSGPRInitBug())
738     MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
739 
740   return std::min(MaxNumSGPRs - ReservedNumSGPRs, MaxAddressableNumSGPRs);
741 }
742 
getMaxNumSGPRs(const MachineFunction & MF) const743 unsigned GCNSubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
744   const Function &F = MF.getFunction();
745   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
746   return getBaseMaxNumSGPRs(F, MFI.getWavesPerEU(), MFI.getNumPreloadedSGPRs(),
747                             getReservedNumSGPRs(MF));
748 }
749 
getMaxNumPreloadedSGPRs()750 static unsigned getMaxNumPreloadedSGPRs() {
751   // Max number of user SGPRs
752   unsigned MaxUserSGPRs = 4 + // private segment buffer
753                           2 + // Dispatch ptr
754                           2 + // queue ptr
755                           2 + // kernel segment ptr
756                           2 + // dispatch ID
757                           2 + // flat scratch init
758                           2;  // Implicit buffer ptr
759 
760   // Max number of system SGPRs
761   unsigned MaxSystemSGPRs = 1 + // WorkGroupIDX
762                             1 + // WorkGroupIDY
763                             1 + // WorkGroupIDZ
764                             1 + // WorkGroupInfo
765                             1;  // private segment wave byte offset
766 
767   // Max number of synthetic SGPRs
768   unsigned SyntheticSGPRs = 1; // LDSKernelId
769 
770   return MaxUserSGPRs + MaxSystemSGPRs + SyntheticSGPRs;
771 }
772 
getMaxNumSGPRs(const Function & F) const773 unsigned GCNSubtarget::getMaxNumSGPRs(const Function &F) const {
774   return getBaseMaxNumSGPRs(F, getWavesPerEU(F), getMaxNumPreloadedSGPRs(),
775                             getReservedNumSGPRs(F));
776 }
777 
getBaseMaxNumVGPRs(const Function & F,std::pair<unsigned,unsigned> WavesPerEU) const778 unsigned GCNSubtarget::getBaseMaxNumVGPRs(
779     const Function &F, std::pair<unsigned, unsigned> WavesPerEU) const {
780   // Compute maximum number of VGPRs function can use using default/requested
781   // minimum number of waves per execution unit.
782   unsigned MaxNumVGPRs = getMaxNumVGPRs(WavesPerEU.first);
783 
784   // Check if maximum number of VGPRs was explicitly requested using
785   // "amdgpu-num-vgpr" attribute.
786   if (F.hasFnAttribute("amdgpu-num-vgpr")) {
787     unsigned Requested =
788         F.getFnAttributeAsParsedInteger("amdgpu-num-vgpr", MaxNumVGPRs);
789 
790     if (hasGFX90AInsts())
791       Requested *= 2;
792 
793     // Make sure requested value is compatible with values implied by
794     // default/requested minimum/maximum number of waves per execution unit.
795     if (Requested && Requested > getMaxNumVGPRs(WavesPerEU.first))
796       Requested = 0;
797     if (WavesPerEU.second &&
798         Requested && Requested < getMinNumVGPRs(WavesPerEU.second))
799       Requested = 0;
800 
801     if (Requested)
802       MaxNumVGPRs = Requested;
803   }
804 
805   return MaxNumVGPRs;
806 }
807 
getMaxNumVGPRs(const Function & F) const808 unsigned GCNSubtarget::getMaxNumVGPRs(const Function &F) const {
809   return getBaseMaxNumVGPRs(F, getWavesPerEU(F));
810 }
811 
getMaxNumVGPRs(const MachineFunction & MF) const812 unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
813   const Function &F = MF.getFunction();
814   const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
815   return getBaseMaxNumVGPRs(F, MFI.getWavesPerEU());
816 }
817 
adjustSchedDependency(SUnit * Def,int DefOpIdx,SUnit * Use,int UseOpIdx,SDep & Dep) const818 void GCNSubtarget::adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use,
819                                          int UseOpIdx, SDep &Dep) const {
820   if (Dep.getKind() != SDep::Kind::Data || !Dep.getReg() ||
821       !Def->isInstr() || !Use->isInstr())
822     return;
823 
824   MachineInstr *DefI = Def->getInstr();
825   MachineInstr *UseI = Use->getInstr();
826 
827   if (DefI->isBundle()) {
828     const SIRegisterInfo *TRI = getRegisterInfo();
829     auto Reg = Dep.getReg();
830     MachineBasicBlock::const_instr_iterator I(DefI->getIterator());
831     MachineBasicBlock::const_instr_iterator E(DefI->getParent()->instr_end());
832     unsigned Lat = 0;
833     for (++I; I != E && I->isBundledWithPred(); ++I) {
834       if (I->modifiesRegister(Reg, TRI))
835         Lat = InstrInfo.getInstrLatency(getInstrItineraryData(), *I);
836       else if (Lat)
837         --Lat;
838     }
839     Dep.setLatency(Lat);
840   } else if (UseI->isBundle()) {
841     const SIRegisterInfo *TRI = getRegisterInfo();
842     auto Reg = Dep.getReg();
843     MachineBasicBlock::const_instr_iterator I(UseI->getIterator());
844     MachineBasicBlock::const_instr_iterator E(UseI->getParent()->instr_end());
845     unsigned Lat = InstrInfo.getInstrLatency(getInstrItineraryData(), *DefI);
846     for (++I; I != E && I->isBundledWithPred() && Lat; ++I) {
847       if (I->readsRegister(Reg, TRI))
848         break;
849       --Lat;
850     }
851     Dep.setLatency(Lat);
852   } else if (Dep.getLatency() == 0 && Dep.getReg() == AMDGPU::VCC_LO) {
853     // Work around the fact that SIInstrInfo::fixImplicitOperands modifies
854     // implicit operands which come from the MCInstrDesc, which can fool
855     // ScheduleDAGInstrs::addPhysRegDataDeps into treating them as implicit
856     // pseudo operands.
857     Dep.setLatency(InstrInfo.getSchedModel().computeOperandLatency(
858         DefI, DefOpIdx, UseI, UseOpIdx));
859   }
860 }
861 
862 namespace {
863 struct FillMFMAShadowMutation : ScheduleDAGMutation {
864   const SIInstrInfo *TII;
865 
866   ScheduleDAGMI *DAG;
867 
FillMFMAShadowMutation__anon697d588f0111::FillMFMAShadowMutation868   FillMFMAShadowMutation(const SIInstrInfo *tii) : TII(tii) {}
869 
isSALU__anon697d588f0111::FillMFMAShadowMutation870   bool isSALU(const SUnit *SU) const {
871     const MachineInstr *MI = SU->getInstr();
872     return MI && TII->isSALU(*MI) && !MI->isTerminator();
873   }
874 
isVALU__anon697d588f0111::FillMFMAShadowMutation875   bool isVALU(const SUnit *SU) const {
876     const MachineInstr *MI = SU->getInstr();
877     return MI && TII->isVALU(*MI);
878   }
879 
880   // Link as many SALU instructions in chain as possible. Return the size
881   // of the chain. Links up to MaxChain instructions.
linkSALUChain__anon697d588f0111::FillMFMAShadowMutation882   unsigned linkSALUChain(SUnit *From, SUnit *To, unsigned MaxChain,
883                          SmallPtrSetImpl<SUnit *> &Visited) const {
884     SmallVector<SUnit *, 8> Worklist({To});
885     unsigned Linked = 0;
886 
887     while (!Worklist.empty() && MaxChain-- > 0) {
888       SUnit *SU = Worklist.pop_back_val();
889       if (!Visited.insert(SU).second)
890         continue;
891 
892       LLVM_DEBUG(dbgs() << "Inserting edge from\n" ; DAG->dumpNode(*From);
893                  dbgs() << "to\n"; DAG->dumpNode(*SU); dbgs() << '\n');
894 
895       if (SU != From && From != &DAG->ExitSU && DAG->canAddEdge(SU, From))
896         if (DAG->addEdge(SU, SDep(From, SDep::Artificial)))
897           ++Linked;
898 
899       for (SDep &SI : From->Succs) {
900         SUnit *SUv = SI.getSUnit();
901         if (SUv != From && SU != &DAG->ExitSU && isVALU(SUv) &&
902             DAG->canAddEdge(SUv, SU))
903           DAG->addEdge(SUv, SDep(SU, SDep::Artificial));
904       }
905 
906       for (SDep &SI : SU->Succs) {
907         SUnit *Succ = SI.getSUnit();
908         if (Succ != SU && isSALU(Succ))
909           Worklist.push_back(Succ);
910       }
911     }
912 
913     return Linked;
914   }
915 
apply__anon697d588f0111::FillMFMAShadowMutation916   void apply(ScheduleDAGInstrs *DAGInstrs) override {
917     const GCNSubtarget &ST = DAGInstrs->MF.getSubtarget<GCNSubtarget>();
918     if (!ST.hasMAIInsts())
919       return;
920     DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
921     const TargetSchedModel *TSchedModel = DAGInstrs->getSchedModel();
922     if (!TSchedModel || DAG->SUnits.empty())
923       return;
924 
925     // Scan for MFMA long latency instructions and try to add a dependency
926     // of available SALU instructions to give them a chance to fill MFMA
927     // shadow. That is desirable to fill MFMA shadow with SALU instructions
928     // rather than VALU to prevent power consumption bursts and throttle.
929     auto LastSALU = DAG->SUnits.begin();
930     auto E = DAG->SUnits.end();
931     SmallPtrSet<SUnit*, 32> Visited;
932     for (SUnit &SU : DAG->SUnits) {
933       MachineInstr &MAI = *SU.getInstr();
934       if (!TII->isMAI(MAI) ||
935            MAI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
936            MAI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64)
937         continue;
938 
939       unsigned Lat = TSchedModel->computeInstrLatency(&MAI) - 1;
940 
941       LLVM_DEBUG(dbgs() << "Found MFMA: "; DAG->dumpNode(SU);
942                  dbgs() << "Need " << Lat
943                         << " instructions to cover latency.\n");
944 
945       // Find up to Lat independent scalar instructions as early as
946       // possible such that they can be scheduled after this MFMA.
947       for ( ; Lat && LastSALU != E; ++LastSALU) {
948         if (Visited.count(&*LastSALU))
949           continue;
950 
951         if (&SU == &DAG->ExitSU || &SU == &*LastSALU || !isSALU(&*LastSALU) ||
952             !DAG->canAddEdge(&*LastSALU, &SU))
953           continue;
954 
955         Lat -= linkSALUChain(&SU, &*LastSALU, Lat, Visited);
956       }
957     }
958   }
959 };
960 } // namespace
961 
getPostRAMutations(std::vector<std::unique_ptr<ScheduleDAGMutation>> & Mutations) const962 void GCNSubtarget::getPostRAMutations(
963     std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
964   Mutations.push_back(std::make_unique<FillMFMAShadowMutation>(&InstrInfo));
965 }
966 
967 std::unique_ptr<ScheduleDAGMutation>
createFillMFMAShadowMutation(const TargetInstrInfo * TII) const968 GCNSubtarget::createFillMFMAShadowMutation(const TargetInstrInfo *TII) const {
969   return EnablePowerSched ? std::make_unique<FillMFMAShadowMutation>(&InstrInfo)
970                           : nullptr;
971 }
972 
getNSAThreshold(const MachineFunction & MF) const973 unsigned GCNSubtarget::getNSAThreshold(const MachineFunction &MF) const {
974   if (NSAThreshold.getNumOccurrences() > 0)
975     return std::max(NSAThreshold.getValue(), 2u);
976 
977   int Value = MF.getFunction().getFnAttributeAsParsedInteger(
978       "amdgpu-nsa-threshold", -1);
979   if (Value > 0)
980     return std::max(Value, 2);
981 
982   return 3;
983 }
984 
get(const MachineFunction & MF)985 const AMDGPUSubtarget &AMDGPUSubtarget::get(const MachineFunction &MF) {
986   if (MF.getTarget().getTargetTriple().getArch() == Triple::amdgcn)
987     return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<GCNSubtarget>());
988   else
989     return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<R600Subtarget>());
990 }
991 
get(const TargetMachine & TM,const Function & F)992 const AMDGPUSubtarget &AMDGPUSubtarget::get(const TargetMachine &TM, const Function &F) {
993   if (TM.getTargetTriple().getArch() == Triple::amdgcn)
994     return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<GCNSubtarget>(F));
995   else
996     return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<R600Subtarget>(F));
997 }
998