10b57cec5SDimitry Andric //===- AMDGPUTargetTransformInfo.cpp - AMDGPU specific TTI pass -----------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // \file
100b57cec5SDimitry Andric // This file implements a TargetTransformInfo analysis pass specific to the
110b57cec5SDimitry Andric // AMDGPU target machine. It uses the target's detailed information to provide
120b57cec5SDimitry Andric // more precise answers to certain TTI queries, while letting the target
130b57cec5SDimitry Andric // independent and default TTI implementations handle the rest.
140b57cec5SDimitry Andric //
150b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
160b57cec5SDimitry Andric
170b57cec5SDimitry Andric #include "AMDGPUTargetTransformInfo.h"
18e8d8bef9SDimitry Andric #include "AMDGPUTargetMachine.h"
19349cc55cSDimitry Andric #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
2006c3fb27SDimitry Andric #include "SIModeRegisterDefaults.h"
2106c3fb27SDimitry Andric #include "llvm/Analysis/InlineCost.h"
220b57cec5SDimitry Andric #include "llvm/Analysis/LoopInfo.h"
230b57cec5SDimitry Andric #include "llvm/Analysis/ValueTracking.h"
2406c3fb27SDimitry Andric #include "llvm/CodeGen/Analysis.h"
25fe6060f1SDimitry Andric #include "llvm/IR/IRBuilder.h"
26349cc55cSDimitry Andric #include "llvm/IR/IntrinsicsAMDGPU.h"
270b57cec5SDimitry Andric #include "llvm/IR/PatternMatch.h"
28e8d8bef9SDimitry Andric #include "llvm/Support/KnownBits.h"
29bdd1243dSDimitry Andric #include <optional>
300b57cec5SDimitry Andric
310b57cec5SDimitry Andric using namespace llvm;
320b57cec5SDimitry Andric
330b57cec5SDimitry Andric #define DEBUG_TYPE "AMDGPUtti"
340b57cec5SDimitry Andric
350b57cec5SDimitry Andric static cl::opt<unsigned> UnrollThresholdPrivate(
360b57cec5SDimitry Andric "amdgpu-unroll-threshold-private",
370b57cec5SDimitry Andric cl::desc("Unroll threshold for AMDGPU if private memory used in a loop"),
38480093f4SDimitry Andric cl::init(2700), cl::Hidden);
390b57cec5SDimitry Andric
400b57cec5SDimitry Andric static cl::opt<unsigned> UnrollThresholdLocal(
410b57cec5SDimitry Andric "amdgpu-unroll-threshold-local",
420b57cec5SDimitry Andric cl::desc("Unroll threshold for AMDGPU if local memory used in a loop"),
430b57cec5SDimitry Andric cl::init(1000), cl::Hidden);
440b57cec5SDimitry Andric
450b57cec5SDimitry Andric static cl::opt<unsigned> UnrollThresholdIf(
460b57cec5SDimitry Andric "amdgpu-unroll-threshold-if",
470b57cec5SDimitry Andric cl::desc("Unroll threshold increment for AMDGPU for each if statement inside loop"),
48fe6060f1SDimitry Andric cl::init(200), cl::Hidden);
490b57cec5SDimitry Andric
505ffd83dbSDimitry Andric static cl::opt<bool> UnrollRuntimeLocal(
515ffd83dbSDimitry Andric "amdgpu-unroll-runtime-local",
525ffd83dbSDimitry Andric cl::desc("Allow runtime unroll for AMDGPU if local memory used in a loop"),
535ffd83dbSDimitry Andric cl::init(true), cl::Hidden);
545ffd83dbSDimitry Andric
555ffd83dbSDimitry Andric static cl::opt<unsigned> UnrollMaxBlockToAnalyze(
565ffd83dbSDimitry Andric "amdgpu-unroll-max-block-to-analyze",
575ffd83dbSDimitry Andric cl::desc("Inner loop block size threshold to analyze in unroll for AMDGPU"),
58e8d8bef9SDimitry Andric cl::init(32), cl::Hidden);
59e8d8bef9SDimitry Andric
60e8d8bef9SDimitry Andric static cl::opt<unsigned> ArgAllocaCost("amdgpu-inline-arg-alloca-cost",
61e8d8bef9SDimitry Andric cl::Hidden, cl::init(4000),
62e8d8bef9SDimitry Andric cl::desc("Cost of alloca argument"));
63e8d8bef9SDimitry Andric
64e8d8bef9SDimitry Andric // If the amount of scratch memory to eliminate exceeds our ability to allocate
65e8d8bef9SDimitry Andric // it into registers we gain nothing by aggressively inlining functions for that
66e8d8bef9SDimitry Andric // heuristic.
67e8d8bef9SDimitry Andric static cl::opt<unsigned>
68e8d8bef9SDimitry Andric ArgAllocaCutoff("amdgpu-inline-arg-alloca-cutoff", cl::Hidden,
69e8d8bef9SDimitry Andric cl::init(256),
70e8d8bef9SDimitry Andric cl::desc("Maximum alloca size to use for inline cost"));
71e8d8bef9SDimitry Andric
72e8d8bef9SDimitry Andric // Inliner constraint to achieve reasonable compilation time.
73e8d8bef9SDimitry Andric static cl::opt<size_t> InlineMaxBB(
74e8d8bef9SDimitry Andric "amdgpu-inline-max-bb", cl::Hidden, cl::init(1100),
75e8d8bef9SDimitry Andric cl::desc("Maximum number of BBs allowed in a function after inlining"
76e8d8bef9SDimitry Andric " (compile time constraint)"));
775ffd83dbSDimitry Andric
dependsOnLocalPhi(const Loop * L,const Value * Cond,unsigned Depth=0)780b57cec5SDimitry Andric static bool dependsOnLocalPhi(const Loop *L, const Value *Cond,
790b57cec5SDimitry Andric unsigned Depth = 0) {
800b57cec5SDimitry Andric const Instruction *I = dyn_cast<Instruction>(Cond);
810b57cec5SDimitry Andric if (!I)
820b57cec5SDimitry Andric return false;
830b57cec5SDimitry Andric
840b57cec5SDimitry Andric for (const Value *V : I->operand_values()) {
850b57cec5SDimitry Andric if (!L->contains(I))
860b57cec5SDimitry Andric continue;
870b57cec5SDimitry Andric if (const PHINode *PHI = dyn_cast<PHINode>(V)) {
880b57cec5SDimitry Andric if (llvm::none_of(L->getSubLoops(), [PHI](const Loop* SubLoop) {
890b57cec5SDimitry Andric return SubLoop->contains(PHI); }))
900b57cec5SDimitry Andric return true;
910b57cec5SDimitry Andric } else if (Depth < 10 && dependsOnLocalPhi(L, V, Depth+1))
920b57cec5SDimitry Andric return true;
930b57cec5SDimitry Andric }
940b57cec5SDimitry Andric return false;
950b57cec5SDimitry Andric }
960b57cec5SDimitry Andric
AMDGPUTTIImpl(const AMDGPUTargetMachine * TM,const Function & F)97e8d8bef9SDimitry Andric AMDGPUTTIImpl::AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
98e8d8bef9SDimitry Andric : BaseT(TM, F.getParent()->getDataLayout()),
99e8d8bef9SDimitry Andric TargetTriple(TM->getTargetTriple()),
100e8d8bef9SDimitry Andric ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
101e8d8bef9SDimitry Andric TLI(ST->getTargetLowering()) {}
102e8d8bef9SDimitry Andric
getUnrollingPreferences(Loop * L,ScalarEvolution & SE,TTI::UnrollingPreferences & UP,OptimizationRemarkEmitter * ORE)1030b57cec5SDimitry Andric void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
104349cc55cSDimitry Andric TTI::UnrollingPreferences &UP,
105349cc55cSDimitry Andric OptimizationRemarkEmitter *ORE) {
106480093f4SDimitry Andric const Function &F = *L->getHeader()->getParent();
107bdd1243dSDimitry Andric UP.Threshold =
108bdd1243dSDimitry Andric F.getFnAttributeAsParsedInteger("amdgpu-unroll-threshold", 300);
1090b57cec5SDimitry Andric UP.MaxCount = std::numeric_limits<unsigned>::max();
1100b57cec5SDimitry Andric UP.Partial = true;
1110b57cec5SDimitry Andric
112fe6060f1SDimitry Andric // Conditional branch in a loop back edge needs 3 additional exec
113fe6060f1SDimitry Andric // manipulations in average.
114fe6060f1SDimitry Andric UP.BEInsns += 3;
115fe6060f1SDimitry Andric
11606c3fb27SDimitry Andric // We want to run unroll even for the loops which have been vectorized.
11706c3fb27SDimitry Andric UP.UnrollVectorizedLoop = true;
11806c3fb27SDimitry Andric
1190b57cec5SDimitry Andric // TODO: Do we want runtime unrolling?
1200b57cec5SDimitry Andric
1210b57cec5SDimitry Andric // Maximum alloca size than can fit registers. Reserve 16 registers.
1220b57cec5SDimitry Andric const unsigned MaxAlloca = (256 - 16) * 4;
1230b57cec5SDimitry Andric unsigned ThresholdPrivate = UnrollThresholdPrivate;
1240b57cec5SDimitry Andric unsigned ThresholdLocal = UnrollThresholdLocal;
125e8d8bef9SDimitry Andric
126e8d8bef9SDimitry Andric // If this loop has the amdgpu.loop.unroll.threshold metadata we will use the
127e8d8bef9SDimitry Andric // provided threshold value as the default for Threshold
128e8d8bef9SDimitry Andric if (MDNode *LoopUnrollThreshold =
129e8d8bef9SDimitry Andric findOptionMDForLoop(L, "amdgpu.loop.unroll.threshold")) {
130e8d8bef9SDimitry Andric if (LoopUnrollThreshold->getNumOperands() == 2) {
131e8d8bef9SDimitry Andric ConstantInt *MetaThresholdValue = mdconst::extract_or_null<ConstantInt>(
132e8d8bef9SDimitry Andric LoopUnrollThreshold->getOperand(1));
133e8d8bef9SDimitry Andric if (MetaThresholdValue) {
134e8d8bef9SDimitry Andric // We will also use the supplied value for PartialThreshold for now.
135e8d8bef9SDimitry Andric // We may introduce additional metadata if it becomes necessary in the
136e8d8bef9SDimitry Andric // future.
137e8d8bef9SDimitry Andric UP.Threshold = MetaThresholdValue->getSExtValue();
138e8d8bef9SDimitry Andric UP.PartialThreshold = UP.Threshold;
139e8d8bef9SDimitry Andric ThresholdPrivate = std::min(ThresholdPrivate, UP.Threshold);
140e8d8bef9SDimitry Andric ThresholdLocal = std::min(ThresholdLocal, UP.Threshold);
141e8d8bef9SDimitry Andric }
142e8d8bef9SDimitry Andric }
143e8d8bef9SDimitry Andric }
144e8d8bef9SDimitry Andric
1450b57cec5SDimitry Andric unsigned MaxBoost = std::max(ThresholdPrivate, ThresholdLocal);
1460b57cec5SDimitry Andric for (const BasicBlock *BB : L->getBlocks()) {
1470b57cec5SDimitry Andric const DataLayout &DL = BB->getModule()->getDataLayout();
1480b57cec5SDimitry Andric unsigned LocalGEPsSeen = 0;
1490b57cec5SDimitry Andric
1500b57cec5SDimitry Andric if (llvm::any_of(L->getSubLoops(), [BB](const Loop* SubLoop) {
1510b57cec5SDimitry Andric return SubLoop->contains(BB); }))
1520b57cec5SDimitry Andric continue; // Block belongs to an inner loop.
1530b57cec5SDimitry Andric
1540b57cec5SDimitry Andric for (const Instruction &I : *BB) {
1550b57cec5SDimitry Andric // Unroll a loop which contains an "if" statement whose condition
1560b57cec5SDimitry Andric // defined by a PHI belonging to the loop. This may help to eliminate
1570b57cec5SDimitry Andric // if region and potentially even PHI itself, saving on both divergence
1580b57cec5SDimitry Andric // and registers used for the PHI.
1590b57cec5SDimitry Andric // Add a small bonus for each of such "if" statements.
1600b57cec5SDimitry Andric if (const BranchInst *Br = dyn_cast<BranchInst>(&I)) {
1610b57cec5SDimitry Andric if (UP.Threshold < MaxBoost && Br->isConditional()) {
1620b57cec5SDimitry Andric BasicBlock *Succ0 = Br->getSuccessor(0);
1630b57cec5SDimitry Andric BasicBlock *Succ1 = Br->getSuccessor(1);
1640b57cec5SDimitry Andric if ((L->contains(Succ0) && L->isLoopExiting(Succ0)) ||
1650b57cec5SDimitry Andric (L->contains(Succ1) && L->isLoopExiting(Succ1)))
1660b57cec5SDimitry Andric continue;
1670b57cec5SDimitry Andric if (dependsOnLocalPhi(L, Br->getCondition())) {
1680b57cec5SDimitry Andric UP.Threshold += UnrollThresholdIf;
1690b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Set unroll threshold " << UP.Threshold
1700b57cec5SDimitry Andric << " for loop:\n"
1710b57cec5SDimitry Andric << *L << " due to " << *Br << '\n');
1720b57cec5SDimitry Andric if (UP.Threshold >= MaxBoost)
1730b57cec5SDimitry Andric return;
1740b57cec5SDimitry Andric }
1750b57cec5SDimitry Andric }
1760b57cec5SDimitry Andric continue;
1770b57cec5SDimitry Andric }
1780b57cec5SDimitry Andric
1790b57cec5SDimitry Andric const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I);
1800b57cec5SDimitry Andric if (!GEP)
1810b57cec5SDimitry Andric continue;
1820b57cec5SDimitry Andric
1830b57cec5SDimitry Andric unsigned AS = GEP->getAddressSpace();
1840b57cec5SDimitry Andric unsigned Threshold = 0;
1850b57cec5SDimitry Andric if (AS == AMDGPUAS::PRIVATE_ADDRESS)
1860b57cec5SDimitry Andric Threshold = ThresholdPrivate;
1870b57cec5SDimitry Andric else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
1880b57cec5SDimitry Andric Threshold = ThresholdLocal;
1890b57cec5SDimitry Andric else
1900b57cec5SDimitry Andric continue;
1910b57cec5SDimitry Andric
1920b57cec5SDimitry Andric if (UP.Threshold >= Threshold)
1930b57cec5SDimitry Andric continue;
1940b57cec5SDimitry Andric
1950b57cec5SDimitry Andric if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
1960b57cec5SDimitry Andric const Value *Ptr = GEP->getPointerOperand();
1970b57cec5SDimitry Andric const AllocaInst *Alloca =
198e8d8bef9SDimitry Andric dyn_cast<AllocaInst>(getUnderlyingObject(Ptr));
1990b57cec5SDimitry Andric if (!Alloca || !Alloca->isStaticAlloca())
2000b57cec5SDimitry Andric continue;
2010b57cec5SDimitry Andric Type *Ty = Alloca->getAllocatedType();
2020b57cec5SDimitry Andric unsigned AllocaSize = Ty->isSized() ? DL.getTypeAllocSize(Ty) : 0;
2030b57cec5SDimitry Andric if (AllocaSize > MaxAlloca)
2040b57cec5SDimitry Andric continue;
2050b57cec5SDimitry Andric } else if (AS == AMDGPUAS::LOCAL_ADDRESS ||
2060b57cec5SDimitry Andric AS == AMDGPUAS::REGION_ADDRESS) {
2070b57cec5SDimitry Andric LocalGEPsSeen++;
2080b57cec5SDimitry Andric // Inhibit unroll for local memory if we have seen addressing not to
2090b57cec5SDimitry Andric // a variable, most likely we will be unable to combine it.
2100b57cec5SDimitry Andric // Do not unroll too deep inner loops for local memory to give a chance
2110b57cec5SDimitry Andric // to unroll an outer loop for a more important reason.
2120b57cec5SDimitry Andric if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2 ||
2130b57cec5SDimitry Andric (!isa<GlobalVariable>(GEP->getPointerOperand()) &&
2140b57cec5SDimitry Andric !isa<Argument>(GEP->getPointerOperand())))
2150b57cec5SDimitry Andric continue;
2165ffd83dbSDimitry Andric LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n"
2175ffd83dbSDimitry Andric << *L << " due to LDS use.\n");
2185ffd83dbSDimitry Andric UP.Runtime = UnrollRuntimeLocal;
2190b57cec5SDimitry Andric }
2200b57cec5SDimitry Andric
2210b57cec5SDimitry Andric // Check if GEP depends on a value defined by this loop itself.
2220b57cec5SDimitry Andric bool HasLoopDef = false;
2230b57cec5SDimitry Andric for (const Value *Op : GEP->operands()) {
2240b57cec5SDimitry Andric const Instruction *Inst = dyn_cast<Instruction>(Op);
2250b57cec5SDimitry Andric if (!Inst || L->isLoopInvariant(Op))
2260b57cec5SDimitry Andric continue;
2270b57cec5SDimitry Andric
2280b57cec5SDimitry Andric if (llvm::any_of(L->getSubLoops(), [Inst](const Loop* SubLoop) {
2290b57cec5SDimitry Andric return SubLoop->contains(Inst); }))
2300b57cec5SDimitry Andric continue;
2310b57cec5SDimitry Andric HasLoopDef = true;
2320b57cec5SDimitry Andric break;
2330b57cec5SDimitry Andric }
2340b57cec5SDimitry Andric if (!HasLoopDef)
2350b57cec5SDimitry Andric continue;
2360b57cec5SDimitry Andric
2370b57cec5SDimitry Andric // We want to do whatever we can to limit the number of alloca
2380b57cec5SDimitry Andric // instructions that make it through to the code generator. allocas
2390b57cec5SDimitry Andric // require us to use indirect addressing, which is slow and prone to
2400b57cec5SDimitry Andric // compiler bugs. If this loop does an address calculation on an
2410b57cec5SDimitry Andric // alloca ptr, then we want to use a higher than normal loop unroll
2420b57cec5SDimitry Andric // threshold. This will give SROA a better chance to eliminate these
2430b57cec5SDimitry Andric // allocas.
2440b57cec5SDimitry Andric //
2450b57cec5SDimitry Andric // We also want to have more unrolling for local memory to let ds
2460b57cec5SDimitry Andric // instructions with different offsets combine.
2470b57cec5SDimitry Andric //
2480b57cec5SDimitry Andric // Don't use the maximum allowed value here as it will make some
2490b57cec5SDimitry Andric // programs way too big.
2500b57cec5SDimitry Andric UP.Threshold = Threshold;
2510b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Set unroll threshold " << Threshold
2520b57cec5SDimitry Andric << " for loop:\n"
2530b57cec5SDimitry Andric << *L << " due to " << *GEP << '\n');
2540b57cec5SDimitry Andric if (UP.Threshold >= MaxBoost)
2550b57cec5SDimitry Andric return;
2560b57cec5SDimitry Andric }
2575ffd83dbSDimitry Andric
2585ffd83dbSDimitry Andric // If we got a GEP in a small BB from inner loop then increase max trip
2595ffd83dbSDimitry Andric // count to analyze for better estimation cost in unroll
260e8d8bef9SDimitry Andric if (L->isInnermost() && BB->size() < UnrollMaxBlockToAnalyze)
2615ffd83dbSDimitry Andric UP.MaxIterationsCountToAnalyze = 32;
2620b57cec5SDimitry Andric }
2630b57cec5SDimitry Andric }
2640b57cec5SDimitry Andric
getPeelingPreferences(Loop * L,ScalarEvolution & SE,TTI::PeelingPreferences & PP)2655ffd83dbSDimitry Andric void AMDGPUTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
2665ffd83dbSDimitry Andric TTI::PeelingPreferences &PP) {
2675ffd83dbSDimitry Andric BaseT::getPeelingPreferences(L, SE, PP);
2685ffd83dbSDimitry Andric }
269e8d8bef9SDimitry Andric
getMaxMemIntrinsicInlineSizeThreshold() const27006c3fb27SDimitry Andric int64_t AMDGPUTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
27106c3fb27SDimitry Andric return 1024;
27206c3fb27SDimitry Andric }
27306c3fb27SDimitry Andric
274e8d8bef9SDimitry Andric const FeatureBitset GCNTTIImpl::InlineFeatureIgnoreList = {
275e8d8bef9SDimitry Andric // Codegen control options which don't matter.
276e8d8bef9SDimitry Andric AMDGPU::FeatureEnableLoadStoreOpt, AMDGPU::FeatureEnableSIScheduler,
277e8d8bef9SDimitry Andric AMDGPU::FeatureEnableUnsafeDSOffsetFolding, AMDGPU::FeatureFlatForGlobal,
278e8d8bef9SDimitry Andric AMDGPU::FeaturePromoteAlloca, AMDGPU::FeatureUnalignedScratchAccess,
279e8d8bef9SDimitry Andric AMDGPU::FeatureUnalignedAccessMode,
280e8d8bef9SDimitry Andric
281e8d8bef9SDimitry Andric AMDGPU::FeatureAutoWaitcntBeforeBarrier,
282e8d8bef9SDimitry Andric
283e8d8bef9SDimitry Andric // Property of the kernel/environment which can't actually differ.
284e8d8bef9SDimitry Andric AMDGPU::FeatureSGPRInitBug, AMDGPU::FeatureXNACK,
285e8d8bef9SDimitry Andric AMDGPU::FeatureTrapHandler,
286e8d8bef9SDimitry Andric
287e8d8bef9SDimitry Andric // The default assumption needs to be ecc is enabled, but no directly
288e8d8bef9SDimitry Andric // exposed operations depend on it, so it can be safely inlined.
289e8d8bef9SDimitry Andric AMDGPU::FeatureSRAMECC,
290e8d8bef9SDimitry Andric
291e8d8bef9SDimitry Andric // Perf-tuning features
292e8d8bef9SDimitry Andric AMDGPU::FeatureFastFMAF32, AMDGPU::HalfRate64Ops};
293e8d8bef9SDimitry Andric
GCNTTIImpl(const AMDGPUTargetMachine * TM,const Function & F)294e8d8bef9SDimitry Andric GCNTTIImpl::GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
295e8d8bef9SDimitry Andric : BaseT(TM, F.getParent()->getDataLayout()),
296e8d8bef9SDimitry Andric ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
297e8d8bef9SDimitry Andric TLI(ST->getTargetLowering()), CommonTTI(TM, F),
29881ad6265SDimitry Andric IsGraphics(AMDGPU::isGraphics(F.getCallingConv())) {
2995f757f3fSDimitry Andric SIModeRegisterDefaults Mode(F, *ST);
30006c3fb27SDimitry Andric HasFP32Denormals = Mode.FP32Denormals != DenormalMode::getPreserveSign();
30106c3fb27SDimitry Andric HasFP64FP16Denormals =
30206c3fb27SDimitry Andric Mode.FP64FP16Denormals != DenormalMode::getPreserveSign();
30306c3fb27SDimitry Andric }
30406c3fb27SDimitry Andric
hasBranchDivergence(const Function * F) const30506c3fb27SDimitry Andric bool GCNTTIImpl::hasBranchDivergence(const Function *F) const {
30606c3fb27SDimitry Andric return !F || !ST->isSingleLaneExecution(*F);
307e8d8bef9SDimitry Andric }
308e8d8bef9SDimitry Andric
getNumberOfRegisters(unsigned RCID) const30981ad6265SDimitry Andric unsigned GCNTTIImpl::getNumberOfRegisters(unsigned RCID) const {
31081ad6265SDimitry Andric // NB: RCID is not an RCID. In fact it is 0 or 1 for scalar or vector
31181ad6265SDimitry Andric // registers. See getRegisterClassForType for the implementation.
31281ad6265SDimitry Andric // In this case vector registers are not vector in terms of
31381ad6265SDimitry Andric // VGPRs, but those which can hold multiple values.
3140b57cec5SDimitry Andric
3150b57cec5SDimitry Andric // This is really the number of registers to fill when vectorizing /
3160b57cec5SDimitry Andric // interleaving loops, so we lie to avoid trying to use all registers.
31781ad6265SDimitry Andric return 4;
3185ffd83dbSDimitry Andric }
3195ffd83dbSDimitry Andric
320fe6060f1SDimitry Andric TypeSize
getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const321fe6060f1SDimitry Andric GCNTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
322fe6060f1SDimitry Andric switch (K) {
323fe6060f1SDimitry Andric case TargetTransformInfo::RGK_Scalar:
324fe6060f1SDimitry Andric return TypeSize::getFixed(32);
325fe6060f1SDimitry Andric case TargetTransformInfo::RGK_FixedWidthVector:
326fe6060f1SDimitry Andric return TypeSize::getFixed(ST->hasPackedFP32Ops() ? 64 : 32);
327fe6060f1SDimitry Andric case TargetTransformInfo::RGK_ScalableVector:
328fe6060f1SDimitry Andric return TypeSize::getScalable(0);
329fe6060f1SDimitry Andric }
330fe6060f1SDimitry Andric llvm_unreachable("Unsupported register kind");
3310b57cec5SDimitry Andric }
3320b57cec5SDimitry Andric
getMinVectorRegisterBitWidth() const3330b57cec5SDimitry Andric unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const {
3340b57cec5SDimitry Andric return 32;
3350b57cec5SDimitry Andric }
3360b57cec5SDimitry Andric
getMaximumVF(unsigned ElemWidth,unsigned Opcode) const337e8d8bef9SDimitry Andric unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
338e8d8bef9SDimitry Andric if (Opcode == Instruction::Load || Opcode == Instruction::Store)
339e8d8bef9SDimitry Andric return 32 * 4 / ElemWidth;
340fe6060f1SDimitry Andric return (ElemWidth == 16 && ST->has16BitInsts()) ? 2
341fe6060f1SDimitry Andric : (ElemWidth == 32 && ST->hasPackedFP32Ops()) ? 2
342fe6060f1SDimitry Andric : 1;
343e8d8bef9SDimitry Andric }
344e8d8bef9SDimitry Andric
getLoadVectorFactor(unsigned VF,unsigned LoadSize,unsigned ChainSizeInBytes,VectorType * VecTy) const3450b57cec5SDimitry Andric unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize,
3460b57cec5SDimitry Andric unsigned ChainSizeInBytes,
3470b57cec5SDimitry Andric VectorType *VecTy) const {
3480b57cec5SDimitry Andric unsigned VecRegBitWidth = VF * LoadSize;
3490b57cec5SDimitry Andric if (VecRegBitWidth > 128 && VecTy->getScalarSizeInBits() < 32)
3500b57cec5SDimitry Andric // TODO: Support element-size less than 32bit?
3510b57cec5SDimitry Andric return 128 / LoadSize;
3520b57cec5SDimitry Andric
3530b57cec5SDimitry Andric return VF;
3540b57cec5SDimitry Andric }
3550b57cec5SDimitry Andric
getStoreVectorFactor(unsigned VF,unsigned StoreSize,unsigned ChainSizeInBytes,VectorType * VecTy) const3560b57cec5SDimitry Andric unsigned GCNTTIImpl::getStoreVectorFactor(unsigned VF, unsigned StoreSize,
3570b57cec5SDimitry Andric unsigned ChainSizeInBytes,
3580b57cec5SDimitry Andric VectorType *VecTy) const {
3590b57cec5SDimitry Andric unsigned VecRegBitWidth = VF * StoreSize;
3600b57cec5SDimitry Andric if (VecRegBitWidth > 128)
3610b57cec5SDimitry Andric return 128 / StoreSize;
3620b57cec5SDimitry Andric
3630b57cec5SDimitry Andric return VF;
3640b57cec5SDimitry Andric }
3650b57cec5SDimitry Andric
getLoadStoreVecRegBitWidth(unsigned AddrSpace) const3660b57cec5SDimitry Andric unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
3670b57cec5SDimitry Andric if (AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ||
3680b57cec5SDimitry Andric AddrSpace == AMDGPUAS::CONSTANT_ADDRESS ||
3690b57cec5SDimitry Andric AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
37006c3fb27SDimitry Andric AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
3715f757f3fSDimitry Andric AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
3725f757f3fSDimitry Andric AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
3730b57cec5SDimitry Andric return 512;
3740b57cec5SDimitry Andric }
3750b57cec5SDimitry Andric
3760b57cec5SDimitry Andric if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
3770b57cec5SDimitry Andric return 8 * ST->getMaxPrivateElementSize();
3780b57cec5SDimitry Andric
3795ffd83dbSDimitry Andric // Common to flat, global, local and region. Assume for unknown addrspace.
3805ffd83dbSDimitry Andric return 128;
3810b57cec5SDimitry Andric }
3820b57cec5SDimitry Andric
isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,Align Alignment,unsigned AddrSpace) const3830b57cec5SDimitry Andric bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
3845ffd83dbSDimitry Andric Align Alignment,
3850b57cec5SDimitry Andric unsigned AddrSpace) const {
3860b57cec5SDimitry Andric // We allow vectorization of flat stores, even though we may need to decompose
3870b57cec5SDimitry Andric // them later if they may access private memory. We don't have enough context
3880b57cec5SDimitry Andric // here, and legalization can handle it.
3890b57cec5SDimitry Andric if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
3900b57cec5SDimitry Andric return (Alignment >= 4 || ST->hasUnalignedScratchAccess()) &&
3910b57cec5SDimitry Andric ChainSizeInBytes <= ST->getMaxPrivateElementSize();
3920b57cec5SDimitry Andric }
3930b57cec5SDimitry Andric return true;
3940b57cec5SDimitry Andric }
3950b57cec5SDimitry Andric
isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,Align Alignment,unsigned AddrSpace) const3960b57cec5SDimitry Andric bool GCNTTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
3975ffd83dbSDimitry Andric Align Alignment,
3980b57cec5SDimitry Andric unsigned AddrSpace) const {
3990b57cec5SDimitry Andric return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
4000b57cec5SDimitry Andric }
4010b57cec5SDimitry Andric
isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,Align Alignment,unsigned AddrSpace) const4020b57cec5SDimitry Andric bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
4035ffd83dbSDimitry Andric Align Alignment,
4040b57cec5SDimitry Andric unsigned AddrSpace) const {
4050b57cec5SDimitry Andric return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
4060b57cec5SDimitry Andric }
4070b57cec5SDimitry Andric
getMaxMemIntrinsicInlineSizeThreshold() const40806c3fb27SDimitry Andric int64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
40906c3fb27SDimitry Andric return 1024;
41006c3fb27SDimitry Andric }
41106c3fb27SDimitry Andric
4125ffd83dbSDimitry Andric // FIXME: Really we would like to issue multiple 128-bit loads and stores per
4135ffd83dbSDimitry Andric // iteration. Should we report a larger size and let it legalize?
4145ffd83dbSDimitry Andric //
4155ffd83dbSDimitry Andric // FIXME: Should we use narrower types for local/region, or account for when
4165ffd83dbSDimitry Andric // unaligned access is legal?
4175ffd83dbSDimitry Andric //
4185ffd83dbSDimitry Andric // FIXME: This could use fine tuning and microbenchmarks.
getMemcpyLoopLoweringType(LLVMContext & Context,Value * Length,unsigned SrcAddrSpace,unsigned DestAddrSpace,unsigned SrcAlign,unsigned DestAlign,std::optional<uint32_t> AtomicElementSize) const41981ad6265SDimitry Andric Type *GCNTTIImpl::getMemcpyLoopLoweringType(
42081ad6265SDimitry Andric LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
42181ad6265SDimitry Andric unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
422bdd1243dSDimitry Andric std::optional<uint32_t> AtomicElementSize) const {
42381ad6265SDimitry Andric
42481ad6265SDimitry Andric if (AtomicElementSize)
42581ad6265SDimitry Andric return Type::getIntNTy(Context, *AtomicElementSize * 8);
42681ad6265SDimitry Andric
4275ffd83dbSDimitry Andric unsigned MinAlign = std::min(SrcAlign, DestAlign);
4285ffd83dbSDimitry Andric
4295ffd83dbSDimitry Andric // A (multi-)dword access at an address == 2 (mod 4) will be decomposed by the
4305ffd83dbSDimitry Andric // hardware into byte accesses. If you assume all alignments are equally
4315ffd83dbSDimitry Andric // probable, it's more efficient on average to use short accesses for this
4325ffd83dbSDimitry Andric // case.
4335ffd83dbSDimitry Andric if (MinAlign == 2)
4345ffd83dbSDimitry Andric return Type::getInt16Ty(Context);
4355ffd83dbSDimitry Andric
4365ffd83dbSDimitry Andric // Not all subtargets have 128-bit DS instructions, and we currently don't
4375ffd83dbSDimitry Andric // form them by default.
4385ffd83dbSDimitry Andric if (SrcAddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
4395ffd83dbSDimitry Andric SrcAddrSpace == AMDGPUAS::REGION_ADDRESS ||
4405ffd83dbSDimitry Andric DestAddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
4415ffd83dbSDimitry Andric DestAddrSpace == AMDGPUAS::REGION_ADDRESS) {
4425ffd83dbSDimitry Andric return FixedVectorType::get(Type::getInt32Ty(Context), 2);
4435ffd83dbSDimitry Andric }
4445ffd83dbSDimitry Andric
4455ffd83dbSDimitry Andric // Global memory works best with 16-byte accesses. Private memory will also
4465ffd83dbSDimitry Andric // hit this, although they'll be decomposed.
4475ffd83dbSDimitry Andric return FixedVectorType::get(Type::getInt32Ty(Context), 4);
4485ffd83dbSDimitry Andric }
4495ffd83dbSDimitry Andric
getMemcpyLoopResidualLoweringType(SmallVectorImpl<Type * > & OpsOut,LLVMContext & Context,unsigned RemainingBytes,unsigned SrcAddrSpace,unsigned DestAddrSpace,unsigned SrcAlign,unsigned DestAlign,std::optional<uint32_t> AtomicCpySize) const4505ffd83dbSDimitry Andric void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
4515ffd83dbSDimitry Andric SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
4525ffd83dbSDimitry Andric unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
45381ad6265SDimitry Andric unsigned SrcAlign, unsigned DestAlign,
454bdd1243dSDimitry Andric std::optional<uint32_t> AtomicCpySize) const {
4555ffd83dbSDimitry Andric assert(RemainingBytes < 16);
4565ffd83dbSDimitry Andric
45781ad6265SDimitry Andric if (AtomicCpySize)
45881ad6265SDimitry Andric BaseT::getMemcpyLoopResidualLoweringType(
45981ad6265SDimitry Andric OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
46081ad6265SDimitry Andric DestAlign, AtomicCpySize);
46181ad6265SDimitry Andric
4625ffd83dbSDimitry Andric unsigned MinAlign = std::min(SrcAlign, DestAlign);
4635ffd83dbSDimitry Andric
4645ffd83dbSDimitry Andric if (MinAlign != 2) {
4655ffd83dbSDimitry Andric Type *I64Ty = Type::getInt64Ty(Context);
4665ffd83dbSDimitry Andric while (RemainingBytes >= 8) {
4675ffd83dbSDimitry Andric OpsOut.push_back(I64Ty);
4685ffd83dbSDimitry Andric RemainingBytes -= 8;
4695ffd83dbSDimitry Andric }
4705ffd83dbSDimitry Andric
4715ffd83dbSDimitry Andric Type *I32Ty = Type::getInt32Ty(Context);
4725ffd83dbSDimitry Andric while (RemainingBytes >= 4) {
4735ffd83dbSDimitry Andric OpsOut.push_back(I32Ty);
4745ffd83dbSDimitry Andric RemainingBytes -= 4;
4755ffd83dbSDimitry Andric }
4765ffd83dbSDimitry Andric }
4775ffd83dbSDimitry Andric
4785ffd83dbSDimitry Andric Type *I16Ty = Type::getInt16Ty(Context);
4795ffd83dbSDimitry Andric while (RemainingBytes >= 2) {
4805ffd83dbSDimitry Andric OpsOut.push_back(I16Ty);
4815ffd83dbSDimitry Andric RemainingBytes -= 2;
4825ffd83dbSDimitry Andric }
4835ffd83dbSDimitry Andric
4845ffd83dbSDimitry Andric Type *I8Ty = Type::getInt8Ty(Context);
4855ffd83dbSDimitry Andric while (RemainingBytes) {
4865ffd83dbSDimitry Andric OpsOut.push_back(I8Ty);
4875ffd83dbSDimitry Andric --RemainingBytes;
4885ffd83dbSDimitry Andric }
4895ffd83dbSDimitry Andric }
4905ffd83dbSDimitry Andric
getMaxInterleaveFactor(ElementCount VF)49106c3fb27SDimitry Andric unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF) {
4920b57cec5SDimitry Andric // Disable unrolling if the loop is not vectorized.
4930b57cec5SDimitry Andric // TODO: Enable this again.
49406c3fb27SDimitry Andric if (VF.isScalar())
4950b57cec5SDimitry Andric return 1;
4960b57cec5SDimitry Andric
4970b57cec5SDimitry Andric return 8;
4980b57cec5SDimitry Andric }
4990b57cec5SDimitry Andric
getTgtMemIntrinsic(IntrinsicInst * Inst,MemIntrinsicInfo & Info) const5000b57cec5SDimitry Andric bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
5010b57cec5SDimitry Andric MemIntrinsicInfo &Info) const {
5020b57cec5SDimitry Andric switch (Inst->getIntrinsicID()) {
5030b57cec5SDimitry Andric case Intrinsic::amdgcn_ds_ordered_add:
5040b57cec5SDimitry Andric case Intrinsic::amdgcn_ds_ordered_swap:
5050b57cec5SDimitry Andric case Intrinsic::amdgcn_ds_fadd:
5060b57cec5SDimitry Andric case Intrinsic::amdgcn_ds_fmin:
5070b57cec5SDimitry Andric case Intrinsic::amdgcn_ds_fmax: {
5080b57cec5SDimitry Andric auto *Ordering = dyn_cast<ConstantInt>(Inst->getArgOperand(2));
5090b57cec5SDimitry Andric auto *Volatile = dyn_cast<ConstantInt>(Inst->getArgOperand(4));
5100b57cec5SDimitry Andric if (!Ordering || !Volatile)
5110b57cec5SDimitry Andric return false; // Invalid.
5120b57cec5SDimitry Andric
5130b57cec5SDimitry Andric unsigned OrderingVal = Ordering->getZExtValue();
5140b57cec5SDimitry Andric if (OrderingVal > static_cast<unsigned>(AtomicOrdering::SequentiallyConsistent))
5150b57cec5SDimitry Andric return false;
5160b57cec5SDimitry Andric
5170b57cec5SDimitry Andric Info.PtrVal = Inst->getArgOperand(0);
5180b57cec5SDimitry Andric Info.Ordering = static_cast<AtomicOrdering>(OrderingVal);
5190b57cec5SDimitry Andric Info.ReadMem = true;
5200b57cec5SDimitry Andric Info.WriteMem = true;
521349cc55cSDimitry Andric Info.IsVolatile = !Volatile->isZero();
5220b57cec5SDimitry Andric return true;
5230b57cec5SDimitry Andric }
5240b57cec5SDimitry Andric default:
5250b57cec5SDimitry Andric return false;
5260b57cec5SDimitry Andric }
5270b57cec5SDimitry Andric }
5280b57cec5SDimitry Andric
getArithmeticInstrCost(unsigned Opcode,Type * Ty,TTI::TargetCostKind CostKind,TTI::OperandValueInfo Op1Info,TTI::OperandValueInfo Op2Info,ArrayRef<const Value * > Args,const Instruction * CxtI)529fe6060f1SDimitry Andric InstructionCost GCNTTIImpl::getArithmeticInstrCost(
530fe6060f1SDimitry Andric unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
531bdd1243dSDimitry Andric TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
532bdd1243dSDimitry Andric ArrayRef<const Value *> Args,
533480093f4SDimitry Andric const Instruction *CxtI) {
5340b57cec5SDimitry Andric
5350b57cec5SDimitry Andric // Legalize the type.
536bdd1243dSDimitry Andric std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
5370b57cec5SDimitry Andric int ISD = TLI->InstructionOpcodeToISD(Opcode);
5380b57cec5SDimitry Andric
5390b57cec5SDimitry Andric // Because we don't have any legal vector operations, but the legal types, we
5400b57cec5SDimitry Andric // need to account for split vectors.
5410b57cec5SDimitry Andric unsigned NElts = LT.second.isVector() ?
5420b57cec5SDimitry Andric LT.second.getVectorNumElements() : 1;
5430b57cec5SDimitry Andric
5440b57cec5SDimitry Andric MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
5450b57cec5SDimitry Andric
5460b57cec5SDimitry Andric switch (ISD) {
5470b57cec5SDimitry Andric case ISD::SHL:
5480b57cec5SDimitry Andric case ISD::SRL:
5490b57cec5SDimitry Andric case ISD::SRA:
5500b57cec5SDimitry Andric if (SLT == MVT::i64)
551e8d8bef9SDimitry Andric return get64BitInstrCost(CostKind) * LT.first * NElts;
5520b57cec5SDimitry Andric
553480093f4SDimitry Andric if (ST->has16BitInsts() && SLT == MVT::i16)
554480093f4SDimitry Andric NElts = (NElts + 1) / 2;
555480093f4SDimitry Andric
5560b57cec5SDimitry Andric // i32
5570b57cec5SDimitry Andric return getFullRateInstrCost() * LT.first * NElts;
5580b57cec5SDimitry Andric case ISD::ADD:
5590b57cec5SDimitry Andric case ISD::SUB:
5600b57cec5SDimitry Andric case ISD::AND:
5610b57cec5SDimitry Andric case ISD::OR:
5620b57cec5SDimitry Andric case ISD::XOR:
5630b57cec5SDimitry Andric if (SLT == MVT::i64) {
5640b57cec5SDimitry Andric // and, or and xor are typically split into 2 VALU instructions.
5650b57cec5SDimitry Andric return 2 * getFullRateInstrCost() * LT.first * NElts;
5660b57cec5SDimitry Andric }
5670b57cec5SDimitry Andric
568480093f4SDimitry Andric if (ST->has16BitInsts() && SLT == MVT::i16)
569480093f4SDimitry Andric NElts = (NElts + 1) / 2;
570480093f4SDimitry Andric
5710b57cec5SDimitry Andric return LT.first * NElts * getFullRateInstrCost();
5720b57cec5SDimitry Andric case ISD::MUL: {
573e8d8bef9SDimitry Andric const int QuarterRateCost = getQuarterRateInstrCost(CostKind);
5740b57cec5SDimitry Andric if (SLT == MVT::i64) {
5750b57cec5SDimitry Andric const int FullRateCost = getFullRateInstrCost();
5760b57cec5SDimitry Andric return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts;
5770b57cec5SDimitry Andric }
5780b57cec5SDimitry Andric
579480093f4SDimitry Andric if (ST->has16BitInsts() && SLT == MVT::i16)
580480093f4SDimitry Andric NElts = (NElts + 1) / 2;
581480093f4SDimitry Andric
5820b57cec5SDimitry Andric // i32
5830b57cec5SDimitry Andric return QuarterRateCost * NElts * LT.first;
5840b57cec5SDimitry Andric }
585e8d8bef9SDimitry Andric case ISD::FMUL:
586e8d8bef9SDimitry Andric // Check possible fuse {fadd|fsub}(a,fmul(b,c)) and return zero cost for
587e8d8bef9SDimitry Andric // fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole
588e8d8bef9SDimitry Andric // fused operation.
589e8d8bef9SDimitry Andric if (CxtI && CxtI->hasOneUse())
590e8d8bef9SDimitry Andric if (const auto *FAdd = dyn_cast<BinaryOperator>(*CxtI->user_begin())) {
591e8d8bef9SDimitry Andric const int OPC = TLI->InstructionOpcodeToISD(FAdd->getOpcode());
592e8d8bef9SDimitry Andric if (OPC == ISD::FADD || OPC == ISD::FSUB) {
593e8d8bef9SDimitry Andric if (ST->hasMadMacF32Insts() && SLT == MVT::f32 && !HasFP32Denormals)
594e8d8bef9SDimitry Andric return TargetTransformInfo::TCC_Free;
595e8d8bef9SDimitry Andric if (ST->has16BitInsts() && SLT == MVT::f16 && !HasFP64FP16Denormals)
596e8d8bef9SDimitry Andric return TargetTransformInfo::TCC_Free;
597e8d8bef9SDimitry Andric
598e8d8bef9SDimitry Andric // Estimate all types may be fused with contract/unsafe flags
599e8d8bef9SDimitry Andric const TargetOptions &Options = TLI->getTargetMachine().Options;
600e8d8bef9SDimitry Andric if (Options.AllowFPOpFusion == FPOpFusion::Fast ||
601e8d8bef9SDimitry Andric Options.UnsafeFPMath ||
602e8d8bef9SDimitry Andric (FAdd->hasAllowContract() && CxtI->hasAllowContract()))
603e8d8bef9SDimitry Andric return TargetTransformInfo::TCC_Free;
604e8d8bef9SDimitry Andric }
605e8d8bef9SDimitry Andric }
606bdd1243dSDimitry Andric [[fallthrough]];
6070b57cec5SDimitry Andric case ISD::FADD:
6080b57cec5SDimitry Andric case ISD::FSUB:
609fe6060f1SDimitry Andric if (ST->hasPackedFP32Ops() && SLT == MVT::f32)
610fe6060f1SDimitry Andric NElts = (NElts + 1) / 2;
6110b57cec5SDimitry Andric if (SLT == MVT::f64)
612e8d8bef9SDimitry Andric return LT.first * NElts * get64BitInstrCost(CostKind);
6130b57cec5SDimitry Andric
614480093f4SDimitry Andric if (ST->has16BitInsts() && SLT == MVT::f16)
615480093f4SDimitry Andric NElts = (NElts + 1) / 2;
616480093f4SDimitry Andric
6170b57cec5SDimitry Andric if (SLT == MVT::f32 || SLT == MVT::f16)
6180b57cec5SDimitry Andric return LT.first * NElts * getFullRateInstrCost();
6190b57cec5SDimitry Andric break;
6200b57cec5SDimitry Andric case ISD::FDIV:
6210b57cec5SDimitry Andric case ISD::FREM:
6220b57cec5SDimitry Andric // FIXME: frem should be handled separately. The fdiv in it is most of it,
6230b57cec5SDimitry Andric // but the current lowering is also not entirely correct.
6240b57cec5SDimitry Andric if (SLT == MVT::f64) {
625e8d8bef9SDimitry Andric int Cost = 7 * get64BitInstrCost(CostKind) +
626e8d8bef9SDimitry Andric getQuarterRateInstrCost(CostKind) +
627e8d8bef9SDimitry Andric 3 * getHalfRateInstrCost(CostKind);
6280b57cec5SDimitry Andric // Add cost of workaround.
6290b57cec5SDimitry Andric if (!ST->hasUsableDivScaleConditionOutput())
6300b57cec5SDimitry Andric Cost += 3 * getFullRateInstrCost();
6310b57cec5SDimitry Andric
6320b57cec5SDimitry Andric return LT.first * Cost * NElts;
6330b57cec5SDimitry Andric }
6340b57cec5SDimitry Andric
6350b57cec5SDimitry Andric if (!Args.empty() && match(Args[0], PatternMatch::m_FPOne())) {
6360b57cec5SDimitry Andric // TODO: This is more complicated, unsafe flags etc.
637480093f4SDimitry Andric if ((SLT == MVT::f32 && !HasFP32Denormals) ||
6380b57cec5SDimitry Andric (SLT == MVT::f16 && ST->has16BitInsts())) {
639e8d8bef9SDimitry Andric return LT.first * getQuarterRateInstrCost(CostKind) * NElts;
6400b57cec5SDimitry Andric }
6410b57cec5SDimitry Andric }
6420b57cec5SDimitry Andric
6430b57cec5SDimitry Andric if (SLT == MVT::f16 && ST->has16BitInsts()) {
6440b57cec5SDimitry Andric // 2 x v_cvt_f32_f16
6450b57cec5SDimitry Andric // f32 rcp
6460b57cec5SDimitry Andric // f32 fmul
6470b57cec5SDimitry Andric // v_cvt_f16_f32
6480b57cec5SDimitry Andric // f16 div_fixup
649e8d8bef9SDimitry Andric int Cost =
650e8d8bef9SDimitry Andric 4 * getFullRateInstrCost() + 2 * getQuarterRateInstrCost(CostKind);
6510b57cec5SDimitry Andric return LT.first * Cost * NElts;
6520b57cec5SDimitry Andric }
6530b57cec5SDimitry Andric
6545f757f3fSDimitry Andric if (SLT == MVT::f32 && ((CxtI && CxtI->hasApproxFunc()) ||
6555f757f3fSDimitry Andric TLI->getTargetMachine().Options.UnsafeFPMath)) {
6565f757f3fSDimitry Andric // Fast unsafe fdiv lowering:
6575f757f3fSDimitry Andric // f32 rcp
6585f757f3fSDimitry Andric // f32 fmul
6595f757f3fSDimitry Andric int Cost = getQuarterRateInstrCost(CostKind) + getFullRateInstrCost();
6605f757f3fSDimitry Andric return LT.first * Cost * NElts;
6615f757f3fSDimitry Andric }
6625f757f3fSDimitry Andric
6630b57cec5SDimitry Andric if (SLT == MVT::f32 || SLT == MVT::f16) {
664e8d8bef9SDimitry Andric // 4 more v_cvt_* insts without f16 insts support
665e8d8bef9SDimitry Andric int Cost = (SLT == MVT::f16 ? 14 : 10) * getFullRateInstrCost() +
666e8d8bef9SDimitry Andric 1 * getQuarterRateInstrCost(CostKind);
6670b57cec5SDimitry Andric
668480093f4SDimitry Andric if (!HasFP32Denormals) {
6690b57cec5SDimitry Andric // FP mode switches.
6700b57cec5SDimitry Andric Cost += 2 * getFullRateInstrCost();
6710b57cec5SDimitry Andric }
6720b57cec5SDimitry Andric
6730b57cec5SDimitry Andric return LT.first * NElts * Cost;
6740b57cec5SDimitry Andric }
6750b57cec5SDimitry Andric break;
6765ffd83dbSDimitry Andric case ISD::FNEG:
6775ffd83dbSDimitry Andric // Use the backend' estimation. If fneg is not free each element will cost
6785ffd83dbSDimitry Andric // one additional instruction.
6795ffd83dbSDimitry Andric return TLI->isFNegFree(SLT) ? 0 : NElts;
6800b57cec5SDimitry Andric default:
6810b57cec5SDimitry Andric break;
6820b57cec5SDimitry Andric }
6830b57cec5SDimitry Andric
684bdd1243dSDimitry Andric return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
685bdd1243dSDimitry Andric Args, CxtI);
6860b57cec5SDimitry Andric }
6870b57cec5SDimitry Andric
688e8d8bef9SDimitry Andric // Return true if there's a potential benefit from using v2f16/v2i16
689e8d8bef9SDimitry Andric // instructions for an intrinsic, even if it requires nontrivial legalization.
intrinsicHasPackedVectorBenefit(Intrinsic::ID ID)6905ffd83dbSDimitry Andric static bool intrinsicHasPackedVectorBenefit(Intrinsic::ID ID) {
6915ffd83dbSDimitry Andric switch (ID) {
6925ffd83dbSDimitry Andric case Intrinsic::fma: // TODO: fmuladd
6935ffd83dbSDimitry Andric // There's a small benefit to using vector ops in the legalized code.
6945ffd83dbSDimitry Andric case Intrinsic::round:
695e8d8bef9SDimitry Andric case Intrinsic::uadd_sat:
696e8d8bef9SDimitry Andric case Intrinsic::usub_sat:
697e8d8bef9SDimitry Andric case Intrinsic::sadd_sat:
698e8d8bef9SDimitry Andric case Intrinsic::ssub_sat:
6995ffd83dbSDimitry Andric return true;
7005ffd83dbSDimitry Andric default:
7015ffd83dbSDimitry Andric return false;
7025ffd83dbSDimitry Andric }
7035ffd83dbSDimitry Andric }
704480093f4SDimitry Andric
705fe6060f1SDimitry Andric InstructionCost
getIntrinsicInstrCost(const IntrinsicCostAttributes & ICA,TTI::TargetCostKind CostKind)706fe6060f1SDimitry Andric GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
7075ffd83dbSDimitry Andric TTI::TargetCostKind CostKind) {
7085ffd83dbSDimitry Andric if (ICA.getID() == Intrinsic::fabs)
7095ffd83dbSDimitry Andric return 0;
7105ffd83dbSDimitry Andric
7115ffd83dbSDimitry Andric if (!intrinsicHasPackedVectorBenefit(ICA.getID()))
7125ffd83dbSDimitry Andric return BaseT::getIntrinsicInstrCost(ICA, CostKind);
7135ffd83dbSDimitry Andric
7145ffd83dbSDimitry Andric Type *RetTy = ICA.getReturnType();
715480093f4SDimitry Andric
716480093f4SDimitry Andric // Legalize the type.
717bdd1243dSDimitry Andric std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(RetTy);
718480093f4SDimitry Andric
719480093f4SDimitry Andric unsigned NElts = LT.second.isVector() ?
720480093f4SDimitry Andric LT.second.getVectorNumElements() : 1;
721480093f4SDimitry Andric
722480093f4SDimitry Andric MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
723480093f4SDimitry Andric
724480093f4SDimitry Andric if (SLT == MVT::f64)
725e8d8bef9SDimitry Andric return LT.first * NElts * get64BitInstrCost(CostKind);
726480093f4SDimitry Andric
727fe6060f1SDimitry Andric if ((ST->has16BitInsts() && SLT == MVT::f16) ||
728fe6060f1SDimitry Andric (ST->hasPackedFP32Ops() && SLT == MVT::f32))
729480093f4SDimitry Andric NElts = (NElts + 1) / 2;
730480093f4SDimitry Andric
7315ffd83dbSDimitry Andric // TODO: Get more refined intrinsic costs?
732e8d8bef9SDimitry Andric unsigned InstRate = getQuarterRateInstrCost(CostKind);
733fe6060f1SDimitry Andric
734fe6060f1SDimitry Andric switch (ICA.getID()) {
735fe6060f1SDimitry Andric case Intrinsic::fma:
736e8d8bef9SDimitry Andric InstRate = ST->hasFastFMAF32() ? getHalfRateInstrCost(CostKind)
737e8d8bef9SDimitry Andric : getQuarterRateInstrCost(CostKind);
738fe6060f1SDimitry Andric break;
739fe6060f1SDimitry Andric case Intrinsic::uadd_sat:
740fe6060f1SDimitry Andric case Intrinsic::usub_sat:
741fe6060f1SDimitry Andric case Intrinsic::sadd_sat:
742fe6060f1SDimitry Andric case Intrinsic::ssub_sat:
743fe6060f1SDimitry Andric static const auto ValidSatTys = {MVT::v2i16, MVT::v4i16};
744fe6060f1SDimitry Andric if (any_of(ValidSatTys, [<](MVT M) { return M == LT.second; }))
745fe6060f1SDimitry Andric NElts = 1;
746fe6060f1SDimitry Andric break;
747480093f4SDimitry Andric }
748480093f4SDimitry Andric
7495ffd83dbSDimitry Andric return LT.first * NElts * InstRate;
750480093f4SDimitry Andric }
751480093f4SDimitry Andric
getCFInstrCost(unsigned Opcode,TTI::TargetCostKind CostKind,const Instruction * I)752fe6060f1SDimitry Andric InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
753fe6060f1SDimitry Andric TTI::TargetCostKind CostKind,
754fe6060f1SDimitry Andric const Instruction *I) {
755fe6060f1SDimitry Andric assert((I == nullptr || I->getOpcode() == Opcode) &&
756fe6060f1SDimitry Andric "Opcode should reflect passed instruction.");
757fe6060f1SDimitry Andric const bool SCost =
758fe6060f1SDimitry Andric (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency);
759fe6060f1SDimitry Andric const int CBrCost = SCost ? 5 : 7;
7600b57cec5SDimitry Andric switch (Opcode) {
761fe6060f1SDimitry Andric case Instruction::Br: {
762fe6060f1SDimitry Andric // Branch instruction takes about 4 slots on gfx900.
763fe6060f1SDimitry Andric auto BI = dyn_cast_or_null<BranchInst>(I);
764fe6060f1SDimitry Andric if (BI && BI->isUnconditional())
765fe6060f1SDimitry Andric return SCost ? 1 : 4;
766fe6060f1SDimitry Andric // Suppose conditional branch takes additional 3 exec manipulations
767fe6060f1SDimitry Andric // instructions in average.
768fe6060f1SDimitry Andric return CBrCost;
7690b57cec5SDimitry Andric }
770fe6060f1SDimitry Andric case Instruction::Switch: {
771fe6060f1SDimitry Andric auto SI = dyn_cast_or_null<SwitchInst>(I);
772fe6060f1SDimitry Andric // Each case (including default) takes 1 cmp + 1 cbr instructions in
773fe6060f1SDimitry Andric // average.
774fe6060f1SDimitry Andric return (SI ? (SI->getNumCases() + 1) : 4) * (CBrCost + 1);
775fe6060f1SDimitry Andric }
776fe6060f1SDimitry Andric case Instruction::Ret:
777fe6060f1SDimitry Andric return SCost ? 1 : 10;
778fe6060f1SDimitry Andric }
779fe6060f1SDimitry Andric return BaseT::getCFInstrCost(Opcode, CostKind, I);
7800b57cec5SDimitry Andric }
7810b57cec5SDimitry Andric
782fe6060f1SDimitry Andric InstructionCost
getArithmeticReductionCost(unsigned Opcode,VectorType * Ty,std::optional<FastMathFlags> FMF,TTI::TargetCostKind CostKind)783fe6060f1SDimitry Andric GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
784bdd1243dSDimitry Andric std::optional<FastMathFlags> FMF,
7855ffd83dbSDimitry Andric TTI::TargetCostKind CostKind) {
786fe6060f1SDimitry Andric if (TTI::requiresOrderedReduction(FMF))
787fe6060f1SDimitry Andric return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
788fe6060f1SDimitry Andric
7890b57cec5SDimitry Andric EVT OrigTy = TLI->getValueType(DL, Ty);
7900b57cec5SDimitry Andric
7910b57cec5SDimitry Andric // Computes cost on targets that have packed math instructions(which support
7920b57cec5SDimitry Andric // 16-bit types only).
793fe6060f1SDimitry Andric if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)
794fe6060f1SDimitry Andric return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
7950b57cec5SDimitry Andric
796bdd1243dSDimitry Andric std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
7970b57cec5SDimitry Andric return LT.first * getFullRateInstrCost();
7980b57cec5SDimitry Andric }
7990b57cec5SDimitry Andric
800fe6060f1SDimitry Andric InstructionCost
getMinMaxReductionCost(Intrinsic::ID IID,VectorType * Ty,FastMathFlags FMF,TTI::TargetCostKind CostKind)80106c3fb27SDimitry Andric GCNTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
80206c3fb27SDimitry Andric FastMathFlags FMF,
8035ffd83dbSDimitry Andric TTI::TargetCostKind CostKind) {
8040b57cec5SDimitry Andric EVT OrigTy = TLI->getValueType(DL, Ty);
8050b57cec5SDimitry Andric
8060b57cec5SDimitry Andric // Computes cost on targets that have packed math instructions(which support
8070b57cec5SDimitry Andric // 16-bit types only).
808fe6060f1SDimitry Andric if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)
80906c3fb27SDimitry Andric return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
8100b57cec5SDimitry Andric
811bdd1243dSDimitry Andric std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
812e8d8bef9SDimitry Andric return LT.first * getHalfRateInstrCost(CostKind);
8130b57cec5SDimitry Andric }
8140b57cec5SDimitry Andric
getVectorInstrCost(unsigned Opcode,Type * ValTy,TTI::TargetCostKind CostKind,unsigned Index,Value * Op0,Value * Op1)815fe6060f1SDimitry Andric InstructionCost GCNTTIImpl::getVectorInstrCost(unsigned Opcode, Type *ValTy,
816bdd1243dSDimitry Andric TTI::TargetCostKind CostKind,
817bdd1243dSDimitry Andric unsigned Index, Value *Op0,
818bdd1243dSDimitry Andric Value *Op1) {
8190b57cec5SDimitry Andric switch (Opcode) {
8200b57cec5SDimitry Andric case Instruction::ExtractElement:
8210b57cec5SDimitry Andric case Instruction::InsertElement: {
8220b57cec5SDimitry Andric unsigned EltSize
8230b57cec5SDimitry Andric = DL.getTypeSizeInBits(cast<VectorType>(ValTy)->getElementType());
8240b57cec5SDimitry Andric if (EltSize < 32) {
8250b57cec5SDimitry Andric if (EltSize == 16 && Index == 0 && ST->has16BitInsts())
8260b57cec5SDimitry Andric return 0;
827bdd1243dSDimitry Andric return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0,
828bdd1243dSDimitry Andric Op1);
8290b57cec5SDimitry Andric }
8300b57cec5SDimitry Andric
8310b57cec5SDimitry Andric // Extracts are just reads of a subregister, so are free. Inserts are
8320b57cec5SDimitry Andric // considered free because we don't want to have any cost for scalarizing
8330b57cec5SDimitry Andric // operations, and we don't have to copy into a different register class.
8340b57cec5SDimitry Andric
8350b57cec5SDimitry Andric // Dynamic indexing isn't free and is best avoided.
8360b57cec5SDimitry Andric return Index == ~0u ? 2 : 0;
8370b57cec5SDimitry Andric }
8380b57cec5SDimitry Andric default:
839bdd1243dSDimitry Andric return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1);
8400b57cec5SDimitry Andric }
8410b57cec5SDimitry Andric }
8420b57cec5SDimitry Andric
8435ffd83dbSDimitry Andric /// Analyze if the results of inline asm are divergent. If \p Indices is empty,
8445ffd83dbSDimitry Andric /// this is analyzing the collective result of all output registers. Otherwise,
8455ffd83dbSDimitry Andric /// this is only querying a specific result index if this returns multiple
8465ffd83dbSDimitry Andric /// registers in a struct.
isInlineAsmSourceOfDivergence(const CallInst * CI,ArrayRef<unsigned> Indices) const8475ffd83dbSDimitry Andric bool GCNTTIImpl::isInlineAsmSourceOfDivergence(
8485ffd83dbSDimitry Andric const CallInst *CI, ArrayRef<unsigned> Indices) const {
8495ffd83dbSDimitry Andric // TODO: Handle complex extract indices
8505ffd83dbSDimitry Andric if (Indices.size() > 1)
8515ffd83dbSDimitry Andric return true;
8525ffd83dbSDimitry Andric
8535ffd83dbSDimitry Andric const DataLayout &DL = CI->getModule()->getDataLayout();
8545ffd83dbSDimitry Andric const SIRegisterInfo *TRI = ST->getRegisterInfo();
8555ffd83dbSDimitry Andric TargetLowering::AsmOperandInfoVector TargetConstraints =
8565ffd83dbSDimitry Andric TLI->ParseConstraints(DL, ST->getRegisterInfo(), *CI);
8575ffd83dbSDimitry Andric
8585ffd83dbSDimitry Andric const int TargetOutputIdx = Indices.empty() ? -1 : Indices[0];
8595ffd83dbSDimitry Andric
8605ffd83dbSDimitry Andric int OutputIdx = 0;
8615ffd83dbSDimitry Andric for (auto &TC : TargetConstraints) {
8625ffd83dbSDimitry Andric if (TC.Type != InlineAsm::isOutput)
8635ffd83dbSDimitry Andric continue;
8645ffd83dbSDimitry Andric
8655ffd83dbSDimitry Andric // Skip outputs we don't care about.
8665ffd83dbSDimitry Andric if (TargetOutputIdx != -1 && TargetOutputIdx != OutputIdx++)
8675ffd83dbSDimitry Andric continue;
8685ffd83dbSDimitry Andric
8695ffd83dbSDimitry Andric TLI->ComputeConstraintToUse(TC, SDValue());
8705ffd83dbSDimitry Andric
87104eeddc0SDimitry Andric const TargetRegisterClass *RC = TLI->getRegForInlineAsmConstraint(
87204eeddc0SDimitry Andric TRI, TC.ConstraintCode, TC.ConstraintVT).second;
8735ffd83dbSDimitry Andric
8745ffd83dbSDimitry Andric // For AGPR constraints null is returned on subtargets without AGPRs, so
8755ffd83dbSDimitry Andric // assume divergent for null.
8765ffd83dbSDimitry Andric if (!RC || !TRI->isSGPRClass(RC))
8775ffd83dbSDimitry Andric return true;
8785ffd83dbSDimitry Andric }
8795ffd83dbSDimitry Andric
8805ffd83dbSDimitry Andric return false;
8815ffd83dbSDimitry Andric }
8825ffd83dbSDimitry Andric
isReadRegisterSourceOfDivergence(const IntrinsicInst * ReadReg) const883bdd1243dSDimitry Andric bool GCNTTIImpl::isReadRegisterSourceOfDivergence(
884bdd1243dSDimitry Andric const IntrinsicInst *ReadReg) const {
885bdd1243dSDimitry Andric Metadata *MD =
886bdd1243dSDimitry Andric cast<MetadataAsValue>(ReadReg->getArgOperand(0))->getMetadata();
887bdd1243dSDimitry Andric StringRef RegName =
888bdd1243dSDimitry Andric cast<MDString>(cast<MDNode>(MD)->getOperand(0))->getString();
889bdd1243dSDimitry Andric
890bdd1243dSDimitry Andric // Special case registers that look like VCC.
891bdd1243dSDimitry Andric MVT VT = MVT::getVT(ReadReg->getType());
892bdd1243dSDimitry Andric if (VT == MVT::i1)
893bdd1243dSDimitry Andric return true;
894bdd1243dSDimitry Andric
895bdd1243dSDimitry Andric // Special case scalar registers that start with 'v'.
8965f757f3fSDimitry Andric if (RegName.starts_with("vcc") || RegName.empty())
897bdd1243dSDimitry Andric return false;
898bdd1243dSDimitry Andric
899bdd1243dSDimitry Andric // VGPR or AGPR is divergent. There aren't any specially named vector
900bdd1243dSDimitry Andric // registers.
901bdd1243dSDimitry Andric return RegName[0] == 'v' || RegName[0] == 'a';
902bdd1243dSDimitry Andric }
903bdd1243dSDimitry Andric
9040b57cec5SDimitry Andric /// \returns true if the result of the value could potentially be
9050b57cec5SDimitry Andric /// different across workitems in a wavefront.
isSourceOfDivergence(const Value * V) const9060b57cec5SDimitry Andric bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const {
9070b57cec5SDimitry Andric if (const Argument *A = dyn_cast<Argument>(V))
908e8d8bef9SDimitry Andric return !AMDGPU::isArgPassedInSGPR(A);
9090b57cec5SDimitry Andric
9100b57cec5SDimitry Andric // Loads from the private and flat address spaces are divergent, because
9110b57cec5SDimitry Andric // threads can execute the load instruction with the same inputs and get
9120b57cec5SDimitry Andric // different results.
9130b57cec5SDimitry Andric //
9140b57cec5SDimitry Andric // All other loads are not divergent, because if threads issue loads with the
9150b57cec5SDimitry Andric // same arguments, they will always get the same result.
9160b57cec5SDimitry Andric if (const LoadInst *Load = dyn_cast<LoadInst>(V))
9170b57cec5SDimitry Andric return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
9180b57cec5SDimitry Andric Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS;
9190b57cec5SDimitry Andric
9200b57cec5SDimitry Andric // Atomics are divergent because they are executed sequentially: when an
9210b57cec5SDimitry Andric // atomic operation refers to the same address in each thread, then each
9220b57cec5SDimitry Andric // thread after the first sees the value written by the previous thread as
9230b57cec5SDimitry Andric // original value.
9240b57cec5SDimitry Andric if (isa<AtomicRMWInst>(V) || isa<AtomicCmpXchgInst>(V))
9250b57cec5SDimitry Andric return true;
9260b57cec5SDimitry Andric
927bdd1243dSDimitry Andric if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) {
928bdd1243dSDimitry Andric if (Intrinsic->getIntrinsicID() == Intrinsic::read_register)
929bdd1243dSDimitry Andric return isReadRegisterSourceOfDivergence(Intrinsic);
930bdd1243dSDimitry Andric
9310b57cec5SDimitry Andric return AMDGPU::isIntrinsicSourceOfDivergence(Intrinsic->getIntrinsicID());
932bdd1243dSDimitry Andric }
9330b57cec5SDimitry Andric
9340b57cec5SDimitry Andric // Assume all function calls are a source of divergence.
9355ffd83dbSDimitry Andric if (const CallInst *CI = dyn_cast<CallInst>(V)) {
9365ffd83dbSDimitry Andric if (CI->isInlineAsm())
9375ffd83dbSDimitry Andric return isInlineAsmSourceOfDivergence(CI);
9385ffd83dbSDimitry Andric return true;
9395ffd83dbSDimitry Andric }
9405ffd83dbSDimitry Andric
9415ffd83dbSDimitry Andric // Assume all function calls are a source of divergence.
9425ffd83dbSDimitry Andric if (isa<InvokeInst>(V))
9430b57cec5SDimitry Andric return true;
9440b57cec5SDimitry Andric
9450b57cec5SDimitry Andric return false;
9460b57cec5SDimitry Andric }
9470b57cec5SDimitry Andric
isAlwaysUniform(const Value * V) const9480b57cec5SDimitry Andric bool GCNTTIImpl::isAlwaysUniform(const Value *V) const {
94906c3fb27SDimitry Andric if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V))
95006c3fb27SDimitry Andric return AMDGPU::isIntrinsicAlwaysUniform(Intrinsic->getIntrinsicID());
9515ffd83dbSDimitry Andric
9525ffd83dbSDimitry Andric if (const CallInst *CI = dyn_cast<CallInst>(V)) {
9535ffd83dbSDimitry Andric if (CI->isInlineAsm())
9545ffd83dbSDimitry Andric return !isInlineAsmSourceOfDivergence(CI);
9555ffd83dbSDimitry Andric return false;
9565ffd83dbSDimitry Andric }
9575ffd83dbSDimitry Andric
958bdd1243dSDimitry Andric // In most cases TID / wavefrontsize is uniform.
959bdd1243dSDimitry Andric //
960bdd1243dSDimitry Andric // However, if a kernel has uneven dimesions we can have a value of
961bdd1243dSDimitry Andric // workitem-id-x divided by the wavefrontsize non-uniform. For example
962bdd1243dSDimitry Andric // dimensions (65, 2) will have workitems with address (64, 0) and (0, 1)
963bdd1243dSDimitry Andric // packed into a same wave which gives 1 and 0 after the division by 64
964bdd1243dSDimitry Andric // respectively.
965bdd1243dSDimitry Andric //
966bdd1243dSDimitry Andric // FIXME: limit it to 1D kernels only, although that shall be possible
967bdd1243dSDimitry Andric // to perform this optimization is the size of the X dimension is a power
968bdd1243dSDimitry Andric // of 2, we just do not currently have infrastructure to query it.
969bdd1243dSDimitry Andric using namespace llvm::PatternMatch;
970bdd1243dSDimitry Andric uint64_t C;
971bdd1243dSDimitry Andric if (match(V, m_LShr(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
972bdd1243dSDimitry Andric m_ConstantInt(C))) ||
973bdd1243dSDimitry Andric match(V, m_AShr(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
974bdd1243dSDimitry Andric m_ConstantInt(C)))) {
975bdd1243dSDimitry Andric const Function *F = cast<Instruction>(V)->getFunction();
976bdd1243dSDimitry Andric return C >= ST->getWavefrontSizeLog2() &&
977bdd1243dSDimitry Andric ST->getMaxWorkitemID(*F, 1) == 0 && ST->getMaxWorkitemID(*F, 2) == 0;
978bdd1243dSDimitry Andric }
979bdd1243dSDimitry Andric
980bdd1243dSDimitry Andric Value *Mask;
981bdd1243dSDimitry Andric if (match(V, m_c_And(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
982bdd1243dSDimitry Andric m_Value(Mask)))) {
983bdd1243dSDimitry Andric const Function *F = cast<Instruction>(V)->getFunction();
984bdd1243dSDimitry Andric const DataLayout &DL = F->getParent()->getDataLayout();
985bdd1243dSDimitry Andric return computeKnownBits(Mask, DL).countMinTrailingZeros() >=
986bdd1243dSDimitry Andric ST->getWavefrontSizeLog2() &&
987bdd1243dSDimitry Andric ST->getMaxWorkitemID(*F, 1) == 0 && ST->getMaxWorkitemID(*F, 2) == 0;
988bdd1243dSDimitry Andric }
989bdd1243dSDimitry Andric
9905ffd83dbSDimitry Andric const ExtractValueInst *ExtValue = dyn_cast<ExtractValueInst>(V);
9915ffd83dbSDimitry Andric if (!ExtValue)
9925ffd83dbSDimitry Andric return false;
9935ffd83dbSDimitry Andric
9945ffd83dbSDimitry Andric const CallInst *CI = dyn_cast<CallInst>(ExtValue->getOperand(0));
9955ffd83dbSDimitry Andric if (!CI)
9965ffd83dbSDimitry Andric return false;
9975ffd83dbSDimitry Andric
9985ffd83dbSDimitry Andric if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(CI)) {
9995ffd83dbSDimitry Andric switch (Intrinsic->getIntrinsicID()) {
10005ffd83dbSDimitry Andric default:
10015ffd83dbSDimitry Andric return false;
10025ffd83dbSDimitry Andric case Intrinsic::amdgcn_if:
10035ffd83dbSDimitry Andric case Intrinsic::amdgcn_else: {
10045ffd83dbSDimitry Andric ArrayRef<unsigned> Indices = ExtValue->getIndices();
10055ffd83dbSDimitry Andric return Indices.size() == 1 && Indices[0] == 1;
10065ffd83dbSDimitry Andric }
10075ffd83dbSDimitry Andric }
10085ffd83dbSDimitry Andric }
10095ffd83dbSDimitry Andric
10105ffd83dbSDimitry Andric // If we have inline asm returning mixed SGPR and VGPR results, we inferred
10115ffd83dbSDimitry Andric // divergent for the overall struct return. We need to override it in the
10125ffd83dbSDimitry Andric // case we're extracting an SGPR component here.
10135ffd83dbSDimitry Andric if (CI->isInlineAsm())
10145ffd83dbSDimitry Andric return !isInlineAsmSourceOfDivergence(CI, ExtValue->getIndices());
10155ffd83dbSDimitry Andric
10160b57cec5SDimitry Andric return false;
10170b57cec5SDimitry Andric }
10180b57cec5SDimitry Andric
collectFlatAddressOperands(SmallVectorImpl<int> & OpIndexes,Intrinsic::ID IID) const10198bcb0991SDimitry Andric bool GCNTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
10208bcb0991SDimitry Andric Intrinsic::ID IID) const {
10218bcb0991SDimitry Andric switch (IID) {
10228bcb0991SDimitry Andric case Intrinsic::amdgcn_ds_fadd:
10238bcb0991SDimitry Andric case Intrinsic::amdgcn_ds_fmin:
10248bcb0991SDimitry Andric case Intrinsic::amdgcn_ds_fmax:
10258bcb0991SDimitry Andric case Intrinsic::amdgcn_is_shared:
10268bcb0991SDimitry Andric case Intrinsic::amdgcn_is_private:
1027bdd1243dSDimitry Andric case Intrinsic::amdgcn_flat_atomic_fadd:
1028bdd1243dSDimitry Andric case Intrinsic::amdgcn_flat_atomic_fmax:
1029bdd1243dSDimitry Andric case Intrinsic::amdgcn_flat_atomic_fmin:
10305f757f3fSDimitry Andric case Intrinsic::amdgcn_flat_atomic_fmax_num:
10315f757f3fSDimitry Andric case Intrinsic::amdgcn_flat_atomic_fmin_num:
10328bcb0991SDimitry Andric OpIndexes.push_back(0);
10338bcb0991SDimitry Andric return true;
10348bcb0991SDimitry Andric default:
10358bcb0991SDimitry Andric return false;
10368bcb0991SDimitry Andric }
10378bcb0991SDimitry Andric }
10388bcb0991SDimitry Andric
rewriteIntrinsicWithAddressSpace(IntrinsicInst * II,Value * OldV,Value * NewV) const10395ffd83dbSDimitry Andric Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
10405ffd83dbSDimitry Andric Value *OldV,
10415ffd83dbSDimitry Andric Value *NewV) const {
10428bcb0991SDimitry Andric auto IntrID = II->getIntrinsicID();
10438bcb0991SDimitry Andric switch (IntrID) {
10448bcb0991SDimitry Andric case Intrinsic::amdgcn_ds_fadd:
10458bcb0991SDimitry Andric case Intrinsic::amdgcn_ds_fmin:
10468bcb0991SDimitry Andric case Intrinsic::amdgcn_ds_fmax: {
10478bcb0991SDimitry Andric const ConstantInt *IsVolatile = cast<ConstantInt>(II->getArgOperand(4));
10488bcb0991SDimitry Andric if (!IsVolatile->isZero())
10495ffd83dbSDimitry Andric return nullptr;
10508bcb0991SDimitry Andric Module *M = II->getParent()->getParent()->getParent();
10518bcb0991SDimitry Andric Type *DestTy = II->getType();
10528bcb0991SDimitry Andric Type *SrcTy = NewV->getType();
10538bcb0991SDimitry Andric Function *NewDecl =
10548bcb0991SDimitry Andric Intrinsic::getDeclaration(M, II->getIntrinsicID(), {DestTy, SrcTy});
10558bcb0991SDimitry Andric II->setArgOperand(0, NewV);
10568bcb0991SDimitry Andric II->setCalledFunction(NewDecl);
10575ffd83dbSDimitry Andric return II;
10588bcb0991SDimitry Andric }
10598bcb0991SDimitry Andric case Intrinsic::amdgcn_is_shared:
10608bcb0991SDimitry Andric case Intrinsic::amdgcn_is_private: {
10618bcb0991SDimitry Andric unsigned TrueAS = IntrID == Intrinsic::amdgcn_is_shared ?
10628bcb0991SDimitry Andric AMDGPUAS::LOCAL_ADDRESS : AMDGPUAS::PRIVATE_ADDRESS;
10638bcb0991SDimitry Andric unsigned NewAS = NewV->getType()->getPointerAddressSpace();
10648bcb0991SDimitry Andric LLVMContext &Ctx = NewV->getType()->getContext();
10658bcb0991SDimitry Andric ConstantInt *NewVal = (TrueAS == NewAS) ?
10668bcb0991SDimitry Andric ConstantInt::getTrue(Ctx) : ConstantInt::getFalse(Ctx);
10675ffd83dbSDimitry Andric return NewVal;
10685ffd83dbSDimitry Andric }
10695ffd83dbSDimitry Andric case Intrinsic::ptrmask: {
10705ffd83dbSDimitry Andric unsigned OldAS = OldV->getType()->getPointerAddressSpace();
10715ffd83dbSDimitry Andric unsigned NewAS = NewV->getType()->getPointerAddressSpace();
10725ffd83dbSDimitry Andric Value *MaskOp = II->getArgOperand(1);
10735ffd83dbSDimitry Andric Type *MaskTy = MaskOp->getType();
10745ffd83dbSDimitry Andric
10755ffd83dbSDimitry Andric bool DoTruncate = false;
1076e8d8bef9SDimitry Andric
1077e8d8bef9SDimitry Andric const GCNTargetMachine &TM =
1078e8d8bef9SDimitry Andric static_cast<const GCNTargetMachine &>(getTLI()->getTargetMachine());
1079e8d8bef9SDimitry Andric if (!TM.isNoopAddrSpaceCast(OldAS, NewAS)) {
10805ffd83dbSDimitry Andric // All valid 64-bit to 32-bit casts work by chopping off the high
10815ffd83dbSDimitry Andric // bits. Any masking only clearing the low bits will also apply in the new
10825ffd83dbSDimitry Andric // address space.
10835ffd83dbSDimitry Andric if (DL.getPointerSizeInBits(OldAS) != 64 ||
10845ffd83dbSDimitry Andric DL.getPointerSizeInBits(NewAS) != 32)
10855ffd83dbSDimitry Andric return nullptr;
10865ffd83dbSDimitry Andric
10875ffd83dbSDimitry Andric // TODO: Do we need to thread more context in here?
10885ffd83dbSDimitry Andric KnownBits Known = computeKnownBits(MaskOp, DL, 0, nullptr, II);
10895ffd83dbSDimitry Andric if (Known.countMinLeadingOnes() < 32)
10905ffd83dbSDimitry Andric return nullptr;
10915ffd83dbSDimitry Andric
10925ffd83dbSDimitry Andric DoTruncate = true;
10935ffd83dbSDimitry Andric }
10945ffd83dbSDimitry Andric
10955ffd83dbSDimitry Andric IRBuilder<> B(II);
10965ffd83dbSDimitry Andric if (DoTruncate) {
10975ffd83dbSDimitry Andric MaskTy = B.getInt32Ty();
10985ffd83dbSDimitry Andric MaskOp = B.CreateTrunc(MaskOp, MaskTy);
10995ffd83dbSDimitry Andric }
11005ffd83dbSDimitry Andric
11015ffd83dbSDimitry Andric return B.CreateIntrinsic(Intrinsic::ptrmask, {NewV->getType(), MaskTy},
11025ffd83dbSDimitry Andric {NewV, MaskOp});
11038bcb0991SDimitry Andric }
1104bdd1243dSDimitry Andric case Intrinsic::amdgcn_flat_atomic_fadd:
1105bdd1243dSDimitry Andric case Intrinsic::amdgcn_flat_atomic_fmax:
11065f757f3fSDimitry Andric case Intrinsic::amdgcn_flat_atomic_fmin:
11075f757f3fSDimitry Andric case Intrinsic::amdgcn_flat_atomic_fmax_num:
11085f757f3fSDimitry Andric case Intrinsic::amdgcn_flat_atomic_fmin_num: {
1109bdd1243dSDimitry Andric Type *DestTy = II->getType();
1110bdd1243dSDimitry Andric Type *SrcTy = NewV->getType();
111106c3fb27SDimitry Andric unsigned NewAS = SrcTy->getPointerAddressSpace();
111206c3fb27SDimitry Andric if (!AMDGPU::isExtendedGlobalAddrSpace(NewAS))
111306c3fb27SDimitry Andric return nullptr;
111406c3fb27SDimitry Andric Module *M = II->getModule();
1115bdd1243dSDimitry Andric Function *NewDecl = Intrinsic::getDeclaration(M, II->getIntrinsicID(),
1116bdd1243dSDimitry Andric {DestTy, SrcTy, DestTy});
1117bdd1243dSDimitry Andric II->setArgOperand(0, NewV);
1118bdd1243dSDimitry Andric II->setCalledFunction(NewDecl);
1119bdd1243dSDimitry Andric return II;
1120bdd1243dSDimitry Andric }
11218bcb0991SDimitry Andric default:
11225ffd83dbSDimitry Andric return nullptr;
11238bcb0991SDimitry Andric }
11248bcb0991SDimitry Andric }
11258bcb0991SDimitry Andric
getShuffleCost(TTI::ShuffleKind Kind,VectorType * VT,ArrayRef<int> Mask,TTI::TargetCostKind CostKind,int Index,VectorType * SubTp,ArrayRef<const Value * > Args)1126fe6060f1SDimitry Andric InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
1127fe6060f1SDimitry Andric VectorType *VT, ArrayRef<int> Mask,
1128bdd1243dSDimitry Andric TTI::TargetCostKind CostKind,
112981ad6265SDimitry Andric int Index, VectorType *SubTp,
113081ad6265SDimitry Andric ArrayRef<const Value *> Args) {
11315f757f3fSDimitry Andric Kind = improveShuffleKindFromMask(Kind, Mask, VT, Index, SubTp);
11325f757f3fSDimitry Andric
11330b57cec5SDimitry Andric if (ST->hasVOP3PInsts()) {
11345ffd83dbSDimitry Andric if (cast<FixedVectorType>(VT)->getNumElements() == 2 &&
11350b57cec5SDimitry Andric DL.getTypeSizeInBits(VT->getElementType()) == 16) {
11360b57cec5SDimitry Andric // With op_sel VOP3P instructions freely can access the low half or high
11370b57cec5SDimitry Andric // half of a register, so any swizzle is free.
11380b57cec5SDimitry Andric
11390b57cec5SDimitry Andric switch (Kind) {
11400b57cec5SDimitry Andric case TTI::SK_Broadcast:
11410b57cec5SDimitry Andric case TTI::SK_Reverse:
11420b57cec5SDimitry Andric case TTI::SK_PermuteSingleSrc:
11430b57cec5SDimitry Andric return 0;
11440b57cec5SDimitry Andric default:
11450b57cec5SDimitry Andric break;
11460b57cec5SDimitry Andric }
11470b57cec5SDimitry Andric }
11480b57cec5SDimitry Andric }
11490b57cec5SDimitry Andric
1150bdd1243dSDimitry Andric return BaseT::getShuffleCost(Kind, VT, Mask, CostKind, Index, SubTp);
11510b57cec5SDimitry Andric }
11520b57cec5SDimitry Andric
areInlineCompatible(const Function * Caller,const Function * Callee) const11530b57cec5SDimitry Andric bool GCNTTIImpl::areInlineCompatible(const Function *Caller,
11540b57cec5SDimitry Andric const Function *Callee) const {
11550b57cec5SDimitry Andric const TargetMachine &TM = getTLI()->getTargetMachine();
1156480093f4SDimitry Andric const GCNSubtarget *CallerST
1157480093f4SDimitry Andric = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Caller));
1158480093f4SDimitry Andric const GCNSubtarget *CalleeST
1159480093f4SDimitry Andric = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Callee));
1160480093f4SDimitry Andric
1161480093f4SDimitry Andric const FeatureBitset &CallerBits = CallerST->getFeatureBits();
1162480093f4SDimitry Andric const FeatureBitset &CalleeBits = CalleeST->getFeatureBits();
11630b57cec5SDimitry Andric
11640b57cec5SDimitry Andric FeatureBitset RealCallerBits = CallerBits & ~InlineFeatureIgnoreList;
11650b57cec5SDimitry Andric FeatureBitset RealCalleeBits = CalleeBits & ~InlineFeatureIgnoreList;
11660b57cec5SDimitry Andric if ((RealCallerBits & RealCalleeBits) != RealCalleeBits)
11670b57cec5SDimitry Andric return false;
11680b57cec5SDimitry Andric
11690b57cec5SDimitry Andric // FIXME: dx10_clamp can just take the caller setting, but there seems to be
11700b57cec5SDimitry Andric // no way to support merge for backend defined attributes.
11715f757f3fSDimitry Andric SIModeRegisterDefaults CallerMode(*Caller, *CallerST);
11725f757f3fSDimitry Andric SIModeRegisterDefaults CalleeMode(*Callee, *CalleeST);
1173e8d8bef9SDimitry Andric if (!CallerMode.isInlineCompatible(CalleeMode))
1174e8d8bef9SDimitry Andric return false;
1175e8d8bef9SDimitry Andric
1176fe6060f1SDimitry Andric if (Callee->hasFnAttribute(Attribute::AlwaysInline) ||
1177fe6060f1SDimitry Andric Callee->hasFnAttribute(Attribute::InlineHint))
1178fe6060f1SDimitry Andric return true;
1179fe6060f1SDimitry Andric
1180e8d8bef9SDimitry Andric // Hack to make compile times reasonable.
1181fe6060f1SDimitry Andric if (InlineMaxBB) {
1182fe6060f1SDimitry Andric // Single BB does not increase total BB amount.
1183fe6060f1SDimitry Andric if (Callee->size() == 1)
1184fe6060f1SDimitry Andric return true;
1185e8d8bef9SDimitry Andric size_t BBSize = Caller->size() + Callee->size() - 1;
1186e8d8bef9SDimitry Andric return BBSize <= InlineMaxBB;
1187e8d8bef9SDimitry Andric }
1188e8d8bef9SDimitry Andric
1189e8d8bef9SDimitry Andric return true;
1190e8d8bef9SDimitry Andric }
1191e8d8bef9SDimitry Andric
adjustInliningThresholdUsingCallee(const CallBase * CB,const SITargetLowering * TLI,const GCNTTIImpl * TTIImpl)119206c3fb27SDimitry Andric static unsigned adjustInliningThresholdUsingCallee(const CallBase *CB,
119306c3fb27SDimitry Andric const SITargetLowering *TLI,
119406c3fb27SDimitry Andric const GCNTTIImpl *TTIImpl) {
119506c3fb27SDimitry Andric const int NrOfSGPRUntilSpill = 26;
119606c3fb27SDimitry Andric const int NrOfVGPRUntilSpill = 32;
119706c3fb27SDimitry Andric
119806c3fb27SDimitry Andric const DataLayout &DL = TTIImpl->getDataLayout();
119906c3fb27SDimitry Andric
120006c3fb27SDimitry Andric unsigned adjustThreshold = 0;
120106c3fb27SDimitry Andric int SGPRsInUse = 0;
120206c3fb27SDimitry Andric int VGPRsInUse = 0;
120306c3fb27SDimitry Andric for (const Use &A : CB->args()) {
120406c3fb27SDimitry Andric SmallVector<EVT, 4> ValueVTs;
120506c3fb27SDimitry Andric ComputeValueVTs(*TLI, DL, A.get()->getType(), ValueVTs);
120606c3fb27SDimitry Andric for (auto ArgVT : ValueVTs) {
120706c3fb27SDimitry Andric unsigned CCRegNum = TLI->getNumRegistersForCallingConv(
120806c3fb27SDimitry Andric CB->getContext(), CB->getCallingConv(), ArgVT);
120906c3fb27SDimitry Andric if (AMDGPU::isArgPassedInSGPR(CB, CB->getArgOperandNo(&A)))
121006c3fb27SDimitry Andric SGPRsInUse += CCRegNum;
121106c3fb27SDimitry Andric else
121206c3fb27SDimitry Andric VGPRsInUse += CCRegNum;
121306c3fb27SDimitry Andric }
121406c3fb27SDimitry Andric }
121506c3fb27SDimitry Andric
121606c3fb27SDimitry Andric // The cost of passing function arguments through the stack:
121706c3fb27SDimitry Andric // 1 instruction to put a function argument on the stack in the caller.
121806c3fb27SDimitry Andric // 1 instruction to take a function argument from the stack in callee.
121906c3fb27SDimitry Andric // 1 instruction is explicitly take care of data dependencies in callee
122006c3fb27SDimitry Andric // function.
122106c3fb27SDimitry Andric InstructionCost ArgStackCost(1);
122206c3fb27SDimitry Andric ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(
122306c3fb27SDimitry Andric Instruction::Store, Type::getInt32Ty(CB->getContext()), Align(4),
122406c3fb27SDimitry Andric AMDGPUAS::PRIVATE_ADDRESS, TTI::TCK_SizeAndLatency);
122506c3fb27SDimitry Andric ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(
122606c3fb27SDimitry Andric Instruction::Load, Type::getInt32Ty(CB->getContext()), Align(4),
122706c3fb27SDimitry Andric AMDGPUAS::PRIVATE_ADDRESS, TTI::TCK_SizeAndLatency);
122806c3fb27SDimitry Andric
122906c3fb27SDimitry Andric // The penalty cost is computed relative to the cost of instructions and does
123006c3fb27SDimitry Andric // not model any storage costs.
123106c3fb27SDimitry Andric adjustThreshold += std::max(0, SGPRsInUse - NrOfSGPRUntilSpill) *
123206c3fb27SDimitry Andric *ArgStackCost.getValue() * InlineConstants::getInstrCost();
123306c3fb27SDimitry Andric adjustThreshold += std::max(0, VGPRsInUse - NrOfVGPRUntilSpill) *
123406c3fb27SDimitry Andric *ArgStackCost.getValue() * InlineConstants::getInstrCost();
123506c3fb27SDimitry Andric return adjustThreshold;
123606c3fb27SDimitry Andric }
123706c3fb27SDimitry Andric
getCallArgsTotalAllocaSize(const CallBase * CB,const DataLayout & DL)123806c3fb27SDimitry Andric static unsigned getCallArgsTotalAllocaSize(const CallBase *CB,
123906c3fb27SDimitry Andric const DataLayout &DL) {
124006c3fb27SDimitry Andric // If we have a pointer to a private array passed into a function
1241e8d8bef9SDimitry Andric // it will not be optimized out, leaving scratch usage.
124206c3fb27SDimitry Andric // This function calculates the total size in bytes of the memory that would
124306c3fb27SDimitry Andric // end in scratch if the call was not inlined.
124406c3fb27SDimitry Andric unsigned AllocaSize = 0;
1245e8d8bef9SDimitry Andric SmallPtrSet<const AllocaInst *, 8> AIVisited;
1246e8d8bef9SDimitry Andric for (Value *PtrArg : CB->args()) {
1247e8d8bef9SDimitry Andric PointerType *Ty = dyn_cast<PointerType>(PtrArg->getType());
124806c3fb27SDimitry Andric if (!Ty)
1249e8d8bef9SDimitry Andric continue;
1250e8d8bef9SDimitry Andric
125106c3fb27SDimitry Andric unsigned AddrSpace = Ty->getAddressSpace();
125206c3fb27SDimitry Andric if (AddrSpace != AMDGPUAS::FLAT_ADDRESS &&
125306c3fb27SDimitry Andric AddrSpace != AMDGPUAS::PRIVATE_ADDRESS)
1254e8d8bef9SDimitry Andric continue;
125506c3fb27SDimitry Andric
125606c3fb27SDimitry Andric const AllocaInst *AI = dyn_cast<AllocaInst>(getUnderlyingObject(PtrArg));
125706c3fb27SDimitry Andric if (!AI || !AI->isStaticAlloca() || !AIVisited.insert(AI).second)
125806c3fb27SDimitry Andric continue;
125906c3fb27SDimitry Andric
1260e8d8bef9SDimitry Andric AllocaSize += DL.getTypeAllocSize(AI->getAllocatedType());
1261e8d8bef9SDimitry Andric }
126206c3fb27SDimitry Andric return AllocaSize;
1263e8d8bef9SDimitry Andric }
126406c3fb27SDimitry Andric
adjustInliningThreshold(const CallBase * CB) const126506c3fb27SDimitry Andric unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const {
126606c3fb27SDimitry Andric unsigned Threshold = adjustInliningThresholdUsingCallee(CB, TLI, this);
126706c3fb27SDimitry Andric
126806c3fb27SDimitry Andric // Private object passed as arguments may end up in scratch usage if the call
126906c3fb27SDimitry Andric // is not inlined. Increase the inline threshold to promote inlining.
127006c3fb27SDimitry Andric unsigned AllocaSize = getCallArgsTotalAllocaSize(CB, DL);
127106c3fb27SDimitry Andric if (AllocaSize > 0)
127206c3fb27SDimitry Andric Threshold += ArgAllocaCost;
127306c3fb27SDimitry Andric return Threshold;
1274e8d8bef9SDimitry Andric }
127506c3fb27SDimitry Andric
getCallerAllocaCost(const CallBase * CB,const AllocaInst * AI) const127606c3fb27SDimitry Andric unsigned GCNTTIImpl::getCallerAllocaCost(const CallBase *CB,
127706c3fb27SDimitry Andric const AllocaInst *AI) const {
127806c3fb27SDimitry Andric
127906c3fb27SDimitry Andric // Below the cutoff, assume that the private memory objects would be
128006c3fb27SDimitry Andric // optimized
128106c3fb27SDimitry Andric auto AllocaSize = getCallArgsTotalAllocaSize(CB, DL);
128206c3fb27SDimitry Andric if (AllocaSize <= ArgAllocaCutoff)
1283e8d8bef9SDimitry Andric return 0;
128406c3fb27SDimitry Andric
128506c3fb27SDimitry Andric // Above the cutoff, we give a cost to each private memory object
128606c3fb27SDimitry Andric // depending its size. If the array can be optimized by SROA this cost is not
128706c3fb27SDimitry Andric // added to the total-cost in the inliner cost analysis.
128806c3fb27SDimitry Andric //
128906c3fb27SDimitry Andric // We choose the total cost of the alloca such that their sum cancels the
129006c3fb27SDimitry Andric // bonus given in the threshold (ArgAllocaCost).
129106c3fb27SDimitry Andric //
129206c3fb27SDimitry Andric // Cost_Alloca_0 + ... + Cost_Alloca_N == ArgAllocaCost
129306c3fb27SDimitry Andric //
129406c3fb27SDimitry Andric // Awkwardly, the ArgAllocaCost bonus is multiplied by threshold-multiplier,
129506c3fb27SDimitry Andric // the single-bb bonus and the vector-bonus.
129606c3fb27SDimitry Andric //
129706c3fb27SDimitry Andric // We compensate the first two multipliers, by repeating logic from the
129806c3fb27SDimitry Andric // inliner-cost in here. The vector-bonus is 0 on AMDGPU.
129906c3fb27SDimitry Andric static_assert(InlinerVectorBonusPercent == 0, "vector bonus assumed to be 0");
130006c3fb27SDimitry Andric unsigned Threshold = ArgAllocaCost * getInliningThresholdMultiplier();
130106c3fb27SDimitry Andric
130206c3fb27SDimitry Andric bool SingleBB = none_of(*CB->getCalledFunction(), [](const BasicBlock &BB) {
130306c3fb27SDimitry Andric return BB.getTerminator()->getNumSuccessors() > 1;
130406c3fb27SDimitry Andric });
130506c3fb27SDimitry Andric if (SingleBB) {
130606c3fb27SDimitry Andric Threshold += Threshold / 2;
130706c3fb27SDimitry Andric }
130806c3fb27SDimitry Andric
130906c3fb27SDimitry Andric auto ArgAllocaSize = DL.getTypeAllocSize(AI->getAllocatedType());
131006c3fb27SDimitry Andric
131106c3fb27SDimitry Andric // Attribute the bonus proportionally to the alloca size
131206c3fb27SDimitry Andric unsigned AllocaThresholdBonus = (Threshold * ArgAllocaSize) / AllocaSize;
131306c3fb27SDimitry Andric
131406c3fb27SDimitry Andric return AllocaThresholdBonus;
13150b57cec5SDimitry Andric }
13160b57cec5SDimitry Andric
getUnrollingPreferences(Loop * L,ScalarEvolution & SE,TTI::UnrollingPreferences & UP,OptimizationRemarkEmitter * ORE)13170b57cec5SDimitry Andric void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
1318349cc55cSDimitry Andric TTI::UnrollingPreferences &UP,
1319349cc55cSDimitry Andric OptimizationRemarkEmitter *ORE) {
1320349cc55cSDimitry Andric CommonTTI.getUnrollingPreferences(L, SE, UP, ORE);
13210b57cec5SDimitry Andric }
13220b57cec5SDimitry Andric
getPeelingPreferences(Loop * L,ScalarEvolution & SE,TTI::PeelingPreferences & PP)13235ffd83dbSDimitry Andric void GCNTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
13245ffd83dbSDimitry Andric TTI::PeelingPreferences &PP) {
13255ffd83dbSDimitry Andric CommonTTI.getPeelingPreferences(L, SE, PP);
13268bcb0991SDimitry Andric }
13278bcb0991SDimitry Andric
get64BitInstrCost(TTI::TargetCostKind CostKind) const1328e8d8bef9SDimitry Andric int GCNTTIImpl::get64BitInstrCost(TTI::TargetCostKind CostKind) const {
1329fe6060f1SDimitry Andric return ST->hasFullRate64Ops()
1330fe6060f1SDimitry Andric ? getFullRateInstrCost()
1331fe6060f1SDimitry Andric : ST->hasHalfRate64Ops() ? getHalfRateInstrCost(CostKind)
1332e8d8bef9SDimitry Andric : getQuarterRateInstrCost(CostKind);
1333e8d8bef9SDimitry Andric }
1334bdd1243dSDimitry Andric
1335bdd1243dSDimitry Andric std::pair<InstructionCost, MVT>
getTypeLegalizationCost(Type * Ty) const1336bdd1243dSDimitry Andric GCNTTIImpl::getTypeLegalizationCost(Type *Ty) const {
1337bdd1243dSDimitry Andric std::pair<InstructionCost, MVT> Cost = BaseT::getTypeLegalizationCost(Ty);
1338bdd1243dSDimitry Andric auto Size = DL.getTypeSizeInBits(Ty);
1339bdd1243dSDimitry Andric // Maximum load or store can handle 8 dwords for scalar and 4 for
1340bdd1243dSDimitry Andric // vector ALU. Let's assume anything above 8 dwords is expensive
1341bdd1243dSDimitry Andric // even if legal.
1342bdd1243dSDimitry Andric if (Size <= 256)
1343bdd1243dSDimitry Andric return Cost;
1344bdd1243dSDimitry Andric
1345bdd1243dSDimitry Andric Cost.first += (Size + 255) / 256;
1346bdd1243dSDimitry Andric return Cost;
1347bdd1243dSDimitry Andric }
1348cb14a3feSDimitry Andric
getPrefetchDistance() const1349cb14a3feSDimitry Andric unsigned GCNTTIImpl::getPrefetchDistance() const {
1350cb14a3feSDimitry Andric return ST->hasPrefetch() ? 128 : 0;
1351cb14a3feSDimitry Andric }
1352cb14a3feSDimitry Andric
shouldPrefetchAddressSpace(unsigned AS) const1353cb14a3feSDimitry Andric bool GCNTTIImpl::shouldPrefetchAddressSpace(unsigned AS) const {
1354cb14a3feSDimitry Andric return AMDGPU::isFlatGlobalAddrSpace(AS);
1355cb14a3feSDimitry Andric }
1356