1 //===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/Analysis/TargetTransformInfo.h"
10 #include "llvm/Analysis/CFG.h"
11 #include "llvm/Analysis/LoopIterator.h"
12 #include "llvm/Analysis/TargetTransformInfoImpl.h"
13 #include "llvm/IR/CFG.h"
14 #include "llvm/IR/DataLayout.h"
15 #include "llvm/IR/Dominators.h"
16 #include "llvm/IR/Instruction.h"
17 #include "llvm/IR/Instructions.h"
18 #include "llvm/IR/IntrinsicInst.h"
19 #include "llvm/IR/Module.h"
20 #include "llvm/IR/Operator.h"
21 #include "llvm/IR/PatternMatch.h"
22 #include "llvm/InitializePasses.h"
23 #include "llvm/Support/CommandLine.h"
24 #include "llvm/Support/ErrorHandling.h"
25 #include <utility>
26
27 using namespace llvm;
28 using namespace PatternMatch;
29
30 #define DEBUG_TYPE "tti"
31
32 static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false),
33 cl::Hidden,
34 cl::desc("Recognize reduction patterns."));
35
36 namespace {
37 /// No-op implementation of the TTI interface using the utility base
38 /// classes.
39 ///
40 /// This is used when no target specific information is available.
41 struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> {
NoTTIImpl__anondbd2b52e0111::NoTTIImpl42 explicit NoTTIImpl(const DataLayout &DL)
43 : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {}
44 };
45 } // namespace
46
canAnalyze(LoopInfo & LI)47 bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) {
48 // If the loop has irreducible control flow, it can not be converted to
49 // Hardware loop.
50 LoopBlocksRPO RPOT(L);
51 RPOT.perform(&LI);
52 if (containsIrreducibleCFG<const BasicBlock *>(RPOT, LI))
53 return false;
54 return true;
55 }
56
IntrinsicCostAttributes(Intrinsic::ID Id,const CallBase & CI,InstructionCost ScalarizationCost)57 IntrinsicCostAttributes::IntrinsicCostAttributes(
58 Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost)
59 : II(dyn_cast<IntrinsicInst>(&CI)), RetTy(CI.getType()), IID(Id),
60 ScalarizationCost(ScalarizationCost) {
61
62 if (const auto *FPMO = dyn_cast<FPMathOperator>(&CI))
63 FMF = FPMO->getFastMathFlags();
64
65 Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end());
66 FunctionType *FTy = CI.getCalledFunction()->getFunctionType();
67 ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end());
68 }
69
IntrinsicCostAttributes(Intrinsic::ID Id,Type * RTy,ArrayRef<Type * > Tys,FastMathFlags Flags,const IntrinsicInst * I,InstructionCost ScalarCost)70 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
71 ArrayRef<Type *> Tys,
72 FastMathFlags Flags,
73 const IntrinsicInst *I,
74 InstructionCost ScalarCost)
75 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {
76 ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end());
77 }
78
IntrinsicCostAttributes(Intrinsic::ID Id,Type * Ty,ArrayRef<const Value * > Args)79 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty,
80 ArrayRef<const Value *> Args)
81 : RetTy(Ty), IID(Id) {
82
83 Arguments.insert(Arguments.begin(), Args.begin(), Args.end());
84 ParamTys.reserve(Arguments.size());
85 for (unsigned Idx = 0, Size = Arguments.size(); Idx != Size; ++Idx)
86 ParamTys.push_back(Arguments[Idx]->getType());
87 }
88
IntrinsicCostAttributes(Intrinsic::ID Id,Type * RTy,ArrayRef<const Value * > Args,ArrayRef<Type * > Tys,FastMathFlags Flags,const IntrinsicInst * I,InstructionCost ScalarCost)89 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
90 ArrayRef<const Value *> Args,
91 ArrayRef<Type *> Tys,
92 FastMathFlags Flags,
93 const IntrinsicInst *I,
94 InstructionCost ScalarCost)
95 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {
96 ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end());
97 Arguments.insert(Arguments.begin(), Args.begin(), Args.end());
98 }
99
isHardwareLoopCandidate(ScalarEvolution & SE,LoopInfo & LI,DominatorTree & DT,bool ForceNestedLoop,bool ForceHardwareLoopPHI)100 bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE,
101 LoopInfo &LI, DominatorTree &DT,
102 bool ForceNestedLoop,
103 bool ForceHardwareLoopPHI) {
104 SmallVector<BasicBlock *, 4> ExitingBlocks;
105 L->getExitingBlocks(ExitingBlocks);
106
107 for (BasicBlock *BB : ExitingBlocks) {
108 // If we pass the updated counter back through a phi, we need to know
109 // which latch the updated value will be coming from.
110 if (!L->isLoopLatch(BB)) {
111 if (ForceHardwareLoopPHI || CounterInReg)
112 continue;
113 }
114
115 const SCEV *EC = SE.getExitCount(L, BB);
116 if (isa<SCEVCouldNotCompute>(EC))
117 continue;
118 if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(EC)) {
119 if (ConstEC->getValue()->isZero())
120 continue;
121 } else if (!SE.isLoopInvariant(EC, L))
122 continue;
123
124 if (SE.getTypeSizeInBits(EC->getType()) > CountType->getBitWidth())
125 continue;
126
127 // If this exiting block is contained in a nested loop, it is not eligible
128 // for insertion of the branch-and-decrement since the inner loop would
129 // end up messing up the value in the CTR.
130 if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop)
131 continue;
132
133 // We now have a loop-invariant count of loop iterations (which is not the
134 // constant zero) for which we know that this loop will not exit via this
135 // existing block.
136
137 // We need to make sure that this block will run on every loop iteration.
138 // For this to be true, we must dominate all blocks with backedges. Such
139 // blocks are in-loop predecessors to the header block.
140 bool NotAlways = false;
141 for (BasicBlock *Pred : predecessors(L->getHeader())) {
142 if (!L->contains(Pred))
143 continue;
144
145 if (!DT.dominates(BB, Pred)) {
146 NotAlways = true;
147 break;
148 }
149 }
150
151 if (NotAlways)
152 continue;
153
154 // Make sure this blocks ends with a conditional branch.
155 Instruction *TI = BB->getTerminator();
156 if (!TI)
157 continue;
158
159 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
160 if (!BI->isConditional())
161 continue;
162
163 ExitBranch = BI;
164 } else
165 continue;
166
167 // Note that this block may not be the loop latch block, even if the loop
168 // has a latch block.
169 ExitBlock = BB;
170 ExitCount = EC;
171 break;
172 }
173
174 if (!ExitBlock)
175 return false;
176 return true;
177 }
178
TargetTransformInfo(const DataLayout & DL)179 TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)
180 : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {}
181
~TargetTransformInfo()182 TargetTransformInfo::~TargetTransformInfo() {}
183
TargetTransformInfo(TargetTransformInfo && Arg)184 TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg)
185 : TTIImpl(std::move(Arg.TTIImpl)) {}
186
operator =(TargetTransformInfo && RHS)187 TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) {
188 TTIImpl = std::move(RHS.TTIImpl);
189 return *this;
190 }
191
getInliningThresholdMultiplier() const192 unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {
193 return TTIImpl->getInliningThresholdMultiplier();
194 }
195
196 unsigned
adjustInliningThreshold(const CallBase * CB) const197 TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const {
198 return TTIImpl->adjustInliningThreshold(CB);
199 }
200
getInlinerVectorBonusPercent() const201 int TargetTransformInfo::getInlinerVectorBonusPercent() const {
202 return TTIImpl->getInlinerVectorBonusPercent();
203 }
204
205 InstructionCost
getGEPCost(Type * PointeeType,const Value * Ptr,ArrayRef<const Value * > Operands,TTI::TargetCostKind CostKind) const206 TargetTransformInfo::getGEPCost(Type *PointeeType, const Value *Ptr,
207 ArrayRef<const Value *> Operands,
208 TTI::TargetCostKind CostKind) const {
209 return TTIImpl->getGEPCost(PointeeType, Ptr, Operands, CostKind);
210 }
211
getEstimatedNumberOfCaseClusters(const SwitchInst & SI,unsigned & JTSize,ProfileSummaryInfo * PSI,BlockFrequencyInfo * BFI) const212 unsigned TargetTransformInfo::getEstimatedNumberOfCaseClusters(
213 const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI,
214 BlockFrequencyInfo *BFI) const {
215 return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI);
216 }
217
218 InstructionCost
getUserCost(const User * U,ArrayRef<const Value * > Operands,enum TargetCostKind CostKind) const219 TargetTransformInfo::getUserCost(const User *U,
220 ArrayRef<const Value *> Operands,
221 enum TargetCostKind CostKind) const {
222 InstructionCost Cost = TTIImpl->getUserCost(U, Operands, CostKind);
223 assert((CostKind == TTI::TCK_RecipThroughput || Cost >= 0) &&
224 "TTI should not produce negative costs!");
225 return Cost;
226 }
227
getPredictableBranchThreshold() const228 BranchProbability TargetTransformInfo::getPredictableBranchThreshold() const {
229 return TTIImpl->getPredictableBranchThreshold();
230 }
231
hasBranchDivergence() const232 bool TargetTransformInfo::hasBranchDivergence() const {
233 return TTIImpl->hasBranchDivergence();
234 }
235
useGPUDivergenceAnalysis() const236 bool TargetTransformInfo::useGPUDivergenceAnalysis() const {
237 return TTIImpl->useGPUDivergenceAnalysis();
238 }
239
isSourceOfDivergence(const Value * V) const240 bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const {
241 return TTIImpl->isSourceOfDivergence(V);
242 }
243
isAlwaysUniform(const Value * V) const244 bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const {
245 return TTIImpl->isAlwaysUniform(V);
246 }
247
getFlatAddressSpace() const248 unsigned TargetTransformInfo::getFlatAddressSpace() const {
249 return TTIImpl->getFlatAddressSpace();
250 }
251
collectFlatAddressOperands(SmallVectorImpl<int> & OpIndexes,Intrinsic::ID IID) const252 bool TargetTransformInfo::collectFlatAddressOperands(
253 SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const {
254 return TTIImpl->collectFlatAddressOperands(OpIndexes, IID);
255 }
256
isNoopAddrSpaceCast(unsigned FromAS,unsigned ToAS) const257 bool TargetTransformInfo::isNoopAddrSpaceCast(unsigned FromAS,
258 unsigned ToAS) const {
259 return TTIImpl->isNoopAddrSpaceCast(FromAS, ToAS);
260 }
261
getAssumedAddrSpace(const Value * V) const262 unsigned TargetTransformInfo::getAssumedAddrSpace(const Value *V) const {
263 return TTIImpl->getAssumedAddrSpace(V);
264 }
265
rewriteIntrinsicWithAddressSpace(IntrinsicInst * II,Value * OldV,Value * NewV) const266 Value *TargetTransformInfo::rewriteIntrinsicWithAddressSpace(
267 IntrinsicInst *II, Value *OldV, Value *NewV) const {
268 return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV);
269 }
270
isLoweredToCall(const Function * F) const271 bool TargetTransformInfo::isLoweredToCall(const Function *F) const {
272 return TTIImpl->isLoweredToCall(F);
273 }
274
isHardwareLoopProfitable(Loop * L,ScalarEvolution & SE,AssumptionCache & AC,TargetLibraryInfo * LibInfo,HardwareLoopInfo & HWLoopInfo) const275 bool TargetTransformInfo::isHardwareLoopProfitable(
276 Loop *L, ScalarEvolution &SE, AssumptionCache &AC,
277 TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const {
278 return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
279 }
280
preferPredicateOverEpilogue(Loop * L,LoopInfo * LI,ScalarEvolution & SE,AssumptionCache & AC,TargetLibraryInfo * TLI,DominatorTree * DT,const LoopAccessInfo * LAI) const281 bool TargetTransformInfo::preferPredicateOverEpilogue(
282 Loop *L, LoopInfo *LI, ScalarEvolution &SE, AssumptionCache &AC,
283 TargetLibraryInfo *TLI, DominatorTree *DT,
284 const LoopAccessInfo *LAI) const {
285 return TTIImpl->preferPredicateOverEpilogue(L, LI, SE, AC, TLI, DT, LAI);
286 }
287
emitGetActiveLaneMask() const288 bool TargetTransformInfo::emitGetActiveLaneMask() const {
289 return TTIImpl->emitGetActiveLaneMask();
290 }
291
292 Optional<Instruction *>
instCombineIntrinsic(InstCombiner & IC,IntrinsicInst & II) const293 TargetTransformInfo::instCombineIntrinsic(InstCombiner &IC,
294 IntrinsicInst &II) const {
295 return TTIImpl->instCombineIntrinsic(IC, II);
296 }
297
simplifyDemandedUseBitsIntrinsic(InstCombiner & IC,IntrinsicInst & II,APInt DemandedMask,KnownBits & Known,bool & KnownBitsComputed) const298 Optional<Value *> TargetTransformInfo::simplifyDemandedUseBitsIntrinsic(
299 InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,
300 bool &KnownBitsComputed) const {
301 return TTIImpl->simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,
302 KnownBitsComputed);
303 }
304
simplifyDemandedVectorEltsIntrinsic(InstCombiner & IC,IntrinsicInst & II,APInt DemandedElts,APInt & UndefElts,APInt & UndefElts2,APInt & UndefElts3,std::function<void (Instruction *,unsigned,APInt,APInt &)> SimplifyAndSetOp) const305 Optional<Value *> TargetTransformInfo::simplifyDemandedVectorEltsIntrinsic(
306 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
307 APInt &UndefElts2, APInt &UndefElts3,
308 std::function<void(Instruction *, unsigned, APInt, APInt &)>
309 SimplifyAndSetOp) const {
310 return TTIImpl->simplifyDemandedVectorEltsIntrinsic(
311 IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
312 SimplifyAndSetOp);
313 }
314
getUnrollingPreferences(Loop * L,ScalarEvolution & SE,UnrollingPreferences & UP) const315 void TargetTransformInfo::getUnrollingPreferences(
316 Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP) const {
317 return TTIImpl->getUnrollingPreferences(L, SE, UP);
318 }
319
getPeelingPreferences(Loop * L,ScalarEvolution & SE,PeelingPreferences & PP) const320 void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
321 PeelingPreferences &PP) const {
322 return TTIImpl->getPeelingPreferences(L, SE, PP);
323 }
324
isLegalAddImmediate(int64_t Imm) const325 bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const {
326 return TTIImpl->isLegalAddImmediate(Imm);
327 }
328
isLegalICmpImmediate(int64_t Imm) const329 bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const {
330 return TTIImpl->isLegalICmpImmediate(Imm);
331 }
332
isLegalAddressingMode(Type * Ty,GlobalValue * BaseGV,int64_t BaseOffset,bool HasBaseReg,int64_t Scale,unsigned AddrSpace,Instruction * I) const333 bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
334 int64_t BaseOffset,
335 bool HasBaseReg, int64_t Scale,
336 unsigned AddrSpace,
337 Instruction *I) const {
338 return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,
339 Scale, AddrSpace, I);
340 }
341
isLSRCostLess(LSRCost & C1,LSRCost & C2) const342 bool TargetTransformInfo::isLSRCostLess(LSRCost &C1, LSRCost &C2) const {
343 return TTIImpl->isLSRCostLess(C1, C2);
344 }
345
isNumRegsMajorCostOfLSR() const346 bool TargetTransformInfo::isNumRegsMajorCostOfLSR() const {
347 return TTIImpl->isNumRegsMajorCostOfLSR();
348 }
349
isProfitableLSRChainElement(Instruction * I) const350 bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const {
351 return TTIImpl->isProfitableLSRChainElement(I);
352 }
353
canMacroFuseCmp() const354 bool TargetTransformInfo::canMacroFuseCmp() const {
355 return TTIImpl->canMacroFuseCmp();
356 }
357
canSaveCmp(Loop * L,BranchInst ** BI,ScalarEvolution * SE,LoopInfo * LI,DominatorTree * DT,AssumptionCache * AC,TargetLibraryInfo * LibInfo) const358 bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI,
359 ScalarEvolution *SE, LoopInfo *LI,
360 DominatorTree *DT, AssumptionCache *AC,
361 TargetLibraryInfo *LibInfo) const {
362 return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo);
363 }
364
365 TTI::AddressingModeKind
getPreferredAddressingMode(const Loop * L,ScalarEvolution * SE) const366 TargetTransformInfo::getPreferredAddressingMode(const Loop *L,
367 ScalarEvolution *SE) const {
368 return TTIImpl->getPreferredAddressingMode(L, SE);
369 }
370
isLegalMaskedStore(Type * DataType,Align Alignment) const371 bool TargetTransformInfo::isLegalMaskedStore(Type *DataType,
372 Align Alignment) const {
373 return TTIImpl->isLegalMaskedStore(DataType, Alignment);
374 }
375
isLegalMaskedLoad(Type * DataType,Align Alignment) const376 bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType,
377 Align Alignment) const {
378 return TTIImpl->isLegalMaskedLoad(DataType, Alignment);
379 }
380
isLegalNTStore(Type * DataType,Align Alignment) const381 bool TargetTransformInfo::isLegalNTStore(Type *DataType,
382 Align Alignment) const {
383 return TTIImpl->isLegalNTStore(DataType, Alignment);
384 }
385
isLegalNTLoad(Type * DataType,Align Alignment) const386 bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const {
387 return TTIImpl->isLegalNTLoad(DataType, Alignment);
388 }
389
isLegalMaskedGather(Type * DataType,Align Alignment) const390 bool TargetTransformInfo::isLegalMaskedGather(Type *DataType,
391 Align Alignment) const {
392 return TTIImpl->isLegalMaskedGather(DataType, Alignment);
393 }
394
isLegalMaskedScatter(Type * DataType,Align Alignment) const395 bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType,
396 Align Alignment) const {
397 return TTIImpl->isLegalMaskedScatter(DataType, Alignment);
398 }
399
isLegalMaskedCompressStore(Type * DataType) const400 bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType) const {
401 return TTIImpl->isLegalMaskedCompressStore(DataType);
402 }
403
isLegalMaskedExpandLoad(Type * DataType) const404 bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType) const {
405 return TTIImpl->isLegalMaskedExpandLoad(DataType);
406 }
407
hasDivRemOp(Type * DataType,bool IsSigned) const408 bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const {
409 return TTIImpl->hasDivRemOp(DataType, IsSigned);
410 }
411
hasVolatileVariant(Instruction * I,unsigned AddrSpace) const412 bool TargetTransformInfo::hasVolatileVariant(Instruction *I,
413 unsigned AddrSpace) const {
414 return TTIImpl->hasVolatileVariant(I, AddrSpace);
415 }
416
prefersVectorizedAddressing() const417 bool TargetTransformInfo::prefersVectorizedAddressing() const {
418 return TTIImpl->prefersVectorizedAddressing();
419 }
420
getScalingFactorCost(Type * Ty,GlobalValue * BaseGV,int64_t BaseOffset,bool HasBaseReg,int64_t Scale,unsigned AddrSpace) const421 InstructionCost TargetTransformInfo::getScalingFactorCost(
422 Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg,
423 int64_t Scale, unsigned AddrSpace) const {
424 InstructionCost Cost = TTIImpl->getScalingFactorCost(
425 Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace);
426 assert(Cost >= 0 && "TTI should not produce negative costs!");
427 return Cost;
428 }
429
LSRWithInstrQueries() const430 bool TargetTransformInfo::LSRWithInstrQueries() const {
431 return TTIImpl->LSRWithInstrQueries();
432 }
433
isTruncateFree(Type * Ty1,Type * Ty2) const434 bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const {
435 return TTIImpl->isTruncateFree(Ty1, Ty2);
436 }
437
isProfitableToHoist(Instruction * I) const438 bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const {
439 return TTIImpl->isProfitableToHoist(I);
440 }
441
useAA() const442 bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); }
443
isTypeLegal(Type * Ty) const444 bool TargetTransformInfo::isTypeLegal(Type *Ty) const {
445 return TTIImpl->isTypeLegal(Ty);
446 }
447
getRegUsageForType(Type * Ty) const448 InstructionCost TargetTransformInfo::getRegUsageForType(Type *Ty) const {
449 return TTIImpl->getRegUsageForType(Ty);
450 }
451
shouldBuildLookupTables() const452 bool TargetTransformInfo::shouldBuildLookupTables() const {
453 return TTIImpl->shouldBuildLookupTables();
454 }
455
shouldBuildLookupTablesForConstant(Constant * C) const456 bool TargetTransformInfo::shouldBuildLookupTablesForConstant(
457 Constant *C) const {
458 return TTIImpl->shouldBuildLookupTablesForConstant(C);
459 }
460
shouldBuildRelLookupTables() const461 bool TargetTransformInfo::shouldBuildRelLookupTables() const {
462 return TTIImpl->shouldBuildRelLookupTables();
463 }
464
useColdCCForColdCall(Function & F) const465 bool TargetTransformInfo::useColdCCForColdCall(Function &F) const {
466 return TTIImpl->useColdCCForColdCall(F);
467 }
468
469 InstructionCost
getScalarizationOverhead(VectorType * Ty,const APInt & DemandedElts,bool Insert,bool Extract) const470 TargetTransformInfo::getScalarizationOverhead(VectorType *Ty,
471 const APInt &DemandedElts,
472 bool Insert, bool Extract) const {
473 return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract);
474 }
475
getOperandsScalarizationOverhead(ArrayRef<const Value * > Args,ArrayRef<Type * > Tys) const476 InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(
477 ArrayRef<const Value *> Args, ArrayRef<Type *> Tys) const {
478 return TTIImpl->getOperandsScalarizationOverhead(Args, Tys);
479 }
480
supportsEfficientVectorElementLoadStore() const481 bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {
482 return TTIImpl->supportsEfficientVectorElementLoadStore();
483 }
484
enableAggressiveInterleaving(bool LoopHasReductions) const485 bool TargetTransformInfo::enableAggressiveInterleaving(
486 bool LoopHasReductions) const {
487 return TTIImpl->enableAggressiveInterleaving(LoopHasReductions);
488 }
489
490 TargetTransformInfo::MemCmpExpansionOptions
enableMemCmpExpansion(bool OptSize,bool IsZeroCmp) const491 TargetTransformInfo::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
492 return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp);
493 }
494
enableInterleavedAccessVectorization() const495 bool TargetTransformInfo::enableInterleavedAccessVectorization() const {
496 return TTIImpl->enableInterleavedAccessVectorization();
497 }
498
enableMaskedInterleavedAccessVectorization() const499 bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const {
500 return TTIImpl->enableMaskedInterleavedAccessVectorization();
501 }
502
isFPVectorizationPotentiallyUnsafe() const503 bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const {
504 return TTIImpl->isFPVectorizationPotentiallyUnsafe();
505 }
506
allowsMisalignedMemoryAccesses(LLVMContext & Context,unsigned BitWidth,unsigned AddressSpace,Align Alignment,bool * Fast) const507 bool TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context,
508 unsigned BitWidth,
509 unsigned AddressSpace,
510 Align Alignment,
511 bool *Fast) const {
512 return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth,
513 AddressSpace, Alignment, Fast);
514 }
515
516 TargetTransformInfo::PopcntSupportKind
getPopcntSupport(unsigned IntTyWidthInBit) const517 TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const {
518 return TTIImpl->getPopcntSupport(IntTyWidthInBit);
519 }
520
haveFastSqrt(Type * Ty) const521 bool TargetTransformInfo::haveFastSqrt(Type *Ty) const {
522 return TTIImpl->haveFastSqrt(Ty);
523 }
524
isFCmpOrdCheaperThanFCmpZero(Type * Ty) const525 bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const {
526 return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty);
527 }
528
getFPOpCost(Type * Ty) const529 InstructionCost TargetTransformInfo::getFPOpCost(Type *Ty) const {
530 InstructionCost Cost = TTIImpl->getFPOpCost(Ty);
531 assert(Cost >= 0 && "TTI should not produce negative costs!");
532 return Cost;
533 }
534
getIntImmCodeSizeCost(unsigned Opcode,unsigned Idx,const APInt & Imm,Type * Ty) const535 InstructionCost TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode,
536 unsigned Idx,
537 const APInt &Imm,
538 Type *Ty) const {
539 InstructionCost Cost = TTIImpl->getIntImmCodeSizeCost(Opcode, Idx, Imm, Ty);
540 assert(Cost >= 0 && "TTI should not produce negative costs!");
541 return Cost;
542 }
543
544 InstructionCost
getIntImmCost(const APInt & Imm,Type * Ty,TTI::TargetCostKind CostKind) const545 TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty,
546 TTI::TargetCostKind CostKind) const {
547 InstructionCost Cost = TTIImpl->getIntImmCost(Imm, Ty, CostKind);
548 assert(Cost >= 0 && "TTI should not produce negative costs!");
549 return Cost;
550 }
551
getIntImmCostInst(unsigned Opcode,unsigned Idx,const APInt & Imm,Type * Ty,TTI::TargetCostKind CostKind,Instruction * Inst) const552 InstructionCost TargetTransformInfo::getIntImmCostInst(
553 unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty,
554 TTI::TargetCostKind CostKind, Instruction *Inst) const {
555 InstructionCost Cost =
556 TTIImpl->getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind, Inst);
557 assert(Cost >= 0 && "TTI should not produce negative costs!");
558 return Cost;
559 }
560
561 InstructionCost
getIntImmCostIntrin(Intrinsic::ID IID,unsigned Idx,const APInt & Imm,Type * Ty,TTI::TargetCostKind CostKind) const562 TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
563 const APInt &Imm, Type *Ty,
564 TTI::TargetCostKind CostKind) const {
565 InstructionCost Cost =
566 TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind);
567 assert(Cost >= 0 && "TTI should not produce negative costs!");
568 return Cost;
569 }
570
getNumberOfRegisters(unsigned ClassID) const571 unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const {
572 return TTIImpl->getNumberOfRegisters(ClassID);
573 }
574
getRegisterClassForType(bool Vector,Type * Ty) const575 unsigned TargetTransformInfo::getRegisterClassForType(bool Vector,
576 Type *Ty) const {
577 return TTIImpl->getRegisterClassForType(Vector, Ty);
578 }
579
getRegisterClassName(unsigned ClassID) const580 const char *TargetTransformInfo::getRegisterClassName(unsigned ClassID) const {
581 return TTIImpl->getRegisterClassName(ClassID);
582 }
583
getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const584 TypeSize TargetTransformInfo::getRegisterBitWidth(
585 TargetTransformInfo::RegisterKind K) const {
586 return TTIImpl->getRegisterBitWidth(K);
587 }
588
getMinVectorRegisterBitWidth() const589 unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {
590 return TTIImpl->getMinVectorRegisterBitWidth();
591 }
592
getMaxVScale() const593 Optional<unsigned> TargetTransformInfo::getMaxVScale() const {
594 return TTIImpl->getMaxVScale();
595 }
596
shouldMaximizeVectorBandwidth() const597 bool TargetTransformInfo::shouldMaximizeVectorBandwidth() const {
598 return TTIImpl->shouldMaximizeVectorBandwidth();
599 }
600
getMinimumVF(unsigned ElemWidth,bool IsScalable) const601 ElementCount TargetTransformInfo::getMinimumVF(unsigned ElemWidth,
602 bool IsScalable) const {
603 return TTIImpl->getMinimumVF(ElemWidth, IsScalable);
604 }
605
getMaximumVF(unsigned ElemWidth,unsigned Opcode) const606 unsigned TargetTransformInfo::getMaximumVF(unsigned ElemWidth,
607 unsigned Opcode) const {
608 return TTIImpl->getMaximumVF(ElemWidth, Opcode);
609 }
610
shouldConsiderAddressTypePromotion(const Instruction & I,bool & AllowPromotionWithoutCommonHeader) const611 bool TargetTransformInfo::shouldConsiderAddressTypePromotion(
612 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
613 return TTIImpl->shouldConsiderAddressTypePromotion(
614 I, AllowPromotionWithoutCommonHeader);
615 }
616
getCacheLineSize() const617 unsigned TargetTransformInfo::getCacheLineSize() const {
618 return TTIImpl->getCacheLineSize();
619 }
620
621 llvm::Optional<unsigned>
getCacheSize(CacheLevel Level) const622 TargetTransformInfo::getCacheSize(CacheLevel Level) const {
623 return TTIImpl->getCacheSize(Level);
624 }
625
626 llvm::Optional<unsigned>
getCacheAssociativity(CacheLevel Level) const627 TargetTransformInfo::getCacheAssociativity(CacheLevel Level) const {
628 return TTIImpl->getCacheAssociativity(Level);
629 }
630
getPrefetchDistance() const631 unsigned TargetTransformInfo::getPrefetchDistance() const {
632 return TTIImpl->getPrefetchDistance();
633 }
634
getMinPrefetchStride(unsigned NumMemAccesses,unsigned NumStridedMemAccesses,unsigned NumPrefetches,bool HasCall) const635 unsigned TargetTransformInfo::getMinPrefetchStride(
636 unsigned NumMemAccesses, unsigned NumStridedMemAccesses,
637 unsigned NumPrefetches, bool HasCall) const {
638 return TTIImpl->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,
639 NumPrefetches, HasCall);
640 }
641
getMaxPrefetchIterationsAhead() const642 unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const {
643 return TTIImpl->getMaxPrefetchIterationsAhead();
644 }
645
enableWritePrefetching() const646 bool TargetTransformInfo::enableWritePrefetching() const {
647 return TTIImpl->enableWritePrefetching();
648 }
649
getMaxInterleaveFactor(unsigned VF) const650 unsigned TargetTransformInfo::getMaxInterleaveFactor(unsigned VF) const {
651 return TTIImpl->getMaxInterleaveFactor(VF);
652 }
653
654 TargetTransformInfo::OperandValueKind
getOperandInfo(const Value * V,OperandValueProperties & OpProps)655 TargetTransformInfo::getOperandInfo(const Value *V,
656 OperandValueProperties &OpProps) {
657 OperandValueKind OpInfo = OK_AnyValue;
658 OpProps = OP_None;
659
660 if (const auto *CI = dyn_cast<ConstantInt>(V)) {
661 if (CI->getValue().isPowerOf2())
662 OpProps = OP_PowerOf2;
663 return OK_UniformConstantValue;
664 }
665
666 // A broadcast shuffle creates a uniform value.
667 // TODO: Add support for non-zero index broadcasts.
668 // TODO: Add support for different source vector width.
669 if (const auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V))
670 if (ShuffleInst->isZeroEltSplat())
671 OpInfo = OK_UniformValue;
672
673 const Value *Splat = getSplatValue(V);
674
675 // Check for a splat of a constant or for a non uniform vector of constants
676 // and check if the constant(s) are all powers of two.
677 if (isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) {
678 OpInfo = OK_NonUniformConstantValue;
679 if (Splat) {
680 OpInfo = OK_UniformConstantValue;
681 if (auto *CI = dyn_cast<ConstantInt>(Splat))
682 if (CI->getValue().isPowerOf2())
683 OpProps = OP_PowerOf2;
684 } else if (const auto *CDS = dyn_cast<ConstantDataSequential>(V)) {
685 OpProps = OP_PowerOf2;
686 for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) {
687 if (auto *CI = dyn_cast<ConstantInt>(CDS->getElementAsConstant(I)))
688 if (CI->getValue().isPowerOf2())
689 continue;
690 OpProps = OP_None;
691 break;
692 }
693 }
694 }
695
696 // Check for a splat of a uniform value. This is not loop aware, so return
697 // true only for the obviously uniform cases (argument, globalvalue)
698 if (Splat && (isa<Argument>(Splat) || isa<GlobalValue>(Splat)))
699 OpInfo = OK_UniformValue;
700
701 return OpInfo;
702 }
703
getArithmeticInstrCost(unsigned Opcode,Type * Ty,TTI::TargetCostKind CostKind,OperandValueKind Opd1Info,OperandValueKind Opd2Info,OperandValueProperties Opd1PropInfo,OperandValueProperties Opd2PropInfo,ArrayRef<const Value * > Args,const Instruction * CxtI) const704 InstructionCost TargetTransformInfo::getArithmeticInstrCost(
705 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
706 OperandValueKind Opd1Info, OperandValueKind Opd2Info,
707 OperandValueProperties Opd1PropInfo, OperandValueProperties Opd2PropInfo,
708 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
709 InstructionCost Cost =
710 TTIImpl->getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info, Opd2Info,
711 Opd1PropInfo, Opd2PropInfo, Args, CxtI);
712 assert(Cost >= 0 && "TTI should not produce negative costs!");
713 return Cost;
714 }
715
getShuffleCost(ShuffleKind Kind,VectorType * Ty,ArrayRef<int> Mask,int Index,VectorType * SubTp) const716 InstructionCost TargetTransformInfo::getShuffleCost(ShuffleKind Kind,
717 VectorType *Ty,
718 ArrayRef<int> Mask,
719 int Index,
720 VectorType *SubTp) const {
721 InstructionCost Cost = TTIImpl->getShuffleCost(Kind, Ty, Mask, Index, SubTp);
722 assert(Cost >= 0 && "TTI should not produce negative costs!");
723 return Cost;
724 }
725
726 TTI::CastContextHint
getCastContextHint(const Instruction * I)727 TargetTransformInfo::getCastContextHint(const Instruction *I) {
728 if (!I)
729 return CastContextHint::None;
730
731 auto getLoadStoreKind = [](const Value *V, unsigned LdStOp, unsigned MaskedOp,
732 unsigned GatScatOp) {
733 const Instruction *I = dyn_cast<Instruction>(V);
734 if (!I)
735 return CastContextHint::None;
736
737 if (I->getOpcode() == LdStOp)
738 return CastContextHint::Normal;
739
740 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
741 if (II->getIntrinsicID() == MaskedOp)
742 return TTI::CastContextHint::Masked;
743 if (II->getIntrinsicID() == GatScatOp)
744 return TTI::CastContextHint::GatherScatter;
745 }
746
747 return TTI::CastContextHint::None;
748 };
749
750 switch (I->getOpcode()) {
751 case Instruction::ZExt:
752 case Instruction::SExt:
753 case Instruction::FPExt:
754 return getLoadStoreKind(I->getOperand(0), Instruction::Load,
755 Intrinsic::masked_load, Intrinsic::masked_gather);
756 case Instruction::Trunc:
757 case Instruction::FPTrunc:
758 if (I->hasOneUse())
759 return getLoadStoreKind(*I->user_begin(), Instruction::Store,
760 Intrinsic::masked_store,
761 Intrinsic::masked_scatter);
762 break;
763 default:
764 return CastContextHint::None;
765 }
766
767 return TTI::CastContextHint::None;
768 }
769
getCastInstrCost(unsigned Opcode,Type * Dst,Type * Src,CastContextHint CCH,TTI::TargetCostKind CostKind,const Instruction * I) const770 InstructionCost TargetTransformInfo::getCastInstrCost(
771 unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH,
772 TTI::TargetCostKind CostKind, const Instruction *I) const {
773 assert((I == nullptr || I->getOpcode() == Opcode) &&
774 "Opcode should reflect passed instruction.");
775 InstructionCost Cost =
776 TTIImpl->getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
777 assert(Cost >= 0 && "TTI should not produce negative costs!");
778 return Cost;
779 }
780
getExtractWithExtendCost(unsigned Opcode,Type * Dst,VectorType * VecTy,unsigned Index) const781 InstructionCost TargetTransformInfo::getExtractWithExtendCost(
782 unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index) const {
783 InstructionCost Cost =
784 TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index);
785 assert(Cost >= 0 && "TTI should not produce negative costs!");
786 return Cost;
787 }
788
getCFInstrCost(unsigned Opcode,TTI::TargetCostKind CostKind,const Instruction * I) const789 InstructionCost TargetTransformInfo::getCFInstrCost(
790 unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) const {
791 assert((I == nullptr || I->getOpcode() == Opcode) &&
792 "Opcode should reflect passed instruction.");
793 InstructionCost Cost = TTIImpl->getCFInstrCost(Opcode, CostKind, I);
794 assert(Cost >= 0 && "TTI should not produce negative costs!");
795 return Cost;
796 }
797
getCmpSelInstrCost(unsigned Opcode,Type * ValTy,Type * CondTy,CmpInst::Predicate VecPred,TTI::TargetCostKind CostKind,const Instruction * I) const798 InstructionCost TargetTransformInfo::getCmpSelInstrCost(
799 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
800 TTI::TargetCostKind CostKind, const Instruction *I) const {
801 assert((I == nullptr || I->getOpcode() == Opcode) &&
802 "Opcode should reflect passed instruction.");
803 InstructionCost Cost =
804 TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
805 assert(Cost >= 0 && "TTI should not produce negative costs!");
806 return Cost;
807 }
808
getVectorInstrCost(unsigned Opcode,Type * Val,unsigned Index) const809 InstructionCost TargetTransformInfo::getVectorInstrCost(unsigned Opcode,
810 Type *Val,
811 unsigned Index) const {
812 InstructionCost Cost = TTIImpl->getVectorInstrCost(Opcode, Val, Index);
813 assert(Cost >= 0 && "TTI should not produce negative costs!");
814 return Cost;
815 }
816
getMemoryOpCost(unsigned Opcode,Type * Src,Align Alignment,unsigned AddressSpace,TTI::TargetCostKind CostKind,const Instruction * I) const817 InstructionCost TargetTransformInfo::getMemoryOpCost(
818 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
819 TTI::TargetCostKind CostKind, const Instruction *I) const {
820 assert((I == nullptr || I->getOpcode() == Opcode) &&
821 "Opcode should reflect passed instruction.");
822 InstructionCost Cost = TTIImpl->getMemoryOpCost(Opcode, Src, Alignment,
823 AddressSpace, CostKind, I);
824 assert(Cost >= 0 && "TTI should not produce negative costs!");
825 return Cost;
826 }
827
getMaskedMemoryOpCost(unsigned Opcode,Type * Src,Align Alignment,unsigned AddressSpace,TTI::TargetCostKind CostKind) const828 InstructionCost TargetTransformInfo::getMaskedMemoryOpCost(
829 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
830 TTI::TargetCostKind CostKind) const {
831 InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment,
832 AddressSpace, CostKind);
833 assert(Cost >= 0 && "TTI should not produce negative costs!");
834 return Cost;
835 }
836
getGatherScatterOpCost(unsigned Opcode,Type * DataTy,const Value * Ptr,bool VariableMask,Align Alignment,TTI::TargetCostKind CostKind,const Instruction * I) const837 InstructionCost TargetTransformInfo::getGatherScatterOpCost(
838 unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
839 Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
840 InstructionCost Cost = TTIImpl->getGatherScatterOpCost(
841 Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
842 assert(Cost >= 0 && "TTI should not produce negative costs!");
843 return Cost;
844 }
845
getInterleavedMemoryOpCost(unsigned Opcode,Type * VecTy,unsigned Factor,ArrayRef<unsigned> Indices,Align Alignment,unsigned AddressSpace,TTI::TargetCostKind CostKind,bool UseMaskForCond,bool UseMaskForGaps) const846 InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost(
847 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
848 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
849 bool UseMaskForCond, bool UseMaskForGaps) const {
850 InstructionCost Cost = TTIImpl->getInterleavedMemoryOpCost(
851 Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind,
852 UseMaskForCond, UseMaskForGaps);
853 assert(Cost >= 0 && "TTI should not produce negative costs!");
854 return Cost;
855 }
856
857 InstructionCost
getIntrinsicInstrCost(const IntrinsicCostAttributes & ICA,TTI::TargetCostKind CostKind) const858 TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
859 TTI::TargetCostKind CostKind) const {
860 InstructionCost Cost = TTIImpl->getIntrinsicInstrCost(ICA, CostKind);
861 assert(Cost >= 0 && "TTI should not produce negative costs!");
862 return Cost;
863 }
864
865 InstructionCost
getCallInstrCost(Function * F,Type * RetTy,ArrayRef<Type * > Tys,TTI::TargetCostKind CostKind) const866 TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy,
867 ArrayRef<Type *> Tys,
868 TTI::TargetCostKind CostKind) const {
869 InstructionCost Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys, CostKind);
870 assert(Cost >= 0 && "TTI should not produce negative costs!");
871 return Cost;
872 }
873
getNumberOfParts(Type * Tp) const874 unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const {
875 return TTIImpl->getNumberOfParts(Tp);
876 }
877
878 InstructionCost
getAddressComputationCost(Type * Tp,ScalarEvolution * SE,const SCEV * Ptr) const879 TargetTransformInfo::getAddressComputationCost(Type *Tp, ScalarEvolution *SE,
880 const SCEV *Ptr) const {
881 InstructionCost Cost = TTIImpl->getAddressComputationCost(Tp, SE, Ptr);
882 assert(Cost >= 0 && "TTI should not produce negative costs!");
883 return Cost;
884 }
885
getMemcpyCost(const Instruction * I) const886 InstructionCost TargetTransformInfo::getMemcpyCost(const Instruction *I) const {
887 InstructionCost Cost = TTIImpl->getMemcpyCost(I);
888 assert(Cost >= 0 && "TTI should not produce negative costs!");
889 return Cost;
890 }
891
getArithmeticReductionCost(unsigned Opcode,VectorType * Ty,Optional<FastMathFlags> FMF,TTI::TargetCostKind CostKind) const892 InstructionCost TargetTransformInfo::getArithmeticReductionCost(
893 unsigned Opcode, VectorType *Ty, Optional<FastMathFlags> FMF,
894 TTI::TargetCostKind CostKind) const {
895 InstructionCost Cost =
896 TTIImpl->getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
897 assert(Cost >= 0 && "TTI should not produce negative costs!");
898 return Cost;
899 }
900
getMinMaxReductionCost(VectorType * Ty,VectorType * CondTy,bool IsUnsigned,TTI::TargetCostKind CostKind) const901 InstructionCost TargetTransformInfo::getMinMaxReductionCost(
902 VectorType *Ty, VectorType *CondTy, bool IsUnsigned,
903 TTI::TargetCostKind CostKind) const {
904 InstructionCost Cost =
905 TTIImpl->getMinMaxReductionCost(Ty, CondTy, IsUnsigned, CostKind);
906 assert(Cost >= 0 && "TTI should not produce negative costs!");
907 return Cost;
908 }
909
getExtendedAddReductionCost(bool IsMLA,bool IsUnsigned,Type * ResTy,VectorType * Ty,TTI::TargetCostKind CostKind) const910 InstructionCost TargetTransformInfo::getExtendedAddReductionCost(
911 bool IsMLA, bool IsUnsigned, Type *ResTy, VectorType *Ty,
912 TTI::TargetCostKind CostKind) const {
913 return TTIImpl->getExtendedAddReductionCost(IsMLA, IsUnsigned, ResTy, Ty,
914 CostKind);
915 }
916
917 InstructionCost
getCostOfKeepingLiveOverCall(ArrayRef<Type * > Tys) const918 TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
919 return TTIImpl->getCostOfKeepingLiveOverCall(Tys);
920 }
921
getTgtMemIntrinsic(IntrinsicInst * Inst,MemIntrinsicInfo & Info) const922 bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,
923 MemIntrinsicInfo &Info) const {
924 return TTIImpl->getTgtMemIntrinsic(Inst, Info);
925 }
926
getAtomicMemIntrinsicMaxElementSize() const927 unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
928 return TTIImpl->getAtomicMemIntrinsicMaxElementSize();
929 }
930
getOrCreateResultFromMemIntrinsic(IntrinsicInst * Inst,Type * ExpectedType) const931 Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
932 IntrinsicInst *Inst, Type *ExpectedType) const {
933 return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
934 }
935
getMemcpyLoopLoweringType(LLVMContext & Context,Value * Length,unsigned SrcAddrSpace,unsigned DestAddrSpace,unsigned SrcAlign,unsigned DestAlign) const936 Type *TargetTransformInfo::getMemcpyLoopLoweringType(
937 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
938 unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign) const {
939 return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace,
940 DestAddrSpace, SrcAlign, DestAlign);
941 }
942
getMemcpyLoopResidualLoweringType(SmallVectorImpl<Type * > & OpsOut,LLVMContext & Context,unsigned RemainingBytes,unsigned SrcAddrSpace,unsigned DestAddrSpace,unsigned SrcAlign,unsigned DestAlign) const943 void TargetTransformInfo::getMemcpyLoopResidualLoweringType(
944 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
945 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
946 unsigned SrcAlign, unsigned DestAlign) const {
947 TTIImpl->getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes,
948 SrcAddrSpace, DestAddrSpace,
949 SrcAlign, DestAlign);
950 }
951
areInlineCompatible(const Function * Caller,const Function * Callee) const952 bool TargetTransformInfo::areInlineCompatible(const Function *Caller,
953 const Function *Callee) const {
954 return TTIImpl->areInlineCompatible(Caller, Callee);
955 }
956
areFunctionArgsABICompatible(const Function * Caller,const Function * Callee,SmallPtrSetImpl<Argument * > & Args) const957 bool TargetTransformInfo::areFunctionArgsABICompatible(
958 const Function *Caller, const Function *Callee,
959 SmallPtrSetImpl<Argument *> &Args) const {
960 return TTIImpl->areFunctionArgsABICompatible(Caller, Callee, Args);
961 }
962
isIndexedLoadLegal(MemIndexedMode Mode,Type * Ty) const963 bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode,
964 Type *Ty) const {
965 return TTIImpl->isIndexedLoadLegal(Mode, Ty);
966 }
967
isIndexedStoreLegal(MemIndexedMode Mode,Type * Ty) const968 bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode,
969 Type *Ty) const {
970 return TTIImpl->isIndexedStoreLegal(Mode, Ty);
971 }
972
getLoadStoreVecRegBitWidth(unsigned AS) const973 unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const {
974 return TTIImpl->getLoadStoreVecRegBitWidth(AS);
975 }
976
isLegalToVectorizeLoad(LoadInst * LI) const977 bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const {
978 return TTIImpl->isLegalToVectorizeLoad(LI);
979 }
980
isLegalToVectorizeStore(StoreInst * SI) const981 bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const {
982 return TTIImpl->isLegalToVectorizeStore(SI);
983 }
984
isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,Align Alignment,unsigned AddrSpace) const985 bool TargetTransformInfo::isLegalToVectorizeLoadChain(
986 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
987 return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment,
988 AddrSpace);
989 }
990
isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,Align Alignment,unsigned AddrSpace) const991 bool TargetTransformInfo::isLegalToVectorizeStoreChain(
992 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
993 return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment,
994 AddrSpace);
995 }
996
isLegalToVectorizeReduction(const RecurrenceDescriptor & RdxDesc,ElementCount VF) const997 bool TargetTransformInfo::isLegalToVectorizeReduction(
998 const RecurrenceDescriptor &RdxDesc, ElementCount VF) const {
999 return TTIImpl->isLegalToVectorizeReduction(RdxDesc, VF);
1000 }
1001
isElementTypeLegalForScalableVector(Type * Ty) const1002 bool TargetTransformInfo::isElementTypeLegalForScalableVector(Type *Ty) const {
1003 return TTIImpl->isElementTypeLegalForScalableVector(Ty);
1004 }
1005
getLoadVectorFactor(unsigned VF,unsigned LoadSize,unsigned ChainSizeInBytes,VectorType * VecTy) const1006 unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF,
1007 unsigned LoadSize,
1008 unsigned ChainSizeInBytes,
1009 VectorType *VecTy) const {
1010 return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy);
1011 }
1012
getStoreVectorFactor(unsigned VF,unsigned StoreSize,unsigned ChainSizeInBytes,VectorType * VecTy) const1013 unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,
1014 unsigned StoreSize,
1015 unsigned ChainSizeInBytes,
1016 VectorType *VecTy) const {
1017 return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
1018 }
1019
preferInLoopReduction(unsigned Opcode,Type * Ty,ReductionFlags Flags) const1020 bool TargetTransformInfo::preferInLoopReduction(unsigned Opcode, Type *Ty,
1021 ReductionFlags Flags) const {
1022 return TTIImpl->preferInLoopReduction(Opcode, Ty, Flags);
1023 }
1024
preferPredicatedReductionSelect(unsigned Opcode,Type * Ty,ReductionFlags Flags) const1025 bool TargetTransformInfo::preferPredicatedReductionSelect(
1026 unsigned Opcode, Type *Ty, ReductionFlags Flags) const {
1027 return TTIImpl->preferPredicatedReductionSelect(Opcode, Ty, Flags);
1028 }
1029
1030 TargetTransformInfo::VPLegalization
getVPLegalizationStrategy(const VPIntrinsic & VPI) const1031 TargetTransformInfo::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
1032 return TTIImpl->getVPLegalizationStrategy(VPI);
1033 }
1034
shouldExpandReduction(const IntrinsicInst * II) const1035 bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {
1036 return TTIImpl->shouldExpandReduction(II);
1037 }
1038
getGISelRematGlobalCost() const1039 unsigned TargetTransformInfo::getGISelRematGlobalCost() const {
1040 return TTIImpl->getGISelRematGlobalCost();
1041 }
1042
supportsScalableVectors() const1043 bool TargetTransformInfo::supportsScalableVectors() const {
1044 return TTIImpl->supportsScalableVectors();
1045 }
1046
hasActiveVectorLength() const1047 bool TargetTransformInfo::hasActiveVectorLength() const {
1048 return TTIImpl->hasActiveVectorLength();
1049 }
1050
1051 InstructionCost
getInstructionLatency(const Instruction * I) const1052 TargetTransformInfo::getInstructionLatency(const Instruction *I) const {
1053 return TTIImpl->getInstructionLatency(I);
1054 }
1055
1056 InstructionCost
getInstructionThroughput(const Instruction * I) const1057 TargetTransformInfo::getInstructionThroughput(const Instruction *I) const {
1058 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
1059
1060 switch (I->getOpcode()) {
1061 case Instruction::GetElementPtr:
1062 case Instruction::Ret:
1063 case Instruction::PHI:
1064 case Instruction::Br:
1065 case Instruction::Add:
1066 case Instruction::FAdd:
1067 case Instruction::Sub:
1068 case Instruction::FSub:
1069 case Instruction::Mul:
1070 case Instruction::FMul:
1071 case Instruction::UDiv:
1072 case Instruction::SDiv:
1073 case Instruction::FDiv:
1074 case Instruction::URem:
1075 case Instruction::SRem:
1076 case Instruction::FRem:
1077 case Instruction::Shl:
1078 case Instruction::LShr:
1079 case Instruction::AShr:
1080 case Instruction::And:
1081 case Instruction::Or:
1082 case Instruction::Xor:
1083 case Instruction::FNeg:
1084 case Instruction::Select:
1085 case Instruction::ICmp:
1086 case Instruction::FCmp:
1087 case Instruction::Store:
1088 case Instruction::Load:
1089 case Instruction::ZExt:
1090 case Instruction::SExt:
1091 case Instruction::FPToUI:
1092 case Instruction::FPToSI:
1093 case Instruction::FPExt:
1094 case Instruction::PtrToInt:
1095 case Instruction::IntToPtr:
1096 case Instruction::SIToFP:
1097 case Instruction::UIToFP:
1098 case Instruction::Trunc:
1099 case Instruction::FPTrunc:
1100 case Instruction::BitCast:
1101 case Instruction::AddrSpaceCast:
1102 case Instruction::ExtractElement:
1103 case Instruction::InsertElement:
1104 case Instruction::ExtractValue:
1105 case Instruction::ShuffleVector:
1106 case Instruction::Call:
1107 case Instruction::Switch:
1108 return getUserCost(I, CostKind);
1109 default:
1110 // We don't have any information on this instruction.
1111 return -1;
1112 }
1113 }
1114
~Concept()1115 TargetTransformInfo::Concept::~Concept() {}
1116
TargetIRAnalysis()1117 TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
1118
TargetIRAnalysis(std::function<Result (const Function &)> TTICallback)1119 TargetIRAnalysis::TargetIRAnalysis(
1120 std::function<Result(const Function &)> TTICallback)
1121 : TTICallback(std::move(TTICallback)) {}
1122
run(const Function & F,FunctionAnalysisManager &)1123 TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F,
1124 FunctionAnalysisManager &) {
1125 return TTICallback(F);
1126 }
1127
1128 AnalysisKey TargetIRAnalysis::Key;
1129
getDefaultTTI(const Function & F)1130 TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) {
1131 return Result(F.getParent()->getDataLayout());
1132 }
1133
1134 // Register the basic pass.
1135 INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti",
1136 "Target Transform Information", false, true)
1137 char TargetTransformInfoWrapperPass::ID = 0;
1138
anchor()1139 void TargetTransformInfoWrapperPass::anchor() {}
1140
TargetTransformInfoWrapperPass()1141 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass()
1142 : ImmutablePass(ID) {
1143 initializeTargetTransformInfoWrapperPassPass(
1144 *PassRegistry::getPassRegistry());
1145 }
1146
TargetTransformInfoWrapperPass(TargetIRAnalysis TIRA)1147 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass(
1148 TargetIRAnalysis TIRA)
1149 : ImmutablePass(ID), TIRA(std::move(TIRA)) {
1150 initializeTargetTransformInfoWrapperPassPass(
1151 *PassRegistry::getPassRegistry());
1152 }
1153
getTTI(const Function & F)1154 TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) {
1155 FunctionAnalysisManager DummyFAM;
1156 TTI = TIRA.run(F, DummyFAM);
1157 return *TTI;
1158 }
1159
1160 ImmutablePass *
createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA)1161 llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) {
1162 return new TargetTransformInfoWrapperPass(std::move(TIRA));
1163 }
1164