1 //===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/Analysis/TargetTransformInfo.h" 10 #include "llvm/Analysis/CFG.h" 11 #include "llvm/Analysis/LoopIterator.h" 12 #include "llvm/Analysis/TargetTransformInfoImpl.h" 13 #include "llvm/IR/CFG.h" 14 #include "llvm/IR/Dominators.h" 15 #include "llvm/IR/Instruction.h" 16 #include "llvm/IR/Instructions.h" 17 #include "llvm/IR/IntrinsicInst.h" 18 #include "llvm/IR/Module.h" 19 #include "llvm/IR/Operator.h" 20 #include "llvm/IR/PatternMatch.h" 21 #include "llvm/InitializePasses.h" 22 #include "llvm/Support/CommandLine.h" 23 #include <optional> 24 #include <utility> 25 26 using namespace llvm; 27 using namespace PatternMatch; 28 29 #define DEBUG_TYPE "tti" 30 31 static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false), 32 cl::Hidden, 33 cl::desc("Recognize reduction patterns.")); 34 35 static cl::opt<unsigned> CacheLineSize( 36 "cache-line-size", cl::init(0), cl::Hidden, 37 cl::desc("Use this to override the target cache line size when " 38 "specified by the user.")); 39 40 static cl::opt<unsigned> PredictableBranchThreshold( 41 "predictable-branch-threshold", cl::init(99), cl::Hidden, 42 cl::desc( 43 "Use this to override the target's predictable branch threshold (%).")); 44 45 namespace { 46 /// No-op implementation of the TTI interface using the utility base 47 /// classes. 48 /// 49 /// This is used when no target specific information is available. 50 struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> { 51 explicit NoTTIImpl(const DataLayout &DL) 52 : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {} 53 }; 54 } // namespace 55 56 bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) { 57 // If the loop has irreducible control flow, it can not be converted to 58 // Hardware loop. 59 LoopBlocksRPO RPOT(L); 60 RPOT.perform(&LI); 61 if (containsIrreducibleCFG<const BasicBlock *>(RPOT, LI)) 62 return false; 63 return true; 64 } 65 66 IntrinsicCostAttributes::IntrinsicCostAttributes( 67 Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost, 68 bool TypeBasedOnly) 69 : II(dyn_cast<IntrinsicInst>(&CI)), RetTy(CI.getType()), IID(Id), 70 ScalarizationCost(ScalarizationCost) { 71 72 if (const auto *FPMO = dyn_cast<FPMathOperator>(&CI)) 73 FMF = FPMO->getFastMathFlags(); 74 75 if (!TypeBasedOnly) 76 Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end()); 77 FunctionType *FTy = CI.getCalledFunction()->getFunctionType(); 78 ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end()); 79 } 80 81 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy, 82 ArrayRef<Type *> Tys, 83 FastMathFlags Flags, 84 const IntrinsicInst *I, 85 InstructionCost ScalarCost) 86 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) { 87 ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end()); 88 } 89 90 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty, 91 ArrayRef<const Value *> Args) 92 : RetTy(Ty), IID(Id) { 93 94 Arguments.insert(Arguments.begin(), Args.begin(), Args.end()); 95 ParamTys.reserve(Arguments.size()); 96 for (unsigned Idx = 0, Size = Arguments.size(); Idx != Size; ++Idx) 97 ParamTys.push_back(Arguments[Idx]->getType()); 98 } 99 100 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy, 101 ArrayRef<const Value *> Args, 102 ArrayRef<Type *> Tys, 103 FastMathFlags Flags, 104 const IntrinsicInst *I, 105 InstructionCost ScalarCost) 106 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) { 107 ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end()); 108 Arguments.insert(Arguments.begin(), Args.begin(), Args.end()); 109 } 110 111 HardwareLoopInfo::HardwareLoopInfo(Loop *L) : L(L) { 112 // Match default options: 113 // - hardware-loop-counter-bitwidth = 32 114 // - hardware-loop-decrement = 1 115 CountType = Type::getInt32Ty(L->getHeader()->getContext()); 116 LoopDecrement = ConstantInt::get(CountType, 1); 117 } 118 119 bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE, 120 LoopInfo &LI, DominatorTree &DT, 121 bool ForceNestedLoop, 122 bool ForceHardwareLoopPHI) { 123 SmallVector<BasicBlock *, 4> ExitingBlocks; 124 L->getExitingBlocks(ExitingBlocks); 125 126 for (BasicBlock *BB : ExitingBlocks) { 127 // If we pass the updated counter back through a phi, we need to know 128 // which latch the updated value will be coming from. 129 if (!L->isLoopLatch(BB)) { 130 if (ForceHardwareLoopPHI || CounterInReg) 131 continue; 132 } 133 134 const SCEV *EC = SE.getExitCount(L, BB); 135 if (isa<SCEVCouldNotCompute>(EC)) 136 continue; 137 if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(EC)) { 138 if (ConstEC->getValue()->isZero()) 139 continue; 140 } else if (!SE.isLoopInvariant(EC, L)) 141 continue; 142 143 if (SE.getTypeSizeInBits(EC->getType()) > CountType->getBitWidth()) 144 continue; 145 146 // If this exiting block is contained in a nested loop, it is not eligible 147 // for insertion of the branch-and-decrement since the inner loop would 148 // end up messing up the value in the CTR. 149 if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop) 150 continue; 151 152 // We now have a loop-invariant count of loop iterations (which is not the 153 // constant zero) for which we know that this loop will not exit via this 154 // existing block. 155 156 // We need to make sure that this block will run on every loop iteration. 157 // For this to be true, we must dominate all blocks with backedges. Such 158 // blocks are in-loop predecessors to the header block. 159 bool NotAlways = false; 160 for (BasicBlock *Pred : predecessors(L->getHeader())) { 161 if (!L->contains(Pred)) 162 continue; 163 164 if (!DT.dominates(BB, Pred)) { 165 NotAlways = true; 166 break; 167 } 168 } 169 170 if (NotAlways) 171 continue; 172 173 // Make sure this blocks ends with a conditional branch. 174 Instruction *TI = BB->getTerminator(); 175 if (!TI) 176 continue; 177 178 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { 179 if (!BI->isConditional()) 180 continue; 181 182 ExitBranch = BI; 183 } else 184 continue; 185 186 // Note that this block may not be the loop latch block, even if the loop 187 // has a latch block. 188 ExitBlock = BB; 189 ExitCount = EC; 190 break; 191 } 192 193 if (!ExitBlock) 194 return false; 195 return true; 196 } 197 198 TargetTransformInfo::TargetTransformInfo(const DataLayout &DL) 199 : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {} 200 201 TargetTransformInfo::~TargetTransformInfo() = default; 202 203 TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg) 204 : TTIImpl(std::move(Arg.TTIImpl)) {} 205 206 TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) { 207 TTIImpl = std::move(RHS.TTIImpl); 208 return *this; 209 } 210 211 unsigned TargetTransformInfo::getInliningThresholdMultiplier() const { 212 return TTIImpl->getInliningThresholdMultiplier(); 213 } 214 215 unsigned 216 TargetTransformInfo::getInliningCostBenefitAnalysisSavingsMultiplier() const { 217 return TTIImpl->getInliningCostBenefitAnalysisSavingsMultiplier(); 218 } 219 220 unsigned 221 TargetTransformInfo::getInliningCostBenefitAnalysisProfitableMultiplier() 222 const { 223 return TTIImpl->getInliningCostBenefitAnalysisProfitableMultiplier(); 224 } 225 226 unsigned 227 TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const { 228 return TTIImpl->adjustInliningThreshold(CB); 229 } 230 231 unsigned TargetTransformInfo::getCallerAllocaCost(const CallBase *CB, 232 const AllocaInst *AI) const { 233 return TTIImpl->getCallerAllocaCost(CB, AI); 234 } 235 236 int TargetTransformInfo::getInlinerVectorBonusPercent() const { 237 return TTIImpl->getInlinerVectorBonusPercent(); 238 } 239 240 InstructionCost TargetTransformInfo::getGEPCost( 241 Type *PointeeType, const Value *Ptr, ArrayRef<const Value *> Operands, 242 Type *AccessType, TTI::TargetCostKind CostKind) const { 243 return TTIImpl->getGEPCost(PointeeType, Ptr, Operands, AccessType, CostKind); 244 } 245 246 InstructionCost TargetTransformInfo::getPointersChainCost( 247 ArrayRef<const Value *> Ptrs, const Value *Base, 248 const TTI::PointersChainInfo &Info, Type *AccessTy, 249 TTI::TargetCostKind CostKind) const { 250 assert((Base || !Info.isSameBase()) && 251 "If pointers have same base address it has to be provided."); 252 return TTIImpl->getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind); 253 } 254 255 unsigned TargetTransformInfo::getEstimatedNumberOfCaseClusters( 256 const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI, 257 BlockFrequencyInfo *BFI) const { 258 return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI); 259 } 260 261 InstructionCost 262 TargetTransformInfo::getInstructionCost(const User *U, 263 ArrayRef<const Value *> Operands, 264 enum TargetCostKind CostKind) const { 265 InstructionCost Cost = TTIImpl->getInstructionCost(U, Operands, CostKind); 266 assert((CostKind == TTI::TCK_RecipThroughput || Cost >= 0) && 267 "TTI should not produce negative costs!"); 268 return Cost; 269 } 270 271 BranchProbability TargetTransformInfo::getPredictableBranchThreshold() const { 272 return PredictableBranchThreshold.getNumOccurrences() > 0 273 ? BranchProbability(PredictableBranchThreshold, 100) 274 : TTIImpl->getPredictableBranchThreshold(); 275 } 276 277 bool TargetTransformInfo::hasBranchDivergence(const Function *F) const { 278 return TTIImpl->hasBranchDivergence(F); 279 } 280 281 bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const { 282 return TTIImpl->isSourceOfDivergence(V); 283 } 284 285 bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const { 286 return TTIImpl->isAlwaysUniform(V); 287 } 288 289 bool llvm::TargetTransformInfo::isValidAddrSpaceCast(unsigned FromAS, 290 unsigned ToAS) const { 291 return TTIImpl->isValidAddrSpaceCast(FromAS, ToAS); 292 } 293 294 bool llvm::TargetTransformInfo::addrspacesMayAlias(unsigned FromAS, 295 unsigned ToAS) const { 296 return TTIImpl->addrspacesMayAlias(FromAS, ToAS); 297 } 298 299 unsigned TargetTransformInfo::getFlatAddressSpace() const { 300 return TTIImpl->getFlatAddressSpace(); 301 } 302 303 bool TargetTransformInfo::collectFlatAddressOperands( 304 SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const { 305 return TTIImpl->collectFlatAddressOperands(OpIndexes, IID); 306 } 307 308 bool TargetTransformInfo::isNoopAddrSpaceCast(unsigned FromAS, 309 unsigned ToAS) const { 310 return TTIImpl->isNoopAddrSpaceCast(FromAS, ToAS); 311 } 312 313 bool TargetTransformInfo::canHaveNonUndefGlobalInitializerInAddressSpace( 314 unsigned AS) const { 315 return TTIImpl->canHaveNonUndefGlobalInitializerInAddressSpace(AS); 316 } 317 318 unsigned TargetTransformInfo::getAssumedAddrSpace(const Value *V) const { 319 return TTIImpl->getAssumedAddrSpace(V); 320 } 321 322 bool TargetTransformInfo::isSingleThreaded() const { 323 return TTIImpl->isSingleThreaded(); 324 } 325 326 std::pair<const Value *, unsigned> 327 TargetTransformInfo::getPredicatedAddrSpace(const Value *V) const { 328 return TTIImpl->getPredicatedAddrSpace(V); 329 } 330 331 Value *TargetTransformInfo::rewriteIntrinsicWithAddressSpace( 332 IntrinsicInst *II, Value *OldV, Value *NewV) const { 333 return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV); 334 } 335 336 bool TargetTransformInfo::isLoweredToCall(const Function *F) const { 337 return TTIImpl->isLoweredToCall(F); 338 } 339 340 bool TargetTransformInfo::isHardwareLoopProfitable( 341 Loop *L, ScalarEvolution &SE, AssumptionCache &AC, 342 TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const { 343 return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo); 344 } 345 346 bool TargetTransformInfo::preferPredicateOverEpilogue( 347 TailFoldingInfo *TFI) const { 348 return TTIImpl->preferPredicateOverEpilogue(TFI); 349 } 350 351 TailFoldingStyle TargetTransformInfo::getPreferredTailFoldingStyle( 352 bool IVUpdateMayOverflow) const { 353 return TTIImpl->getPreferredTailFoldingStyle(IVUpdateMayOverflow); 354 } 355 356 std::optional<Instruction *> 357 TargetTransformInfo::instCombineIntrinsic(InstCombiner &IC, 358 IntrinsicInst &II) const { 359 return TTIImpl->instCombineIntrinsic(IC, II); 360 } 361 362 std::optional<Value *> TargetTransformInfo::simplifyDemandedUseBitsIntrinsic( 363 InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known, 364 bool &KnownBitsComputed) const { 365 return TTIImpl->simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known, 366 KnownBitsComputed); 367 } 368 369 std::optional<Value *> TargetTransformInfo::simplifyDemandedVectorEltsIntrinsic( 370 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, 371 APInt &UndefElts2, APInt &UndefElts3, 372 std::function<void(Instruction *, unsigned, APInt, APInt &)> 373 SimplifyAndSetOp) const { 374 return TTIImpl->simplifyDemandedVectorEltsIntrinsic( 375 IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3, 376 SimplifyAndSetOp); 377 } 378 379 void TargetTransformInfo::getUnrollingPreferences( 380 Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP, 381 OptimizationRemarkEmitter *ORE) const { 382 return TTIImpl->getUnrollingPreferences(L, SE, UP, ORE); 383 } 384 385 void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE, 386 PeelingPreferences &PP) const { 387 return TTIImpl->getPeelingPreferences(L, SE, PP); 388 } 389 390 bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const { 391 return TTIImpl->isLegalAddImmediate(Imm); 392 } 393 394 bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const { 395 return TTIImpl->isLegalICmpImmediate(Imm); 396 } 397 398 bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, 399 int64_t BaseOffset, 400 bool HasBaseReg, int64_t Scale, 401 unsigned AddrSpace, 402 Instruction *I) const { 403 return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, 404 Scale, AddrSpace, I); 405 } 406 407 bool TargetTransformInfo::isLSRCostLess(const LSRCost &C1, 408 const LSRCost &C2) const { 409 return TTIImpl->isLSRCostLess(C1, C2); 410 } 411 412 bool TargetTransformInfo::isNumRegsMajorCostOfLSR() const { 413 return TTIImpl->isNumRegsMajorCostOfLSR(); 414 } 415 416 bool TargetTransformInfo::shouldFoldTerminatingConditionAfterLSR() const { 417 return TTIImpl->shouldFoldTerminatingConditionAfterLSR(); 418 } 419 420 bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const { 421 return TTIImpl->isProfitableLSRChainElement(I); 422 } 423 424 bool TargetTransformInfo::canMacroFuseCmp() const { 425 return TTIImpl->canMacroFuseCmp(); 426 } 427 428 bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI, 429 ScalarEvolution *SE, LoopInfo *LI, 430 DominatorTree *DT, AssumptionCache *AC, 431 TargetLibraryInfo *LibInfo) const { 432 return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo); 433 } 434 435 TTI::AddressingModeKind 436 TargetTransformInfo::getPreferredAddressingMode(const Loop *L, 437 ScalarEvolution *SE) const { 438 return TTIImpl->getPreferredAddressingMode(L, SE); 439 } 440 441 bool TargetTransformInfo::isLegalMaskedStore(Type *DataType, 442 Align Alignment) const { 443 return TTIImpl->isLegalMaskedStore(DataType, Alignment); 444 } 445 446 bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType, 447 Align Alignment) const { 448 return TTIImpl->isLegalMaskedLoad(DataType, Alignment); 449 } 450 451 bool TargetTransformInfo::isLegalNTStore(Type *DataType, 452 Align Alignment) const { 453 return TTIImpl->isLegalNTStore(DataType, Alignment); 454 } 455 456 bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const { 457 return TTIImpl->isLegalNTLoad(DataType, Alignment); 458 } 459 460 bool TargetTransformInfo::isLegalBroadcastLoad(Type *ElementTy, 461 ElementCount NumElements) const { 462 return TTIImpl->isLegalBroadcastLoad(ElementTy, NumElements); 463 } 464 465 bool TargetTransformInfo::isLegalMaskedGather(Type *DataType, 466 Align Alignment) const { 467 return TTIImpl->isLegalMaskedGather(DataType, Alignment); 468 } 469 470 bool TargetTransformInfo::isLegalAltInstr( 471 VectorType *VecTy, unsigned Opcode0, unsigned Opcode1, 472 const SmallBitVector &OpcodeMask) const { 473 return TTIImpl->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask); 474 } 475 476 bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType, 477 Align Alignment) const { 478 return TTIImpl->isLegalMaskedScatter(DataType, Alignment); 479 } 480 481 bool TargetTransformInfo::forceScalarizeMaskedGather(VectorType *DataType, 482 Align Alignment) const { 483 return TTIImpl->forceScalarizeMaskedGather(DataType, Alignment); 484 } 485 486 bool TargetTransformInfo::forceScalarizeMaskedScatter(VectorType *DataType, 487 Align Alignment) const { 488 return TTIImpl->forceScalarizeMaskedScatter(DataType, Alignment); 489 } 490 491 bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType) const { 492 return TTIImpl->isLegalMaskedCompressStore(DataType); 493 } 494 495 bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType) const { 496 return TTIImpl->isLegalMaskedExpandLoad(DataType); 497 } 498 499 bool TargetTransformInfo::enableOrderedReductions() const { 500 return TTIImpl->enableOrderedReductions(); 501 } 502 503 bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const { 504 return TTIImpl->hasDivRemOp(DataType, IsSigned); 505 } 506 507 bool TargetTransformInfo::hasVolatileVariant(Instruction *I, 508 unsigned AddrSpace) const { 509 return TTIImpl->hasVolatileVariant(I, AddrSpace); 510 } 511 512 bool TargetTransformInfo::prefersVectorizedAddressing() const { 513 return TTIImpl->prefersVectorizedAddressing(); 514 } 515 516 InstructionCost TargetTransformInfo::getScalingFactorCost( 517 Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, 518 int64_t Scale, unsigned AddrSpace) const { 519 InstructionCost Cost = TTIImpl->getScalingFactorCost( 520 Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace); 521 assert(Cost >= 0 && "TTI should not produce negative costs!"); 522 return Cost; 523 } 524 525 bool TargetTransformInfo::LSRWithInstrQueries() const { 526 return TTIImpl->LSRWithInstrQueries(); 527 } 528 529 bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const { 530 return TTIImpl->isTruncateFree(Ty1, Ty2); 531 } 532 533 bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const { 534 return TTIImpl->isProfitableToHoist(I); 535 } 536 537 bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); } 538 539 bool TargetTransformInfo::isTypeLegal(Type *Ty) const { 540 return TTIImpl->isTypeLegal(Ty); 541 } 542 543 unsigned TargetTransformInfo::getRegUsageForType(Type *Ty) const { 544 return TTIImpl->getRegUsageForType(Ty); 545 } 546 547 bool TargetTransformInfo::shouldBuildLookupTables() const { 548 return TTIImpl->shouldBuildLookupTables(); 549 } 550 551 bool TargetTransformInfo::shouldBuildLookupTablesForConstant( 552 Constant *C) const { 553 return TTIImpl->shouldBuildLookupTablesForConstant(C); 554 } 555 556 bool TargetTransformInfo::shouldBuildRelLookupTables() const { 557 return TTIImpl->shouldBuildRelLookupTables(); 558 } 559 560 bool TargetTransformInfo::useColdCCForColdCall(Function &F) const { 561 return TTIImpl->useColdCCForColdCall(F); 562 } 563 564 InstructionCost TargetTransformInfo::getScalarizationOverhead( 565 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, 566 TTI::TargetCostKind CostKind) const { 567 return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract, 568 CostKind); 569 } 570 571 InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead( 572 ArrayRef<const Value *> Args, ArrayRef<Type *> Tys, 573 TTI::TargetCostKind CostKind) const { 574 return TTIImpl->getOperandsScalarizationOverhead(Args, Tys, CostKind); 575 } 576 577 bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const { 578 return TTIImpl->supportsEfficientVectorElementLoadStore(); 579 } 580 581 bool TargetTransformInfo::supportsTailCalls() const { 582 return TTIImpl->supportsTailCalls(); 583 } 584 585 bool TargetTransformInfo::supportsTailCallFor(const CallBase *CB) const { 586 return TTIImpl->supportsTailCallFor(CB); 587 } 588 589 bool TargetTransformInfo::enableAggressiveInterleaving( 590 bool LoopHasReductions) const { 591 return TTIImpl->enableAggressiveInterleaving(LoopHasReductions); 592 } 593 594 TargetTransformInfo::MemCmpExpansionOptions 595 TargetTransformInfo::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const { 596 return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp); 597 } 598 599 bool TargetTransformInfo::enableSelectOptimize() const { 600 return TTIImpl->enableSelectOptimize(); 601 } 602 603 bool TargetTransformInfo::enableInterleavedAccessVectorization() const { 604 return TTIImpl->enableInterleavedAccessVectorization(); 605 } 606 607 bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const { 608 return TTIImpl->enableMaskedInterleavedAccessVectorization(); 609 } 610 611 bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const { 612 return TTIImpl->isFPVectorizationPotentiallyUnsafe(); 613 } 614 615 bool 616 TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context, 617 unsigned BitWidth, 618 unsigned AddressSpace, 619 Align Alignment, 620 unsigned *Fast) const { 621 return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth, 622 AddressSpace, Alignment, Fast); 623 } 624 625 TargetTransformInfo::PopcntSupportKind 626 TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const { 627 return TTIImpl->getPopcntSupport(IntTyWidthInBit); 628 } 629 630 bool TargetTransformInfo::haveFastSqrt(Type *Ty) const { 631 return TTIImpl->haveFastSqrt(Ty); 632 } 633 634 bool TargetTransformInfo::isExpensiveToSpeculativelyExecute( 635 const Instruction *I) const { 636 return TTIImpl->isExpensiveToSpeculativelyExecute(I); 637 } 638 639 bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const { 640 return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty); 641 } 642 643 InstructionCost TargetTransformInfo::getFPOpCost(Type *Ty) const { 644 InstructionCost Cost = TTIImpl->getFPOpCost(Ty); 645 assert(Cost >= 0 && "TTI should not produce negative costs!"); 646 return Cost; 647 } 648 649 InstructionCost TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode, 650 unsigned Idx, 651 const APInt &Imm, 652 Type *Ty) const { 653 InstructionCost Cost = TTIImpl->getIntImmCodeSizeCost(Opcode, Idx, Imm, Ty); 654 assert(Cost >= 0 && "TTI should not produce negative costs!"); 655 return Cost; 656 } 657 658 InstructionCost 659 TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty, 660 TTI::TargetCostKind CostKind) const { 661 InstructionCost Cost = TTIImpl->getIntImmCost(Imm, Ty, CostKind); 662 assert(Cost >= 0 && "TTI should not produce negative costs!"); 663 return Cost; 664 } 665 666 InstructionCost TargetTransformInfo::getIntImmCostInst( 667 unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty, 668 TTI::TargetCostKind CostKind, Instruction *Inst) const { 669 InstructionCost Cost = 670 TTIImpl->getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind, Inst); 671 assert(Cost >= 0 && "TTI should not produce negative costs!"); 672 return Cost; 673 } 674 675 InstructionCost 676 TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, 677 const APInt &Imm, Type *Ty, 678 TTI::TargetCostKind CostKind) const { 679 InstructionCost Cost = 680 TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind); 681 assert(Cost >= 0 && "TTI should not produce negative costs!"); 682 return Cost; 683 } 684 685 bool TargetTransformInfo::preferToKeepConstantsAttached( 686 const Instruction &Inst, const Function &Fn) const { 687 return TTIImpl->preferToKeepConstantsAttached(Inst, Fn); 688 } 689 690 unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const { 691 return TTIImpl->getNumberOfRegisters(ClassID); 692 } 693 694 unsigned TargetTransformInfo::getRegisterClassForType(bool Vector, 695 Type *Ty) const { 696 return TTIImpl->getRegisterClassForType(Vector, Ty); 697 } 698 699 const char *TargetTransformInfo::getRegisterClassName(unsigned ClassID) const { 700 return TTIImpl->getRegisterClassName(ClassID); 701 } 702 703 TypeSize TargetTransformInfo::getRegisterBitWidth( 704 TargetTransformInfo::RegisterKind K) const { 705 return TTIImpl->getRegisterBitWidth(K); 706 } 707 708 unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const { 709 return TTIImpl->getMinVectorRegisterBitWidth(); 710 } 711 712 std::optional<unsigned> TargetTransformInfo::getMaxVScale() const { 713 return TTIImpl->getMaxVScale(); 714 } 715 716 std::optional<unsigned> TargetTransformInfo::getVScaleForTuning() const { 717 return TTIImpl->getVScaleForTuning(); 718 } 719 720 bool TargetTransformInfo::isVScaleKnownToBeAPowerOfTwo() const { 721 return TTIImpl->isVScaleKnownToBeAPowerOfTwo(); 722 } 723 724 bool TargetTransformInfo::shouldMaximizeVectorBandwidth( 725 TargetTransformInfo::RegisterKind K) const { 726 return TTIImpl->shouldMaximizeVectorBandwidth(K); 727 } 728 729 ElementCount TargetTransformInfo::getMinimumVF(unsigned ElemWidth, 730 bool IsScalable) const { 731 return TTIImpl->getMinimumVF(ElemWidth, IsScalable); 732 } 733 734 unsigned TargetTransformInfo::getMaximumVF(unsigned ElemWidth, 735 unsigned Opcode) const { 736 return TTIImpl->getMaximumVF(ElemWidth, Opcode); 737 } 738 739 unsigned TargetTransformInfo::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy, 740 Type *ScalarValTy) const { 741 return TTIImpl->getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy); 742 } 743 744 bool TargetTransformInfo::shouldConsiderAddressTypePromotion( 745 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const { 746 return TTIImpl->shouldConsiderAddressTypePromotion( 747 I, AllowPromotionWithoutCommonHeader); 748 } 749 750 unsigned TargetTransformInfo::getCacheLineSize() const { 751 return CacheLineSize.getNumOccurrences() > 0 ? CacheLineSize 752 : TTIImpl->getCacheLineSize(); 753 } 754 755 std::optional<unsigned> 756 TargetTransformInfo::getCacheSize(CacheLevel Level) const { 757 return TTIImpl->getCacheSize(Level); 758 } 759 760 std::optional<unsigned> 761 TargetTransformInfo::getCacheAssociativity(CacheLevel Level) const { 762 return TTIImpl->getCacheAssociativity(Level); 763 } 764 765 unsigned TargetTransformInfo::getPrefetchDistance() const { 766 return TTIImpl->getPrefetchDistance(); 767 } 768 769 unsigned TargetTransformInfo::getMinPrefetchStride( 770 unsigned NumMemAccesses, unsigned NumStridedMemAccesses, 771 unsigned NumPrefetches, bool HasCall) const { 772 return TTIImpl->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses, 773 NumPrefetches, HasCall); 774 } 775 776 unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const { 777 return TTIImpl->getMaxPrefetchIterationsAhead(); 778 } 779 780 bool TargetTransformInfo::enableWritePrefetching() const { 781 return TTIImpl->enableWritePrefetching(); 782 } 783 784 bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const { 785 return TTIImpl->shouldPrefetchAddressSpace(AS); 786 } 787 788 unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const { 789 return TTIImpl->getMaxInterleaveFactor(VF); 790 } 791 792 TargetTransformInfo::OperandValueInfo 793 TargetTransformInfo::getOperandInfo(const Value *V) { 794 OperandValueKind OpInfo = OK_AnyValue; 795 OperandValueProperties OpProps = OP_None; 796 797 if (isa<ConstantInt>(V) || isa<ConstantFP>(V)) { 798 if (const auto *CI = dyn_cast<ConstantInt>(V)) { 799 if (CI->getValue().isPowerOf2()) 800 OpProps = OP_PowerOf2; 801 else if (CI->getValue().isNegatedPowerOf2()) 802 OpProps = OP_NegatedPowerOf2; 803 } 804 return {OK_UniformConstantValue, OpProps}; 805 } 806 807 // A broadcast shuffle creates a uniform value. 808 // TODO: Add support for non-zero index broadcasts. 809 // TODO: Add support for different source vector width. 810 if (const auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V)) 811 if (ShuffleInst->isZeroEltSplat()) 812 OpInfo = OK_UniformValue; 813 814 const Value *Splat = getSplatValue(V); 815 816 // Check for a splat of a constant or for a non uniform vector of constants 817 // and check if the constant(s) are all powers of two. 818 if (isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) { 819 OpInfo = OK_NonUniformConstantValue; 820 if (Splat) { 821 OpInfo = OK_UniformConstantValue; 822 if (auto *CI = dyn_cast<ConstantInt>(Splat)) { 823 if (CI->getValue().isPowerOf2()) 824 OpProps = OP_PowerOf2; 825 else if (CI->getValue().isNegatedPowerOf2()) 826 OpProps = OP_NegatedPowerOf2; 827 } 828 } else if (const auto *CDS = dyn_cast<ConstantDataSequential>(V)) { 829 bool AllPow2 = true, AllNegPow2 = true; 830 for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) { 831 if (auto *CI = dyn_cast<ConstantInt>(CDS->getElementAsConstant(I))) { 832 AllPow2 &= CI->getValue().isPowerOf2(); 833 AllNegPow2 &= CI->getValue().isNegatedPowerOf2(); 834 if (AllPow2 || AllNegPow2) 835 continue; 836 } 837 AllPow2 = AllNegPow2 = false; 838 break; 839 } 840 OpProps = AllPow2 ? OP_PowerOf2 : OpProps; 841 OpProps = AllNegPow2 ? OP_NegatedPowerOf2 : OpProps; 842 } 843 } 844 845 // Check for a splat of a uniform value. This is not loop aware, so return 846 // true only for the obviously uniform cases (argument, globalvalue) 847 if (Splat && (isa<Argument>(Splat) || isa<GlobalValue>(Splat))) 848 OpInfo = OK_UniformValue; 849 850 return {OpInfo, OpProps}; 851 } 852 853 InstructionCost TargetTransformInfo::getArithmeticInstrCost( 854 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, 855 OperandValueInfo Op1Info, OperandValueInfo Op2Info, 856 ArrayRef<const Value *> Args, const Instruction *CxtI) const { 857 InstructionCost Cost = 858 TTIImpl->getArithmeticInstrCost(Opcode, Ty, CostKind, 859 Op1Info, Op2Info, 860 Args, CxtI); 861 assert(Cost >= 0 && "TTI should not produce negative costs!"); 862 return Cost; 863 } 864 865 InstructionCost TargetTransformInfo::getShuffleCost( 866 ShuffleKind Kind, VectorType *Ty, ArrayRef<int> Mask, 867 TTI::TargetCostKind CostKind, int Index, VectorType *SubTp, 868 ArrayRef<const Value *> Args) const { 869 InstructionCost Cost = 870 TTIImpl->getShuffleCost(Kind, Ty, Mask, CostKind, Index, SubTp, Args); 871 assert(Cost >= 0 && "TTI should not produce negative costs!"); 872 return Cost; 873 } 874 875 TTI::CastContextHint 876 TargetTransformInfo::getCastContextHint(const Instruction *I) { 877 if (!I) 878 return CastContextHint::None; 879 880 auto getLoadStoreKind = [](const Value *V, unsigned LdStOp, unsigned MaskedOp, 881 unsigned GatScatOp) { 882 const Instruction *I = dyn_cast<Instruction>(V); 883 if (!I) 884 return CastContextHint::None; 885 886 if (I->getOpcode() == LdStOp) 887 return CastContextHint::Normal; 888 889 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) { 890 if (II->getIntrinsicID() == MaskedOp) 891 return TTI::CastContextHint::Masked; 892 if (II->getIntrinsicID() == GatScatOp) 893 return TTI::CastContextHint::GatherScatter; 894 } 895 896 return TTI::CastContextHint::None; 897 }; 898 899 switch (I->getOpcode()) { 900 case Instruction::ZExt: 901 case Instruction::SExt: 902 case Instruction::FPExt: 903 return getLoadStoreKind(I->getOperand(0), Instruction::Load, 904 Intrinsic::masked_load, Intrinsic::masked_gather); 905 case Instruction::Trunc: 906 case Instruction::FPTrunc: 907 if (I->hasOneUse()) 908 return getLoadStoreKind(*I->user_begin(), Instruction::Store, 909 Intrinsic::masked_store, 910 Intrinsic::masked_scatter); 911 break; 912 default: 913 return CastContextHint::None; 914 } 915 916 return TTI::CastContextHint::None; 917 } 918 919 InstructionCost TargetTransformInfo::getCastInstrCost( 920 unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH, 921 TTI::TargetCostKind CostKind, const Instruction *I) const { 922 assert((I == nullptr || I->getOpcode() == Opcode) && 923 "Opcode should reflect passed instruction."); 924 InstructionCost Cost = 925 TTIImpl->getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); 926 assert(Cost >= 0 && "TTI should not produce negative costs!"); 927 return Cost; 928 } 929 930 InstructionCost TargetTransformInfo::getExtractWithExtendCost( 931 unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index) const { 932 InstructionCost Cost = 933 TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index); 934 assert(Cost >= 0 && "TTI should not produce negative costs!"); 935 return Cost; 936 } 937 938 InstructionCost TargetTransformInfo::getCFInstrCost( 939 unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) const { 940 assert((I == nullptr || I->getOpcode() == Opcode) && 941 "Opcode should reflect passed instruction."); 942 InstructionCost Cost = TTIImpl->getCFInstrCost(Opcode, CostKind, I); 943 assert(Cost >= 0 && "TTI should not produce negative costs!"); 944 return Cost; 945 } 946 947 InstructionCost TargetTransformInfo::getCmpSelInstrCost( 948 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, 949 TTI::TargetCostKind CostKind, const Instruction *I) const { 950 assert((I == nullptr || I->getOpcode() == Opcode) && 951 "Opcode should reflect passed instruction."); 952 InstructionCost Cost = 953 TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); 954 assert(Cost >= 0 && "TTI should not produce negative costs!"); 955 return Cost; 956 } 957 958 InstructionCost TargetTransformInfo::getVectorInstrCost( 959 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, 960 Value *Op0, Value *Op1) const { 961 // FIXME: Assert that Opcode is either InsertElement or ExtractElement. 962 // This is mentioned in the interface description and respected by all 963 // callers, but never asserted upon. 964 InstructionCost Cost = 965 TTIImpl->getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1); 966 assert(Cost >= 0 && "TTI should not produce negative costs!"); 967 return Cost; 968 } 969 970 InstructionCost 971 TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val, 972 TTI::TargetCostKind CostKind, 973 unsigned Index) const { 974 // FIXME: Assert that Opcode is either InsertElement or ExtractElement. 975 // This is mentioned in the interface description and respected by all 976 // callers, but never asserted upon. 977 InstructionCost Cost = TTIImpl->getVectorInstrCost(I, Val, CostKind, Index); 978 assert(Cost >= 0 && "TTI should not produce negative costs!"); 979 return Cost; 980 } 981 982 InstructionCost TargetTransformInfo::getReplicationShuffleCost( 983 Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, 984 TTI::TargetCostKind CostKind) { 985 InstructionCost Cost = TTIImpl->getReplicationShuffleCost( 986 EltTy, ReplicationFactor, VF, DemandedDstElts, CostKind); 987 assert(Cost >= 0 && "TTI should not produce negative costs!"); 988 return Cost; 989 } 990 991 InstructionCost TargetTransformInfo::getMemoryOpCost( 992 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, 993 TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo, 994 const Instruction *I) const { 995 assert((I == nullptr || I->getOpcode() == Opcode) && 996 "Opcode should reflect passed instruction."); 997 InstructionCost Cost = TTIImpl->getMemoryOpCost( 998 Opcode, Src, Alignment, AddressSpace, CostKind, OpInfo, I); 999 assert(Cost >= 0 && "TTI should not produce negative costs!"); 1000 return Cost; 1001 } 1002 1003 InstructionCost TargetTransformInfo::getMaskedMemoryOpCost( 1004 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, 1005 TTI::TargetCostKind CostKind) const { 1006 InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, 1007 AddressSpace, CostKind); 1008 assert(Cost >= 0 && "TTI should not produce negative costs!"); 1009 return Cost; 1010 } 1011 1012 InstructionCost TargetTransformInfo::getGatherScatterOpCost( 1013 unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, 1014 Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { 1015 InstructionCost Cost = TTIImpl->getGatherScatterOpCost( 1016 Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I); 1017 assert(Cost >= 0 && "TTI should not produce negative costs!"); 1018 return Cost; 1019 } 1020 1021 InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost( 1022 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, 1023 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, 1024 bool UseMaskForCond, bool UseMaskForGaps) const { 1025 InstructionCost Cost = TTIImpl->getInterleavedMemoryOpCost( 1026 Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind, 1027 UseMaskForCond, UseMaskForGaps); 1028 assert(Cost >= 0 && "TTI should not produce negative costs!"); 1029 return Cost; 1030 } 1031 1032 InstructionCost 1033 TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, 1034 TTI::TargetCostKind CostKind) const { 1035 InstructionCost Cost = TTIImpl->getIntrinsicInstrCost(ICA, CostKind); 1036 assert(Cost >= 0 && "TTI should not produce negative costs!"); 1037 return Cost; 1038 } 1039 1040 InstructionCost 1041 TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy, 1042 ArrayRef<Type *> Tys, 1043 TTI::TargetCostKind CostKind) const { 1044 InstructionCost Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys, CostKind); 1045 assert(Cost >= 0 && "TTI should not produce negative costs!"); 1046 return Cost; 1047 } 1048 1049 unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const { 1050 return TTIImpl->getNumberOfParts(Tp); 1051 } 1052 1053 InstructionCost 1054 TargetTransformInfo::getAddressComputationCost(Type *Tp, ScalarEvolution *SE, 1055 const SCEV *Ptr) const { 1056 InstructionCost Cost = TTIImpl->getAddressComputationCost(Tp, SE, Ptr); 1057 assert(Cost >= 0 && "TTI should not produce negative costs!"); 1058 return Cost; 1059 } 1060 1061 InstructionCost TargetTransformInfo::getMemcpyCost(const Instruction *I) const { 1062 InstructionCost Cost = TTIImpl->getMemcpyCost(I); 1063 assert(Cost >= 0 && "TTI should not produce negative costs!"); 1064 return Cost; 1065 } 1066 1067 uint64_t TargetTransformInfo::getMaxMemIntrinsicInlineSizeThreshold() const { 1068 return TTIImpl->getMaxMemIntrinsicInlineSizeThreshold(); 1069 } 1070 1071 InstructionCost TargetTransformInfo::getArithmeticReductionCost( 1072 unsigned Opcode, VectorType *Ty, std::optional<FastMathFlags> FMF, 1073 TTI::TargetCostKind CostKind) const { 1074 InstructionCost Cost = 1075 TTIImpl->getArithmeticReductionCost(Opcode, Ty, FMF, CostKind); 1076 assert(Cost >= 0 && "TTI should not produce negative costs!"); 1077 return Cost; 1078 } 1079 1080 InstructionCost TargetTransformInfo::getMinMaxReductionCost( 1081 Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, 1082 TTI::TargetCostKind CostKind) const { 1083 InstructionCost Cost = 1084 TTIImpl->getMinMaxReductionCost(IID, Ty, FMF, CostKind); 1085 assert(Cost >= 0 && "TTI should not produce negative costs!"); 1086 return Cost; 1087 } 1088 1089 InstructionCost TargetTransformInfo::getExtendedReductionCost( 1090 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty, 1091 FastMathFlags FMF, TTI::TargetCostKind CostKind) const { 1092 return TTIImpl->getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty, FMF, 1093 CostKind); 1094 } 1095 1096 InstructionCost TargetTransformInfo::getMulAccReductionCost( 1097 bool IsUnsigned, Type *ResTy, VectorType *Ty, 1098 TTI::TargetCostKind CostKind) const { 1099 return TTIImpl->getMulAccReductionCost(IsUnsigned, ResTy, Ty, CostKind); 1100 } 1101 1102 InstructionCost 1103 TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const { 1104 return TTIImpl->getCostOfKeepingLiveOverCall(Tys); 1105 } 1106 1107 bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst, 1108 MemIntrinsicInfo &Info) const { 1109 return TTIImpl->getTgtMemIntrinsic(Inst, Info); 1110 } 1111 1112 unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const { 1113 return TTIImpl->getAtomicMemIntrinsicMaxElementSize(); 1114 } 1115 1116 Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic( 1117 IntrinsicInst *Inst, Type *ExpectedType) const { 1118 return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType); 1119 } 1120 1121 Type *TargetTransformInfo::getMemcpyLoopLoweringType( 1122 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace, 1123 unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign, 1124 std::optional<uint32_t> AtomicElementSize) const { 1125 return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace, 1126 DestAddrSpace, SrcAlign, DestAlign, 1127 AtomicElementSize); 1128 } 1129 1130 void TargetTransformInfo::getMemcpyLoopResidualLoweringType( 1131 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, 1132 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, 1133 unsigned SrcAlign, unsigned DestAlign, 1134 std::optional<uint32_t> AtomicCpySize) const { 1135 TTIImpl->getMemcpyLoopResidualLoweringType( 1136 OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign, 1137 DestAlign, AtomicCpySize); 1138 } 1139 1140 bool TargetTransformInfo::areInlineCompatible(const Function *Caller, 1141 const Function *Callee) const { 1142 return TTIImpl->areInlineCompatible(Caller, Callee); 1143 } 1144 1145 unsigned 1146 TargetTransformInfo::getInlineCallPenalty(const Function *F, 1147 const CallBase &Call, 1148 unsigned DefaultCallPenalty) const { 1149 return TTIImpl->getInlineCallPenalty(F, Call, DefaultCallPenalty); 1150 } 1151 1152 bool TargetTransformInfo::areTypesABICompatible( 1153 const Function *Caller, const Function *Callee, 1154 const ArrayRef<Type *> &Types) const { 1155 return TTIImpl->areTypesABICompatible(Caller, Callee, Types); 1156 } 1157 1158 bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode, 1159 Type *Ty) const { 1160 return TTIImpl->isIndexedLoadLegal(Mode, Ty); 1161 } 1162 1163 bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode, 1164 Type *Ty) const { 1165 return TTIImpl->isIndexedStoreLegal(Mode, Ty); 1166 } 1167 1168 unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const { 1169 return TTIImpl->getLoadStoreVecRegBitWidth(AS); 1170 } 1171 1172 bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const { 1173 return TTIImpl->isLegalToVectorizeLoad(LI); 1174 } 1175 1176 bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const { 1177 return TTIImpl->isLegalToVectorizeStore(SI); 1178 } 1179 1180 bool TargetTransformInfo::isLegalToVectorizeLoadChain( 1181 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const { 1182 return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment, 1183 AddrSpace); 1184 } 1185 1186 bool TargetTransformInfo::isLegalToVectorizeStoreChain( 1187 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const { 1188 return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment, 1189 AddrSpace); 1190 } 1191 1192 bool TargetTransformInfo::isLegalToVectorizeReduction( 1193 const RecurrenceDescriptor &RdxDesc, ElementCount VF) const { 1194 return TTIImpl->isLegalToVectorizeReduction(RdxDesc, VF); 1195 } 1196 1197 bool TargetTransformInfo::isElementTypeLegalForScalableVector(Type *Ty) const { 1198 return TTIImpl->isElementTypeLegalForScalableVector(Ty); 1199 } 1200 1201 unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF, 1202 unsigned LoadSize, 1203 unsigned ChainSizeInBytes, 1204 VectorType *VecTy) const { 1205 return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy); 1206 } 1207 1208 unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF, 1209 unsigned StoreSize, 1210 unsigned ChainSizeInBytes, 1211 VectorType *VecTy) const { 1212 return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy); 1213 } 1214 1215 bool TargetTransformInfo::preferInLoopReduction(unsigned Opcode, Type *Ty, 1216 ReductionFlags Flags) const { 1217 return TTIImpl->preferInLoopReduction(Opcode, Ty, Flags); 1218 } 1219 1220 bool TargetTransformInfo::preferPredicatedReductionSelect( 1221 unsigned Opcode, Type *Ty, ReductionFlags Flags) const { 1222 return TTIImpl->preferPredicatedReductionSelect(Opcode, Ty, Flags); 1223 } 1224 1225 bool TargetTransformInfo::preferEpilogueVectorization() const { 1226 return TTIImpl->preferEpilogueVectorization(); 1227 } 1228 1229 TargetTransformInfo::VPLegalization 1230 TargetTransformInfo::getVPLegalizationStrategy(const VPIntrinsic &VPI) const { 1231 return TTIImpl->getVPLegalizationStrategy(VPI); 1232 } 1233 1234 bool TargetTransformInfo::hasArmWideBranch(bool Thumb) const { 1235 return TTIImpl->hasArmWideBranch(Thumb); 1236 } 1237 1238 unsigned TargetTransformInfo::getMaxNumArgs() const { 1239 return TTIImpl->getMaxNumArgs(); 1240 } 1241 1242 bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const { 1243 return TTIImpl->shouldExpandReduction(II); 1244 } 1245 1246 unsigned TargetTransformInfo::getGISelRematGlobalCost() const { 1247 return TTIImpl->getGISelRematGlobalCost(); 1248 } 1249 1250 unsigned TargetTransformInfo::getMinTripCountTailFoldingThreshold() const { 1251 return TTIImpl->getMinTripCountTailFoldingThreshold(); 1252 } 1253 1254 bool TargetTransformInfo::supportsScalableVectors() const { 1255 return TTIImpl->supportsScalableVectors(); 1256 } 1257 1258 bool TargetTransformInfo::enableScalableVectorization() const { 1259 return TTIImpl->enableScalableVectorization(); 1260 } 1261 1262 bool TargetTransformInfo::hasActiveVectorLength(unsigned Opcode, Type *DataType, 1263 Align Alignment) const { 1264 return TTIImpl->hasActiveVectorLength(Opcode, DataType, Alignment); 1265 } 1266 1267 TargetTransformInfo::Concept::~Concept() = default; 1268 1269 TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {} 1270 1271 TargetIRAnalysis::TargetIRAnalysis( 1272 std::function<Result(const Function &)> TTICallback) 1273 : TTICallback(std::move(TTICallback)) {} 1274 1275 TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F, 1276 FunctionAnalysisManager &) { 1277 return TTICallback(F); 1278 } 1279 1280 AnalysisKey TargetIRAnalysis::Key; 1281 1282 TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) { 1283 return Result(F.getParent()->getDataLayout()); 1284 } 1285 1286 // Register the basic pass. 1287 INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti", 1288 "Target Transform Information", false, true) 1289 char TargetTransformInfoWrapperPass::ID = 0; 1290 1291 void TargetTransformInfoWrapperPass::anchor() {} 1292 1293 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass() 1294 : ImmutablePass(ID) { 1295 initializeTargetTransformInfoWrapperPassPass( 1296 *PassRegistry::getPassRegistry()); 1297 } 1298 1299 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass( 1300 TargetIRAnalysis TIRA) 1301 : ImmutablePass(ID), TIRA(std::move(TIRA)) { 1302 initializeTargetTransformInfoWrapperPassPass( 1303 *PassRegistry::getPassRegistry()); 1304 } 1305 1306 TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) { 1307 FunctionAnalysisManager DummyFAM; 1308 TTI = TIRA.run(F, DummyFAM); 1309 return *TTI; 1310 } 1311 1312 ImmutablePass * 1313 llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) { 1314 return new TargetTransformInfoWrapperPass(std::move(TIRA)); 1315 } 1316