1 //===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops 10 // and generates target-independent LLVM-IR. 11 // The vectorizer uses the TargetTransformInfo analysis to estimate the costs 12 // of instructions in order to estimate the profitability of vectorization. 13 // 14 // The loop vectorizer combines consecutive loop iterations into a single 15 // 'wide' iteration. After this transformation the index is incremented 16 // by the SIMD vector width, and not by one. 17 // 18 // This pass has three parts: 19 // 1. The main loop pass that drives the different parts. 20 // 2. LoopVectorizationLegality - A unit that checks for the legality 21 // of the vectorization. 22 // 3. InnerLoopVectorizer - A unit that performs the actual 23 // widening of instructions. 24 // 4. LoopVectorizationCostModel - A unit that checks for the profitability 25 // of vectorization. It decides on the optimal vector width, which 26 // can be one, if vectorization is not profitable. 27 // 28 // There is a development effort going on to migrate loop vectorizer to the 29 // VPlan infrastructure and to introduce outer loop vectorization support (see 30 // docs/Proposal/VectorizationPlan.rst and 31 // http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this 32 // purpose, we temporarily introduced the VPlan-native vectorization path: an 33 // alternative vectorization path that is natively implemented on top of the 34 // VPlan infrastructure. See EnableVPlanNativePath for enabling. 35 // 36 //===----------------------------------------------------------------------===// 37 // 38 // The reduction-variable vectorization is based on the paper: 39 // D. Nuzman and R. Henderson. Multi-platform Auto-vectorization. 40 // 41 // Variable uniformity checks are inspired by: 42 // Karrenberg, R. and Hack, S. Whole Function Vectorization. 43 // 44 // The interleaved access vectorization is based on the paper: 45 // Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved 46 // Data for SIMD 47 // 48 // Other ideas/concepts are from: 49 // A. Zaks and D. Nuzman. Autovectorization in GCC-two years later. 50 // 51 // S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of 52 // Vectorizing Compilers. 53 // 54 //===----------------------------------------------------------------------===// 55 56 #include "llvm/Transforms/Vectorize/LoopVectorize.h" 57 #include "LoopVectorizationPlanner.h" 58 #include "VPRecipeBuilder.h" 59 #include "VPlan.h" 60 #include "VPlanHCFGBuilder.h" 61 #include "VPlanPredicator.h" 62 #include "VPlanTransforms.h" 63 #include "llvm/ADT/APInt.h" 64 #include "llvm/ADT/ArrayRef.h" 65 #include "llvm/ADT/DenseMap.h" 66 #include "llvm/ADT/DenseMapInfo.h" 67 #include "llvm/ADT/Hashing.h" 68 #include "llvm/ADT/MapVector.h" 69 #include "llvm/ADT/None.h" 70 #include "llvm/ADT/Optional.h" 71 #include "llvm/ADT/STLExtras.h" 72 #include "llvm/ADT/SetVector.h" 73 #include "llvm/ADT/SmallPtrSet.h" 74 #include "llvm/ADT/SmallVector.h" 75 #include "llvm/ADT/Statistic.h" 76 #include "llvm/ADT/StringRef.h" 77 #include "llvm/ADT/Twine.h" 78 #include "llvm/ADT/iterator_range.h" 79 #include "llvm/Analysis/AssumptionCache.h" 80 #include "llvm/Analysis/BasicAliasAnalysis.h" 81 #include "llvm/Analysis/BlockFrequencyInfo.h" 82 #include "llvm/Analysis/CFG.h" 83 #include "llvm/Analysis/CodeMetrics.h" 84 #include "llvm/Analysis/DemandedBits.h" 85 #include "llvm/Analysis/GlobalsModRef.h" 86 #include "llvm/Analysis/LoopAccessAnalysis.h" 87 #include "llvm/Analysis/LoopAnalysisManager.h" 88 #include "llvm/Analysis/LoopInfo.h" 89 #include "llvm/Analysis/LoopIterator.h" 90 #include "llvm/Analysis/MemorySSA.h" 91 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 92 #include "llvm/Analysis/ProfileSummaryInfo.h" 93 #include "llvm/Analysis/ScalarEvolution.h" 94 #include "llvm/Analysis/ScalarEvolutionExpressions.h" 95 #include "llvm/Analysis/TargetLibraryInfo.h" 96 #include "llvm/Analysis/TargetTransformInfo.h" 97 #include "llvm/Analysis/VectorUtils.h" 98 #include "llvm/IR/Attributes.h" 99 #include "llvm/IR/BasicBlock.h" 100 #include "llvm/IR/CFG.h" 101 #include "llvm/IR/Constant.h" 102 #include "llvm/IR/Constants.h" 103 #include "llvm/IR/DataLayout.h" 104 #include "llvm/IR/DebugInfoMetadata.h" 105 #include "llvm/IR/DebugLoc.h" 106 #include "llvm/IR/DerivedTypes.h" 107 #include "llvm/IR/DiagnosticInfo.h" 108 #include "llvm/IR/Dominators.h" 109 #include "llvm/IR/Function.h" 110 #include "llvm/IR/IRBuilder.h" 111 #include "llvm/IR/InstrTypes.h" 112 #include "llvm/IR/Instruction.h" 113 #include "llvm/IR/Instructions.h" 114 #include "llvm/IR/IntrinsicInst.h" 115 #include "llvm/IR/Intrinsics.h" 116 #include "llvm/IR/LLVMContext.h" 117 #include "llvm/IR/Metadata.h" 118 #include "llvm/IR/Module.h" 119 #include "llvm/IR/Operator.h" 120 #include "llvm/IR/Type.h" 121 #include "llvm/IR/Use.h" 122 #include "llvm/IR/User.h" 123 #include "llvm/IR/Value.h" 124 #include "llvm/IR/ValueHandle.h" 125 #include "llvm/IR/Verifier.h" 126 #include "llvm/InitializePasses.h" 127 #include "llvm/Pass.h" 128 #include "llvm/Support/Casting.h" 129 #include "llvm/Support/CommandLine.h" 130 #include "llvm/Support/Compiler.h" 131 #include "llvm/Support/Debug.h" 132 #include "llvm/Support/ErrorHandling.h" 133 #include "llvm/Support/InstructionCost.h" 134 #include "llvm/Support/MathExtras.h" 135 #include "llvm/Support/raw_ostream.h" 136 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 137 #include "llvm/Transforms/Utils/InjectTLIMappings.h" 138 #include "llvm/Transforms/Utils/LoopSimplify.h" 139 #include "llvm/Transforms/Utils/LoopUtils.h" 140 #include "llvm/Transforms/Utils/LoopVersioning.h" 141 #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h" 142 #include "llvm/Transforms/Utils/SizeOpts.h" 143 #include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h" 144 #include <algorithm> 145 #include <cassert> 146 #include <cstdint> 147 #include <cstdlib> 148 #include <functional> 149 #include <iterator> 150 #include <limits> 151 #include <memory> 152 #include <string> 153 #include <tuple> 154 #include <utility> 155 156 using namespace llvm; 157 158 #define LV_NAME "loop-vectorize" 159 #define DEBUG_TYPE LV_NAME 160 161 #ifndef NDEBUG 162 const char VerboseDebug[] = DEBUG_TYPE "-verbose"; 163 #endif 164 165 /// @{ 166 /// Metadata attribute names 167 const char LLVMLoopVectorizeFollowupAll[] = "llvm.loop.vectorize.followup_all"; 168 const char LLVMLoopVectorizeFollowupVectorized[] = 169 "llvm.loop.vectorize.followup_vectorized"; 170 const char LLVMLoopVectorizeFollowupEpilogue[] = 171 "llvm.loop.vectorize.followup_epilogue"; 172 /// @} 173 174 STATISTIC(LoopsVectorized, "Number of loops vectorized"); 175 STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization"); 176 STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized"); 177 178 static cl::opt<bool> EnableEpilogueVectorization( 179 "enable-epilogue-vectorization", cl::init(true), cl::Hidden, 180 cl::desc("Enable vectorization of epilogue loops.")); 181 182 static cl::opt<unsigned> EpilogueVectorizationForceVF( 183 "epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, 184 cl::desc("When epilogue vectorization is enabled, and a value greater than " 185 "1 is specified, forces the given VF for all applicable epilogue " 186 "loops.")); 187 188 static cl::opt<unsigned> EpilogueVectorizationMinVF( 189 "epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden, 190 cl::desc("Only loops with vectorization factor equal to or larger than " 191 "the specified value are considered for epilogue vectorization.")); 192 193 /// Loops with a known constant trip count below this number are vectorized only 194 /// if no scalar iteration overheads are incurred. 195 static cl::opt<unsigned> TinyTripCountVectorThreshold( 196 "vectorizer-min-trip-count", cl::init(16), cl::Hidden, 197 cl::desc("Loops with a constant trip count that is smaller than this " 198 "value are vectorized only if no scalar iteration overheads " 199 "are incurred.")); 200 201 // Option prefer-predicate-over-epilogue indicates that an epilogue is undesired, 202 // that predication is preferred, and this lists all options. I.e., the 203 // vectorizer will try to fold the tail-loop (epilogue) into the vector body 204 // and predicate the instructions accordingly. If tail-folding fails, there are 205 // different fallback strategies depending on these values: 206 namespace PreferPredicateTy { 207 enum Option { 208 ScalarEpilogue = 0, 209 PredicateElseScalarEpilogue, 210 PredicateOrDontVectorize 211 }; 212 } // namespace PreferPredicateTy 213 214 static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue( 215 "prefer-predicate-over-epilogue", 216 cl::init(PreferPredicateTy::ScalarEpilogue), 217 cl::Hidden, 218 cl::desc("Tail-folding and predication preferences over creating a scalar " 219 "epilogue loop."), 220 cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, 221 "scalar-epilogue", 222 "Don't tail-predicate loops, create scalar epilogue"), 223 clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, 224 "predicate-else-scalar-epilogue", 225 "prefer tail-folding, create scalar epilogue if tail " 226 "folding fails."), 227 clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, 228 "predicate-dont-vectorize", 229 "prefers tail-folding, don't attempt vectorization if " 230 "tail-folding fails."))); 231 232 static cl::opt<bool> MaximizeBandwidth( 233 "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, 234 cl::desc("Maximize bandwidth when selecting vectorization factor which " 235 "will be determined by the smallest type in loop.")); 236 237 static cl::opt<bool> EnableInterleavedMemAccesses( 238 "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, 239 cl::desc("Enable vectorization on interleaved memory accesses in a loop")); 240 241 /// An interleave-group may need masking if it resides in a block that needs 242 /// predication, or in order to mask away gaps. 243 static cl::opt<bool> EnableMaskedInterleavedMemAccesses( 244 "enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, 245 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop")); 246 247 static cl::opt<unsigned> TinyTripCountInterleaveThreshold( 248 "tiny-trip-count-interleave-threshold", cl::init(128), cl::Hidden, 249 cl::desc("We don't interleave loops with a estimated constant trip count " 250 "below this number")); 251 252 static cl::opt<unsigned> ForceTargetNumScalarRegs( 253 "force-target-num-scalar-regs", cl::init(0), cl::Hidden, 254 cl::desc("A flag that overrides the target's number of scalar registers.")); 255 256 static cl::opt<unsigned> ForceTargetNumVectorRegs( 257 "force-target-num-vector-regs", cl::init(0), cl::Hidden, 258 cl::desc("A flag that overrides the target's number of vector registers.")); 259 260 static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor( 261 "force-target-max-scalar-interleave", cl::init(0), cl::Hidden, 262 cl::desc("A flag that overrides the target's max interleave factor for " 263 "scalar loops.")); 264 265 static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor( 266 "force-target-max-vector-interleave", cl::init(0), cl::Hidden, 267 cl::desc("A flag that overrides the target's max interleave factor for " 268 "vectorized loops.")); 269 270 static cl::opt<unsigned> ForceTargetInstructionCost( 271 "force-target-instruction-cost", cl::init(0), cl::Hidden, 272 cl::desc("A flag that overrides the target's expected cost for " 273 "an instruction to a single constant value. Mostly " 274 "useful for getting consistent testing.")); 275 276 static cl::opt<bool> ForceTargetSupportsScalableVectors( 277 "force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, 278 cl::desc( 279 "Pretend that scalable vectors are supported, even if the target does " 280 "not support them. This flag should only be used for testing.")); 281 282 static cl::opt<unsigned> SmallLoopCost( 283 "small-loop-cost", cl::init(20), cl::Hidden, 284 cl::desc( 285 "The cost of a loop that is considered 'small' by the interleaver.")); 286 287 static cl::opt<bool> LoopVectorizeWithBlockFrequency( 288 "loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, 289 cl::desc("Enable the use of the block frequency analysis to access PGO " 290 "heuristics minimizing code growth in cold regions and being more " 291 "aggressive in hot regions.")); 292 293 // Runtime interleave loops for load/store throughput. 294 static cl::opt<bool> EnableLoadStoreRuntimeInterleave( 295 "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, 296 cl::desc( 297 "Enable runtime interleaving until load/store ports are saturated")); 298 299 /// Interleave small loops with scalar reductions. 300 static cl::opt<bool> InterleaveSmallLoopScalarReduction( 301 "interleave-small-loop-scalar-reduction", cl::init(false), cl::Hidden, 302 cl::desc("Enable interleaving for loops with small iteration counts that " 303 "contain scalar reductions to expose ILP.")); 304 305 /// The number of stores in a loop that are allowed to need predication. 306 static cl::opt<unsigned> NumberOfStoresToPredicate( 307 "vectorize-num-stores-pred", cl::init(1), cl::Hidden, 308 cl::desc("Max number of stores to be predicated behind an if.")); 309 310 static cl::opt<bool> EnableIndVarRegisterHeur( 311 "enable-ind-var-reg-heur", cl::init(true), cl::Hidden, 312 cl::desc("Count the induction variable only once when interleaving")); 313 314 static cl::opt<bool> EnableCondStoresVectorization( 315 "enable-cond-stores-vec", cl::init(true), cl::Hidden, 316 cl::desc("Enable if predication of stores during vectorization.")); 317 318 static cl::opt<unsigned> MaxNestedScalarReductionIC( 319 "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, 320 cl::desc("The maximum interleave count to use when interleaving a scalar " 321 "reduction in a nested loop.")); 322 323 static cl::opt<bool> 324 PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), 325 cl::Hidden, 326 cl::desc("Prefer in-loop vector reductions, " 327 "overriding the targets preference.")); 328 329 static cl::opt<bool> PreferPredicatedReductionSelect( 330 "prefer-predicated-reduction-select", cl::init(false), cl::Hidden, 331 cl::desc( 332 "Prefer predicating a reduction operation over an after loop select.")); 333 334 cl::opt<bool> EnableVPlanNativePath( 335 "enable-vplan-native-path", cl::init(false), cl::Hidden, 336 cl::desc("Enable VPlan-native vectorization path with " 337 "support for outer loop vectorization.")); 338 339 // FIXME: Remove this switch once we have divergence analysis. Currently we 340 // assume divergent non-backedge branches when this switch is true. 341 cl::opt<bool> EnableVPlanPredication( 342 "enable-vplan-predication", cl::init(false), cl::Hidden, 343 cl::desc("Enable VPlan-native vectorization path predicator with " 344 "support for outer loop vectorization.")); 345 346 // This flag enables the stress testing of the VPlan H-CFG construction in the 347 // VPlan-native vectorization path. It must be used in conjuction with 348 // -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the 349 // verification of the H-CFGs built. 350 static cl::opt<bool> VPlanBuildStressTest( 351 "vplan-build-stress-test", cl::init(false), cl::Hidden, 352 cl::desc( 353 "Build VPlan for every supported loop nest in the function and bail " 354 "out right after the build (stress test the VPlan H-CFG construction " 355 "in the VPlan-native vectorization path).")); 356 357 cl::opt<bool> llvm::EnableLoopInterleaving( 358 "interleave-loops", cl::init(true), cl::Hidden, 359 cl::desc("Enable loop interleaving in Loop vectorization passes")); 360 cl::opt<bool> llvm::EnableLoopVectorization( 361 "vectorize-loops", cl::init(true), cl::Hidden, 362 cl::desc("Run the Loop vectorization passes")); 363 364 /// A helper function that returns the type of loaded or stored value. 365 static Type *getMemInstValueType(Value *I) { 366 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && 367 "Expected Load or Store instruction"); 368 if (auto *LI = dyn_cast<LoadInst>(I)) 369 return LI->getType(); 370 return cast<StoreInst>(I)->getValueOperand()->getType(); 371 } 372 373 /// A helper function that returns true if the given type is irregular. The 374 /// type is irregular if its allocated size doesn't equal the store size of an 375 /// element of the corresponding vector type. 376 static bool hasIrregularType(Type *Ty, const DataLayout &DL) { 377 // Determine if an array of N elements of type Ty is "bitcast compatible" 378 // with a <N x Ty> vector. 379 // This is only true if there is no padding between the array elements. 380 return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty); 381 } 382 383 /// A helper function that returns the reciprocal of the block probability of 384 /// predicated blocks. If we return X, we are assuming the predicated block 385 /// will execute once for every X iterations of the loop header. 386 /// 387 /// TODO: We should use actual block probability here, if available. Currently, 388 /// we always assume predicated blocks have a 50% chance of executing. 389 static unsigned getReciprocalPredBlockProb() { return 2; } 390 391 /// A helper function that adds a 'fast' flag to floating-point operations. 392 static Value *addFastMathFlag(Value *V) { 393 if (isa<FPMathOperator>(V)) 394 cast<Instruction>(V)->setFastMathFlags(FastMathFlags::getFast()); 395 return V; 396 } 397 398 static Value *addFastMathFlag(Value *V, FastMathFlags FMF) { 399 if (isa<FPMathOperator>(V)) 400 cast<Instruction>(V)->setFastMathFlags(FMF); 401 return V; 402 } 403 404 /// A helper function that returns an integer or floating-point constant with 405 /// value C. 406 static Constant *getSignedIntOrFpConstant(Type *Ty, int64_t C) { 407 return Ty->isIntegerTy() ? ConstantInt::getSigned(Ty, C) 408 : ConstantFP::get(Ty, C); 409 } 410 411 /// Returns "best known" trip count for the specified loop \p L as defined by 412 /// the following procedure: 413 /// 1) Returns exact trip count if it is known. 414 /// 2) Returns expected trip count according to profile data if any. 415 /// 3) Returns upper bound estimate if it is known. 416 /// 4) Returns None if all of the above failed. 417 static Optional<unsigned> getSmallBestKnownTC(ScalarEvolution &SE, Loop *L) { 418 // Check if exact trip count is known. 419 if (unsigned ExpectedTC = SE.getSmallConstantTripCount(L)) 420 return ExpectedTC; 421 422 // Check if there is an expected trip count available from profile data. 423 if (LoopVectorizeWithBlockFrequency) 424 if (auto EstimatedTC = getLoopEstimatedTripCount(L)) 425 return EstimatedTC; 426 427 // Check if upper bound estimate is known. 428 if (unsigned ExpectedTC = SE.getSmallConstantMaxTripCount(L)) 429 return ExpectedTC; 430 431 return None; 432 } 433 434 namespace llvm { 435 436 /// InnerLoopVectorizer vectorizes loops which contain only one basic 437 /// block to a specified vectorization factor (VF). 438 /// This class performs the widening of scalars into vectors, or multiple 439 /// scalars. This class also implements the following features: 440 /// * It inserts an epilogue loop for handling loops that don't have iteration 441 /// counts that are known to be a multiple of the vectorization factor. 442 /// * It handles the code generation for reduction variables. 443 /// * Scalarization (implementation using scalars) of un-vectorizable 444 /// instructions. 445 /// InnerLoopVectorizer does not perform any vectorization-legality 446 /// checks, and relies on the caller to check for the different legality 447 /// aspects. The InnerLoopVectorizer relies on the 448 /// LoopVectorizationLegality class to provide information about the induction 449 /// and reduction variables that were found to a given vectorization factor. 450 class InnerLoopVectorizer { 451 public: 452 InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, 453 LoopInfo *LI, DominatorTree *DT, 454 const TargetLibraryInfo *TLI, 455 const TargetTransformInfo *TTI, AssumptionCache *AC, 456 OptimizationRemarkEmitter *ORE, ElementCount VecWidth, 457 unsigned UnrollFactor, LoopVectorizationLegality *LVL, 458 LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, 459 ProfileSummaryInfo *PSI) 460 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI), 461 AC(AC), ORE(ORE), VF(VecWidth), UF(UnrollFactor), 462 Builder(PSE.getSE()->getContext()), 463 VectorLoopValueMap(UnrollFactor, VecWidth), Legal(LVL), Cost(CM), 464 BFI(BFI), PSI(PSI) { 465 // Query this against the original loop and save it here because the profile 466 // of the original loop header may change as the transformation happens. 467 OptForSizeBasedOnProfile = llvm::shouldOptimizeForSize( 468 OrigLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass); 469 } 470 471 virtual ~InnerLoopVectorizer() = default; 472 473 /// Create a new empty loop that will contain vectorized instructions later 474 /// on, while the old loop will be used as the scalar remainder. Control flow 475 /// is generated around the vectorized (and scalar epilogue) loops consisting 476 /// of various checks and bypasses. Return the pre-header block of the new 477 /// loop. 478 /// In the case of epilogue vectorization, this function is overriden to 479 /// handle the more complex control flow around the loops. 480 virtual BasicBlock *createVectorizedLoopSkeleton(); 481 482 /// Widen a single instruction within the innermost loop. 483 void widenInstruction(Instruction &I, VPValue *Def, VPUser &Operands, 484 VPTransformState &State); 485 486 /// Widen a single call instruction within the innermost loop. 487 void widenCallInstruction(CallInst &I, VPValue *Def, VPUser &ArgOperands, 488 VPTransformState &State); 489 490 /// Widen a single select instruction within the innermost loop. 491 void widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands, 492 bool InvariantCond, VPTransformState &State); 493 494 /// Fix the vectorized code, taking care of header phi's, live-outs, and more. 495 void fixVectorizedLoop(); 496 497 // Return true if any runtime check is added. 498 bool areSafetyChecksAdded() { return AddedSafetyChecks; } 499 500 /// A type for vectorized values in the new loop. Each value from the 501 /// original loop, when vectorized, is represented by UF vector values in the 502 /// new unrolled loop, where UF is the unroll factor. 503 using VectorParts = SmallVector<Value *, 2>; 504 505 /// Vectorize a single GetElementPtrInst based on information gathered and 506 /// decisions taken during planning. 507 void widenGEP(GetElementPtrInst *GEP, VPValue *VPDef, VPUser &Indices, 508 unsigned UF, ElementCount VF, bool IsPtrLoopInvariant, 509 SmallBitVector &IsIndexLoopInvariant, VPTransformState &State); 510 511 /// Vectorize a single PHINode in a block. This method handles the induction 512 /// variable canonicalization. It supports both VF = 1 for unrolled loops and 513 /// arbitrary length vectors. 514 void widenPHIInstruction(Instruction *PN, RecurrenceDescriptor *RdxDesc, 515 Value *StartV, unsigned UF, ElementCount VF); 516 517 /// A helper function to scalarize a single Instruction in the innermost loop. 518 /// Generates a sequence of scalar instances for each lane between \p MinLane 519 /// and \p MaxLane, times each part between \p MinPart and \p MaxPart, 520 /// inclusive. Uses the VPValue operands from \p Operands instead of \p 521 /// Instr's operands. 522 void scalarizeInstruction(Instruction *Instr, VPUser &Operands, 523 const VPIteration &Instance, bool IfPredicateInstr, 524 VPTransformState &State); 525 526 /// Widen an integer or floating-point induction variable \p IV. If \p Trunc 527 /// is provided, the integer induction variable will first be truncated to 528 /// the corresponding type. 529 void widenIntOrFpInduction(PHINode *IV, Value *Start, 530 TruncInst *Trunc = nullptr); 531 532 /// getOrCreateVectorValue and getOrCreateScalarValue coordinate to generate a 533 /// vector or scalar value on-demand if one is not yet available. When 534 /// vectorizing a loop, we visit the definition of an instruction before its 535 /// uses. When visiting the definition, we either vectorize or scalarize the 536 /// instruction, creating an entry for it in the corresponding map. (In some 537 /// cases, such as induction variables, we will create both vector and scalar 538 /// entries.) Then, as we encounter uses of the definition, we derive values 539 /// for each scalar or vector use unless such a value is already available. 540 /// For example, if we scalarize a definition and one of its uses is vector, 541 /// we build the required vector on-demand with an insertelement sequence 542 /// when visiting the use. Otherwise, if the use is scalar, we can use the 543 /// existing scalar definition. 544 /// 545 /// Return a value in the new loop corresponding to \p V from the original 546 /// loop at unroll index \p Part. If the value has already been vectorized, 547 /// the corresponding vector entry in VectorLoopValueMap is returned. If, 548 /// however, the value has a scalar entry in VectorLoopValueMap, we construct 549 /// a new vector value on-demand by inserting the scalar values into a vector 550 /// with an insertelement sequence. If the value has been neither vectorized 551 /// nor scalarized, it must be loop invariant, so we simply broadcast the 552 /// value into a vector. 553 Value *getOrCreateVectorValue(Value *V, unsigned Part); 554 555 void setVectorValue(Value *Scalar, unsigned Part, Value *Vector) { 556 VectorLoopValueMap.setVectorValue(Scalar, Part, Vector); 557 } 558 559 /// Return a value in the new loop corresponding to \p V from the original 560 /// loop at unroll and vector indices \p Instance. If the value has been 561 /// vectorized but not scalarized, the necessary extractelement instruction 562 /// will be generated. 563 Value *getOrCreateScalarValue(Value *V, const VPIteration &Instance); 564 565 /// Construct the vector value of a scalarized value \p V one lane at a time. 566 void packScalarIntoVectorValue(Value *V, const VPIteration &Instance); 567 568 /// Try to vectorize interleaved access group \p Group with the base address 569 /// given in \p Addr, optionally masking the vector operations if \p 570 /// BlockInMask is non-null. Use \p State to translate given VPValues to IR 571 /// values in the vectorized loop. 572 void vectorizeInterleaveGroup(const InterleaveGroup<Instruction> *Group, 573 ArrayRef<VPValue *> VPDefs, 574 VPTransformState &State, VPValue *Addr, 575 ArrayRef<VPValue *> StoredValues, 576 VPValue *BlockInMask = nullptr); 577 578 /// Vectorize Load and Store instructions with the base address given in \p 579 /// Addr, optionally masking the vector operations if \p BlockInMask is 580 /// non-null. Use \p State to translate given VPValues to IR values in the 581 /// vectorized loop. 582 void vectorizeMemoryInstruction(Instruction *Instr, VPTransformState &State, 583 VPValue *Def, VPValue *Addr, 584 VPValue *StoredValue, VPValue *BlockInMask); 585 586 /// Set the debug location in the builder using the debug location in 587 /// the instruction. 588 void setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr); 589 590 /// Fix the non-induction PHIs in the OrigPHIsToFix vector. 591 void fixNonInductionPHIs(void); 592 593 protected: 594 friend class LoopVectorizationPlanner; 595 596 /// A small list of PHINodes. 597 using PhiVector = SmallVector<PHINode *, 4>; 598 599 /// A type for scalarized values in the new loop. Each value from the 600 /// original loop, when scalarized, is represented by UF x VF scalar values 601 /// in the new unrolled loop, where UF is the unroll factor and VF is the 602 /// vectorization factor. 603 using ScalarParts = SmallVector<SmallVector<Value *, 4>, 2>; 604 605 /// Set up the values of the IVs correctly when exiting the vector loop. 606 void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II, 607 Value *CountRoundDown, Value *EndValue, 608 BasicBlock *MiddleBlock); 609 610 /// Create a new induction variable inside L. 611 PHINode *createInductionVariable(Loop *L, Value *Start, Value *End, 612 Value *Step, Instruction *DL); 613 614 /// Handle all cross-iteration phis in the header. 615 void fixCrossIterationPHIs(); 616 617 /// Fix a first-order recurrence. This is the second phase of vectorizing 618 /// this phi node. 619 void fixFirstOrderRecurrence(PHINode *Phi); 620 621 /// Fix a reduction cross-iteration phi. This is the second phase of 622 /// vectorizing this phi node. 623 void fixReduction(PHINode *Phi); 624 625 /// Clear NSW/NUW flags from reduction instructions if necessary. 626 void clearReductionWrapFlags(RecurrenceDescriptor &RdxDesc); 627 628 /// Fixup the LCSSA phi nodes in the unique exit block. This simply 629 /// means we need to add the appropriate incoming value from the middle 630 /// block as exiting edges from the scalar epilogue loop (if present) are 631 /// already in place, and we exit the vector loop exclusively to the middle 632 /// block. 633 void fixLCSSAPHIs(); 634 635 /// Iteratively sink the scalarized operands of a predicated instruction into 636 /// the block that was created for it. 637 void sinkScalarOperands(Instruction *PredInst); 638 639 /// Shrinks vector element sizes to the smallest bitwidth they can be legally 640 /// represented as. 641 void truncateToMinimalBitwidths(); 642 643 /// Create a broadcast instruction. This method generates a broadcast 644 /// instruction (shuffle) for loop invariant values and for the induction 645 /// value. If this is the induction variable then we extend it to N, N+1, ... 646 /// this is needed because each iteration in the loop corresponds to a SIMD 647 /// element. 648 virtual Value *getBroadcastInstrs(Value *V); 649 650 /// This function adds (StartIdx, StartIdx + Step, StartIdx + 2*Step, ...) 651 /// to each vector element of Val. The sequence starts at StartIndex. 652 /// \p Opcode is relevant for FP induction variable. 653 virtual Value *getStepVector(Value *Val, int StartIdx, Value *Step, 654 Instruction::BinaryOps Opcode = 655 Instruction::BinaryOpsEnd); 656 657 /// Compute scalar induction steps. \p ScalarIV is the scalar induction 658 /// variable on which to base the steps, \p Step is the size of the step, and 659 /// \p EntryVal is the value from the original loop that maps to the steps. 660 /// Note that \p EntryVal doesn't have to be an induction variable - it 661 /// can also be a truncate instruction. 662 void buildScalarSteps(Value *ScalarIV, Value *Step, Instruction *EntryVal, 663 const InductionDescriptor &ID); 664 665 /// Create a vector induction phi node based on an existing scalar one. \p 666 /// EntryVal is the value from the original loop that maps to the vector phi 667 /// node, and \p Step is the loop-invariant step. If \p EntryVal is a 668 /// truncate instruction, instead of widening the original IV, we widen a 669 /// version of the IV truncated to \p EntryVal's type. 670 void createVectorIntOrFpInductionPHI(const InductionDescriptor &II, 671 Value *Step, Value *Start, 672 Instruction *EntryVal); 673 674 /// Returns true if an instruction \p I should be scalarized instead of 675 /// vectorized for the chosen vectorization factor. 676 bool shouldScalarizeInstruction(Instruction *I) const; 677 678 /// Returns true if we should generate a scalar version of \p IV. 679 bool needsScalarInduction(Instruction *IV) const; 680 681 /// If there is a cast involved in the induction variable \p ID, which should 682 /// be ignored in the vectorized loop body, this function records the 683 /// VectorLoopValue of the respective Phi also as the VectorLoopValue of the 684 /// cast. We had already proved that the casted Phi is equal to the uncasted 685 /// Phi in the vectorized loop (under a runtime guard), and therefore 686 /// there is no need to vectorize the cast - the same value can be used in the 687 /// vector loop for both the Phi and the cast. 688 /// If \p VectorLoopValue is a scalarized value, \p Lane is also specified, 689 /// Otherwise, \p VectorLoopValue is a widened/vectorized value. 690 /// 691 /// \p EntryVal is the value from the original loop that maps to the vector 692 /// phi node and is used to distinguish what is the IV currently being 693 /// processed - original one (if \p EntryVal is a phi corresponding to the 694 /// original IV) or the "newly-created" one based on the proof mentioned above 695 /// (see also buildScalarSteps() and createVectorIntOrFPInductionPHI()). In the 696 /// latter case \p EntryVal is a TruncInst and we must not record anything for 697 /// that IV, but it's error-prone to expect callers of this routine to care 698 /// about that, hence this explicit parameter. 699 void recordVectorLoopValueForInductionCast(const InductionDescriptor &ID, 700 const Instruction *EntryVal, 701 Value *VectorLoopValue, 702 unsigned Part, 703 unsigned Lane = UINT_MAX); 704 705 /// Generate a shuffle sequence that will reverse the vector Vec. 706 virtual Value *reverseVector(Value *Vec); 707 708 /// Returns (and creates if needed) the original loop trip count. 709 Value *getOrCreateTripCount(Loop *NewLoop); 710 711 /// Returns (and creates if needed) the trip count of the widened loop. 712 Value *getOrCreateVectorTripCount(Loop *NewLoop); 713 714 /// Returns a bitcasted value to the requested vector type. 715 /// Also handles bitcasts of vector<float> <-> vector<pointer> types. 716 Value *createBitOrPointerCast(Value *V, VectorType *DstVTy, 717 const DataLayout &DL); 718 719 /// Emit a bypass check to see if the vector trip count is zero, including if 720 /// it overflows. 721 void emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass); 722 723 /// Emit a bypass check to see if all of the SCEV assumptions we've 724 /// had to make are correct. 725 void emitSCEVChecks(Loop *L, BasicBlock *Bypass); 726 727 /// Emit bypass checks to check any memory assumptions we may have made. 728 void emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass); 729 730 /// Compute the transformed value of Index at offset StartValue using step 731 /// StepValue. 732 /// For integer induction, returns StartValue + Index * StepValue. 733 /// For pointer induction, returns StartValue[Index * StepValue]. 734 /// FIXME: The newly created binary instructions should contain nsw/nuw 735 /// flags, which can be found from the original scalar operations. 736 Value *emitTransformedIndex(IRBuilder<> &B, Value *Index, ScalarEvolution *SE, 737 const DataLayout &DL, 738 const InductionDescriptor &ID) const; 739 740 /// Emit basic blocks (prefixed with \p Prefix) for the iteration check, 741 /// vector loop preheader, middle block and scalar preheader. Also 742 /// allocate a loop object for the new vector loop and return it. 743 Loop *createVectorLoopSkeleton(StringRef Prefix); 744 745 /// Create new phi nodes for the induction variables to resume iteration count 746 /// in the scalar epilogue, from where the vectorized loop left off (given by 747 /// \p VectorTripCount). 748 /// In cases where the loop skeleton is more complicated (eg. epilogue 749 /// vectorization) and the resume values can come from an additional bypass 750 /// block, the \p AdditionalBypass pair provides information about the bypass 751 /// block and the end value on the edge from bypass to this loop. 752 void createInductionResumeValues( 753 Loop *L, Value *VectorTripCount, 754 std::pair<BasicBlock *, Value *> AdditionalBypass = {nullptr, nullptr}); 755 756 /// Complete the loop skeleton by adding debug MDs, creating appropriate 757 /// conditional branches in the middle block, preparing the builder and 758 /// running the verifier. Take in the vector loop \p L as argument, and return 759 /// the preheader of the completed vector loop. 760 BasicBlock *completeLoopSkeleton(Loop *L, MDNode *OrigLoopID); 761 762 /// Add additional metadata to \p To that was not present on \p Orig. 763 /// 764 /// Currently this is used to add the noalias annotations based on the 765 /// inserted memchecks. Use this for instructions that are *cloned* into the 766 /// vector loop. 767 void addNewMetadata(Instruction *To, const Instruction *Orig); 768 769 /// Add metadata from one instruction to another. 770 /// 771 /// This includes both the original MDs from \p From and additional ones (\see 772 /// addNewMetadata). Use this for *newly created* instructions in the vector 773 /// loop. 774 void addMetadata(Instruction *To, Instruction *From); 775 776 /// Similar to the previous function but it adds the metadata to a 777 /// vector of instructions. 778 void addMetadata(ArrayRef<Value *> To, Instruction *From); 779 780 /// Allow subclasses to override and print debug traces before/after vplan 781 /// execution, when trace information is requested. 782 virtual void printDebugTracesAtStart(){}; 783 virtual void printDebugTracesAtEnd(){}; 784 785 /// The original loop. 786 Loop *OrigLoop; 787 788 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies 789 /// dynamic knowledge to simplify SCEV expressions and converts them to a 790 /// more usable form. 791 PredicatedScalarEvolution &PSE; 792 793 /// Loop Info. 794 LoopInfo *LI; 795 796 /// Dominator Tree. 797 DominatorTree *DT; 798 799 /// Alias Analysis. 800 AAResults *AA; 801 802 /// Target Library Info. 803 const TargetLibraryInfo *TLI; 804 805 /// Target Transform Info. 806 const TargetTransformInfo *TTI; 807 808 /// Assumption Cache. 809 AssumptionCache *AC; 810 811 /// Interface to emit optimization remarks. 812 OptimizationRemarkEmitter *ORE; 813 814 /// LoopVersioning. It's only set up (non-null) if memchecks were 815 /// used. 816 /// 817 /// This is currently only used to add no-alias metadata based on the 818 /// memchecks. The actually versioning is performed manually. 819 std::unique_ptr<LoopVersioning> LVer; 820 821 /// The vectorization SIMD factor to use. Each vector will have this many 822 /// vector elements. 823 ElementCount VF; 824 825 /// The vectorization unroll factor to use. Each scalar is vectorized to this 826 /// many different vector instructions. 827 unsigned UF; 828 829 /// The builder that we use 830 IRBuilder<> Builder; 831 832 // --- Vectorization state --- 833 834 /// The vector-loop preheader. 835 BasicBlock *LoopVectorPreHeader; 836 837 /// The scalar-loop preheader. 838 BasicBlock *LoopScalarPreHeader; 839 840 /// Middle Block between the vector and the scalar. 841 BasicBlock *LoopMiddleBlock; 842 843 /// The (unique) ExitBlock of the scalar loop. Note that 844 /// there can be multiple exiting edges reaching this block. 845 BasicBlock *LoopExitBlock; 846 847 /// The vector loop body. 848 BasicBlock *LoopVectorBody; 849 850 /// The scalar loop body. 851 BasicBlock *LoopScalarBody; 852 853 /// A list of all bypass blocks. The first block is the entry of the loop. 854 SmallVector<BasicBlock *, 4> LoopBypassBlocks; 855 856 /// The new Induction variable which was added to the new block. 857 PHINode *Induction = nullptr; 858 859 /// The induction variable of the old basic block. 860 PHINode *OldInduction = nullptr; 861 862 /// Maps values from the original loop to their corresponding values in the 863 /// vectorized loop. A key value can map to either vector values, scalar 864 /// values or both kinds of values, depending on whether the key was 865 /// vectorized and scalarized. 866 VectorizerValueMap VectorLoopValueMap; 867 868 /// Store instructions that were predicated. 869 SmallVector<Instruction *, 4> PredicatedInstructions; 870 871 /// Trip count of the original loop. 872 Value *TripCount = nullptr; 873 874 /// Trip count of the widened loop (TripCount - TripCount % (VF*UF)) 875 Value *VectorTripCount = nullptr; 876 877 /// The legality analysis. 878 LoopVectorizationLegality *Legal; 879 880 /// The profitablity analysis. 881 LoopVectorizationCostModel *Cost; 882 883 // Record whether runtime checks are added. 884 bool AddedSafetyChecks = false; 885 886 // Holds the end values for each induction variable. We save the end values 887 // so we can later fix-up the external users of the induction variables. 888 DenseMap<PHINode *, Value *> IVEndValues; 889 890 // Vector of original scalar PHIs whose corresponding widened PHIs need to be 891 // fixed up at the end of vector code generation. 892 SmallVector<PHINode *, 8> OrigPHIsToFix; 893 894 /// BFI and PSI are used to check for profile guided size optimizations. 895 BlockFrequencyInfo *BFI; 896 ProfileSummaryInfo *PSI; 897 898 // Whether this loop should be optimized for size based on profile guided size 899 // optimizatios. 900 bool OptForSizeBasedOnProfile; 901 }; 902 903 class InnerLoopUnroller : public InnerLoopVectorizer { 904 public: 905 InnerLoopUnroller(Loop *OrigLoop, PredicatedScalarEvolution &PSE, 906 LoopInfo *LI, DominatorTree *DT, 907 const TargetLibraryInfo *TLI, 908 const TargetTransformInfo *TTI, AssumptionCache *AC, 909 OptimizationRemarkEmitter *ORE, unsigned UnrollFactor, 910 LoopVectorizationLegality *LVL, 911 LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, 912 ProfileSummaryInfo *PSI) 913 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE, 914 ElementCount::getFixed(1), UnrollFactor, LVL, CM, 915 BFI, PSI) {} 916 917 private: 918 Value *getBroadcastInstrs(Value *V) override; 919 Value *getStepVector(Value *Val, int StartIdx, Value *Step, 920 Instruction::BinaryOps Opcode = 921 Instruction::BinaryOpsEnd) override; 922 Value *reverseVector(Value *Vec) override; 923 }; 924 925 /// Encapsulate information regarding vectorization of a loop and its epilogue. 926 /// This information is meant to be updated and used across two stages of 927 /// epilogue vectorization. 928 struct EpilogueLoopVectorizationInfo { 929 ElementCount MainLoopVF = ElementCount::getFixed(0); 930 unsigned MainLoopUF = 0; 931 ElementCount EpilogueVF = ElementCount::getFixed(0); 932 unsigned EpilogueUF = 0; 933 BasicBlock *MainLoopIterationCountCheck = nullptr; 934 BasicBlock *EpilogueIterationCountCheck = nullptr; 935 BasicBlock *SCEVSafetyCheck = nullptr; 936 BasicBlock *MemSafetyCheck = nullptr; 937 Value *TripCount = nullptr; 938 Value *VectorTripCount = nullptr; 939 940 EpilogueLoopVectorizationInfo(unsigned MVF, unsigned MUF, unsigned EVF, 941 unsigned EUF) 942 : MainLoopVF(ElementCount::getFixed(MVF)), MainLoopUF(MUF), 943 EpilogueVF(ElementCount::getFixed(EVF)), EpilogueUF(EUF) { 944 assert(EUF == 1 && 945 "A high UF for the epilogue loop is likely not beneficial."); 946 } 947 }; 948 949 /// An extension of the inner loop vectorizer that creates a skeleton for a 950 /// vectorized loop that has its epilogue (residual) also vectorized. 951 /// The idea is to run the vplan on a given loop twice, firstly to setup the 952 /// skeleton and vectorize the main loop, and secondly to complete the skeleton 953 /// from the first step and vectorize the epilogue. This is achieved by 954 /// deriving two concrete strategy classes from this base class and invoking 955 /// them in succession from the loop vectorizer planner. 956 class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer { 957 public: 958 InnerLoopAndEpilogueVectorizer( 959 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, 960 DominatorTree *DT, const TargetLibraryInfo *TLI, 961 const TargetTransformInfo *TTI, AssumptionCache *AC, 962 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI, 963 LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM, 964 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI) 965 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE, 966 EPI.MainLoopVF, EPI.MainLoopUF, LVL, CM, BFI, PSI), 967 EPI(EPI) {} 968 969 // Override this function to handle the more complex control flow around the 970 // three loops. 971 BasicBlock *createVectorizedLoopSkeleton() final override { 972 return createEpilogueVectorizedLoopSkeleton(); 973 } 974 975 /// The interface for creating a vectorized skeleton using one of two 976 /// different strategies, each corresponding to one execution of the vplan 977 /// as described above. 978 virtual BasicBlock *createEpilogueVectorizedLoopSkeleton() = 0; 979 980 /// Holds and updates state information required to vectorize the main loop 981 /// and its epilogue in two separate passes. This setup helps us avoid 982 /// regenerating and recomputing runtime safety checks. It also helps us to 983 /// shorten the iteration-count-check path length for the cases where the 984 /// iteration count of the loop is so small that the main vector loop is 985 /// completely skipped. 986 EpilogueLoopVectorizationInfo &EPI; 987 }; 988 989 /// A specialized derived class of inner loop vectorizer that performs 990 /// vectorization of *main* loops in the process of vectorizing loops and their 991 /// epilogues. 992 class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer { 993 public: 994 EpilogueVectorizerMainLoop( 995 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, 996 DominatorTree *DT, const TargetLibraryInfo *TLI, 997 const TargetTransformInfo *TTI, AssumptionCache *AC, 998 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI, 999 LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM, 1000 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI) 1001 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE, 1002 EPI, LVL, CM, BFI, PSI) {} 1003 /// Implements the interface for creating a vectorized skeleton using the 1004 /// *main loop* strategy (ie the first pass of vplan execution). 1005 BasicBlock *createEpilogueVectorizedLoopSkeleton() final override; 1006 1007 protected: 1008 /// Emits an iteration count bypass check once for the main loop (when \p 1009 /// ForEpilogue is false) and once for the epilogue loop (when \p 1010 /// ForEpilogue is true). 1011 BasicBlock *emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass, 1012 bool ForEpilogue); 1013 void printDebugTracesAtStart() override; 1014 void printDebugTracesAtEnd() override; 1015 }; 1016 1017 // A specialized derived class of inner loop vectorizer that performs 1018 // vectorization of *epilogue* loops in the process of vectorizing loops and 1019 // their epilogues. 1020 class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer { 1021 public: 1022 EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, 1023 LoopInfo *LI, DominatorTree *DT, 1024 const TargetLibraryInfo *TLI, 1025 const TargetTransformInfo *TTI, AssumptionCache *AC, 1026 OptimizationRemarkEmitter *ORE, 1027 EpilogueLoopVectorizationInfo &EPI, 1028 LoopVectorizationLegality *LVL, 1029 llvm::LoopVectorizationCostModel *CM, 1030 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI) 1031 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE, 1032 EPI, LVL, CM, BFI, PSI) {} 1033 /// Implements the interface for creating a vectorized skeleton using the 1034 /// *epilogue loop* strategy (ie the second pass of vplan execution). 1035 BasicBlock *createEpilogueVectorizedLoopSkeleton() final override; 1036 1037 protected: 1038 /// Emits an iteration count bypass check after the main vector loop has 1039 /// finished to see if there are any iterations left to execute by either 1040 /// the vector epilogue or the scalar epilogue. 1041 BasicBlock *emitMinimumVectorEpilogueIterCountCheck(Loop *L, 1042 BasicBlock *Bypass, 1043 BasicBlock *Insert); 1044 void printDebugTracesAtStart() override; 1045 void printDebugTracesAtEnd() override; 1046 }; 1047 } // end namespace llvm 1048 1049 /// Look for a meaningful debug location on the instruction or it's 1050 /// operands. 1051 static Instruction *getDebugLocFromInstOrOperands(Instruction *I) { 1052 if (!I) 1053 return I; 1054 1055 DebugLoc Empty; 1056 if (I->getDebugLoc() != Empty) 1057 return I; 1058 1059 for (User::op_iterator OI = I->op_begin(), OE = I->op_end(); OI != OE; ++OI) { 1060 if (Instruction *OpInst = dyn_cast<Instruction>(*OI)) 1061 if (OpInst->getDebugLoc() != Empty) 1062 return OpInst; 1063 } 1064 1065 return I; 1066 } 1067 1068 void InnerLoopVectorizer::setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr) { 1069 if (const Instruction *Inst = dyn_cast_or_null<Instruction>(Ptr)) { 1070 const DILocation *DIL = Inst->getDebugLoc(); 1071 if (DIL && Inst->getFunction()->isDebugInfoForProfiling() && 1072 !isa<DbgInfoIntrinsic>(Inst)) { 1073 assert(!VF.isScalable() && "scalable vectors not yet supported."); 1074 auto NewDIL = 1075 DIL->cloneByMultiplyingDuplicationFactor(UF * VF.getKnownMinValue()); 1076 if (NewDIL) 1077 B.SetCurrentDebugLocation(NewDIL.getValue()); 1078 else 1079 LLVM_DEBUG(dbgs() 1080 << "Failed to create new discriminator: " 1081 << DIL->getFilename() << " Line: " << DIL->getLine()); 1082 } 1083 else 1084 B.SetCurrentDebugLocation(DIL); 1085 } else 1086 B.SetCurrentDebugLocation(DebugLoc()); 1087 } 1088 1089 /// Write a record \p DebugMsg about vectorization failure to the debug 1090 /// output stream. If \p I is passed, it is an instruction that prevents 1091 /// vectorization. 1092 #ifndef NDEBUG 1093 static void debugVectorizationFailure(const StringRef DebugMsg, 1094 Instruction *I) { 1095 dbgs() << "LV: Not vectorizing: " << DebugMsg; 1096 if (I != nullptr) 1097 dbgs() << " " << *I; 1098 else 1099 dbgs() << '.'; 1100 dbgs() << '\n'; 1101 } 1102 #endif 1103 1104 /// Create an analysis remark that explains why vectorization failed 1105 /// 1106 /// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p 1107 /// RemarkName is the identifier for the remark. If \p I is passed it is an 1108 /// instruction that prevents vectorization. Otherwise \p TheLoop is used for 1109 /// the location of the remark. \return the remark object that can be 1110 /// streamed to. 1111 static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName, 1112 StringRef RemarkName, Loop *TheLoop, Instruction *I) { 1113 Value *CodeRegion = TheLoop->getHeader(); 1114 DebugLoc DL = TheLoop->getStartLoc(); 1115 1116 if (I) { 1117 CodeRegion = I->getParent(); 1118 // If there is no debug location attached to the instruction, revert back to 1119 // using the loop's. 1120 if (I->getDebugLoc()) 1121 DL = I->getDebugLoc(); 1122 } 1123 1124 OptimizationRemarkAnalysis R(PassName, RemarkName, DL, CodeRegion); 1125 R << "loop not vectorized: "; 1126 return R; 1127 } 1128 1129 /// Return a value for Step multiplied by VF. 1130 static Value *createStepForVF(IRBuilder<> &B, Constant *Step, ElementCount VF) { 1131 assert(isa<ConstantInt>(Step) && "Expected an integer step"); 1132 Constant *StepVal = ConstantInt::get( 1133 Step->getType(), 1134 cast<ConstantInt>(Step)->getSExtValue() * VF.getKnownMinValue()); 1135 return VF.isScalable() ? B.CreateVScale(StepVal) : StepVal; 1136 } 1137 1138 namespace llvm { 1139 1140 void reportVectorizationFailure(const StringRef DebugMsg, 1141 const StringRef OREMsg, const StringRef ORETag, 1142 OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I) { 1143 LLVM_DEBUG(debugVectorizationFailure(DebugMsg, I)); 1144 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE); 1145 ORE->emit(createLVAnalysis(Hints.vectorizeAnalysisPassName(), 1146 ORETag, TheLoop, I) << OREMsg); 1147 } 1148 1149 } // end namespace llvm 1150 1151 #ifndef NDEBUG 1152 /// \return string containing a file name and a line # for the given loop. 1153 static std::string getDebugLocString(const Loop *L) { 1154 std::string Result; 1155 if (L) { 1156 raw_string_ostream OS(Result); 1157 if (const DebugLoc LoopDbgLoc = L->getStartLoc()) 1158 LoopDbgLoc.print(OS); 1159 else 1160 // Just print the module name. 1161 OS << L->getHeader()->getParent()->getParent()->getModuleIdentifier(); 1162 OS.flush(); 1163 } 1164 return Result; 1165 } 1166 #endif 1167 1168 void InnerLoopVectorizer::addNewMetadata(Instruction *To, 1169 const Instruction *Orig) { 1170 // If the loop was versioned with memchecks, add the corresponding no-alias 1171 // metadata. 1172 if (LVer && (isa<LoadInst>(Orig) || isa<StoreInst>(Orig))) 1173 LVer->annotateInstWithNoAlias(To, Orig); 1174 } 1175 1176 void InnerLoopVectorizer::addMetadata(Instruction *To, 1177 Instruction *From) { 1178 propagateMetadata(To, From); 1179 addNewMetadata(To, From); 1180 } 1181 1182 void InnerLoopVectorizer::addMetadata(ArrayRef<Value *> To, 1183 Instruction *From) { 1184 for (Value *V : To) { 1185 if (Instruction *I = dyn_cast<Instruction>(V)) 1186 addMetadata(I, From); 1187 } 1188 } 1189 1190 namespace llvm { 1191 1192 // Loop vectorization cost-model hints how the scalar epilogue loop should be 1193 // lowered. 1194 enum ScalarEpilogueLowering { 1195 1196 // The default: allowing scalar epilogues. 1197 CM_ScalarEpilogueAllowed, 1198 1199 // Vectorization with OptForSize: don't allow epilogues. 1200 CM_ScalarEpilogueNotAllowedOptSize, 1201 1202 // A special case of vectorisation with OptForSize: loops with a very small 1203 // trip count are considered for vectorization under OptForSize, thereby 1204 // making sure the cost of their loop body is dominant, free of runtime 1205 // guards and scalar iteration overheads. 1206 CM_ScalarEpilogueNotAllowedLowTripLoop, 1207 1208 // Loop hint predicate indicating an epilogue is undesired. 1209 CM_ScalarEpilogueNotNeededUsePredicate, 1210 1211 // Directive indicating we must either tail fold or not vectorize 1212 CM_ScalarEpilogueNotAllowedUsePredicate 1213 }; 1214 1215 /// LoopVectorizationCostModel - estimates the expected speedups due to 1216 /// vectorization. 1217 /// In many cases vectorization is not profitable. This can happen because of 1218 /// a number of reasons. In this class we mainly attempt to predict the 1219 /// expected speedup/slowdowns due to the supported instruction set. We use the 1220 /// TargetTransformInfo to query the different backends for the cost of 1221 /// different operations. 1222 class LoopVectorizationCostModel { 1223 public: 1224 LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L, 1225 PredicatedScalarEvolution &PSE, LoopInfo *LI, 1226 LoopVectorizationLegality *Legal, 1227 const TargetTransformInfo &TTI, 1228 const TargetLibraryInfo *TLI, DemandedBits *DB, 1229 AssumptionCache *AC, 1230 OptimizationRemarkEmitter *ORE, const Function *F, 1231 const LoopVectorizeHints *Hints, 1232 InterleavedAccessInfo &IAI) 1233 : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal), 1234 TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F), 1235 Hints(Hints), InterleaveInfo(IAI) {} 1236 1237 /// \return An upper bound for the vectorization factor, or None if 1238 /// vectorization and interleaving should be avoided up front. 1239 Optional<ElementCount> computeMaxVF(ElementCount UserVF, unsigned UserIC); 1240 1241 /// \return True if runtime checks are required for vectorization, and false 1242 /// otherwise. 1243 bool runtimeChecksRequired(); 1244 1245 /// \return The most profitable vectorization factor and the cost of that VF. 1246 /// This method checks every power of two up to MaxVF. If UserVF is not ZERO 1247 /// then this vectorization factor will be selected if vectorization is 1248 /// possible. 1249 VectorizationFactor selectVectorizationFactor(ElementCount MaxVF); 1250 VectorizationFactor 1251 selectEpilogueVectorizationFactor(const ElementCount MaxVF, 1252 const LoopVectorizationPlanner &LVP); 1253 1254 /// Setup cost-based decisions for user vectorization factor. 1255 void selectUserVectorizationFactor(ElementCount UserVF) { 1256 collectUniformsAndScalars(UserVF); 1257 collectInstsToScalarize(UserVF); 1258 } 1259 1260 /// \return The size (in bits) of the smallest and widest types in the code 1261 /// that needs to be vectorized. We ignore values that remain scalar such as 1262 /// 64 bit loop indices. 1263 std::pair<unsigned, unsigned> getSmallestAndWidestTypes(); 1264 1265 /// \return The desired interleave count. 1266 /// If interleave count has been specified by metadata it will be returned. 1267 /// Otherwise, the interleave count is computed and returned. VF and LoopCost 1268 /// are the selected vectorization factor and the cost of the selected VF. 1269 unsigned selectInterleaveCount(ElementCount VF, unsigned LoopCost); 1270 1271 /// Memory access instruction may be vectorized in more than one way. 1272 /// Form of instruction after vectorization depends on cost. 1273 /// This function takes cost-based decisions for Load/Store instructions 1274 /// and collects them in a map. This decisions map is used for building 1275 /// the lists of loop-uniform and loop-scalar instructions. 1276 /// The calculated cost is saved with widening decision in order to 1277 /// avoid redundant calculations. 1278 void setCostBasedWideningDecision(ElementCount VF); 1279 1280 /// A struct that represents some properties of the register usage 1281 /// of a loop. 1282 struct RegisterUsage { 1283 /// Holds the number of loop invariant values that are used in the loop. 1284 /// The key is ClassID of target-provided register class. 1285 SmallMapVector<unsigned, unsigned, 4> LoopInvariantRegs; 1286 /// Holds the maximum number of concurrent live intervals in the loop. 1287 /// The key is ClassID of target-provided register class. 1288 SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers; 1289 }; 1290 1291 /// \return Returns information about the register usages of the loop for the 1292 /// given vectorization factors. 1293 SmallVector<RegisterUsage, 8> 1294 calculateRegisterUsage(ArrayRef<ElementCount> VFs); 1295 1296 /// Collect values we want to ignore in the cost model. 1297 void collectValuesToIgnore(); 1298 1299 /// Split reductions into those that happen in the loop, and those that happen 1300 /// outside. In loop reductions are collected into InLoopReductionChains. 1301 void collectInLoopReductions(); 1302 1303 /// \returns The smallest bitwidth each instruction can be represented with. 1304 /// The vector equivalents of these instructions should be truncated to this 1305 /// type. 1306 const MapVector<Instruction *, uint64_t> &getMinimalBitwidths() const { 1307 return MinBWs; 1308 } 1309 1310 /// \returns True if it is more profitable to scalarize instruction \p I for 1311 /// vectorization factor \p VF. 1312 bool isProfitableToScalarize(Instruction *I, ElementCount VF) const { 1313 assert(VF.isVector() && 1314 "Profitable to scalarize relevant only for VF > 1."); 1315 1316 // Cost model is not run in the VPlan-native path - return conservative 1317 // result until this changes. 1318 if (EnableVPlanNativePath) 1319 return false; 1320 1321 auto Scalars = InstsToScalarize.find(VF); 1322 assert(Scalars != InstsToScalarize.end() && 1323 "VF not yet analyzed for scalarization profitability"); 1324 return Scalars->second.find(I) != Scalars->second.end(); 1325 } 1326 1327 /// Returns true if \p I is known to be uniform after vectorization. 1328 bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const { 1329 if (VF.isScalar()) 1330 return true; 1331 1332 // Cost model is not run in the VPlan-native path - return conservative 1333 // result until this changes. 1334 if (EnableVPlanNativePath) 1335 return false; 1336 1337 auto UniformsPerVF = Uniforms.find(VF); 1338 assert(UniformsPerVF != Uniforms.end() && 1339 "VF not yet analyzed for uniformity"); 1340 return UniformsPerVF->second.count(I); 1341 } 1342 1343 /// Returns true if \p I is known to be scalar after vectorization. 1344 bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const { 1345 if (VF.isScalar()) 1346 return true; 1347 1348 // Cost model is not run in the VPlan-native path - return conservative 1349 // result until this changes. 1350 if (EnableVPlanNativePath) 1351 return false; 1352 1353 auto ScalarsPerVF = Scalars.find(VF); 1354 assert(ScalarsPerVF != Scalars.end() && 1355 "Scalar values are not calculated for VF"); 1356 return ScalarsPerVF->second.count(I); 1357 } 1358 1359 /// \returns True if instruction \p I can be truncated to a smaller bitwidth 1360 /// for vectorization factor \p VF. 1361 bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const { 1362 return VF.isVector() && MinBWs.find(I) != MinBWs.end() && 1363 !isProfitableToScalarize(I, VF) && 1364 !isScalarAfterVectorization(I, VF); 1365 } 1366 1367 /// Decision that was taken during cost calculation for memory instruction. 1368 enum InstWidening { 1369 CM_Unknown, 1370 CM_Widen, // For consecutive accesses with stride +1. 1371 CM_Widen_Reverse, // For consecutive accesses with stride -1. 1372 CM_Interleave, 1373 CM_GatherScatter, 1374 CM_Scalarize 1375 }; 1376 1377 /// Save vectorization decision \p W and \p Cost taken by the cost model for 1378 /// instruction \p I and vector width \p VF. 1379 void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W, 1380 InstructionCost Cost) { 1381 assert(VF.isVector() && "Expected VF >=2"); 1382 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost); 1383 } 1384 1385 /// Save vectorization decision \p W and \p Cost taken by the cost model for 1386 /// interleaving group \p Grp and vector width \p VF. 1387 void setWideningDecision(const InterleaveGroup<Instruction> *Grp, 1388 ElementCount VF, InstWidening W, 1389 InstructionCost Cost) { 1390 assert(VF.isVector() && "Expected VF >=2"); 1391 /// Broadcast this decicion to all instructions inside the group. 1392 /// But the cost will be assigned to one instruction only. 1393 for (unsigned i = 0; i < Grp->getFactor(); ++i) { 1394 if (auto *I = Grp->getMember(i)) { 1395 if (Grp->getInsertPos() == I) 1396 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost); 1397 else 1398 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, 0); 1399 } 1400 } 1401 } 1402 1403 /// Return the cost model decision for the given instruction \p I and vector 1404 /// width \p VF. Return CM_Unknown if this instruction did not pass 1405 /// through the cost modeling. 1406 InstWidening getWideningDecision(Instruction *I, ElementCount VF) { 1407 assert(VF.isVector() && "Expected VF to be a vector VF"); 1408 // Cost model is not run in the VPlan-native path - return conservative 1409 // result until this changes. 1410 if (EnableVPlanNativePath) 1411 return CM_GatherScatter; 1412 1413 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF); 1414 auto Itr = WideningDecisions.find(InstOnVF); 1415 if (Itr == WideningDecisions.end()) 1416 return CM_Unknown; 1417 return Itr->second.first; 1418 } 1419 1420 /// Return the vectorization cost for the given instruction \p I and vector 1421 /// width \p VF. 1422 InstructionCost getWideningCost(Instruction *I, ElementCount VF) { 1423 assert(VF.isVector() && "Expected VF >=2"); 1424 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF); 1425 assert(WideningDecisions.find(InstOnVF) != WideningDecisions.end() && 1426 "The cost is not calculated"); 1427 return WideningDecisions[InstOnVF].second; 1428 } 1429 1430 /// Return True if instruction \p I is an optimizable truncate whose operand 1431 /// is an induction variable. Such a truncate will be removed by adding a new 1432 /// induction variable with the destination type. 1433 bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) { 1434 // If the instruction is not a truncate, return false. 1435 auto *Trunc = dyn_cast<TruncInst>(I); 1436 if (!Trunc) 1437 return false; 1438 1439 // Get the source and destination types of the truncate. 1440 Type *SrcTy = ToVectorTy(cast<CastInst>(I)->getSrcTy(), VF); 1441 Type *DestTy = ToVectorTy(cast<CastInst>(I)->getDestTy(), VF); 1442 1443 // If the truncate is free for the given types, return false. Replacing a 1444 // free truncate with an induction variable would add an induction variable 1445 // update instruction to each iteration of the loop. We exclude from this 1446 // check the primary induction variable since it will need an update 1447 // instruction regardless. 1448 Value *Op = Trunc->getOperand(0); 1449 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy)) 1450 return false; 1451 1452 // If the truncated value is not an induction variable, return false. 1453 return Legal->isInductionPhi(Op); 1454 } 1455 1456 /// Collects the instructions to scalarize for each predicated instruction in 1457 /// the loop. 1458 void collectInstsToScalarize(ElementCount VF); 1459 1460 /// Collect Uniform and Scalar values for the given \p VF. 1461 /// The sets depend on CM decision for Load/Store instructions 1462 /// that may be vectorized as interleave, gather-scatter or scalarized. 1463 void collectUniformsAndScalars(ElementCount VF) { 1464 // Do the analysis once. 1465 if (VF.isScalar() || Uniforms.find(VF) != Uniforms.end()) 1466 return; 1467 setCostBasedWideningDecision(VF); 1468 collectLoopUniforms(VF); 1469 collectLoopScalars(VF); 1470 } 1471 1472 /// Returns true if the target machine supports masked store operation 1473 /// for the given \p DataType and kind of access to \p Ptr. 1474 bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) { 1475 return Legal->isConsecutivePtr(Ptr) && 1476 TTI.isLegalMaskedStore(DataType, Alignment); 1477 } 1478 1479 /// Returns true if the target machine supports masked load operation 1480 /// for the given \p DataType and kind of access to \p Ptr. 1481 bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) { 1482 return Legal->isConsecutivePtr(Ptr) && 1483 TTI.isLegalMaskedLoad(DataType, Alignment); 1484 } 1485 1486 /// Returns true if the target machine supports masked scatter operation 1487 /// for the given \p DataType. 1488 bool isLegalMaskedScatter(Type *DataType, Align Alignment) { 1489 return TTI.isLegalMaskedScatter(DataType, Alignment); 1490 } 1491 1492 /// Returns true if the target machine supports masked gather operation 1493 /// for the given \p DataType. 1494 bool isLegalMaskedGather(Type *DataType, Align Alignment) { 1495 return TTI.isLegalMaskedGather(DataType, Alignment); 1496 } 1497 1498 /// Returns true if the target machine can represent \p V as a masked gather 1499 /// or scatter operation. 1500 bool isLegalGatherOrScatter(Value *V) { 1501 bool LI = isa<LoadInst>(V); 1502 bool SI = isa<StoreInst>(V); 1503 if (!LI && !SI) 1504 return false; 1505 auto *Ty = getMemInstValueType(V); 1506 Align Align = getLoadStoreAlignment(V); 1507 return (LI && isLegalMaskedGather(Ty, Align)) || 1508 (SI && isLegalMaskedScatter(Ty, Align)); 1509 } 1510 1511 /// Returns true if \p I is an instruction that will be scalarized with 1512 /// predication. Such instructions include conditional stores and 1513 /// instructions that may divide by zero. 1514 /// If a non-zero VF has been calculated, we check if I will be scalarized 1515 /// predication for that VF. 1516 bool isScalarWithPredication(Instruction *I, 1517 ElementCount VF = ElementCount::getFixed(1)); 1518 1519 // Returns true if \p I is an instruction that will be predicated either 1520 // through scalar predication or masked load/store or masked gather/scatter. 1521 // Superset of instructions that return true for isScalarWithPredication. 1522 bool isPredicatedInst(Instruction *I) { 1523 if (!blockNeedsPredication(I->getParent())) 1524 return false; 1525 // Loads and stores that need some form of masked operation are predicated 1526 // instructions. 1527 if (isa<LoadInst>(I) || isa<StoreInst>(I)) 1528 return Legal->isMaskRequired(I); 1529 return isScalarWithPredication(I); 1530 } 1531 1532 /// Returns true if \p I is a memory instruction with consecutive memory 1533 /// access that can be widened. 1534 bool 1535 memoryInstructionCanBeWidened(Instruction *I, 1536 ElementCount VF = ElementCount::getFixed(1)); 1537 1538 /// Returns true if \p I is a memory instruction in an interleaved-group 1539 /// of memory accesses that can be vectorized with wide vector loads/stores 1540 /// and shuffles. 1541 bool 1542 interleavedAccessCanBeWidened(Instruction *I, 1543 ElementCount VF = ElementCount::getFixed(1)); 1544 1545 /// Check if \p Instr belongs to any interleaved access group. 1546 bool isAccessInterleaved(Instruction *Instr) { 1547 return InterleaveInfo.isInterleaved(Instr); 1548 } 1549 1550 /// Get the interleaved access group that \p Instr belongs to. 1551 const InterleaveGroup<Instruction> * 1552 getInterleavedAccessGroup(Instruction *Instr) { 1553 return InterleaveInfo.getInterleaveGroup(Instr); 1554 } 1555 1556 /// Returns true if we're required to use a scalar epilogue for at least 1557 /// the final iteration of the original loop. 1558 bool requiresScalarEpilogue() const { 1559 if (!isScalarEpilogueAllowed()) 1560 return false; 1561 // If we might exit from anywhere but the latch, must run the exiting 1562 // iteration in scalar form. 1563 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) 1564 return true; 1565 return InterleaveInfo.requiresScalarEpilogue(); 1566 } 1567 1568 /// Returns true if a scalar epilogue is not allowed due to optsize or a 1569 /// loop hint annotation. 1570 bool isScalarEpilogueAllowed() const { 1571 return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed; 1572 } 1573 1574 /// Returns true if all loop blocks should be masked to fold tail loop. 1575 bool foldTailByMasking() const { return FoldTailByMasking; } 1576 1577 bool blockNeedsPredication(BasicBlock *BB) { 1578 return foldTailByMasking() || Legal->blockNeedsPredication(BB); 1579 } 1580 1581 /// A SmallMapVector to store the InLoop reduction op chains, mapping phi 1582 /// nodes to the chain of instructions representing the reductions. Uses a 1583 /// MapVector to ensure deterministic iteration order. 1584 using ReductionChainMap = 1585 SmallMapVector<PHINode *, SmallVector<Instruction *, 4>, 4>; 1586 1587 /// Return the chain of instructions representing an inloop reduction. 1588 const ReductionChainMap &getInLoopReductionChains() const { 1589 return InLoopReductionChains; 1590 } 1591 1592 /// Returns true if the Phi is part of an inloop reduction. 1593 bool isInLoopReduction(PHINode *Phi) const { 1594 return InLoopReductionChains.count(Phi); 1595 } 1596 1597 /// Estimate cost of an intrinsic call instruction CI if it were vectorized 1598 /// with factor VF. Return the cost of the instruction, including 1599 /// scalarization overhead if it's needed. 1600 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF); 1601 1602 /// Estimate cost of a call instruction CI if it were vectorized with factor 1603 /// VF. Return the cost of the instruction, including scalarization overhead 1604 /// if it's needed. The flag NeedToScalarize shows if the call needs to be 1605 /// scalarized - 1606 /// i.e. either vector version isn't available, or is too expensive. 1607 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF, 1608 bool &NeedToScalarize); 1609 1610 /// Invalidates decisions already taken by the cost model. 1611 void invalidateCostModelingDecisions() { 1612 WideningDecisions.clear(); 1613 Uniforms.clear(); 1614 Scalars.clear(); 1615 } 1616 1617 private: 1618 unsigned NumPredStores = 0; 1619 1620 /// \return An upper bound for the vectorization factor, a power-of-2 larger 1621 /// than zero. One is returned if vectorization should best be avoided due 1622 /// to cost. 1623 ElementCount computeFeasibleMaxVF(unsigned ConstTripCount, 1624 ElementCount UserVF); 1625 1626 /// The vectorization cost is a combination of the cost itself and a boolean 1627 /// indicating whether any of the contributing operations will actually 1628 /// operate on 1629 /// vector values after type legalization in the backend. If this latter value 1630 /// is 1631 /// false, then all operations will be scalarized (i.e. no vectorization has 1632 /// actually taken place). 1633 using VectorizationCostTy = std::pair<InstructionCost, bool>; 1634 1635 /// Returns the expected execution cost. The unit of the cost does 1636 /// not matter because we use the 'cost' units to compare different 1637 /// vector widths. The cost that is returned is *not* normalized by 1638 /// the factor width. 1639 VectorizationCostTy expectedCost(ElementCount VF); 1640 1641 /// Returns the execution time cost of an instruction for a given vector 1642 /// width. Vector width of one means scalar. 1643 VectorizationCostTy getInstructionCost(Instruction *I, ElementCount VF); 1644 1645 /// The cost-computation logic from getInstructionCost which provides 1646 /// the vector type as an output parameter. 1647 InstructionCost getInstructionCost(Instruction *I, ElementCount VF, 1648 Type *&VectorTy); 1649 1650 /// Return the cost of instructions in an inloop reduction pattern, if I is 1651 /// part of that pattern. 1652 InstructionCost getReductionPatternCost(Instruction *I, ElementCount VF, 1653 Type *VectorTy, 1654 TTI::TargetCostKind CostKind); 1655 1656 /// Calculate vectorization cost of memory instruction \p I. 1657 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF); 1658 1659 /// The cost computation for scalarized memory instruction. 1660 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF); 1661 1662 /// The cost computation for interleaving group of memory instructions. 1663 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF); 1664 1665 /// The cost computation for Gather/Scatter instruction. 1666 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF); 1667 1668 /// The cost computation for widening instruction \p I with consecutive 1669 /// memory access. 1670 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF); 1671 1672 /// The cost calculation for Load/Store instruction \p I with uniform pointer - 1673 /// Load: scalar load + broadcast. 1674 /// Store: scalar store + (loop invariant value stored? 0 : extract of last 1675 /// element) 1676 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF); 1677 1678 /// Estimate the overhead of scalarizing an instruction. This is a 1679 /// convenience wrapper for the type-based getScalarizationOverhead API. 1680 InstructionCost getScalarizationOverhead(Instruction *I, ElementCount VF); 1681 1682 /// Returns whether the instruction is a load or store and will be a emitted 1683 /// as a vector operation. 1684 bool isConsecutiveLoadOrStore(Instruction *I); 1685 1686 /// Returns true if an artificially high cost for emulated masked memrefs 1687 /// should be used. 1688 bool useEmulatedMaskMemRefHack(Instruction *I); 1689 1690 /// Map of scalar integer values to the smallest bitwidth they can be legally 1691 /// represented as. The vector equivalents of these values should be truncated 1692 /// to this type. 1693 MapVector<Instruction *, uint64_t> MinBWs; 1694 1695 /// A type representing the costs for instructions if they were to be 1696 /// scalarized rather than vectorized. The entries are Instruction-Cost 1697 /// pairs. 1698 using ScalarCostsTy = DenseMap<Instruction *, InstructionCost>; 1699 1700 /// A set containing all BasicBlocks that are known to present after 1701 /// vectorization as a predicated block. 1702 SmallPtrSet<BasicBlock *, 4> PredicatedBBsAfterVectorization; 1703 1704 /// Records whether it is allowed to have the original scalar loop execute at 1705 /// least once. This may be needed as a fallback loop in case runtime 1706 /// aliasing/dependence checks fail, or to handle the tail/remainder 1707 /// iterations when the trip count is unknown or doesn't divide by the VF, 1708 /// or as a peel-loop to handle gaps in interleave-groups. 1709 /// Under optsize and when the trip count is very small we don't allow any 1710 /// iterations to execute in the scalar loop. 1711 ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed; 1712 1713 /// All blocks of loop are to be masked to fold tail of scalar iterations. 1714 bool FoldTailByMasking = false; 1715 1716 /// A map holding scalar costs for different vectorization factors. The 1717 /// presence of a cost for an instruction in the mapping indicates that the 1718 /// instruction will be scalarized when vectorizing with the associated 1719 /// vectorization factor. The entries are VF-ScalarCostTy pairs. 1720 DenseMap<ElementCount, ScalarCostsTy> InstsToScalarize; 1721 1722 /// Holds the instructions known to be uniform after vectorization. 1723 /// The data is collected per VF. 1724 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms; 1725 1726 /// Holds the instructions known to be scalar after vectorization. 1727 /// The data is collected per VF. 1728 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars; 1729 1730 /// Holds the instructions (address computations) that are forced to be 1731 /// scalarized. 1732 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars; 1733 1734 /// PHINodes of the reductions that should be expanded in-loop along with 1735 /// their associated chains of reduction operations, in program order from top 1736 /// (PHI) to bottom 1737 ReductionChainMap InLoopReductionChains; 1738 1739 /// A Map of inloop reduction operations and their immediate chain operand. 1740 /// FIXME: This can be removed once reductions can be costed correctly in 1741 /// vplan. This was added to allow quick lookup to the inloop operations, 1742 /// without having to loop through InLoopReductionChains. 1743 DenseMap<Instruction *, Instruction *> InLoopReductionImmediateChains; 1744 1745 /// Returns the expected difference in cost from scalarizing the expression 1746 /// feeding a predicated instruction \p PredInst. The instructions to 1747 /// scalarize and their scalar costs are collected in \p ScalarCosts. A 1748 /// non-negative return value implies the expression will be scalarized. 1749 /// Currently, only single-use chains are considered for scalarization. 1750 int computePredInstDiscount(Instruction *PredInst, ScalarCostsTy &ScalarCosts, 1751 ElementCount VF); 1752 1753 /// Collect the instructions that are uniform after vectorization. An 1754 /// instruction is uniform if we represent it with a single scalar value in 1755 /// the vectorized loop corresponding to each vector iteration. Examples of 1756 /// uniform instructions include pointer operands of consecutive or 1757 /// interleaved memory accesses. Note that although uniformity implies an 1758 /// instruction will be scalar, the reverse is not true. In general, a 1759 /// scalarized instruction will be represented by VF scalar values in the 1760 /// vectorized loop, each corresponding to an iteration of the original 1761 /// scalar loop. 1762 void collectLoopUniforms(ElementCount VF); 1763 1764 /// Collect the instructions that are scalar after vectorization. An 1765 /// instruction is scalar if it is known to be uniform or will be scalarized 1766 /// during vectorization. Non-uniform scalarized instructions will be 1767 /// represented by VF values in the vectorized loop, each corresponding to an 1768 /// iteration of the original scalar loop. 1769 void collectLoopScalars(ElementCount VF); 1770 1771 /// Keeps cost model vectorization decision and cost for instructions. 1772 /// Right now it is used for memory instructions only. 1773 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>, 1774 std::pair<InstWidening, InstructionCost>>; 1775 1776 DecisionList WideningDecisions; 1777 1778 /// Returns true if \p V is expected to be vectorized and it needs to be 1779 /// extracted. 1780 bool needsExtract(Value *V, ElementCount VF) const { 1781 Instruction *I = dyn_cast<Instruction>(V); 1782 if (VF.isScalar() || !I || !TheLoop->contains(I) || 1783 TheLoop->isLoopInvariant(I)) 1784 return false; 1785 1786 // Assume we can vectorize V (and hence we need extraction) if the 1787 // scalars are not computed yet. This can happen, because it is called 1788 // via getScalarizationOverhead from setCostBasedWideningDecision, before 1789 // the scalars are collected. That should be a safe assumption in most 1790 // cases, because we check if the operands have vectorizable types 1791 // beforehand in LoopVectorizationLegality. 1792 return Scalars.find(VF) == Scalars.end() || 1793 !isScalarAfterVectorization(I, VF); 1794 }; 1795 1796 /// Returns a range containing only operands needing to be extracted. 1797 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops, 1798 ElementCount VF) { 1799 return SmallVector<Value *, 4>(make_filter_range( 1800 Ops, [this, VF](Value *V) { return this->needsExtract(V, VF); })); 1801 } 1802 1803 /// Determines if we have the infrastructure to vectorize loop \p L and its 1804 /// epilogue, assuming the main loop is vectorized by \p VF. 1805 bool isCandidateForEpilogueVectorization(const Loop &L, 1806 const ElementCount VF) const; 1807 1808 /// Returns true if epilogue vectorization is considered profitable, and 1809 /// false otherwise. 1810 /// \p VF is the vectorization factor chosen for the original loop. 1811 bool isEpilogueVectorizationProfitable(const ElementCount VF) const; 1812 1813 public: 1814 /// The loop that we evaluate. 1815 Loop *TheLoop; 1816 1817 /// Predicated scalar evolution analysis. 1818 PredicatedScalarEvolution &PSE; 1819 1820 /// Loop Info analysis. 1821 LoopInfo *LI; 1822 1823 /// Vectorization legality. 1824 LoopVectorizationLegality *Legal; 1825 1826 /// Vector target information. 1827 const TargetTransformInfo &TTI; 1828 1829 /// Target Library Info. 1830 const TargetLibraryInfo *TLI; 1831 1832 /// Demanded bits analysis. 1833 DemandedBits *DB; 1834 1835 /// Assumption cache. 1836 AssumptionCache *AC; 1837 1838 /// Interface to emit optimization remarks. 1839 OptimizationRemarkEmitter *ORE; 1840 1841 const Function *TheFunction; 1842 1843 /// Loop Vectorize Hint. 1844 const LoopVectorizeHints *Hints; 1845 1846 /// The interleave access information contains groups of interleaved accesses 1847 /// with the same stride and close to each other. 1848 InterleavedAccessInfo &InterleaveInfo; 1849 1850 /// Values to ignore in the cost model. 1851 SmallPtrSet<const Value *, 16> ValuesToIgnore; 1852 1853 /// Values to ignore in the cost model when VF > 1. 1854 SmallPtrSet<const Value *, 16> VecValuesToIgnore; 1855 1856 /// Profitable vector factors. 1857 SmallVector<VectorizationFactor, 8> ProfitableVFs; 1858 }; 1859 1860 } // end namespace llvm 1861 1862 // Return true if \p OuterLp is an outer loop annotated with hints for explicit 1863 // vectorization. The loop needs to be annotated with #pragma omp simd 1864 // simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the 1865 // vector length information is not provided, vectorization is not considered 1866 // explicit. Interleave hints are not allowed either. These limitations will be 1867 // relaxed in the future. 1868 // Please, note that we are currently forced to abuse the pragma 'clang 1869 // vectorize' semantics. This pragma provides *auto-vectorization hints* 1870 // (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd' 1871 // provides *explicit vectorization hints* (LV can bypass legal checks and 1872 // assume that vectorization is legal). However, both hints are implemented 1873 // using the same metadata (llvm.loop.vectorize, processed by 1874 // LoopVectorizeHints). This will be fixed in the future when the native IR 1875 // representation for pragma 'omp simd' is introduced. 1876 static bool isExplicitVecOuterLoop(Loop *OuterLp, 1877 OptimizationRemarkEmitter *ORE) { 1878 assert(!OuterLp->isInnermost() && "This is not an outer loop"); 1879 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE); 1880 1881 // Only outer loops with an explicit vectorization hint are supported. 1882 // Unannotated outer loops are ignored. 1883 if (Hints.getForce() == LoopVectorizeHints::FK_Undefined) 1884 return false; 1885 1886 Function *Fn = OuterLp->getHeader()->getParent(); 1887 if (!Hints.allowVectorization(Fn, OuterLp, 1888 true /*VectorizeOnlyWhenForced*/)) { 1889 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n"); 1890 return false; 1891 } 1892 1893 if (Hints.getInterleave() > 1) { 1894 // TODO: Interleave support is future work. 1895 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for " 1896 "outer loops.\n"); 1897 Hints.emitRemarkWithHints(); 1898 return false; 1899 } 1900 1901 return true; 1902 } 1903 1904 static void collectSupportedLoops(Loop &L, LoopInfo *LI, 1905 OptimizationRemarkEmitter *ORE, 1906 SmallVectorImpl<Loop *> &V) { 1907 // Collect inner loops and outer loops without irreducible control flow. For 1908 // now, only collect outer loops that have explicit vectorization hints. If we 1909 // are stress testing the VPlan H-CFG construction, we collect the outermost 1910 // loop of every loop nest. 1911 if (L.isInnermost() || VPlanBuildStressTest || 1912 (EnableVPlanNativePath && isExplicitVecOuterLoop(&L, ORE))) { 1913 LoopBlocksRPO RPOT(&L); 1914 RPOT.perform(LI); 1915 if (!containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI)) { 1916 V.push_back(&L); 1917 // TODO: Collect inner loops inside marked outer loops in case 1918 // vectorization fails for the outer loop. Do not invoke 1919 // 'containsIrreducibleCFG' again for inner loops when the outer loop is 1920 // already known to be reducible. We can use an inherited attribute for 1921 // that. 1922 return; 1923 } 1924 } 1925 for (Loop *InnerL : L) 1926 collectSupportedLoops(*InnerL, LI, ORE, V); 1927 } 1928 1929 namespace { 1930 1931 /// The LoopVectorize Pass. 1932 struct LoopVectorize : public FunctionPass { 1933 /// Pass identification, replacement for typeid 1934 static char ID; 1935 1936 LoopVectorizePass Impl; 1937 1938 explicit LoopVectorize(bool InterleaveOnlyWhenForced = false, 1939 bool VectorizeOnlyWhenForced = false) 1940 : FunctionPass(ID), 1941 Impl({InterleaveOnlyWhenForced, VectorizeOnlyWhenForced}) { 1942 initializeLoopVectorizePass(*PassRegistry::getPassRegistry()); 1943 } 1944 1945 bool runOnFunction(Function &F) override { 1946 if (skipFunction(F)) 1947 return false; 1948 1949 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE(); 1950 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); 1951 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 1952 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); 1953 auto *BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI(); 1954 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>(); 1955 auto *TLI = TLIP ? &TLIP->getTLI(F) : nullptr; 1956 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 1957 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); 1958 auto *LAA = &getAnalysis<LoopAccessLegacyAnalysis>(); 1959 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits(); 1960 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE(); 1961 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); 1962 1963 std::function<const LoopAccessInfo &(Loop &)> GetLAA = 1964 [&](Loop &L) -> const LoopAccessInfo & { return LAA->getInfo(&L); }; 1965 1966 return Impl.runImpl(F, *SE, *LI, *TTI, *DT, *BFI, TLI, *DB, *AA, *AC, 1967 GetLAA, *ORE, PSI).MadeAnyChange; 1968 } 1969 1970 void getAnalysisUsage(AnalysisUsage &AU) const override { 1971 AU.addRequired<AssumptionCacheTracker>(); 1972 AU.addRequired<BlockFrequencyInfoWrapperPass>(); 1973 AU.addRequired<DominatorTreeWrapperPass>(); 1974 AU.addRequired<LoopInfoWrapperPass>(); 1975 AU.addRequired<ScalarEvolutionWrapperPass>(); 1976 AU.addRequired<TargetTransformInfoWrapperPass>(); 1977 AU.addRequired<AAResultsWrapperPass>(); 1978 AU.addRequired<LoopAccessLegacyAnalysis>(); 1979 AU.addRequired<DemandedBitsWrapperPass>(); 1980 AU.addRequired<OptimizationRemarkEmitterWrapperPass>(); 1981 AU.addRequired<InjectTLIMappingsLegacy>(); 1982 1983 // We currently do not preserve loopinfo/dominator analyses with outer loop 1984 // vectorization. Until this is addressed, mark these analyses as preserved 1985 // only for non-VPlan-native path. 1986 // TODO: Preserve Loop and Dominator analyses for VPlan-native path. 1987 if (!EnableVPlanNativePath) { 1988 AU.addPreserved<LoopInfoWrapperPass>(); 1989 AU.addPreserved<DominatorTreeWrapperPass>(); 1990 } 1991 1992 AU.addPreserved<BasicAAWrapperPass>(); 1993 AU.addPreserved<GlobalsAAWrapperPass>(); 1994 AU.addRequired<ProfileSummaryInfoWrapperPass>(); 1995 } 1996 }; 1997 1998 } // end anonymous namespace 1999 2000 //===----------------------------------------------------------------------===// 2001 // Implementation of LoopVectorizationLegality, InnerLoopVectorizer and 2002 // LoopVectorizationCostModel and LoopVectorizationPlanner. 2003 //===----------------------------------------------------------------------===// 2004 2005 Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) { 2006 // We need to place the broadcast of invariant variables outside the loop, 2007 // but only if it's proven safe to do so. Else, broadcast will be inside 2008 // vector loop body. 2009 Instruction *Instr = dyn_cast<Instruction>(V); 2010 bool SafeToHoist = OrigLoop->isLoopInvariant(V) && 2011 (!Instr || 2012 DT->dominates(Instr->getParent(), LoopVectorPreHeader)); 2013 // Place the code for broadcasting invariant variables in the new preheader. 2014 IRBuilder<>::InsertPointGuard Guard(Builder); 2015 if (SafeToHoist) 2016 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator()); 2017 2018 // Broadcast the scalar into all locations in the vector. 2019 Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast"); 2020 2021 return Shuf; 2022 } 2023 2024 void InnerLoopVectorizer::createVectorIntOrFpInductionPHI( 2025 const InductionDescriptor &II, Value *Step, Value *Start, 2026 Instruction *EntryVal) { 2027 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && 2028 "Expected either an induction phi-node or a truncate of it!"); 2029 2030 // Construct the initial value of the vector IV in the vector loop preheader 2031 auto CurrIP = Builder.saveIP(); 2032 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator()); 2033 if (isa<TruncInst>(EntryVal)) { 2034 assert(Start->getType()->isIntegerTy() && 2035 "Truncation requires an integer type"); 2036 auto *TruncType = cast<IntegerType>(EntryVal->getType()); 2037 Step = Builder.CreateTrunc(Step, TruncType); 2038 Start = Builder.CreateCast(Instruction::Trunc, Start, TruncType); 2039 } 2040 Value *SplatStart = Builder.CreateVectorSplat(VF, Start); 2041 Value *SteppedStart = 2042 getStepVector(SplatStart, 0, Step, II.getInductionOpcode()); 2043 2044 // We create vector phi nodes for both integer and floating-point induction 2045 // variables. Here, we determine the kind of arithmetic we will perform. 2046 Instruction::BinaryOps AddOp; 2047 Instruction::BinaryOps MulOp; 2048 if (Step->getType()->isIntegerTy()) { 2049 AddOp = Instruction::Add; 2050 MulOp = Instruction::Mul; 2051 } else { 2052 AddOp = II.getInductionOpcode(); 2053 MulOp = Instruction::FMul; 2054 } 2055 2056 // Multiply the vectorization factor by the step using integer or 2057 // floating-point arithmetic as appropriate. 2058 Value *ConstVF = 2059 getSignedIntOrFpConstant(Step->getType(), VF.getKnownMinValue()); 2060 Value *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, Step, ConstVF)); 2061 2062 // Create a vector splat to use in the induction update. 2063 // 2064 // FIXME: If the step is non-constant, we create the vector splat with 2065 // IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't 2066 // handle a constant vector splat. 2067 assert(!VF.isScalable() && "scalable vectors not yet supported."); 2068 Value *SplatVF = isa<Constant>(Mul) 2069 ? ConstantVector::getSplat(VF, cast<Constant>(Mul)) 2070 : Builder.CreateVectorSplat(VF, Mul); 2071 Builder.restoreIP(CurrIP); 2072 2073 // We may need to add the step a number of times, depending on the unroll 2074 // factor. The last of those goes into the PHI. 2075 PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind", 2076 &*LoopVectorBody->getFirstInsertionPt()); 2077 VecInd->setDebugLoc(EntryVal->getDebugLoc()); 2078 Instruction *LastInduction = VecInd; 2079 for (unsigned Part = 0; Part < UF; ++Part) { 2080 VectorLoopValueMap.setVectorValue(EntryVal, Part, LastInduction); 2081 2082 if (isa<TruncInst>(EntryVal)) 2083 addMetadata(LastInduction, EntryVal); 2084 recordVectorLoopValueForInductionCast(II, EntryVal, LastInduction, Part); 2085 2086 LastInduction = cast<Instruction>(addFastMathFlag( 2087 Builder.CreateBinOp(AddOp, LastInduction, SplatVF, "step.add"))); 2088 LastInduction->setDebugLoc(EntryVal->getDebugLoc()); 2089 } 2090 2091 // Move the last step to the end of the latch block. This ensures consistent 2092 // placement of all induction updates. 2093 auto *LoopVectorLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch(); 2094 auto *Br = cast<BranchInst>(LoopVectorLatch->getTerminator()); 2095 auto *ICmp = cast<Instruction>(Br->getCondition()); 2096 LastInduction->moveBefore(ICmp); 2097 LastInduction->setName("vec.ind.next"); 2098 2099 VecInd->addIncoming(SteppedStart, LoopVectorPreHeader); 2100 VecInd->addIncoming(LastInduction, LoopVectorLatch); 2101 } 2102 2103 bool InnerLoopVectorizer::shouldScalarizeInstruction(Instruction *I) const { 2104 return Cost->isScalarAfterVectorization(I, VF) || 2105 Cost->isProfitableToScalarize(I, VF); 2106 } 2107 2108 bool InnerLoopVectorizer::needsScalarInduction(Instruction *IV) const { 2109 if (shouldScalarizeInstruction(IV)) 2110 return true; 2111 auto isScalarInst = [&](User *U) -> bool { 2112 auto *I = cast<Instruction>(U); 2113 return (OrigLoop->contains(I) && shouldScalarizeInstruction(I)); 2114 }; 2115 return llvm::any_of(IV->users(), isScalarInst); 2116 } 2117 2118 void InnerLoopVectorizer::recordVectorLoopValueForInductionCast( 2119 const InductionDescriptor &ID, const Instruction *EntryVal, 2120 Value *VectorLoopVal, unsigned Part, unsigned Lane) { 2121 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && 2122 "Expected either an induction phi-node or a truncate of it!"); 2123 2124 // This induction variable is not the phi from the original loop but the 2125 // newly-created IV based on the proof that casted Phi is equal to the 2126 // uncasted Phi in the vectorized loop (under a runtime guard possibly). It 2127 // re-uses the same InductionDescriptor that original IV uses but we don't 2128 // have to do any recording in this case - that is done when original IV is 2129 // processed. 2130 if (isa<TruncInst>(EntryVal)) 2131 return; 2132 2133 const SmallVectorImpl<Instruction *> &Casts = ID.getCastInsts(); 2134 if (Casts.empty()) 2135 return; 2136 // Only the first Cast instruction in the Casts vector is of interest. 2137 // The rest of the Casts (if exist) have no uses outside the 2138 // induction update chain itself. 2139 Instruction *CastInst = *Casts.begin(); 2140 if (Lane < UINT_MAX) 2141 VectorLoopValueMap.setScalarValue(CastInst, {Part, Lane}, VectorLoopVal); 2142 else 2143 VectorLoopValueMap.setVectorValue(CastInst, Part, VectorLoopVal); 2144 } 2145 2146 void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start, 2147 TruncInst *Trunc) { 2148 assert((IV->getType()->isIntegerTy() || IV != OldInduction) && 2149 "Primary induction variable must have an integer type"); 2150 2151 auto II = Legal->getInductionVars().find(IV); 2152 assert(II != Legal->getInductionVars().end() && "IV is not an induction"); 2153 2154 auto ID = II->second; 2155 assert(IV->getType() == ID.getStartValue()->getType() && "Types must match"); 2156 2157 // The value from the original loop to which we are mapping the new induction 2158 // variable. 2159 Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV; 2160 2161 auto &DL = OrigLoop->getHeader()->getModule()->getDataLayout(); 2162 2163 // Generate code for the induction step. Note that induction steps are 2164 // required to be loop-invariant 2165 auto CreateStepValue = [&](const SCEV *Step) -> Value * { 2166 assert(PSE.getSE()->isLoopInvariant(Step, OrigLoop) && 2167 "Induction step should be loop invariant"); 2168 if (PSE.getSE()->isSCEVable(IV->getType())) { 2169 SCEVExpander Exp(*PSE.getSE(), DL, "induction"); 2170 return Exp.expandCodeFor(Step, Step->getType(), 2171 LoopVectorPreHeader->getTerminator()); 2172 } 2173 return cast<SCEVUnknown>(Step)->getValue(); 2174 }; 2175 2176 // The scalar value to broadcast. This is derived from the canonical 2177 // induction variable. If a truncation type is given, truncate the canonical 2178 // induction variable and step. Otherwise, derive these values from the 2179 // induction descriptor. 2180 auto CreateScalarIV = [&](Value *&Step) -> Value * { 2181 Value *ScalarIV = Induction; 2182 if (IV != OldInduction) { 2183 ScalarIV = IV->getType()->isIntegerTy() 2184 ? Builder.CreateSExtOrTrunc(Induction, IV->getType()) 2185 : Builder.CreateCast(Instruction::SIToFP, Induction, 2186 IV->getType()); 2187 ScalarIV = emitTransformedIndex(Builder, ScalarIV, PSE.getSE(), DL, ID); 2188 ScalarIV->setName("offset.idx"); 2189 } 2190 if (Trunc) { 2191 auto *TruncType = cast<IntegerType>(Trunc->getType()); 2192 assert(Step->getType()->isIntegerTy() && 2193 "Truncation requires an integer step"); 2194 ScalarIV = Builder.CreateTrunc(ScalarIV, TruncType); 2195 Step = Builder.CreateTrunc(Step, TruncType); 2196 } 2197 return ScalarIV; 2198 }; 2199 2200 // Create the vector values from the scalar IV, in the absence of creating a 2201 // vector IV. 2202 auto CreateSplatIV = [&](Value *ScalarIV, Value *Step) { 2203 Value *Broadcasted = getBroadcastInstrs(ScalarIV); 2204 for (unsigned Part = 0; Part < UF; ++Part) { 2205 assert(!VF.isScalable() && "scalable vectors not yet supported."); 2206 Value *EntryPart = 2207 getStepVector(Broadcasted, VF.getKnownMinValue() * Part, Step, 2208 ID.getInductionOpcode()); 2209 VectorLoopValueMap.setVectorValue(EntryVal, Part, EntryPart); 2210 if (Trunc) 2211 addMetadata(EntryPart, Trunc); 2212 recordVectorLoopValueForInductionCast(ID, EntryVal, EntryPart, Part); 2213 } 2214 }; 2215 2216 // Now do the actual transformations, and start with creating the step value. 2217 Value *Step = CreateStepValue(ID.getStep()); 2218 if (VF.isZero() || VF.isScalar()) { 2219 Value *ScalarIV = CreateScalarIV(Step); 2220 CreateSplatIV(ScalarIV, Step); 2221 return; 2222 } 2223 2224 // Determine if we want a scalar version of the induction variable. This is 2225 // true if the induction variable itself is not widened, or if it has at 2226 // least one user in the loop that is not widened. 2227 auto NeedsScalarIV = needsScalarInduction(EntryVal); 2228 if (!NeedsScalarIV) { 2229 createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal); 2230 return; 2231 } 2232 2233 // Try to create a new independent vector induction variable. If we can't 2234 // create the phi node, we will splat the scalar induction variable in each 2235 // loop iteration. 2236 if (!shouldScalarizeInstruction(EntryVal)) { 2237 createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal); 2238 Value *ScalarIV = CreateScalarIV(Step); 2239 // Create scalar steps that can be used by instructions we will later 2240 // scalarize. Note that the addition of the scalar steps will not increase 2241 // the number of instructions in the loop in the common case prior to 2242 // InstCombine. We will be trading one vector extract for each scalar step. 2243 buildScalarSteps(ScalarIV, Step, EntryVal, ID); 2244 return; 2245 } 2246 2247 // All IV users are scalar instructions, so only emit a scalar IV, not a 2248 // vectorised IV. Except when we tail-fold, then the splat IV feeds the 2249 // predicate used by the masked loads/stores. 2250 Value *ScalarIV = CreateScalarIV(Step); 2251 if (!Cost->isScalarEpilogueAllowed()) 2252 CreateSplatIV(ScalarIV, Step); 2253 buildScalarSteps(ScalarIV, Step, EntryVal, ID); 2254 } 2255 2256 Value *InnerLoopVectorizer::getStepVector(Value *Val, int StartIdx, Value *Step, 2257 Instruction::BinaryOps BinOp) { 2258 // Create and check the types. 2259 auto *ValVTy = cast<FixedVectorType>(Val->getType()); 2260 int VLen = ValVTy->getNumElements(); 2261 2262 Type *STy = Val->getType()->getScalarType(); 2263 assert((STy->isIntegerTy() || STy->isFloatingPointTy()) && 2264 "Induction Step must be an integer or FP"); 2265 assert(Step->getType() == STy && "Step has wrong type"); 2266 2267 SmallVector<Constant *, 8> Indices; 2268 2269 if (STy->isIntegerTy()) { 2270 // Create a vector of consecutive numbers from zero to VF. 2271 for (int i = 0; i < VLen; ++i) 2272 Indices.push_back(ConstantInt::get(STy, StartIdx + i)); 2273 2274 // Add the consecutive indices to the vector value. 2275 Constant *Cv = ConstantVector::get(Indices); 2276 assert(Cv->getType() == Val->getType() && "Invalid consecutive vec"); 2277 Step = Builder.CreateVectorSplat(VLen, Step); 2278 assert(Step->getType() == Val->getType() && "Invalid step vec"); 2279 // FIXME: The newly created binary instructions should contain nsw/nuw flags, 2280 // which can be found from the original scalar operations. 2281 Step = Builder.CreateMul(Cv, Step); 2282 return Builder.CreateAdd(Val, Step, "induction"); 2283 } 2284 2285 // Floating point induction. 2286 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) && 2287 "Binary Opcode should be specified for FP induction"); 2288 // Create a vector of consecutive numbers from zero to VF. 2289 for (int i = 0; i < VLen; ++i) 2290 Indices.push_back(ConstantFP::get(STy, (double)(StartIdx + i))); 2291 2292 // Add the consecutive indices to the vector value. 2293 Constant *Cv = ConstantVector::get(Indices); 2294 2295 Step = Builder.CreateVectorSplat(VLen, Step); 2296 2297 // Floating point operations had to be 'fast' to enable the induction. 2298 FastMathFlags Flags; 2299 Flags.setFast(); 2300 2301 Value *MulOp = Builder.CreateFMul(Cv, Step); 2302 if (isa<Instruction>(MulOp)) 2303 // Have to check, MulOp may be a constant 2304 cast<Instruction>(MulOp)->setFastMathFlags(Flags); 2305 2306 Value *BOp = Builder.CreateBinOp(BinOp, Val, MulOp, "induction"); 2307 if (isa<Instruction>(BOp)) 2308 cast<Instruction>(BOp)->setFastMathFlags(Flags); 2309 return BOp; 2310 } 2311 2312 void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step, 2313 Instruction *EntryVal, 2314 const InductionDescriptor &ID) { 2315 // We shouldn't have to build scalar steps if we aren't vectorizing. 2316 assert(VF.isVector() && "VF should be greater than one"); 2317 // Get the value type and ensure it and the step have the same integer type. 2318 Type *ScalarIVTy = ScalarIV->getType()->getScalarType(); 2319 assert(ScalarIVTy == Step->getType() && 2320 "Val and Step should have the same type"); 2321 2322 // We build scalar steps for both integer and floating-point induction 2323 // variables. Here, we determine the kind of arithmetic we will perform. 2324 Instruction::BinaryOps AddOp; 2325 Instruction::BinaryOps MulOp; 2326 if (ScalarIVTy->isIntegerTy()) { 2327 AddOp = Instruction::Add; 2328 MulOp = Instruction::Mul; 2329 } else { 2330 AddOp = ID.getInductionOpcode(); 2331 MulOp = Instruction::FMul; 2332 } 2333 2334 // Determine the number of scalars we need to generate for each unroll 2335 // iteration. If EntryVal is uniform, we only need to generate the first 2336 // lane. Otherwise, we generate all VF values. 2337 unsigned Lanes = 2338 Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), VF) 2339 ? 1 2340 : VF.getKnownMinValue(); 2341 assert((!VF.isScalable() || Lanes == 1) && 2342 "Should never scalarize a scalable vector"); 2343 // Compute the scalar steps and save the results in VectorLoopValueMap. 2344 for (unsigned Part = 0; Part < UF; ++Part) { 2345 for (unsigned Lane = 0; Lane < Lanes; ++Lane) { 2346 auto *IntStepTy = IntegerType::get(ScalarIVTy->getContext(), 2347 ScalarIVTy->getScalarSizeInBits()); 2348 Value *StartIdx = 2349 createStepForVF(Builder, ConstantInt::get(IntStepTy, Part), VF); 2350 if (ScalarIVTy->isFloatingPointTy()) 2351 StartIdx = Builder.CreateSIToFP(StartIdx, ScalarIVTy); 2352 StartIdx = addFastMathFlag(Builder.CreateBinOp( 2353 AddOp, StartIdx, getSignedIntOrFpConstant(ScalarIVTy, Lane))); 2354 // The step returned by `createStepForVF` is a runtime-evaluated value 2355 // when VF is scalable. Otherwise, it should be folded into a Constant. 2356 assert((VF.isScalable() || isa<Constant>(StartIdx)) && 2357 "Expected StartIdx to be folded to a constant when VF is not " 2358 "scalable"); 2359 auto *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, StartIdx, Step)); 2360 auto *Add = addFastMathFlag(Builder.CreateBinOp(AddOp, ScalarIV, Mul)); 2361 VectorLoopValueMap.setScalarValue(EntryVal, {Part, Lane}, Add); 2362 recordVectorLoopValueForInductionCast(ID, EntryVal, Add, Part, Lane); 2363 } 2364 } 2365 } 2366 2367 Value *InnerLoopVectorizer::getOrCreateVectorValue(Value *V, unsigned Part) { 2368 assert(V != Induction && "The new induction variable should not be used."); 2369 assert(!V->getType()->isVectorTy() && "Can't widen a vector"); 2370 assert(!V->getType()->isVoidTy() && "Type does not produce a value"); 2371 2372 // If we have a stride that is replaced by one, do it here. Defer this for 2373 // the VPlan-native path until we start running Legal checks in that path. 2374 if (!EnableVPlanNativePath && Legal->hasStride(V)) 2375 V = ConstantInt::get(V->getType(), 1); 2376 2377 // If we have a vector mapped to this value, return it. 2378 if (VectorLoopValueMap.hasVectorValue(V, Part)) 2379 return VectorLoopValueMap.getVectorValue(V, Part); 2380 2381 // If the value has not been vectorized, check if it has been scalarized 2382 // instead. If it has been scalarized, and we actually need the value in 2383 // vector form, we will construct the vector values on demand. 2384 if (VectorLoopValueMap.hasAnyScalarValue(V)) { 2385 Value *ScalarValue = VectorLoopValueMap.getScalarValue(V, {Part, 0}); 2386 2387 // If we've scalarized a value, that value should be an instruction. 2388 auto *I = cast<Instruction>(V); 2389 2390 // If we aren't vectorizing, we can just copy the scalar map values over to 2391 // the vector map. 2392 if (VF.isScalar()) { 2393 VectorLoopValueMap.setVectorValue(V, Part, ScalarValue); 2394 return ScalarValue; 2395 } 2396 2397 // Get the last scalar instruction we generated for V and Part. If the value 2398 // is known to be uniform after vectorization, this corresponds to lane zero 2399 // of the Part unroll iteration. Otherwise, the last instruction is the one 2400 // we created for the last vector lane of the Part unroll iteration. 2401 unsigned LastLane = Cost->isUniformAfterVectorization(I, VF) 2402 ? 0 2403 : VF.getKnownMinValue() - 1; 2404 assert((!VF.isScalable() || LastLane == 0) && 2405 "Scalable vectorization can't lead to any scalarized values."); 2406 auto *LastInst = cast<Instruction>( 2407 VectorLoopValueMap.getScalarValue(V, {Part, LastLane})); 2408 2409 // Set the insert point after the last scalarized instruction. This ensures 2410 // the insertelement sequence will directly follow the scalar definitions. 2411 auto OldIP = Builder.saveIP(); 2412 auto NewIP = std::next(BasicBlock::iterator(LastInst)); 2413 Builder.SetInsertPoint(&*NewIP); 2414 2415 // However, if we are vectorizing, we need to construct the vector values. 2416 // If the value is known to be uniform after vectorization, we can just 2417 // broadcast the scalar value corresponding to lane zero for each unroll 2418 // iteration. Otherwise, we construct the vector values using insertelement 2419 // instructions. Since the resulting vectors are stored in 2420 // VectorLoopValueMap, we will only generate the insertelements once. 2421 Value *VectorValue = nullptr; 2422 if (Cost->isUniformAfterVectorization(I, VF)) { 2423 VectorValue = getBroadcastInstrs(ScalarValue); 2424 VectorLoopValueMap.setVectorValue(V, Part, VectorValue); 2425 } else { 2426 // Initialize packing with insertelements to start from poison. 2427 assert(!VF.isScalable() && "VF is assumed to be non scalable."); 2428 Value *Poison = PoisonValue::get(VectorType::get(V->getType(), VF)); 2429 VectorLoopValueMap.setVectorValue(V, Part, Poison); 2430 for (unsigned Lane = 0; Lane < VF.getKnownMinValue(); ++Lane) 2431 packScalarIntoVectorValue(V, {Part, Lane}); 2432 VectorValue = VectorLoopValueMap.getVectorValue(V, Part); 2433 } 2434 Builder.restoreIP(OldIP); 2435 return VectorValue; 2436 } 2437 2438 // If this scalar is unknown, assume that it is a constant or that it is 2439 // loop invariant. Broadcast V and save the value for future uses. 2440 Value *B = getBroadcastInstrs(V); 2441 VectorLoopValueMap.setVectorValue(V, Part, B); 2442 return B; 2443 } 2444 2445 Value * 2446 InnerLoopVectorizer::getOrCreateScalarValue(Value *V, 2447 const VPIteration &Instance) { 2448 // If the value is not an instruction contained in the loop, it should 2449 // already be scalar. 2450 if (OrigLoop->isLoopInvariant(V)) 2451 return V; 2452 2453 assert(Instance.Lane > 0 2454 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF) 2455 : true && "Uniform values only have lane zero"); 2456 2457 // If the value from the original loop has not been vectorized, it is 2458 // represented by UF x VF scalar values in the new loop. Return the requested 2459 // scalar value. 2460 if (VectorLoopValueMap.hasScalarValue(V, Instance)) 2461 return VectorLoopValueMap.getScalarValue(V, Instance); 2462 2463 // If the value has not been scalarized, get its entry in VectorLoopValueMap 2464 // for the given unroll part. If this entry is not a vector type (i.e., the 2465 // vectorization factor is one), there is no need to generate an 2466 // extractelement instruction. 2467 auto *U = getOrCreateVectorValue(V, Instance.Part); 2468 if (!U->getType()->isVectorTy()) { 2469 assert(VF.isScalar() && "Value not scalarized has non-vector type"); 2470 return U; 2471 } 2472 2473 // Otherwise, the value from the original loop has been vectorized and is 2474 // represented by UF vector values. Extract and return the requested scalar 2475 // value from the appropriate vector lane. 2476 return Builder.CreateExtractElement(U, Builder.getInt32(Instance.Lane)); 2477 } 2478 2479 void InnerLoopVectorizer::packScalarIntoVectorValue( 2480 Value *V, const VPIteration &Instance) { 2481 assert(V != Induction && "The new induction variable should not be used."); 2482 assert(!V->getType()->isVectorTy() && "Can't pack a vector"); 2483 assert(!V->getType()->isVoidTy() && "Type does not produce a value"); 2484 2485 Value *ScalarInst = VectorLoopValueMap.getScalarValue(V, Instance); 2486 Value *VectorValue = VectorLoopValueMap.getVectorValue(V, Instance.Part); 2487 VectorValue = Builder.CreateInsertElement(VectorValue, ScalarInst, 2488 Builder.getInt32(Instance.Lane)); 2489 VectorLoopValueMap.resetVectorValue(V, Instance.Part, VectorValue); 2490 } 2491 2492 Value *InnerLoopVectorizer::reverseVector(Value *Vec) { 2493 assert(Vec->getType()->isVectorTy() && "Invalid type"); 2494 assert(!VF.isScalable() && "Cannot reverse scalable vectors"); 2495 SmallVector<int, 8> ShuffleMask; 2496 for (unsigned i = 0; i < VF.getKnownMinValue(); ++i) 2497 ShuffleMask.push_back(VF.getKnownMinValue() - i - 1); 2498 2499 return Builder.CreateShuffleVector(Vec, ShuffleMask, "reverse"); 2500 } 2501 2502 // Return whether we allow using masked interleave-groups (for dealing with 2503 // strided loads/stores that reside in predicated blocks, or for dealing 2504 // with gaps). 2505 static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) { 2506 // If an override option has been passed in for interleaved accesses, use it. 2507 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0) 2508 return EnableMaskedInterleavedMemAccesses; 2509 2510 return TTI.enableMaskedInterleavedAccessVectorization(); 2511 } 2512 2513 // Try to vectorize the interleave group that \p Instr belongs to. 2514 // 2515 // E.g. Translate following interleaved load group (factor = 3): 2516 // for (i = 0; i < N; i+=3) { 2517 // R = Pic[i]; // Member of index 0 2518 // G = Pic[i+1]; // Member of index 1 2519 // B = Pic[i+2]; // Member of index 2 2520 // ... // do something to R, G, B 2521 // } 2522 // To: 2523 // %wide.vec = load <12 x i32> ; Read 4 tuples of R,G,B 2524 // %R.vec = shuffle %wide.vec, poison, <0, 3, 6, 9> ; R elements 2525 // %G.vec = shuffle %wide.vec, poison, <1, 4, 7, 10> ; G elements 2526 // %B.vec = shuffle %wide.vec, poison, <2, 5, 8, 11> ; B elements 2527 // 2528 // Or translate following interleaved store group (factor = 3): 2529 // for (i = 0; i < N; i+=3) { 2530 // ... do something to R, G, B 2531 // Pic[i] = R; // Member of index 0 2532 // Pic[i+1] = G; // Member of index 1 2533 // Pic[i+2] = B; // Member of index 2 2534 // } 2535 // To: 2536 // %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7> 2537 // %B_U.vec = shuffle %B.vec, poison, <0, 1, 2, 3, u, u, u, u> 2538 // %interleaved.vec = shuffle %R_G.vec, %B_U.vec, 2539 // <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> ; Interleave R,G,B elements 2540 // store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B 2541 void InnerLoopVectorizer::vectorizeInterleaveGroup( 2542 const InterleaveGroup<Instruction> *Group, ArrayRef<VPValue *> VPDefs, 2543 VPTransformState &State, VPValue *Addr, ArrayRef<VPValue *> StoredValues, 2544 VPValue *BlockInMask) { 2545 Instruction *Instr = Group->getInsertPos(); 2546 const DataLayout &DL = Instr->getModule()->getDataLayout(); 2547 2548 // Prepare for the vector type of the interleaved load/store. 2549 Type *ScalarTy = getMemInstValueType(Instr); 2550 unsigned InterleaveFactor = Group->getFactor(); 2551 assert(!VF.isScalable() && "scalable vectors not yet supported."); 2552 auto *VecTy = VectorType::get(ScalarTy, VF * InterleaveFactor); 2553 2554 // Prepare for the new pointers. 2555 SmallVector<Value *, 2> AddrParts; 2556 unsigned Index = Group->getIndex(Instr); 2557 2558 // TODO: extend the masked interleaved-group support to reversed access. 2559 assert((!BlockInMask || !Group->isReverse()) && 2560 "Reversed masked interleave-group not supported."); 2561 2562 // If the group is reverse, adjust the index to refer to the last vector lane 2563 // instead of the first. We adjust the index from the first vector lane, 2564 // rather than directly getting the pointer for lane VF - 1, because the 2565 // pointer operand of the interleaved access is supposed to be uniform. For 2566 // uniform instructions, we're only required to generate a value for the 2567 // first vector lane in each unroll iteration. 2568 assert(!VF.isScalable() && 2569 "scalable vector reverse operation is not implemented"); 2570 if (Group->isReverse()) 2571 Index += (VF.getKnownMinValue() - 1) * Group->getFactor(); 2572 2573 for (unsigned Part = 0; Part < UF; Part++) { 2574 Value *AddrPart = State.get(Addr, {Part, 0}); 2575 setDebugLocFromInst(Builder, AddrPart); 2576 2577 // Notice current instruction could be any index. Need to adjust the address 2578 // to the member of index 0. 2579 // 2580 // E.g. a = A[i+1]; // Member of index 1 (Current instruction) 2581 // b = A[i]; // Member of index 0 2582 // Current pointer is pointed to A[i+1], adjust it to A[i]. 2583 // 2584 // E.g. A[i+1] = a; // Member of index 1 2585 // A[i] = b; // Member of index 0 2586 // A[i+2] = c; // Member of index 2 (Current instruction) 2587 // Current pointer is pointed to A[i+2], adjust it to A[i]. 2588 2589 bool InBounds = false; 2590 if (auto *gep = dyn_cast<GetElementPtrInst>(AddrPart->stripPointerCasts())) 2591 InBounds = gep->isInBounds(); 2592 AddrPart = Builder.CreateGEP(ScalarTy, AddrPart, Builder.getInt32(-Index)); 2593 cast<GetElementPtrInst>(AddrPart)->setIsInBounds(InBounds); 2594 2595 // Cast to the vector pointer type. 2596 unsigned AddressSpace = AddrPart->getType()->getPointerAddressSpace(); 2597 Type *PtrTy = VecTy->getPointerTo(AddressSpace); 2598 AddrParts.push_back(Builder.CreateBitCast(AddrPart, PtrTy)); 2599 } 2600 2601 setDebugLocFromInst(Builder, Instr); 2602 Value *PoisonVec = PoisonValue::get(VecTy); 2603 2604 Value *MaskForGaps = nullptr; 2605 if (Group->requiresScalarEpilogue() && !Cost->isScalarEpilogueAllowed()) { 2606 assert(!VF.isScalable() && "scalable vectors not yet supported."); 2607 MaskForGaps = createBitMaskForGaps(Builder, VF.getKnownMinValue(), *Group); 2608 assert(MaskForGaps && "Mask for Gaps is required but it is null"); 2609 } 2610 2611 // Vectorize the interleaved load group. 2612 if (isa<LoadInst>(Instr)) { 2613 // For each unroll part, create a wide load for the group. 2614 SmallVector<Value *, 2> NewLoads; 2615 for (unsigned Part = 0; Part < UF; Part++) { 2616 Instruction *NewLoad; 2617 if (BlockInMask || MaskForGaps) { 2618 assert(useMaskedInterleavedAccesses(*TTI) && 2619 "masked interleaved groups are not allowed."); 2620 Value *GroupMask = MaskForGaps; 2621 if (BlockInMask) { 2622 Value *BlockInMaskPart = State.get(BlockInMask, Part); 2623 assert(!VF.isScalable() && "scalable vectors not yet supported."); 2624 Value *ShuffledMask = Builder.CreateShuffleVector( 2625 BlockInMaskPart, 2626 createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()), 2627 "interleaved.mask"); 2628 GroupMask = MaskForGaps 2629 ? Builder.CreateBinOp(Instruction::And, ShuffledMask, 2630 MaskForGaps) 2631 : ShuffledMask; 2632 } 2633 NewLoad = 2634 Builder.CreateMaskedLoad(AddrParts[Part], Group->getAlign(), 2635 GroupMask, PoisonVec, "wide.masked.vec"); 2636 } 2637 else 2638 NewLoad = Builder.CreateAlignedLoad(VecTy, AddrParts[Part], 2639 Group->getAlign(), "wide.vec"); 2640 Group->addMetadata(NewLoad); 2641 NewLoads.push_back(NewLoad); 2642 } 2643 2644 // For each member in the group, shuffle out the appropriate data from the 2645 // wide loads. 2646 unsigned J = 0; 2647 for (unsigned I = 0; I < InterleaveFactor; ++I) { 2648 Instruction *Member = Group->getMember(I); 2649 2650 // Skip the gaps in the group. 2651 if (!Member) 2652 continue; 2653 2654 assert(!VF.isScalable() && "scalable vectors not yet supported."); 2655 auto StrideMask = 2656 createStrideMask(I, InterleaveFactor, VF.getKnownMinValue()); 2657 for (unsigned Part = 0; Part < UF; Part++) { 2658 Value *StridedVec = Builder.CreateShuffleVector( 2659 NewLoads[Part], StrideMask, "strided.vec"); 2660 2661 // If this member has different type, cast the result type. 2662 if (Member->getType() != ScalarTy) { 2663 assert(!VF.isScalable() && "VF is assumed to be non scalable."); 2664 VectorType *OtherVTy = VectorType::get(Member->getType(), VF); 2665 StridedVec = createBitOrPointerCast(StridedVec, OtherVTy, DL); 2666 } 2667 2668 if (Group->isReverse()) 2669 StridedVec = reverseVector(StridedVec); 2670 2671 State.set(VPDefs[J], Member, StridedVec, Part); 2672 } 2673 ++J; 2674 } 2675 return; 2676 } 2677 2678 // The sub vector type for current instruction. 2679 assert(!VF.isScalable() && "VF is assumed to be non scalable."); 2680 auto *SubVT = VectorType::get(ScalarTy, VF); 2681 2682 // Vectorize the interleaved store group. 2683 for (unsigned Part = 0; Part < UF; Part++) { 2684 // Collect the stored vector from each member. 2685 SmallVector<Value *, 4> StoredVecs; 2686 for (unsigned i = 0; i < InterleaveFactor; i++) { 2687 // Interleaved store group doesn't allow a gap, so each index has a member 2688 assert(Group->getMember(i) && "Fail to get a member from an interleaved store group"); 2689 2690 Value *StoredVec = State.get(StoredValues[i], Part); 2691 2692 if (Group->isReverse()) 2693 StoredVec = reverseVector(StoredVec); 2694 2695 // If this member has different type, cast it to a unified type. 2696 2697 if (StoredVec->getType() != SubVT) 2698 StoredVec = createBitOrPointerCast(StoredVec, SubVT, DL); 2699 2700 StoredVecs.push_back(StoredVec); 2701 } 2702 2703 // Concatenate all vectors into a wide vector. 2704 Value *WideVec = concatenateVectors(Builder, StoredVecs); 2705 2706 // Interleave the elements in the wide vector. 2707 assert(!VF.isScalable() && "scalable vectors not yet supported."); 2708 Value *IVec = Builder.CreateShuffleVector( 2709 WideVec, createInterleaveMask(VF.getKnownMinValue(), InterleaveFactor), 2710 "interleaved.vec"); 2711 2712 Instruction *NewStoreInstr; 2713 if (BlockInMask) { 2714 Value *BlockInMaskPart = State.get(BlockInMask, Part); 2715 Value *ShuffledMask = Builder.CreateShuffleVector( 2716 BlockInMaskPart, 2717 createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()), 2718 "interleaved.mask"); 2719 NewStoreInstr = Builder.CreateMaskedStore( 2720 IVec, AddrParts[Part], Group->getAlign(), ShuffledMask); 2721 } 2722 else 2723 NewStoreInstr = 2724 Builder.CreateAlignedStore(IVec, AddrParts[Part], Group->getAlign()); 2725 2726 Group->addMetadata(NewStoreInstr); 2727 } 2728 } 2729 2730 void InnerLoopVectorizer::vectorizeMemoryInstruction( 2731 Instruction *Instr, VPTransformState &State, VPValue *Def, VPValue *Addr, 2732 VPValue *StoredValue, VPValue *BlockInMask) { 2733 // Attempt to issue a wide load. 2734 LoadInst *LI = dyn_cast<LoadInst>(Instr); 2735 StoreInst *SI = dyn_cast<StoreInst>(Instr); 2736 2737 assert((LI || SI) && "Invalid Load/Store instruction"); 2738 assert((!SI || StoredValue) && "No stored value provided for widened store"); 2739 assert((!LI || !StoredValue) && "Stored value provided for widened load"); 2740 2741 LoopVectorizationCostModel::InstWidening Decision = 2742 Cost->getWideningDecision(Instr, VF); 2743 assert((Decision == LoopVectorizationCostModel::CM_Widen || 2744 Decision == LoopVectorizationCostModel::CM_Widen_Reverse || 2745 Decision == LoopVectorizationCostModel::CM_GatherScatter) && 2746 "CM decision is not to widen the memory instruction"); 2747 2748 Type *ScalarDataTy = getMemInstValueType(Instr); 2749 2750 auto *DataTy = VectorType::get(ScalarDataTy, VF); 2751 const Align Alignment = getLoadStoreAlignment(Instr); 2752 2753 // Determine if the pointer operand of the access is either consecutive or 2754 // reverse consecutive. 2755 bool Reverse = (Decision == LoopVectorizationCostModel::CM_Widen_Reverse); 2756 bool ConsecutiveStride = 2757 Reverse || (Decision == LoopVectorizationCostModel::CM_Widen); 2758 bool CreateGatherScatter = 2759 (Decision == LoopVectorizationCostModel::CM_GatherScatter); 2760 2761 // Either Ptr feeds a vector load/store, or a vector GEP should feed a vector 2762 // gather/scatter. Otherwise Decision should have been to Scalarize. 2763 assert((ConsecutiveStride || CreateGatherScatter) && 2764 "The instruction should be scalarized"); 2765 (void)ConsecutiveStride; 2766 2767 VectorParts BlockInMaskParts(UF); 2768 bool isMaskRequired = BlockInMask; 2769 if (isMaskRequired) 2770 for (unsigned Part = 0; Part < UF; ++Part) 2771 BlockInMaskParts[Part] = State.get(BlockInMask, Part); 2772 2773 const auto CreateVecPtr = [&](unsigned Part, Value *Ptr) -> Value * { 2774 // Calculate the pointer for the specific unroll-part. 2775 GetElementPtrInst *PartPtr = nullptr; 2776 2777 bool InBounds = false; 2778 if (auto *gep = dyn_cast<GetElementPtrInst>(Ptr->stripPointerCasts())) 2779 InBounds = gep->isInBounds(); 2780 2781 if (Reverse) { 2782 assert(!VF.isScalable() && 2783 "Reversing vectors is not yet supported for scalable vectors."); 2784 2785 // If the address is consecutive but reversed, then the 2786 // wide store needs to start at the last vector element. 2787 PartPtr = cast<GetElementPtrInst>(Builder.CreateGEP( 2788 ScalarDataTy, Ptr, Builder.getInt32(-Part * VF.getKnownMinValue()))); 2789 PartPtr->setIsInBounds(InBounds); 2790 PartPtr = cast<GetElementPtrInst>(Builder.CreateGEP( 2791 ScalarDataTy, PartPtr, Builder.getInt32(1 - VF.getKnownMinValue()))); 2792 PartPtr->setIsInBounds(InBounds); 2793 if (isMaskRequired) // Reverse of a null all-one mask is a null mask. 2794 BlockInMaskParts[Part] = reverseVector(BlockInMaskParts[Part]); 2795 } else { 2796 Value *Increment = createStepForVF(Builder, Builder.getInt32(Part), VF); 2797 PartPtr = cast<GetElementPtrInst>( 2798 Builder.CreateGEP(ScalarDataTy, Ptr, Increment)); 2799 PartPtr->setIsInBounds(InBounds); 2800 } 2801 2802 unsigned AddressSpace = Ptr->getType()->getPointerAddressSpace(); 2803 return Builder.CreateBitCast(PartPtr, DataTy->getPointerTo(AddressSpace)); 2804 }; 2805 2806 // Handle Stores: 2807 if (SI) { 2808 setDebugLocFromInst(Builder, SI); 2809 2810 for (unsigned Part = 0; Part < UF; ++Part) { 2811 Instruction *NewSI = nullptr; 2812 Value *StoredVal = State.get(StoredValue, Part); 2813 if (CreateGatherScatter) { 2814 Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr; 2815 Value *VectorGep = State.get(Addr, Part); 2816 NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment, 2817 MaskPart); 2818 } else { 2819 if (Reverse) { 2820 // If we store to reverse consecutive memory locations, then we need 2821 // to reverse the order of elements in the stored value. 2822 StoredVal = reverseVector(StoredVal); 2823 // We don't want to update the value in the map as it might be used in 2824 // another expression. So don't call resetVectorValue(StoredVal). 2825 } 2826 auto *VecPtr = CreateVecPtr(Part, State.get(Addr, {0, 0})); 2827 if (isMaskRequired) 2828 NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment, 2829 BlockInMaskParts[Part]); 2830 else 2831 NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment); 2832 } 2833 addMetadata(NewSI, SI); 2834 } 2835 return; 2836 } 2837 2838 // Handle loads. 2839 assert(LI && "Must have a load instruction"); 2840 setDebugLocFromInst(Builder, LI); 2841 for (unsigned Part = 0; Part < UF; ++Part) { 2842 Value *NewLI; 2843 if (CreateGatherScatter) { 2844 Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr; 2845 Value *VectorGep = State.get(Addr, Part); 2846 NewLI = Builder.CreateMaskedGather(VectorGep, Alignment, MaskPart, 2847 nullptr, "wide.masked.gather"); 2848 addMetadata(NewLI, LI); 2849 } else { 2850 auto *VecPtr = CreateVecPtr(Part, State.get(Addr, {0, 0})); 2851 if (isMaskRequired) 2852 NewLI = Builder.CreateMaskedLoad( 2853 VecPtr, Alignment, BlockInMaskParts[Part], PoisonValue::get(DataTy), 2854 "wide.masked.load"); 2855 else 2856 NewLI = 2857 Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load"); 2858 2859 // Add metadata to the load, but setVectorValue to the reverse shuffle. 2860 addMetadata(NewLI, LI); 2861 if (Reverse) 2862 NewLI = reverseVector(NewLI); 2863 } 2864 2865 State.set(Def, Instr, NewLI, Part); 2866 } 2867 } 2868 2869 void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr, VPUser &User, 2870 const VPIteration &Instance, 2871 bool IfPredicateInstr, 2872 VPTransformState &State) { 2873 assert(!Instr->getType()->isAggregateType() && "Can't handle vectors"); 2874 2875 // llvm.experimental.noalias.scope.decl intrinsics must only be duplicated for 2876 // the first lane and part. 2877 if (isa<NoAliasScopeDeclInst>(Instr)) 2878 if (Instance.Lane != 0 || Instance.Part != 0) 2879 return; 2880 2881 setDebugLocFromInst(Builder, Instr); 2882 2883 // Does this instruction return a value ? 2884 bool IsVoidRetTy = Instr->getType()->isVoidTy(); 2885 2886 Instruction *Cloned = Instr->clone(); 2887 if (!IsVoidRetTy) 2888 Cloned->setName(Instr->getName() + ".cloned"); 2889 2890 // Replace the operands of the cloned instructions with their scalar 2891 // equivalents in the new loop. 2892 for (unsigned op = 0, e = User.getNumOperands(); op != e; ++op) { 2893 auto *Operand = dyn_cast<Instruction>(Instr->getOperand(op)); 2894 auto InputInstance = Instance; 2895 if (!Operand || !OrigLoop->contains(Operand) || 2896 (Cost->isUniformAfterVectorization(Operand, State.VF))) 2897 InputInstance.Lane = 0; 2898 auto *NewOp = State.get(User.getOperand(op), InputInstance); 2899 Cloned->setOperand(op, NewOp); 2900 } 2901 addNewMetadata(Cloned, Instr); 2902 2903 // Place the cloned scalar in the new loop. 2904 Builder.Insert(Cloned); 2905 2906 // TODO: Set result for VPValue of VPReciplicateRecipe. This requires 2907 // representing scalar values in VPTransformState. Add the cloned scalar to 2908 // the scalar map entry. 2909 VectorLoopValueMap.setScalarValue(Instr, Instance, Cloned); 2910 2911 // If we just cloned a new assumption, add it the assumption cache. 2912 if (auto *II = dyn_cast<IntrinsicInst>(Cloned)) 2913 if (II->getIntrinsicID() == Intrinsic::assume) 2914 AC->registerAssumption(II); 2915 2916 // End if-block. 2917 if (IfPredicateInstr) 2918 PredicatedInstructions.push_back(Cloned); 2919 } 2920 2921 PHINode *InnerLoopVectorizer::createInductionVariable(Loop *L, Value *Start, 2922 Value *End, Value *Step, 2923 Instruction *DL) { 2924 BasicBlock *Header = L->getHeader(); 2925 BasicBlock *Latch = L->getLoopLatch(); 2926 // As we're just creating this loop, it's possible no latch exists 2927 // yet. If so, use the header as this will be a single block loop. 2928 if (!Latch) 2929 Latch = Header; 2930 2931 IRBuilder<> Builder(&*Header->getFirstInsertionPt()); 2932 Instruction *OldInst = getDebugLocFromInstOrOperands(OldInduction); 2933 setDebugLocFromInst(Builder, OldInst); 2934 auto *Induction = Builder.CreatePHI(Start->getType(), 2, "index"); 2935 2936 Builder.SetInsertPoint(Latch->getTerminator()); 2937 setDebugLocFromInst(Builder, OldInst); 2938 2939 // Create i+1 and fill the PHINode. 2940 Value *Next = Builder.CreateAdd(Induction, Step, "index.next"); 2941 Induction->addIncoming(Start, L->getLoopPreheader()); 2942 Induction->addIncoming(Next, Latch); 2943 // Create the compare. 2944 Value *ICmp = Builder.CreateICmpEQ(Next, End); 2945 Builder.CreateCondBr(ICmp, L->getUniqueExitBlock(), Header); 2946 2947 // Now we have two terminators. Remove the old one from the block. 2948 Latch->getTerminator()->eraseFromParent(); 2949 2950 return Induction; 2951 } 2952 2953 Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) { 2954 if (TripCount) 2955 return TripCount; 2956 2957 assert(L && "Create Trip Count for null loop."); 2958 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator()); 2959 // Find the loop boundaries. 2960 ScalarEvolution *SE = PSE.getSE(); 2961 const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount(); 2962 assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCount) && 2963 "Invalid loop count"); 2964 2965 Type *IdxTy = Legal->getWidestInductionType(); 2966 assert(IdxTy && "No type for induction"); 2967 2968 // The exit count might have the type of i64 while the phi is i32. This can 2969 // happen if we have an induction variable that is sign extended before the 2970 // compare. The only way that we get a backedge taken count is that the 2971 // induction variable was signed and as such will not overflow. In such a case 2972 // truncation is legal. 2973 if (SE->getTypeSizeInBits(BackedgeTakenCount->getType()) > 2974 IdxTy->getPrimitiveSizeInBits()) 2975 BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount, IdxTy); 2976 BackedgeTakenCount = SE->getNoopOrZeroExtend(BackedgeTakenCount, IdxTy); 2977 2978 // Get the total trip count from the count by adding 1. 2979 const SCEV *ExitCount = SE->getAddExpr( 2980 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType())); 2981 2982 const DataLayout &DL = L->getHeader()->getModule()->getDataLayout(); 2983 2984 // Expand the trip count and place the new instructions in the preheader. 2985 // Notice that the pre-header does not change, only the loop body. 2986 SCEVExpander Exp(*SE, DL, "induction"); 2987 2988 // Count holds the overall loop count (N). 2989 TripCount = Exp.expandCodeFor(ExitCount, ExitCount->getType(), 2990 L->getLoopPreheader()->getTerminator()); 2991 2992 if (TripCount->getType()->isPointerTy()) 2993 TripCount = 2994 CastInst::CreatePointerCast(TripCount, IdxTy, "exitcount.ptrcnt.to.int", 2995 L->getLoopPreheader()->getTerminator()); 2996 2997 return TripCount; 2998 } 2999 3000 Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) { 3001 if (VectorTripCount) 3002 return VectorTripCount; 3003 3004 Value *TC = getOrCreateTripCount(L); 3005 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator()); 3006 3007 Type *Ty = TC->getType(); 3008 // This is where we can make the step a runtime constant. 3009 Value *Step = createStepForVF(Builder, ConstantInt::get(Ty, UF), VF); 3010 3011 // If the tail is to be folded by masking, round the number of iterations N 3012 // up to a multiple of Step instead of rounding down. This is done by first 3013 // adding Step-1 and then rounding down. Note that it's ok if this addition 3014 // overflows: the vector induction variable will eventually wrap to zero given 3015 // that it starts at zero and its Step is a power of two; the loop will then 3016 // exit, with the last early-exit vector comparison also producing all-true. 3017 if (Cost->foldTailByMasking()) { 3018 assert(isPowerOf2_32(VF.getKnownMinValue() * UF) && 3019 "VF*UF must be a power of 2 when folding tail by masking"); 3020 assert(!VF.isScalable() && 3021 "Tail folding not yet supported for scalable vectors"); 3022 TC = Builder.CreateAdd( 3023 TC, ConstantInt::get(Ty, VF.getKnownMinValue() * UF - 1), "n.rnd.up"); 3024 } 3025 3026 // Now we need to generate the expression for the part of the loop that the 3027 // vectorized body will execute. This is equal to N - (N % Step) if scalar 3028 // iterations are not required for correctness, or N - Step, otherwise. Step 3029 // is equal to the vectorization factor (number of SIMD elements) times the 3030 // unroll factor (number of SIMD instructions). 3031 Value *R = Builder.CreateURem(TC, Step, "n.mod.vf"); 3032 3033 // There are two cases where we need to ensure (at least) the last iteration 3034 // runs in the scalar remainder loop. Thus, if the step evenly divides 3035 // the trip count, we set the remainder to be equal to the step. If the step 3036 // does not evenly divide the trip count, no adjustment is necessary since 3037 // there will already be scalar iterations. Note that the minimum iterations 3038 // check ensures that N >= Step. The cases are: 3039 // 1) If there is a non-reversed interleaved group that may speculatively 3040 // access memory out-of-bounds. 3041 // 2) If any instruction may follow a conditionally taken exit. That is, if 3042 // the loop contains multiple exiting blocks, or a single exiting block 3043 // which is not the latch. 3044 if (VF.isVector() && Cost->requiresScalarEpilogue()) { 3045 auto *IsZero = Builder.CreateICmpEQ(R, ConstantInt::get(R->getType(), 0)); 3046 R = Builder.CreateSelect(IsZero, Step, R); 3047 } 3048 3049 VectorTripCount = Builder.CreateSub(TC, R, "n.vec"); 3050 3051 return VectorTripCount; 3052 } 3053 3054 Value *InnerLoopVectorizer::createBitOrPointerCast(Value *V, VectorType *DstVTy, 3055 const DataLayout &DL) { 3056 // Verify that V is a vector type with same number of elements as DstVTy. 3057 auto *DstFVTy = cast<FixedVectorType>(DstVTy); 3058 unsigned VF = DstFVTy->getNumElements(); 3059 auto *SrcVecTy = cast<FixedVectorType>(V->getType()); 3060 assert((VF == SrcVecTy->getNumElements()) && "Vector dimensions do not match"); 3061 Type *SrcElemTy = SrcVecTy->getElementType(); 3062 Type *DstElemTy = DstFVTy->getElementType(); 3063 assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) && 3064 "Vector elements must have same size"); 3065 3066 // Do a direct cast if element types are castable. 3067 if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL)) { 3068 return Builder.CreateBitOrPointerCast(V, DstFVTy); 3069 } 3070 // V cannot be directly casted to desired vector type. 3071 // May happen when V is a floating point vector but DstVTy is a vector of 3072 // pointers or vice-versa. Handle this using a two-step bitcast using an 3073 // intermediate Integer type for the bitcast i.e. Ptr <-> Int <-> Float. 3074 assert((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) && 3075 "Only one type should be a pointer type"); 3076 assert((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) && 3077 "Only one type should be a floating point type"); 3078 Type *IntTy = 3079 IntegerType::getIntNTy(V->getContext(), DL.getTypeSizeInBits(SrcElemTy)); 3080 auto *VecIntTy = FixedVectorType::get(IntTy, VF); 3081 Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy); 3082 return Builder.CreateBitOrPointerCast(CastVal, DstFVTy); 3083 } 3084 3085 void InnerLoopVectorizer::emitMinimumIterationCountCheck(Loop *L, 3086 BasicBlock *Bypass) { 3087 Value *Count = getOrCreateTripCount(L); 3088 // Reuse existing vector loop preheader for TC checks. 3089 // Note that new preheader block is generated for vector loop. 3090 BasicBlock *const TCCheckBlock = LoopVectorPreHeader; 3091 IRBuilder<> Builder(TCCheckBlock->getTerminator()); 3092 3093 // Generate code to check if the loop's trip count is less than VF * UF, or 3094 // equal to it in case a scalar epilogue is required; this implies that the 3095 // vector trip count is zero. This check also covers the case where adding one 3096 // to the backedge-taken count overflowed leading to an incorrect trip count 3097 // of zero. In this case we will also jump to the scalar loop. 3098 auto P = Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE 3099 : ICmpInst::ICMP_ULT; 3100 3101 // If tail is to be folded, vector loop takes care of all iterations. 3102 Value *CheckMinIters = Builder.getFalse(); 3103 if (!Cost->foldTailByMasking()) { 3104 Value *Step = 3105 createStepForVF(Builder, ConstantInt::get(Count->getType(), UF), VF); 3106 CheckMinIters = Builder.CreateICmp(P, Count, Step, "min.iters.check"); 3107 } 3108 // Create new preheader for vector loop. 3109 LoopVectorPreHeader = 3110 SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(), DT, LI, nullptr, 3111 "vector.ph"); 3112 3113 assert(DT->properlyDominates(DT->getNode(TCCheckBlock), 3114 DT->getNode(Bypass)->getIDom()) && 3115 "TC check is expected to dominate Bypass"); 3116 3117 // Update dominator for Bypass & LoopExit. 3118 DT->changeImmediateDominator(Bypass, TCCheckBlock); 3119 DT->changeImmediateDominator(LoopExitBlock, TCCheckBlock); 3120 3121 ReplaceInstWithInst( 3122 TCCheckBlock->getTerminator(), 3123 BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters)); 3124 LoopBypassBlocks.push_back(TCCheckBlock); 3125 } 3126 3127 void InnerLoopVectorizer::emitSCEVChecks(Loop *L, BasicBlock *Bypass) { 3128 // Reuse existing vector loop preheader for SCEV checks. 3129 // Note that new preheader block is generated for vector loop. 3130 BasicBlock *const SCEVCheckBlock = LoopVectorPreHeader; 3131 3132 // Generate the code to check that the SCEV assumptions that we made. 3133 // We want the new basic block to start at the first instruction in a 3134 // sequence of instructions that form a check. 3135 SCEVExpander Exp(*PSE.getSE(), Bypass->getModule()->getDataLayout(), 3136 "scev.check"); 3137 Value *SCEVCheck = Exp.expandCodeForPredicate( 3138 &PSE.getUnionPredicate(), SCEVCheckBlock->getTerminator()); 3139 3140 if (auto *C = dyn_cast<ConstantInt>(SCEVCheck)) 3141 if (C->isZero()) 3142 return; 3143 3144 assert(!(SCEVCheckBlock->getParent()->hasOptSize() || 3145 (OptForSizeBasedOnProfile && 3146 Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && 3147 "Cannot SCEV check stride or overflow when optimizing for size"); 3148 3149 SCEVCheckBlock->setName("vector.scevcheck"); 3150 // Create new preheader for vector loop. 3151 LoopVectorPreHeader = 3152 SplitBlock(SCEVCheckBlock, SCEVCheckBlock->getTerminator(), DT, LI, 3153 nullptr, "vector.ph"); 3154 3155 // Update dominator only if this is first RT check. 3156 if (LoopBypassBlocks.empty()) { 3157 DT->changeImmediateDominator(Bypass, SCEVCheckBlock); 3158 DT->changeImmediateDominator(LoopExitBlock, SCEVCheckBlock); 3159 } 3160 3161 ReplaceInstWithInst( 3162 SCEVCheckBlock->getTerminator(), 3163 BranchInst::Create(Bypass, LoopVectorPreHeader, SCEVCheck)); 3164 LoopBypassBlocks.push_back(SCEVCheckBlock); 3165 AddedSafetyChecks = true; 3166 } 3167 3168 void InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass) { 3169 // VPlan-native path does not do any analysis for runtime checks currently. 3170 if (EnableVPlanNativePath) 3171 return; 3172 3173 // Reuse existing vector loop preheader for runtime memory checks. 3174 // Note that new preheader block is generated for vector loop. 3175 BasicBlock *const MemCheckBlock = L->getLoopPreheader(); 3176 3177 // Generate the code that checks in runtime if arrays overlap. We put the 3178 // checks into a separate block to make the more common case of few elements 3179 // faster. 3180 auto *LAI = Legal->getLAI(); 3181 const auto &RtPtrChecking = *LAI->getRuntimePointerChecking(); 3182 if (!RtPtrChecking.Need) 3183 return; 3184 3185 if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) { 3186 assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && 3187 "Cannot emit memory checks when optimizing for size, unless forced " 3188 "to vectorize."); 3189 ORE->emit([&]() { 3190 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize", 3191 L->getStartLoc(), L->getHeader()) 3192 << "Code-size may be reduced by not forcing " 3193 "vectorization, or by source-code modifications " 3194 "eliminating the need for runtime checks " 3195 "(e.g., adding 'restrict')."; 3196 }); 3197 } 3198 3199 MemCheckBlock->setName("vector.memcheck"); 3200 // Create new preheader for vector loop. 3201 LoopVectorPreHeader = 3202 SplitBlock(MemCheckBlock, MemCheckBlock->getTerminator(), DT, LI, nullptr, 3203 "vector.ph"); 3204 3205 auto *CondBranch = cast<BranchInst>( 3206 Builder.CreateCondBr(Builder.getTrue(), Bypass, LoopVectorPreHeader)); 3207 ReplaceInstWithInst(MemCheckBlock->getTerminator(), CondBranch); 3208 LoopBypassBlocks.push_back(MemCheckBlock); 3209 AddedSafetyChecks = true; 3210 3211 // Update dominator only if this is first RT check. 3212 if (LoopBypassBlocks.empty()) { 3213 DT->changeImmediateDominator(Bypass, MemCheckBlock); 3214 DT->changeImmediateDominator(LoopExitBlock, MemCheckBlock); 3215 } 3216 3217 Instruction *FirstCheckInst; 3218 Instruction *MemRuntimeCheck; 3219 std::tie(FirstCheckInst, MemRuntimeCheck) = 3220 addRuntimeChecks(MemCheckBlock->getTerminator(), OrigLoop, 3221 RtPtrChecking.getChecks(), RtPtrChecking.getSE()); 3222 assert(MemRuntimeCheck && "no RT checks generated although RtPtrChecking " 3223 "claimed checks are required"); 3224 CondBranch->setCondition(MemRuntimeCheck); 3225 3226 // We currently don't use LoopVersioning for the actual loop cloning but we 3227 // still use it to add the noalias metadata. 3228 LVer = std::make_unique<LoopVersioning>( 3229 *Legal->getLAI(), 3230 Legal->getLAI()->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, 3231 DT, PSE.getSE()); 3232 LVer->prepareNoAliasMetadata(); 3233 } 3234 3235 Value *InnerLoopVectorizer::emitTransformedIndex( 3236 IRBuilder<> &B, Value *Index, ScalarEvolution *SE, const DataLayout &DL, 3237 const InductionDescriptor &ID) const { 3238 3239 SCEVExpander Exp(*SE, DL, "induction"); 3240 auto Step = ID.getStep(); 3241 auto StartValue = ID.getStartValue(); 3242 assert(Index->getType() == Step->getType() && 3243 "Index type does not match StepValue type"); 3244 3245 // Note: the IR at this point is broken. We cannot use SE to create any new 3246 // SCEV and then expand it, hoping that SCEV's simplification will give us 3247 // a more optimal code. Unfortunately, attempt of doing so on invalid IR may 3248 // lead to various SCEV crashes. So all we can do is to use builder and rely 3249 // on InstCombine for future simplifications. Here we handle some trivial 3250 // cases only. 3251 auto CreateAdd = [&B](Value *X, Value *Y) { 3252 assert(X->getType() == Y->getType() && "Types don't match!"); 3253 if (auto *CX = dyn_cast<ConstantInt>(X)) 3254 if (CX->isZero()) 3255 return Y; 3256 if (auto *CY = dyn_cast<ConstantInt>(Y)) 3257 if (CY->isZero()) 3258 return X; 3259 return B.CreateAdd(X, Y); 3260 }; 3261 3262 auto CreateMul = [&B](Value *X, Value *Y) { 3263 assert(X->getType() == Y->getType() && "Types don't match!"); 3264 if (auto *CX = dyn_cast<ConstantInt>(X)) 3265 if (CX->isOne()) 3266 return Y; 3267 if (auto *CY = dyn_cast<ConstantInt>(Y)) 3268 if (CY->isOne()) 3269 return X; 3270 return B.CreateMul(X, Y); 3271 }; 3272 3273 // Get a suitable insert point for SCEV expansion. For blocks in the vector 3274 // loop, choose the end of the vector loop header (=LoopVectorBody), because 3275 // the DomTree is not kept up-to-date for additional blocks generated in the 3276 // vector loop. By using the header as insertion point, we guarantee that the 3277 // expanded instructions dominate all their uses. 3278 auto GetInsertPoint = [this, &B]() { 3279 BasicBlock *InsertBB = B.GetInsertPoint()->getParent(); 3280 if (InsertBB != LoopVectorBody && 3281 LI->getLoopFor(LoopVectorBody) == LI->getLoopFor(InsertBB)) 3282 return LoopVectorBody->getTerminator(); 3283 return &*B.GetInsertPoint(); 3284 }; 3285 switch (ID.getKind()) { 3286 case InductionDescriptor::IK_IntInduction: { 3287 assert(Index->getType() == StartValue->getType() && 3288 "Index type does not match StartValue type"); 3289 if (ID.getConstIntStepValue() && ID.getConstIntStepValue()->isMinusOne()) 3290 return B.CreateSub(StartValue, Index); 3291 auto *Offset = CreateMul( 3292 Index, Exp.expandCodeFor(Step, Index->getType(), GetInsertPoint())); 3293 return CreateAdd(StartValue, Offset); 3294 } 3295 case InductionDescriptor::IK_PtrInduction: { 3296 assert(isa<SCEVConstant>(Step) && 3297 "Expected constant step for pointer induction"); 3298 return B.CreateGEP( 3299 StartValue->getType()->getPointerElementType(), StartValue, 3300 CreateMul(Index, 3301 Exp.expandCodeFor(Step, Index->getType(), GetInsertPoint()))); 3302 } 3303 case InductionDescriptor::IK_FpInduction: { 3304 assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value"); 3305 auto InductionBinOp = ID.getInductionBinOp(); 3306 assert(InductionBinOp && 3307 (InductionBinOp->getOpcode() == Instruction::FAdd || 3308 InductionBinOp->getOpcode() == Instruction::FSub) && 3309 "Original bin op should be defined for FP induction"); 3310 3311 Value *StepValue = cast<SCEVUnknown>(Step)->getValue(); 3312 3313 // Floating point operations had to be 'fast' to enable the induction. 3314 FastMathFlags Flags; 3315 Flags.setFast(); 3316 3317 Value *MulExp = B.CreateFMul(StepValue, Index); 3318 if (isa<Instruction>(MulExp)) 3319 // We have to check, the MulExp may be a constant. 3320 cast<Instruction>(MulExp)->setFastMathFlags(Flags); 3321 3322 Value *BOp = B.CreateBinOp(InductionBinOp->getOpcode(), StartValue, MulExp, 3323 "induction"); 3324 if (isa<Instruction>(BOp)) 3325 cast<Instruction>(BOp)->setFastMathFlags(Flags); 3326 3327 return BOp; 3328 } 3329 case InductionDescriptor::IK_NoInduction: 3330 return nullptr; 3331 } 3332 llvm_unreachable("invalid enum"); 3333 } 3334 3335 Loop *InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) { 3336 LoopScalarBody = OrigLoop->getHeader(); 3337 LoopVectorPreHeader = OrigLoop->getLoopPreheader(); 3338 LoopExitBlock = OrigLoop->getUniqueExitBlock(); 3339 assert(LoopExitBlock && "Must have an exit block"); 3340 assert(LoopVectorPreHeader && "Invalid loop structure"); 3341 3342 LoopMiddleBlock = 3343 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT, 3344 LI, nullptr, Twine(Prefix) + "middle.block"); 3345 LoopScalarPreHeader = 3346 SplitBlock(LoopMiddleBlock, LoopMiddleBlock->getTerminator(), DT, LI, 3347 nullptr, Twine(Prefix) + "scalar.ph"); 3348 3349 // Set up branch from middle block to the exit and scalar preheader blocks. 3350 // completeLoopSkeleton will update the condition to use an iteration check, 3351 // if required to decide whether to execute the remainder. 3352 BranchInst *BrInst = 3353 BranchInst::Create(LoopExitBlock, LoopScalarPreHeader, Builder.getTrue()); 3354 auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator(); 3355 BrInst->setDebugLoc(ScalarLatchTerm->getDebugLoc()); 3356 ReplaceInstWithInst(LoopMiddleBlock->getTerminator(), BrInst); 3357 3358 // We intentionally don't let SplitBlock to update LoopInfo since 3359 // LoopVectorBody should belong to another loop than LoopVectorPreHeader. 3360 // LoopVectorBody is explicitly added to the correct place few lines later. 3361 LoopVectorBody = 3362 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT, 3363 nullptr, nullptr, Twine(Prefix) + "vector.body"); 3364 3365 // Update dominator for loop exit. 3366 DT->changeImmediateDominator(LoopExitBlock, LoopMiddleBlock); 3367 3368 // Create and register the new vector loop. 3369 Loop *Lp = LI->AllocateLoop(); 3370 Loop *ParentLoop = OrigLoop->getParentLoop(); 3371 3372 // Insert the new loop into the loop nest and register the new basic blocks 3373 // before calling any utilities such as SCEV that require valid LoopInfo. 3374 if (ParentLoop) { 3375 ParentLoop->addChildLoop(Lp); 3376 } else { 3377 LI->addTopLevelLoop(Lp); 3378 } 3379 Lp->addBasicBlockToLoop(LoopVectorBody, *LI); 3380 return Lp; 3381 } 3382 3383 void InnerLoopVectorizer::createInductionResumeValues( 3384 Loop *L, Value *VectorTripCount, 3385 std::pair<BasicBlock *, Value *> AdditionalBypass) { 3386 assert(VectorTripCount && L && "Expected valid arguments"); 3387 assert(((AdditionalBypass.first && AdditionalBypass.second) || 3388 (!AdditionalBypass.first && !AdditionalBypass.second)) && 3389 "Inconsistent information about additional bypass."); 3390 // We are going to resume the execution of the scalar loop. 3391 // Go over all of the induction variables that we found and fix the 3392 // PHIs that are left in the scalar version of the loop. 3393 // The starting values of PHI nodes depend on the counter of the last 3394 // iteration in the vectorized loop. 3395 // If we come from a bypass edge then we need to start from the original 3396 // start value. 3397 for (auto &InductionEntry : Legal->getInductionVars()) { 3398 PHINode *OrigPhi = InductionEntry.first; 3399 InductionDescriptor II = InductionEntry.second; 3400 3401 // Create phi nodes to merge from the backedge-taken check block. 3402 PHINode *BCResumeVal = 3403 PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val", 3404 LoopScalarPreHeader->getTerminator()); 3405 // Copy original phi DL over to the new one. 3406 BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc()); 3407 Value *&EndValue = IVEndValues[OrigPhi]; 3408 Value *EndValueFromAdditionalBypass = AdditionalBypass.second; 3409 if (OrigPhi == OldInduction) { 3410 // We know what the end value is. 3411 EndValue = VectorTripCount; 3412 } else { 3413 IRBuilder<> B(L->getLoopPreheader()->getTerminator()); 3414 Type *StepType = II.getStep()->getType(); 3415 Instruction::CastOps CastOp = 3416 CastInst::getCastOpcode(VectorTripCount, true, StepType, true); 3417 Value *CRD = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.crd"); 3418 const DataLayout &DL = LoopScalarBody->getModule()->getDataLayout(); 3419 EndValue = emitTransformedIndex(B, CRD, PSE.getSE(), DL, II); 3420 EndValue->setName("ind.end"); 3421 3422 // Compute the end value for the additional bypass (if applicable). 3423 if (AdditionalBypass.first) { 3424 B.SetInsertPoint(&(*AdditionalBypass.first->getFirstInsertionPt())); 3425 CastOp = CastInst::getCastOpcode(AdditionalBypass.second, true, 3426 StepType, true); 3427 CRD = 3428 B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.crd"); 3429 EndValueFromAdditionalBypass = 3430 emitTransformedIndex(B, CRD, PSE.getSE(), DL, II); 3431 EndValueFromAdditionalBypass->setName("ind.end"); 3432 } 3433 } 3434 // The new PHI merges the original incoming value, in case of a bypass, 3435 // or the value at the end of the vectorized loop. 3436 BCResumeVal->addIncoming(EndValue, LoopMiddleBlock); 3437 3438 // Fix the scalar body counter (PHI node). 3439 // The old induction's phi node in the scalar body needs the truncated 3440 // value. 3441 for (BasicBlock *BB : LoopBypassBlocks) 3442 BCResumeVal->addIncoming(II.getStartValue(), BB); 3443 3444 if (AdditionalBypass.first) 3445 BCResumeVal->setIncomingValueForBlock(AdditionalBypass.first, 3446 EndValueFromAdditionalBypass); 3447 3448 OrigPhi->setIncomingValueForBlock(LoopScalarPreHeader, BCResumeVal); 3449 } 3450 } 3451 3452 BasicBlock *InnerLoopVectorizer::completeLoopSkeleton(Loop *L, 3453 MDNode *OrigLoopID) { 3454 assert(L && "Expected valid loop."); 3455 3456 // The trip counts should be cached by now. 3457 Value *Count = getOrCreateTripCount(L); 3458 Value *VectorTripCount = getOrCreateVectorTripCount(L); 3459 3460 auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator(); 3461 3462 // Add a check in the middle block to see if we have completed 3463 // all of the iterations in the first vector loop. 3464 // If (N - N%VF) == N, then we *don't* need to run the remainder. 3465 // If tail is to be folded, we know we don't need to run the remainder. 3466 if (!Cost->foldTailByMasking()) { 3467 Instruction *CmpN = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, 3468 Count, VectorTripCount, "cmp.n", 3469 LoopMiddleBlock->getTerminator()); 3470 3471 // Here we use the same DebugLoc as the scalar loop latch terminator instead 3472 // of the corresponding compare because they may have ended up with 3473 // different line numbers and we want to avoid awkward line stepping while 3474 // debugging. Eg. if the compare has got a line number inside the loop. 3475 CmpN->setDebugLoc(ScalarLatchTerm->getDebugLoc()); 3476 cast<BranchInst>(LoopMiddleBlock->getTerminator())->setCondition(CmpN); 3477 } 3478 3479 // Get ready to start creating new instructions into the vectorized body. 3480 assert(LoopVectorPreHeader == L->getLoopPreheader() && 3481 "Inconsistent vector loop preheader"); 3482 Builder.SetInsertPoint(&*LoopVectorBody->getFirstInsertionPt()); 3483 3484 Optional<MDNode *> VectorizedLoopID = 3485 makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll, 3486 LLVMLoopVectorizeFollowupVectorized}); 3487 if (VectorizedLoopID.hasValue()) { 3488 L->setLoopID(VectorizedLoopID.getValue()); 3489 3490 // Do not setAlreadyVectorized if loop attributes have been defined 3491 // explicitly. 3492 return LoopVectorPreHeader; 3493 } 3494 3495 // Keep all loop hints from the original loop on the vector loop (we'll 3496 // replace the vectorizer-specific hints below). 3497 if (MDNode *LID = OrigLoop->getLoopID()) 3498 L->setLoopID(LID); 3499 3500 LoopVectorizeHints Hints(L, true, *ORE); 3501 Hints.setAlreadyVectorized(); 3502 3503 #ifdef EXPENSIVE_CHECKS 3504 assert(DT->verify(DominatorTree::VerificationLevel::Fast)); 3505 LI->verify(*DT); 3506 #endif 3507 3508 return LoopVectorPreHeader; 3509 } 3510 3511 BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() { 3512 /* 3513 In this function we generate a new loop. The new loop will contain 3514 the vectorized instructions while the old loop will continue to run the 3515 scalar remainder. 3516 3517 [ ] <-- loop iteration number check. 3518 / | 3519 / v 3520 | [ ] <-- vector loop bypass (may consist of multiple blocks). 3521 | / | 3522 | / v 3523 || [ ] <-- vector pre header. 3524 |/ | 3525 | v 3526 | [ ] \ 3527 | [ ]_| <-- vector loop. 3528 | | 3529 | v 3530 | -[ ] <--- middle-block. 3531 | / | 3532 | / v 3533 -|- >[ ] <--- new preheader. 3534 | | 3535 | v 3536 | [ ] \ 3537 | [ ]_| <-- old scalar loop to handle remainder. 3538 \ | 3539 \ v 3540 >[ ] <-- exit block. 3541 ... 3542 */ 3543 3544 // Get the metadata of the original loop before it gets modified. 3545 MDNode *OrigLoopID = OrigLoop->getLoopID(); 3546 3547 // Create an empty vector loop, and prepare basic blocks for the runtime 3548 // checks. 3549 Loop *Lp = createVectorLoopSkeleton(""); 3550 3551 // Now, compare the new count to zero. If it is zero skip the vector loop and 3552 // jump to the scalar loop. This check also covers the case where the 3553 // backedge-taken count is uint##_max: adding one to it will overflow leading 3554 // to an incorrect trip count of zero. In this (rare) case we will also jump 3555 // to the scalar loop. 3556 emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader); 3557 3558 // Generate the code to check any assumptions that we've made for SCEV 3559 // expressions. 3560 emitSCEVChecks(Lp, LoopScalarPreHeader); 3561 3562 // Generate the code that checks in runtime if arrays overlap. We put the 3563 // checks into a separate block to make the more common case of few elements 3564 // faster. 3565 emitMemRuntimeChecks(Lp, LoopScalarPreHeader); 3566 3567 // Some loops have a single integer induction variable, while other loops 3568 // don't. One example is c++ iterators that often have multiple pointer 3569 // induction variables. In the code below we also support a case where we 3570 // don't have a single induction variable. 3571 // 3572 // We try to obtain an induction variable from the original loop as hard 3573 // as possible. However if we don't find one that: 3574 // - is an integer 3575 // - counts from zero, stepping by one 3576 // - is the size of the widest induction variable type 3577 // then we create a new one. 3578 OldInduction = Legal->getPrimaryInduction(); 3579 Type *IdxTy = Legal->getWidestInductionType(); 3580 Value *StartIdx = ConstantInt::get(IdxTy, 0); 3581 // The loop step is equal to the vectorization factor (num of SIMD elements) 3582 // times the unroll factor (num of SIMD instructions). 3583 Builder.SetInsertPoint(&*Lp->getHeader()->getFirstInsertionPt()); 3584 Value *Step = createStepForVF(Builder, ConstantInt::get(IdxTy, UF), VF); 3585 Value *CountRoundDown = getOrCreateVectorTripCount(Lp); 3586 Induction = 3587 createInductionVariable(Lp, StartIdx, CountRoundDown, Step, 3588 getDebugLocFromInstOrOperands(OldInduction)); 3589 3590 // Emit phis for the new starting index of the scalar loop. 3591 createInductionResumeValues(Lp, CountRoundDown); 3592 3593 return completeLoopSkeleton(Lp, OrigLoopID); 3594 } 3595 3596 // Fix up external users of the induction variable. At this point, we are 3597 // in LCSSA form, with all external PHIs that use the IV having one input value, 3598 // coming from the remainder loop. We need those PHIs to also have a correct 3599 // value for the IV when arriving directly from the middle block. 3600 void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi, 3601 const InductionDescriptor &II, 3602 Value *CountRoundDown, Value *EndValue, 3603 BasicBlock *MiddleBlock) { 3604 // There are two kinds of external IV usages - those that use the value 3605 // computed in the last iteration (the PHI) and those that use the penultimate 3606 // value (the value that feeds into the phi from the loop latch). 3607 // We allow both, but they, obviously, have different values. 3608 3609 assert(OrigLoop->getUniqueExitBlock() && "Expected a single exit block"); 3610 3611 DenseMap<Value *, Value *> MissingVals; 3612 3613 // An external user of the last iteration's value should see the value that 3614 // the remainder loop uses to initialize its own IV. 3615 Value *PostInc = OrigPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch()); 3616 for (User *U : PostInc->users()) { 3617 Instruction *UI = cast<Instruction>(U); 3618 if (!OrigLoop->contains(UI)) { 3619 assert(isa<PHINode>(UI) && "Expected LCSSA form"); 3620 MissingVals[UI] = EndValue; 3621 } 3622 } 3623 3624 // An external user of the penultimate value need to see EndValue - Step. 3625 // The simplest way to get this is to recompute it from the constituent SCEVs, 3626 // that is Start + (Step * (CRD - 1)). 3627 for (User *U : OrigPhi->users()) { 3628 auto *UI = cast<Instruction>(U); 3629 if (!OrigLoop->contains(UI)) { 3630 const DataLayout &DL = 3631 OrigLoop->getHeader()->getModule()->getDataLayout(); 3632 assert(isa<PHINode>(UI) && "Expected LCSSA form"); 3633 3634 IRBuilder<> B(MiddleBlock->getTerminator()); 3635 Value *CountMinusOne = B.CreateSub( 3636 CountRoundDown, ConstantInt::get(CountRoundDown->getType(), 1)); 3637 Value *CMO = 3638 !II.getStep()->getType()->isIntegerTy() 3639 ? B.CreateCast(Instruction::SIToFP, CountMinusOne, 3640 II.getStep()->getType()) 3641 : B.CreateSExtOrTrunc(CountMinusOne, II.getStep()->getType()); 3642 CMO->setName("cast.cmo"); 3643 Value *Escape = emitTransformedIndex(B, CMO, PSE.getSE(), DL, II); 3644 Escape->setName("ind.escape"); 3645 MissingVals[UI] = Escape; 3646 } 3647 } 3648 3649 for (auto &I : MissingVals) { 3650 PHINode *PHI = cast<PHINode>(I.first); 3651 // One corner case we have to handle is two IVs "chasing" each-other, 3652 // that is %IV2 = phi [...], [ %IV1, %latch ] 3653 // In this case, if IV1 has an external use, we need to avoid adding both 3654 // "last value of IV1" and "penultimate value of IV2". So, verify that we 3655 // don't already have an incoming value for the middle block. 3656 if (PHI->getBasicBlockIndex(MiddleBlock) == -1) 3657 PHI->addIncoming(I.second, MiddleBlock); 3658 } 3659 } 3660 3661 namespace { 3662 3663 struct CSEDenseMapInfo { 3664 static bool canHandle(const Instruction *I) { 3665 return isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) || 3666 isa<ShuffleVectorInst>(I) || isa<GetElementPtrInst>(I); 3667 } 3668 3669 static inline Instruction *getEmptyKey() { 3670 return DenseMapInfo<Instruction *>::getEmptyKey(); 3671 } 3672 3673 static inline Instruction *getTombstoneKey() { 3674 return DenseMapInfo<Instruction *>::getTombstoneKey(); 3675 } 3676 3677 static unsigned getHashValue(const Instruction *I) { 3678 assert(canHandle(I) && "Unknown instruction!"); 3679 return hash_combine(I->getOpcode(), hash_combine_range(I->value_op_begin(), 3680 I->value_op_end())); 3681 } 3682 3683 static bool isEqual(const Instruction *LHS, const Instruction *RHS) { 3684 if (LHS == getEmptyKey() || RHS == getEmptyKey() || 3685 LHS == getTombstoneKey() || RHS == getTombstoneKey()) 3686 return LHS == RHS; 3687 return LHS->isIdenticalTo(RHS); 3688 } 3689 }; 3690 3691 } // end anonymous namespace 3692 3693 ///Perform cse of induction variable instructions. 3694 static void cse(BasicBlock *BB) { 3695 // Perform simple cse. 3696 SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap; 3697 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) { 3698 Instruction *In = &*I++; 3699 3700 if (!CSEDenseMapInfo::canHandle(In)) 3701 continue; 3702 3703 // Check if we can replace this instruction with any of the 3704 // visited instructions. 3705 if (Instruction *V = CSEMap.lookup(In)) { 3706 In->replaceAllUsesWith(V); 3707 In->eraseFromParent(); 3708 continue; 3709 } 3710 3711 CSEMap[In] = In; 3712 } 3713 } 3714 3715 InstructionCost 3716 LoopVectorizationCostModel::getVectorCallCost(CallInst *CI, ElementCount VF, 3717 bool &NeedToScalarize) { 3718 assert(!VF.isScalable() && "scalable vectors not yet supported."); 3719 Function *F = CI->getCalledFunction(); 3720 Type *ScalarRetTy = CI->getType(); 3721 SmallVector<Type *, 4> Tys, ScalarTys; 3722 for (auto &ArgOp : CI->arg_operands()) 3723 ScalarTys.push_back(ArgOp->getType()); 3724 3725 // Estimate cost of scalarized vector call. The source operands are assumed 3726 // to be vectors, so we need to extract individual elements from there, 3727 // execute VF scalar calls, and then gather the result into the vector return 3728 // value. 3729 InstructionCost ScalarCallCost = 3730 TTI.getCallInstrCost(F, ScalarRetTy, ScalarTys, TTI::TCK_RecipThroughput); 3731 if (VF.isScalar()) 3732 return ScalarCallCost; 3733 3734 // Compute corresponding vector type for return value and arguments. 3735 Type *RetTy = ToVectorTy(ScalarRetTy, VF); 3736 for (Type *ScalarTy : ScalarTys) 3737 Tys.push_back(ToVectorTy(ScalarTy, VF)); 3738 3739 // Compute costs of unpacking argument values for the scalar calls and 3740 // packing the return values to a vector. 3741 InstructionCost ScalarizationCost = getScalarizationOverhead(CI, VF); 3742 3743 InstructionCost Cost = 3744 ScalarCallCost * VF.getKnownMinValue() + ScalarizationCost; 3745 3746 // If we can't emit a vector call for this function, then the currently found 3747 // cost is the cost we need to return. 3748 NeedToScalarize = true; 3749 VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/); 3750 Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape); 3751 3752 if (!TLI || CI->isNoBuiltin() || !VecFunc) 3753 return Cost; 3754 3755 // If the corresponding vector cost is cheaper, return its cost. 3756 InstructionCost VectorCallCost = 3757 TTI.getCallInstrCost(nullptr, RetTy, Tys, TTI::TCK_RecipThroughput); 3758 if (VectorCallCost < Cost) { 3759 NeedToScalarize = false; 3760 Cost = VectorCallCost; 3761 } 3762 return Cost; 3763 } 3764 3765 InstructionCost 3766 LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI, 3767 ElementCount VF) { 3768 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 3769 assert(ID && "Expected intrinsic call!"); 3770 3771 IntrinsicCostAttributes CostAttrs(ID, *CI, VF); 3772 return TTI.getIntrinsicInstrCost(CostAttrs, 3773 TargetTransformInfo::TCK_RecipThroughput); 3774 } 3775 3776 static Type *smallestIntegerVectorType(Type *T1, Type *T2) { 3777 auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType()); 3778 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType()); 3779 return I1->getBitWidth() < I2->getBitWidth() ? T1 : T2; 3780 } 3781 3782 static Type *largestIntegerVectorType(Type *T1, Type *T2) { 3783 auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType()); 3784 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType()); 3785 return I1->getBitWidth() > I2->getBitWidth() ? T1 : T2; 3786 } 3787 3788 void InnerLoopVectorizer::truncateToMinimalBitwidths() { 3789 // For every instruction `I` in MinBWs, truncate the operands, create a 3790 // truncated version of `I` and reextend its result. InstCombine runs 3791 // later and will remove any ext/trunc pairs. 3792 SmallPtrSet<Value *, 4> Erased; 3793 for (const auto &KV : Cost->getMinimalBitwidths()) { 3794 // If the value wasn't vectorized, we must maintain the original scalar 3795 // type. The absence of the value from VectorLoopValueMap indicates that it 3796 // wasn't vectorized. 3797 if (!VectorLoopValueMap.hasAnyVectorValue(KV.first)) 3798 continue; 3799 for (unsigned Part = 0; Part < UF; ++Part) { 3800 Value *I = getOrCreateVectorValue(KV.first, Part); 3801 if (Erased.count(I) || I->use_empty() || !isa<Instruction>(I)) 3802 continue; 3803 Type *OriginalTy = I->getType(); 3804 Type *ScalarTruncatedTy = 3805 IntegerType::get(OriginalTy->getContext(), KV.second); 3806 auto *TruncatedTy = FixedVectorType::get( 3807 ScalarTruncatedTy, 3808 cast<FixedVectorType>(OriginalTy)->getNumElements()); 3809 if (TruncatedTy == OriginalTy) 3810 continue; 3811 3812 IRBuilder<> B(cast<Instruction>(I)); 3813 auto ShrinkOperand = [&](Value *V) -> Value * { 3814 if (auto *ZI = dyn_cast<ZExtInst>(V)) 3815 if (ZI->getSrcTy() == TruncatedTy) 3816 return ZI->getOperand(0); 3817 return B.CreateZExtOrTrunc(V, TruncatedTy); 3818 }; 3819 3820 // The actual instruction modification depends on the instruction type, 3821 // unfortunately. 3822 Value *NewI = nullptr; 3823 if (auto *BO = dyn_cast<BinaryOperator>(I)) { 3824 NewI = B.CreateBinOp(BO->getOpcode(), ShrinkOperand(BO->getOperand(0)), 3825 ShrinkOperand(BO->getOperand(1))); 3826 3827 // Any wrapping introduced by shrinking this operation shouldn't be 3828 // considered undefined behavior. So, we can't unconditionally copy 3829 // arithmetic wrapping flags to NewI. 3830 cast<BinaryOperator>(NewI)->copyIRFlags(I, /*IncludeWrapFlags=*/false); 3831 } else if (auto *CI = dyn_cast<ICmpInst>(I)) { 3832 NewI = 3833 B.CreateICmp(CI->getPredicate(), ShrinkOperand(CI->getOperand(0)), 3834 ShrinkOperand(CI->getOperand(1))); 3835 } else if (auto *SI = dyn_cast<SelectInst>(I)) { 3836 NewI = B.CreateSelect(SI->getCondition(), 3837 ShrinkOperand(SI->getTrueValue()), 3838 ShrinkOperand(SI->getFalseValue())); 3839 } else if (auto *CI = dyn_cast<CastInst>(I)) { 3840 switch (CI->getOpcode()) { 3841 default: 3842 llvm_unreachable("Unhandled cast!"); 3843 case Instruction::Trunc: 3844 NewI = ShrinkOperand(CI->getOperand(0)); 3845 break; 3846 case Instruction::SExt: 3847 NewI = B.CreateSExtOrTrunc( 3848 CI->getOperand(0), 3849 smallestIntegerVectorType(OriginalTy, TruncatedTy)); 3850 break; 3851 case Instruction::ZExt: 3852 NewI = B.CreateZExtOrTrunc( 3853 CI->getOperand(0), 3854 smallestIntegerVectorType(OriginalTy, TruncatedTy)); 3855 break; 3856 } 3857 } else if (auto *SI = dyn_cast<ShuffleVectorInst>(I)) { 3858 auto Elements0 = cast<FixedVectorType>(SI->getOperand(0)->getType()) 3859 ->getNumElements(); 3860 auto *O0 = B.CreateZExtOrTrunc( 3861 SI->getOperand(0), 3862 FixedVectorType::get(ScalarTruncatedTy, Elements0)); 3863 auto Elements1 = cast<FixedVectorType>(SI->getOperand(1)->getType()) 3864 ->getNumElements(); 3865 auto *O1 = B.CreateZExtOrTrunc( 3866 SI->getOperand(1), 3867 FixedVectorType::get(ScalarTruncatedTy, Elements1)); 3868 3869 NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask()); 3870 } else if (isa<LoadInst>(I) || isa<PHINode>(I)) { 3871 // Don't do anything with the operands, just extend the result. 3872 continue; 3873 } else if (auto *IE = dyn_cast<InsertElementInst>(I)) { 3874 auto Elements = cast<FixedVectorType>(IE->getOperand(0)->getType()) 3875 ->getNumElements(); 3876 auto *O0 = B.CreateZExtOrTrunc( 3877 IE->getOperand(0), 3878 FixedVectorType::get(ScalarTruncatedTy, Elements)); 3879 auto *O1 = B.CreateZExtOrTrunc(IE->getOperand(1), ScalarTruncatedTy); 3880 NewI = B.CreateInsertElement(O0, O1, IE->getOperand(2)); 3881 } else if (auto *EE = dyn_cast<ExtractElementInst>(I)) { 3882 auto Elements = cast<FixedVectorType>(EE->getOperand(0)->getType()) 3883 ->getNumElements(); 3884 auto *O0 = B.CreateZExtOrTrunc( 3885 EE->getOperand(0), 3886 FixedVectorType::get(ScalarTruncatedTy, Elements)); 3887 NewI = B.CreateExtractElement(O0, EE->getOperand(2)); 3888 } else { 3889 // If we don't know what to do, be conservative and don't do anything. 3890 continue; 3891 } 3892 3893 // Lastly, extend the result. 3894 NewI->takeName(cast<Instruction>(I)); 3895 Value *Res = B.CreateZExtOrTrunc(NewI, OriginalTy); 3896 I->replaceAllUsesWith(Res); 3897 cast<Instruction>(I)->eraseFromParent(); 3898 Erased.insert(I); 3899 VectorLoopValueMap.resetVectorValue(KV.first, Part, Res); 3900 } 3901 } 3902 3903 // We'll have created a bunch of ZExts that are now parentless. Clean up. 3904 for (const auto &KV : Cost->getMinimalBitwidths()) { 3905 // If the value wasn't vectorized, we must maintain the original scalar 3906 // type. The absence of the value from VectorLoopValueMap indicates that it 3907 // wasn't vectorized. 3908 if (!VectorLoopValueMap.hasAnyVectorValue(KV.first)) 3909 continue; 3910 for (unsigned Part = 0; Part < UF; ++Part) { 3911 Value *I = getOrCreateVectorValue(KV.first, Part); 3912 ZExtInst *Inst = dyn_cast<ZExtInst>(I); 3913 if (Inst && Inst->use_empty()) { 3914 Value *NewI = Inst->getOperand(0); 3915 Inst->eraseFromParent(); 3916 VectorLoopValueMap.resetVectorValue(KV.first, Part, NewI); 3917 } 3918 } 3919 } 3920 } 3921 3922 void InnerLoopVectorizer::fixVectorizedLoop() { 3923 // Insert truncates and extends for any truncated instructions as hints to 3924 // InstCombine. 3925 if (VF.isVector()) 3926 truncateToMinimalBitwidths(); 3927 3928 // Fix widened non-induction PHIs by setting up the PHI operands. 3929 if (OrigPHIsToFix.size()) { 3930 assert(EnableVPlanNativePath && 3931 "Unexpected non-induction PHIs for fixup in non VPlan-native path"); 3932 fixNonInductionPHIs(); 3933 } 3934 3935 // At this point every instruction in the original loop is widened to a 3936 // vector form. Now we need to fix the recurrences in the loop. These PHI 3937 // nodes are currently empty because we did not want to introduce cycles. 3938 // This is the second stage of vectorizing recurrences. 3939 fixCrossIterationPHIs(); 3940 3941 // Forget the original basic block. 3942 PSE.getSE()->forgetLoop(OrigLoop); 3943 3944 // Fix-up external users of the induction variables. 3945 for (auto &Entry : Legal->getInductionVars()) 3946 fixupIVUsers(Entry.first, Entry.second, 3947 getOrCreateVectorTripCount(LI->getLoopFor(LoopVectorBody)), 3948 IVEndValues[Entry.first], LoopMiddleBlock); 3949 3950 fixLCSSAPHIs(); 3951 for (Instruction *PI : PredicatedInstructions) 3952 sinkScalarOperands(&*PI); 3953 3954 // Remove redundant induction instructions. 3955 cse(LoopVectorBody); 3956 3957 // Set/update profile weights for the vector and remainder loops as original 3958 // loop iterations are now distributed among them. Note that original loop 3959 // represented by LoopScalarBody becomes remainder loop after vectorization. 3960 // 3961 // For cases like foldTailByMasking() and requiresScalarEpiloque() we may 3962 // end up getting slightly roughened result but that should be OK since 3963 // profile is not inherently precise anyway. Note also possible bypass of 3964 // vector code caused by legality checks is ignored, assigning all the weight 3965 // to the vector loop, optimistically. 3966 // 3967 // For scalable vectorization we can't know at compile time how many iterations 3968 // of the loop are handled in one vector iteration, so instead assume a pessimistic 3969 // vscale of '1'. 3970 setProfileInfoAfterUnrolling( 3971 LI->getLoopFor(LoopScalarBody), LI->getLoopFor(LoopVectorBody), 3972 LI->getLoopFor(LoopScalarBody), VF.getKnownMinValue() * UF); 3973 } 3974 3975 void InnerLoopVectorizer::fixCrossIterationPHIs() { 3976 // In order to support recurrences we need to be able to vectorize Phi nodes. 3977 // Phi nodes have cycles, so we need to vectorize them in two stages. This is 3978 // stage #2: We now need to fix the recurrences by adding incoming edges to 3979 // the currently empty PHI nodes. At this point every instruction in the 3980 // original loop is widened to a vector form so we can use them to construct 3981 // the incoming edges. 3982 for (PHINode &Phi : OrigLoop->getHeader()->phis()) { 3983 // Handle first-order recurrences and reductions that need to be fixed. 3984 if (Legal->isFirstOrderRecurrence(&Phi)) 3985 fixFirstOrderRecurrence(&Phi); 3986 else if (Legal->isReductionVariable(&Phi)) 3987 fixReduction(&Phi); 3988 } 3989 } 3990 3991 void InnerLoopVectorizer::fixFirstOrderRecurrence(PHINode *Phi) { 3992 // This is the second phase of vectorizing first-order recurrences. An 3993 // overview of the transformation is described below. Suppose we have the 3994 // following loop. 3995 // 3996 // for (int i = 0; i < n; ++i) 3997 // b[i] = a[i] - a[i - 1]; 3998 // 3999 // There is a first-order recurrence on "a". For this loop, the shorthand 4000 // scalar IR looks like: 4001 // 4002 // scalar.ph: 4003 // s_init = a[-1] 4004 // br scalar.body 4005 // 4006 // scalar.body: 4007 // i = phi [0, scalar.ph], [i+1, scalar.body] 4008 // s1 = phi [s_init, scalar.ph], [s2, scalar.body] 4009 // s2 = a[i] 4010 // b[i] = s2 - s1 4011 // br cond, scalar.body, ... 4012 // 4013 // In this example, s1 is a recurrence because it's value depends on the 4014 // previous iteration. In the first phase of vectorization, we created a 4015 // temporary value for s1. We now complete the vectorization and produce the 4016 // shorthand vector IR shown below (for VF = 4, UF = 1). 4017 // 4018 // vector.ph: 4019 // v_init = vector(..., ..., ..., a[-1]) 4020 // br vector.body 4021 // 4022 // vector.body 4023 // i = phi [0, vector.ph], [i+4, vector.body] 4024 // v1 = phi [v_init, vector.ph], [v2, vector.body] 4025 // v2 = a[i, i+1, i+2, i+3]; 4026 // v3 = vector(v1(3), v2(0, 1, 2)) 4027 // b[i, i+1, i+2, i+3] = v2 - v3 4028 // br cond, vector.body, middle.block 4029 // 4030 // middle.block: 4031 // x = v2(3) 4032 // br scalar.ph 4033 // 4034 // scalar.ph: 4035 // s_init = phi [x, middle.block], [a[-1], otherwise] 4036 // br scalar.body 4037 // 4038 // After execution completes the vector loop, we extract the next value of 4039 // the recurrence (x) to use as the initial value in the scalar loop. 4040 4041 // Get the original loop preheader and single loop latch. 4042 auto *Preheader = OrigLoop->getLoopPreheader(); 4043 auto *Latch = OrigLoop->getLoopLatch(); 4044 4045 // Get the initial and previous values of the scalar recurrence. 4046 auto *ScalarInit = Phi->getIncomingValueForBlock(Preheader); 4047 auto *Previous = Phi->getIncomingValueForBlock(Latch); 4048 4049 // Create a vector from the initial value. 4050 auto *VectorInit = ScalarInit; 4051 if (VF.isVector()) { 4052 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator()); 4053 assert(!VF.isScalable() && "VF is assumed to be non scalable."); 4054 VectorInit = Builder.CreateInsertElement( 4055 PoisonValue::get(VectorType::get(VectorInit->getType(), VF)), VectorInit, 4056 Builder.getInt32(VF.getKnownMinValue() - 1), "vector.recur.init"); 4057 } 4058 4059 // We constructed a temporary phi node in the first phase of vectorization. 4060 // This phi node will eventually be deleted. 4061 Builder.SetInsertPoint( 4062 cast<Instruction>(VectorLoopValueMap.getVectorValue(Phi, 0))); 4063 4064 // Create a phi node for the new recurrence. The current value will either be 4065 // the initial value inserted into a vector or loop-varying vector value. 4066 auto *VecPhi = Builder.CreatePHI(VectorInit->getType(), 2, "vector.recur"); 4067 VecPhi->addIncoming(VectorInit, LoopVectorPreHeader); 4068 4069 // Get the vectorized previous value of the last part UF - 1. It appears last 4070 // among all unrolled iterations, due to the order of their construction. 4071 Value *PreviousLastPart = getOrCreateVectorValue(Previous, UF - 1); 4072 4073 // Find and set the insertion point after the previous value if it is an 4074 // instruction. 4075 BasicBlock::iterator InsertPt; 4076 // Note that the previous value may have been constant-folded so it is not 4077 // guaranteed to be an instruction in the vector loop. 4078 // FIXME: Loop invariant values do not form recurrences. We should deal with 4079 // them earlier. 4080 if (LI->getLoopFor(LoopVectorBody)->isLoopInvariant(PreviousLastPart)) 4081 InsertPt = LoopVectorBody->getFirstInsertionPt(); 4082 else { 4083 Instruction *PreviousInst = cast<Instruction>(PreviousLastPart); 4084 if (isa<PHINode>(PreviousLastPart)) 4085 // If the previous value is a phi node, we should insert after all the phi 4086 // nodes in the block containing the PHI to avoid breaking basic block 4087 // verification. Note that the basic block may be different to 4088 // LoopVectorBody, in case we predicate the loop. 4089 InsertPt = PreviousInst->getParent()->getFirstInsertionPt(); 4090 else 4091 InsertPt = ++PreviousInst->getIterator(); 4092 } 4093 Builder.SetInsertPoint(&*InsertPt); 4094 4095 // We will construct a vector for the recurrence by combining the values for 4096 // the current and previous iterations. This is the required shuffle mask. 4097 assert(!VF.isScalable()); 4098 SmallVector<int, 8> ShuffleMask(VF.getKnownMinValue()); 4099 ShuffleMask[0] = VF.getKnownMinValue() - 1; 4100 for (unsigned I = 1; I < VF.getKnownMinValue(); ++I) 4101 ShuffleMask[I] = I + VF.getKnownMinValue() - 1; 4102 4103 // The vector from which to take the initial value for the current iteration 4104 // (actual or unrolled). Initially, this is the vector phi node. 4105 Value *Incoming = VecPhi; 4106 4107 // Shuffle the current and previous vector and update the vector parts. 4108 for (unsigned Part = 0; Part < UF; ++Part) { 4109 Value *PreviousPart = getOrCreateVectorValue(Previous, Part); 4110 Value *PhiPart = VectorLoopValueMap.getVectorValue(Phi, Part); 4111 auto *Shuffle = 4112 VF.isVector() 4113 ? Builder.CreateShuffleVector(Incoming, PreviousPart, ShuffleMask) 4114 : Incoming; 4115 PhiPart->replaceAllUsesWith(Shuffle); 4116 cast<Instruction>(PhiPart)->eraseFromParent(); 4117 VectorLoopValueMap.resetVectorValue(Phi, Part, Shuffle); 4118 Incoming = PreviousPart; 4119 } 4120 4121 // Fix the latch value of the new recurrence in the vector loop. 4122 VecPhi->addIncoming(Incoming, LI->getLoopFor(LoopVectorBody)->getLoopLatch()); 4123 4124 // Extract the last vector element in the middle block. This will be the 4125 // initial value for the recurrence when jumping to the scalar loop. 4126 auto *ExtractForScalar = Incoming; 4127 if (VF.isVector()) { 4128 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator()); 4129 ExtractForScalar = Builder.CreateExtractElement( 4130 ExtractForScalar, Builder.getInt32(VF.getKnownMinValue() - 1), 4131 "vector.recur.extract"); 4132 } 4133 // Extract the second last element in the middle block if the 4134 // Phi is used outside the loop. We need to extract the phi itself 4135 // and not the last element (the phi update in the current iteration). This 4136 // will be the value when jumping to the exit block from the LoopMiddleBlock, 4137 // when the scalar loop is not run at all. 4138 Value *ExtractForPhiUsedOutsideLoop = nullptr; 4139 if (VF.isVector()) 4140 ExtractForPhiUsedOutsideLoop = Builder.CreateExtractElement( 4141 Incoming, Builder.getInt32(VF.getKnownMinValue() - 2), 4142 "vector.recur.extract.for.phi"); 4143 // When loop is unrolled without vectorizing, initialize 4144 // ExtractForPhiUsedOutsideLoop with the value just prior to unrolled value of 4145 // `Incoming`. This is analogous to the vectorized case above: extracting the 4146 // second last element when VF > 1. 4147 else if (UF > 1) 4148 ExtractForPhiUsedOutsideLoop = getOrCreateVectorValue(Previous, UF - 2); 4149 4150 // Fix the initial value of the original recurrence in the scalar loop. 4151 Builder.SetInsertPoint(&*LoopScalarPreHeader->begin()); 4152 auto *Start = Builder.CreatePHI(Phi->getType(), 2, "scalar.recur.init"); 4153 for (auto *BB : predecessors(LoopScalarPreHeader)) { 4154 auto *Incoming = BB == LoopMiddleBlock ? ExtractForScalar : ScalarInit; 4155 Start->addIncoming(Incoming, BB); 4156 } 4157 4158 Phi->setIncomingValueForBlock(LoopScalarPreHeader, Start); 4159 Phi->setName("scalar.recur"); 4160 4161 // Finally, fix users of the recurrence outside the loop. The users will need 4162 // either the last value of the scalar recurrence or the last value of the 4163 // vector recurrence we extracted in the middle block. Since the loop is in 4164 // LCSSA form, we just need to find all the phi nodes for the original scalar 4165 // recurrence in the exit block, and then add an edge for the middle block. 4166 // Note that LCSSA does not imply single entry when the original scalar loop 4167 // had multiple exiting edges (as we always run the last iteration in the 4168 // scalar epilogue); in that case, the exiting path through middle will be 4169 // dynamically dead and the value picked for the phi doesn't matter. 4170 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) 4171 if (any_of(LCSSAPhi.incoming_values(), 4172 [Phi](Value *V) { return V == Phi; })) 4173 LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock); 4174 } 4175 4176 void InnerLoopVectorizer::fixReduction(PHINode *Phi) { 4177 // Get it's reduction variable descriptor. 4178 assert(Legal->isReductionVariable(Phi) && 4179 "Unable to find the reduction variable"); 4180 RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi]; 4181 4182 RecurKind RK = RdxDesc.getRecurrenceKind(); 4183 TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue(); 4184 Instruction *LoopExitInst = RdxDesc.getLoopExitInstr(); 4185 setDebugLocFromInst(Builder, ReductionStartValue); 4186 bool IsInLoopReductionPhi = Cost->isInLoopReduction(Phi); 4187 4188 // This is the vector-clone of the value that leaves the loop. 4189 Type *VecTy = getOrCreateVectorValue(LoopExitInst, 0)->getType(); 4190 4191 // Wrap flags are in general invalid after vectorization, clear them. 4192 clearReductionWrapFlags(RdxDesc); 4193 4194 // Fix the vector-loop phi. 4195 4196 // Reductions do not have to start at zero. They can start with 4197 // any loop invariant values. 4198 BasicBlock *Latch = OrigLoop->getLoopLatch(); 4199 Value *LoopVal = Phi->getIncomingValueForBlock(Latch); 4200 4201 for (unsigned Part = 0; Part < UF; ++Part) { 4202 Value *VecRdxPhi = getOrCreateVectorValue(Phi, Part); 4203 Value *Val = getOrCreateVectorValue(LoopVal, Part); 4204 cast<PHINode>(VecRdxPhi) 4205 ->addIncoming(Val, LI->getLoopFor(LoopVectorBody)->getLoopLatch()); 4206 } 4207 4208 // Before each round, move the insertion point right between 4209 // the PHIs and the values we are going to write. 4210 // This allows us to write both PHINodes and the extractelement 4211 // instructions. 4212 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt()); 4213 4214 setDebugLocFromInst(Builder, LoopExitInst); 4215 4216 // If tail is folded by masking, the vector value to leave the loop should be 4217 // a Select choosing between the vectorized LoopExitInst and vectorized Phi, 4218 // instead of the former. For an inloop reduction the reduction will already 4219 // be predicated, and does not need to be handled here. 4220 if (Cost->foldTailByMasking() && !IsInLoopReductionPhi) { 4221 for (unsigned Part = 0; Part < UF; ++Part) { 4222 Value *VecLoopExitInst = 4223 VectorLoopValueMap.getVectorValue(LoopExitInst, Part); 4224 Value *Sel = nullptr; 4225 for (User *U : VecLoopExitInst->users()) { 4226 if (isa<SelectInst>(U)) { 4227 assert(!Sel && "Reduction exit feeding two selects"); 4228 Sel = U; 4229 } else 4230 assert(isa<PHINode>(U) && "Reduction exit must feed Phi's or select"); 4231 } 4232 assert(Sel && "Reduction exit feeds no select"); 4233 VectorLoopValueMap.resetVectorValue(LoopExitInst, Part, Sel); 4234 4235 // If the target can create a predicated operator for the reduction at no 4236 // extra cost in the loop (for example a predicated vadd), it can be 4237 // cheaper for the select to remain in the loop than be sunk out of it, 4238 // and so use the select value for the phi instead of the old 4239 // LoopExitValue. 4240 RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi]; 4241 if (PreferPredicatedReductionSelect || 4242 TTI->preferPredicatedReductionSelect( 4243 RdxDesc.getOpcode(), Phi->getType(), 4244 TargetTransformInfo::ReductionFlags())) { 4245 auto *VecRdxPhi = cast<PHINode>(getOrCreateVectorValue(Phi, Part)); 4246 VecRdxPhi->setIncomingValueForBlock( 4247 LI->getLoopFor(LoopVectorBody)->getLoopLatch(), Sel); 4248 } 4249 } 4250 } 4251 4252 // If the vector reduction can be performed in a smaller type, we truncate 4253 // then extend the loop exit value to enable InstCombine to evaluate the 4254 // entire expression in the smaller type. 4255 if (VF.isVector() && Phi->getType() != RdxDesc.getRecurrenceType()) { 4256 assert(!IsInLoopReductionPhi && "Unexpected truncated inloop reduction!"); 4257 assert(!VF.isScalable() && "scalable vectors not yet supported."); 4258 Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF); 4259 Builder.SetInsertPoint( 4260 LI->getLoopFor(LoopVectorBody)->getLoopLatch()->getTerminator()); 4261 VectorParts RdxParts(UF); 4262 for (unsigned Part = 0; Part < UF; ++Part) { 4263 RdxParts[Part] = VectorLoopValueMap.getVectorValue(LoopExitInst, Part); 4264 Value *Trunc = Builder.CreateTrunc(RdxParts[Part], RdxVecTy); 4265 Value *Extnd = RdxDesc.isSigned() ? Builder.CreateSExt(Trunc, VecTy) 4266 : Builder.CreateZExt(Trunc, VecTy); 4267 for (Value::user_iterator UI = RdxParts[Part]->user_begin(); 4268 UI != RdxParts[Part]->user_end();) 4269 if (*UI != Trunc) { 4270 (*UI++)->replaceUsesOfWith(RdxParts[Part], Extnd); 4271 RdxParts[Part] = Extnd; 4272 } else { 4273 ++UI; 4274 } 4275 } 4276 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt()); 4277 for (unsigned Part = 0; Part < UF; ++Part) { 4278 RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy); 4279 VectorLoopValueMap.resetVectorValue(LoopExitInst, Part, RdxParts[Part]); 4280 } 4281 } 4282 4283 // Reduce all of the unrolled parts into a single vector. 4284 Value *ReducedPartRdx = VectorLoopValueMap.getVectorValue(LoopExitInst, 0); 4285 unsigned Op = RecurrenceDescriptor::getOpcode(RK); 4286 4287 // The middle block terminator has already been assigned a DebugLoc here (the 4288 // OrigLoop's single latch terminator). We want the whole middle block to 4289 // appear to execute on this line because: (a) it is all compiler generated, 4290 // (b) these instructions are always executed after evaluating the latch 4291 // conditional branch, and (c) other passes may add new predecessors which 4292 // terminate on this line. This is the easiest way to ensure we don't 4293 // accidentally cause an extra step back into the loop while debugging. 4294 setDebugLocFromInst(Builder, LoopMiddleBlock->getTerminator()); 4295 for (unsigned Part = 1; Part < UF; ++Part) { 4296 Value *RdxPart = VectorLoopValueMap.getVectorValue(LoopExitInst, Part); 4297 if (Op != Instruction::ICmp && Op != Instruction::FCmp) 4298 // Floating point operations had to be 'fast' to enable the reduction. 4299 ReducedPartRdx = addFastMathFlag( 4300 Builder.CreateBinOp((Instruction::BinaryOps)Op, RdxPart, 4301 ReducedPartRdx, "bin.rdx"), 4302 RdxDesc.getFastMathFlags()); 4303 else 4304 ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart); 4305 } 4306 4307 // Create the reduction after the loop. Note that inloop reductions create the 4308 // target reduction in the loop using a Reduction recipe. 4309 if (VF.isVector() && !IsInLoopReductionPhi) { 4310 ReducedPartRdx = 4311 createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx); 4312 // If the reduction can be performed in a smaller type, we need to extend 4313 // the reduction to the wider type before we branch to the original loop. 4314 if (Phi->getType() != RdxDesc.getRecurrenceType()) 4315 ReducedPartRdx = 4316 RdxDesc.isSigned() 4317 ? Builder.CreateSExt(ReducedPartRdx, Phi->getType()) 4318 : Builder.CreateZExt(ReducedPartRdx, Phi->getType()); 4319 } 4320 4321 // Create a phi node that merges control-flow from the backedge-taken check 4322 // block and the middle block. 4323 PHINode *BCBlockPhi = PHINode::Create(Phi->getType(), 2, "bc.merge.rdx", 4324 LoopScalarPreHeader->getTerminator()); 4325 for (unsigned I = 0, E = LoopBypassBlocks.size(); I != E; ++I) 4326 BCBlockPhi->addIncoming(ReductionStartValue, LoopBypassBlocks[I]); 4327 BCBlockPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock); 4328 4329 // Now, we need to fix the users of the reduction variable 4330 // inside and outside of the scalar remainder loop. 4331 4332 // We know that the loop is in LCSSA form. We need to update the PHI nodes 4333 // in the exit blocks. See comment on analogous loop in 4334 // fixFirstOrderRecurrence for a more complete explaination of the logic. 4335 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) 4336 if (any_of(LCSSAPhi.incoming_values(), 4337 [LoopExitInst](Value *V) { return V == LoopExitInst; })) 4338 LCSSAPhi.addIncoming(ReducedPartRdx, LoopMiddleBlock); 4339 4340 // Fix the scalar loop reduction variable with the incoming reduction sum 4341 // from the vector body and from the backedge value. 4342 int IncomingEdgeBlockIdx = 4343 Phi->getBasicBlockIndex(OrigLoop->getLoopLatch()); 4344 assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index"); 4345 // Pick the other block. 4346 int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1); 4347 Phi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi); 4348 Phi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst); 4349 } 4350 4351 void InnerLoopVectorizer::clearReductionWrapFlags( 4352 RecurrenceDescriptor &RdxDesc) { 4353 RecurKind RK = RdxDesc.getRecurrenceKind(); 4354 if (RK != RecurKind::Add && RK != RecurKind::Mul) 4355 return; 4356 4357 Instruction *LoopExitInstr = RdxDesc.getLoopExitInstr(); 4358 assert(LoopExitInstr && "null loop exit instruction"); 4359 SmallVector<Instruction *, 8> Worklist; 4360 SmallPtrSet<Instruction *, 8> Visited; 4361 Worklist.push_back(LoopExitInstr); 4362 Visited.insert(LoopExitInstr); 4363 4364 while (!Worklist.empty()) { 4365 Instruction *Cur = Worklist.pop_back_val(); 4366 if (isa<OverflowingBinaryOperator>(Cur)) 4367 for (unsigned Part = 0; Part < UF; ++Part) { 4368 Value *V = getOrCreateVectorValue(Cur, Part); 4369 cast<Instruction>(V)->dropPoisonGeneratingFlags(); 4370 } 4371 4372 for (User *U : Cur->users()) { 4373 Instruction *UI = cast<Instruction>(U); 4374 if ((Cur != LoopExitInstr || OrigLoop->contains(UI->getParent())) && 4375 Visited.insert(UI).second) 4376 Worklist.push_back(UI); 4377 } 4378 } 4379 } 4380 4381 void InnerLoopVectorizer::fixLCSSAPHIs() { 4382 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) { 4383 if (LCSSAPhi.getBasicBlockIndex(LoopMiddleBlock) != -1) 4384 // Some phis were already hand updated by the reduction and recurrence 4385 // code above, leave them alone. 4386 continue; 4387 4388 auto *IncomingValue = LCSSAPhi.getIncomingValue(0); 4389 // Non-instruction incoming values will have only one value. 4390 unsigned LastLane = 0; 4391 if (isa<Instruction>(IncomingValue)) 4392 LastLane = Cost->isUniformAfterVectorization( 4393 cast<Instruction>(IncomingValue), VF) 4394 ? 0 4395 : VF.getKnownMinValue() - 1; 4396 assert((!VF.isScalable() || LastLane == 0) && 4397 "scalable vectors dont support non-uniform scalars yet"); 4398 // Can be a loop invariant incoming value or the last scalar value to be 4399 // extracted from the vectorized loop. 4400 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator()); 4401 Value *lastIncomingValue = 4402 getOrCreateScalarValue(IncomingValue, { UF - 1, LastLane }); 4403 LCSSAPhi.addIncoming(lastIncomingValue, LoopMiddleBlock); 4404 } 4405 } 4406 4407 void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) { 4408 // The basic block and loop containing the predicated instruction. 4409 auto *PredBB = PredInst->getParent(); 4410 auto *VectorLoop = LI->getLoopFor(PredBB); 4411 4412 // Initialize a worklist with the operands of the predicated instruction. 4413 SetVector<Value *> Worklist(PredInst->op_begin(), PredInst->op_end()); 4414 4415 // Holds instructions that we need to analyze again. An instruction may be 4416 // reanalyzed if we don't yet know if we can sink it or not. 4417 SmallVector<Instruction *, 8> InstsToReanalyze; 4418 4419 // Returns true if a given use occurs in the predicated block. Phi nodes use 4420 // their operands in their corresponding predecessor blocks. 4421 auto isBlockOfUsePredicated = [&](Use &U) -> bool { 4422 auto *I = cast<Instruction>(U.getUser()); 4423 BasicBlock *BB = I->getParent(); 4424 if (auto *Phi = dyn_cast<PHINode>(I)) 4425 BB = Phi->getIncomingBlock( 4426 PHINode::getIncomingValueNumForOperand(U.getOperandNo())); 4427 return BB == PredBB; 4428 }; 4429 4430 // Iteratively sink the scalarized operands of the predicated instruction 4431 // into the block we created for it. When an instruction is sunk, it's 4432 // operands are then added to the worklist. The algorithm ends after one pass 4433 // through the worklist doesn't sink a single instruction. 4434 bool Changed; 4435 do { 4436 // Add the instructions that need to be reanalyzed to the worklist, and 4437 // reset the changed indicator. 4438 Worklist.insert(InstsToReanalyze.begin(), InstsToReanalyze.end()); 4439 InstsToReanalyze.clear(); 4440 Changed = false; 4441 4442 while (!Worklist.empty()) { 4443 auto *I = dyn_cast<Instruction>(Worklist.pop_back_val()); 4444 4445 // We can't sink an instruction if it is a phi node, is already in the 4446 // predicated block, is not in the loop, or may have side effects. 4447 if (!I || isa<PHINode>(I) || I->getParent() == PredBB || 4448 !VectorLoop->contains(I) || I->mayHaveSideEffects()) 4449 continue; 4450 4451 // It's legal to sink the instruction if all its uses occur in the 4452 // predicated block. Otherwise, there's nothing to do yet, and we may 4453 // need to reanalyze the instruction. 4454 if (!llvm::all_of(I->uses(), isBlockOfUsePredicated)) { 4455 InstsToReanalyze.push_back(I); 4456 continue; 4457 } 4458 4459 // Move the instruction to the beginning of the predicated block, and add 4460 // it's operands to the worklist. 4461 I->moveBefore(&*PredBB->getFirstInsertionPt()); 4462 Worklist.insert(I->op_begin(), I->op_end()); 4463 4464 // The sinking may have enabled other instructions to be sunk, so we will 4465 // need to iterate. 4466 Changed = true; 4467 } 4468 } while (Changed); 4469 } 4470 4471 void InnerLoopVectorizer::fixNonInductionPHIs() { 4472 for (PHINode *OrigPhi : OrigPHIsToFix) { 4473 PHINode *NewPhi = 4474 cast<PHINode>(VectorLoopValueMap.getVectorValue(OrigPhi, 0)); 4475 unsigned NumIncomingValues = OrigPhi->getNumIncomingValues(); 4476 4477 SmallVector<BasicBlock *, 2> ScalarBBPredecessors( 4478 predecessors(OrigPhi->getParent())); 4479 SmallVector<BasicBlock *, 2> VectorBBPredecessors( 4480 predecessors(NewPhi->getParent())); 4481 assert(ScalarBBPredecessors.size() == VectorBBPredecessors.size() && 4482 "Scalar and Vector BB should have the same number of predecessors"); 4483 4484 // The insertion point in Builder may be invalidated by the time we get 4485 // here. Force the Builder insertion point to something valid so that we do 4486 // not run into issues during insertion point restore in 4487 // getOrCreateVectorValue calls below. 4488 Builder.SetInsertPoint(NewPhi); 4489 4490 // The predecessor order is preserved and we can rely on mapping between 4491 // scalar and vector block predecessors. 4492 for (unsigned i = 0; i < NumIncomingValues; ++i) { 4493 BasicBlock *NewPredBB = VectorBBPredecessors[i]; 4494 4495 // When looking up the new scalar/vector values to fix up, use incoming 4496 // values from original phi. 4497 Value *ScIncV = 4498 OrigPhi->getIncomingValueForBlock(ScalarBBPredecessors[i]); 4499 4500 // Scalar incoming value may need a broadcast 4501 Value *NewIncV = getOrCreateVectorValue(ScIncV, 0); 4502 NewPhi->addIncoming(NewIncV, NewPredBB); 4503 } 4504 } 4505 } 4506 4507 void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPValue *VPDef, 4508 VPUser &Operands, unsigned UF, 4509 ElementCount VF, bool IsPtrLoopInvariant, 4510 SmallBitVector &IsIndexLoopInvariant, 4511 VPTransformState &State) { 4512 // Construct a vector GEP by widening the operands of the scalar GEP as 4513 // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP 4514 // results in a vector of pointers when at least one operand of the GEP 4515 // is vector-typed. Thus, to keep the representation compact, we only use 4516 // vector-typed operands for loop-varying values. 4517 4518 if (VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) { 4519 // If we are vectorizing, but the GEP has only loop-invariant operands, 4520 // the GEP we build (by only using vector-typed operands for 4521 // loop-varying values) would be a scalar pointer. Thus, to ensure we 4522 // produce a vector of pointers, we need to either arbitrarily pick an 4523 // operand to broadcast, or broadcast a clone of the original GEP. 4524 // Here, we broadcast a clone of the original. 4525 // 4526 // TODO: If at some point we decide to scalarize instructions having 4527 // loop-invariant operands, this special case will no longer be 4528 // required. We would add the scalarization decision to 4529 // collectLoopScalars() and teach getVectorValue() to broadcast 4530 // the lane-zero scalar value. 4531 auto *Clone = Builder.Insert(GEP->clone()); 4532 for (unsigned Part = 0; Part < UF; ++Part) { 4533 Value *EntryPart = Builder.CreateVectorSplat(VF, Clone); 4534 State.set(VPDef, GEP, EntryPart, Part); 4535 addMetadata(EntryPart, GEP); 4536 } 4537 } else { 4538 // If the GEP has at least one loop-varying operand, we are sure to 4539 // produce a vector of pointers. But if we are only unrolling, we want 4540 // to produce a scalar GEP for each unroll part. Thus, the GEP we 4541 // produce with the code below will be scalar (if VF == 1) or vector 4542 // (otherwise). Note that for the unroll-only case, we still maintain 4543 // values in the vector mapping with initVector, as we do for other 4544 // instructions. 4545 for (unsigned Part = 0; Part < UF; ++Part) { 4546 // The pointer operand of the new GEP. If it's loop-invariant, we 4547 // won't broadcast it. 4548 auto *Ptr = IsPtrLoopInvariant ? State.get(Operands.getOperand(0), {0, 0}) 4549 : State.get(Operands.getOperand(0), Part); 4550 4551 // Collect all the indices for the new GEP. If any index is 4552 // loop-invariant, we won't broadcast it. 4553 SmallVector<Value *, 4> Indices; 4554 for (unsigned I = 1, E = Operands.getNumOperands(); I < E; I++) { 4555 VPValue *Operand = Operands.getOperand(I); 4556 if (IsIndexLoopInvariant[I - 1]) 4557 Indices.push_back(State.get(Operand, {0, 0})); 4558 else 4559 Indices.push_back(State.get(Operand, Part)); 4560 } 4561 4562 // Create the new GEP. Note that this GEP may be a scalar if VF == 1, 4563 // but it should be a vector, otherwise. 4564 auto *NewGEP = 4565 GEP->isInBounds() 4566 ? Builder.CreateInBoundsGEP(GEP->getSourceElementType(), Ptr, 4567 Indices) 4568 : Builder.CreateGEP(GEP->getSourceElementType(), Ptr, Indices); 4569 assert((VF.isScalar() || NewGEP->getType()->isVectorTy()) && 4570 "NewGEP is not a pointer vector"); 4571 State.set(VPDef, GEP, NewGEP, Part); 4572 addMetadata(NewGEP, GEP); 4573 } 4574 } 4575 } 4576 4577 void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN, 4578 RecurrenceDescriptor *RdxDesc, 4579 Value *StartV, unsigned UF, 4580 ElementCount VF) { 4581 assert(!VF.isScalable() && "scalable vectors not yet supported."); 4582 PHINode *P = cast<PHINode>(PN); 4583 if (EnableVPlanNativePath) { 4584 // Currently we enter here in the VPlan-native path for non-induction 4585 // PHIs where all control flow is uniform. We simply widen these PHIs. 4586 // Create a vector phi with no operands - the vector phi operands will be 4587 // set at the end of vector code generation. 4588 Type *VecTy = 4589 (VF.isScalar()) ? PN->getType() : VectorType::get(PN->getType(), VF); 4590 Value *VecPhi = Builder.CreatePHI(VecTy, PN->getNumOperands(), "vec.phi"); 4591 VectorLoopValueMap.setVectorValue(P, 0, VecPhi); 4592 OrigPHIsToFix.push_back(P); 4593 4594 return; 4595 } 4596 4597 assert(PN->getParent() == OrigLoop->getHeader() && 4598 "Non-header phis should have been handled elsewhere"); 4599 4600 // In order to support recurrences we need to be able to vectorize Phi nodes. 4601 // Phi nodes have cycles, so we need to vectorize them in two stages. This is 4602 // stage #1: We create a new vector PHI node with no incoming edges. We'll use 4603 // this value when we vectorize all of the instructions that use the PHI. 4604 if (RdxDesc || Legal->isFirstOrderRecurrence(P)) { 4605 Value *Iden = nullptr; 4606 bool ScalarPHI = 4607 (VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN)); 4608 Type *VecTy = 4609 ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), VF); 4610 4611 if (RdxDesc) { 4612 assert(Legal->isReductionVariable(P) && StartV && 4613 "RdxDesc should only be set for reduction variables; in that case " 4614 "a StartV is also required"); 4615 RecurKind RK = RdxDesc->getRecurrenceKind(); 4616 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) { 4617 // MinMax reduction have the start value as their identify. 4618 if (ScalarPHI) { 4619 Iden = StartV; 4620 } else { 4621 IRBuilderBase::InsertPointGuard IPBuilder(Builder); 4622 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator()); 4623 StartV = Iden = Builder.CreateVectorSplat(VF, StartV, "minmax.ident"); 4624 } 4625 } else { 4626 Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity( 4627 RK, VecTy->getScalarType()); 4628 Iden = IdenC; 4629 4630 if (!ScalarPHI) { 4631 Iden = ConstantVector::getSplat(VF, IdenC); 4632 IRBuilderBase::InsertPointGuard IPBuilder(Builder); 4633 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator()); 4634 Constant *Zero = Builder.getInt32(0); 4635 StartV = Builder.CreateInsertElement(Iden, StartV, Zero); 4636 } 4637 } 4638 } 4639 4640 for (unsigned Part = 0; Part < UF; ++Part) { 4641 // This is phase one of vectorizing PHIs. 4642 Value *EntryPart = PHINode::Create( 4643 VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt()); 4644 VectorLoopValueMap.setVectorValue(P, Part, EntryPart); 4645 if (StartV) { 4646 // Make sure to add the reduction start value only to the 4647 // first unroll part. 4648 Value *StartVal = (Part == 0) ? StartV : Iden; 4649 cast<PHINode>(EntryPart)->addIncoming(StartVal, LoopVectorPreHeader); 4650 } 4651 } 4652 return; 4653 } 4654 4655 assert(!Legal->isReductionVariable(P) && 4656 "reductions should be handled above"); 4657 4658 setDebugLocFromInst(Builder, P); 4659 4660 // This PHINode must be an induction variable. 4661 // Make sure that we know about it. 4662 assert(Legal->getInductionVars().count(P) && "Not an induction variable"); 4663 4664 InductionDescriptor II = Legal->getInductionVars().lookup(P); 4665 const DataLayout &DL = OrigLoop->getHeader()->getModule()->getDataLayout(); 4666 4667 // FIXME: The newly created binary instructions should contain nsw/nuw flags, 4668 // which can be found from the original scalar operations. 4669 switch (II.getKind()) { 4670 case InductionDescriptor::IK_NoInduction: 4671 llvm_unreachable("Unknown induction"); 4672 case InductionDescriptor::IK_IntInduction: 4673 case InductionDescriptor::IK_FpInduction: 4674 llvm_unreachable("Integer/fp induction is handled elsewhere."); 4675 case InductionDescriptor::IK_PtrInduction: { 4676 // Handle the pointer induction variable case. 4677 assert(P->getType()->isPointerTy() && "Unexpected type."); 4678 4679 if (Cost->isScalarAfterVectorization(P, VF)) { 4680 // This is the normalized GEP that starts counting at zero. 4681 Value *PtrInd = 4682 Builder.CreateSExtOrTrunc(Induction, II.getStep()->getType()); 4683 // Determine the number of scalars we need to generate for each unroll 4684 // iteration. If the instruction is uniform, we only need to generate the 4685 // first lane. Otherwise, we generate all VF values. 4686 unsigned Lanes = 4687 Cost->isUniformAfterVectorization(P, VF) ? 1 : VF.getKnownMinValue(); 4688 for (unsigned Part = 0; Part < UF; ++Part) { 4689 for (unsigned Lane = 0; Lane < Lanes; ++Lane) { 4690 Constant *Idx = ConstantInt::get(PtrInd->getType(), 4691 Lane + Part * VF.getKnownMinValue()); 4692 Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx); 4693 Value *SclrGep = 4694 emitTransformedIndex(Builder, GlobalIdx, PSE.getSE(), DL, II); 4695 SclrGep->setName("next.gep"); 4696 VectorLoopValueMap.setScalarValue(P, {Part, Lane}, SclrGep); 4697 } 4698 } 4699 return; 4700 } 4701 assert(isa<SCEVConstant>(II.getStep()) && 4702 "Induction step not a SCEV constant!"); 4703 Type *PhiType = II.getStep()->getType(); 4704 4705 // Build a pointer phi 4706 Value *ScalarStartValue = II.getStartValue(); 4707 Type *ScStValueType = ScalarStartValue->getType(); 4708 PHINode *NewPointerPhi = 4709 PHINode::Create(ScStValueType, 2, "pointer.phi", Induction); 4710 NewPointerPhi->addIncoming(ScalarStartValue, LoopVectorPreHeader); 4711 4712 // A pointer induction, performed by using a gep 4713 BasicBlock *LoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch(); 4714 Instruction *InductionLoc = LoopLatch->getTerminator(); 4715 const SCEV *ScalarStep = II.getStep(); 4716 SCEVExpander Exp(*PSE.getSE(), DL, "induction"); 4717 Value *ScalarStepValue = 4718 Exp.expandCodeFor(ScalarStep, PhiType, InductionLoc); 4719 Value *InductionGEP = GetElementPtrInst::Create( 4720 ScStValueType->getPointerElementType(), NewPointerPhi, 4721 Builder.CreateMul( 4722 ScalarStepValue, 4723 ConstantInt::get(PhiType, VF.getKnownMinValue() * UF)), 4724 "ptr.ind", InductionLoc); 4725 NewPointerPhi->addIncoming(InductionGEP, LoopLatch); 4726 4727 // Create UF many actual address geps that use the pointer 4728 // phi as base and a vectorized version of the step value 4729 // (<step*0, ..., step*N>) as offset. 4730 for (unsigned Part = 0; Part < UF; ++Part) { 4731 SmallVector<Constant *, 8> Indices; 4732 // Create a vector of consecutive numbers from zero to VF. 4733 for (unsigned i = 0; i < VF.getKnownMinValue(); ++i) 4734 Indices.push_back( 4735 ConstantInt::get(PhiType, i + Part * VF.getKnownMinValue())); 4736 Constant *StartOffset = ConstantVector::get(Indices); 4737 4738 Value *GEP = Builder.CreateGEP( 4739 ScStValueType->getPointerElementType(), NewPointerPhi, 4740 Builder.CreateMul( 4741 StartOffset, 4742 Builder.CreateVectorSplat(VF.getKnownMinValue(), ScalarStepValue), 4743 "vector.gep")); 4744 VectorLoopValueMap.setVectorValue(P, Part, GEP); 4745 } 4746 } 4747 } 4748 } 4749 4750 /// A helper function for checking whether an integer division-related 4751 /// instruction may divide by zero (in which case it must be predicated if 4752 /// executed conditionally in the scalar code). 4753 /// TODO: It may be worthwhile to generalize and check isKnownNonZero(). 4754 /// Non-zero divisors that are non compile-time constants will not be 4755 /// converted into multiplication, so we will still end up scalarizing 4756 /// the division, but can do so w/o predication. 4757 static bool mayDivideByZero(Instruction &I) { 4758 assert((I.getOpcode() == Instruction::UDiv || 4759 I.getOpcode() == Instruction::SDiv || 4760 I.getOpcode() == Instruction::URem || 4761 I.getOpcode() == Instruction::SRem) && 4762 "Unexpected instruction"); 4763 Value *Divisor = I.getOperand(1); 4764 auto *CInt = dyn_cast<ConstantInt>(Divisor); 4765 return !CInt || CInt->isZero(); 4766 } 4767 4768 void InnerLoopVectorizer::widenInstruction(Instruction &I, VPValue *Def, 4769 VPUser &User, 4770 VPTransformState &State) { 4771 switch (I.getOpcode()) { 4772 case Instruction::Call: 4773 case Instruction::Br: 4774 case Instruction::PHI: 4775 case Instruction::GetElementPtr: 4776 case Instruction::Select: 4777 llvm_unreachable("This instruction is handled by a different recipe."); 4778 case Instruction::UDiv: 4779 case Instruction::SDiv: 4780 case Instruction::SRem: 4781 case Instruction::URem: 4782 case Instruction::Add: 4783 case Instruction::FAdd: 4784 case Instruction::Sub: 4785 case Instruction::FSub: 4786 case Instruction::FNeg: 4787 case Instruction::Mul: 4788 case Instruction::FMul: 4789 case Instruction::FDiv: 4790 case Instruction::FRem: 4791 case Instruction::Shl: 4792 case Instruction::LShr: 4793 case Instruction::AShr: 4794 case Instruction::And: 4795 case Instruction::Or: 4796 case Instruction::Xor: { 4797 // Just widen unops and binops. 4798 setDebugLocFromInst(Builder, &I); 4799 4800 for (unsigned Part = 0; Part < UF; ++Part) { 4801 SmallVector<Value *, 2> Ops; 4802 for (VPValue *VPOp : User.operands()) 4803 Ops.push_back(State.get(VPOp, Part)); 4804 4805 Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops); 4806 4807 if (auto *VecOp = dyn_cast<Instruction>(V)) 4808 VecOp->copyIRFlags(&I); 4809 4810 // Use this vector value for all users of the original instruction. 4811 State.set(Def, &I, V, Part); 4812 addMetadata(V, &I); 4813 } 4814 4815 break; 4816 } 4817 case Instruction::ICmp: 4818 case Instruction::FCmp: { 4819 // Widen compares. Generate vector compares. 4820 bool FCmp = (I.getOpcode() == Instruction::FCmp); 4821 auto *Cmp = cast<CmpInst>(&I); 4822 setDebugLocFromInst(Builder, Cmp); 4823 for (unsigned Part = 0; Part < UF; ++Part) { 4824 Value *A = State.get(User.getOperand(0), Part); 4825 Value *B = State.get(User.getOperand(1), Part); 4826 Value *C = nullptr; 4827 if (FCmp) { 4828 // Propagate fast math flags. 4829 IRBuilder<>::FastMathFlagGuard FMFG(Builder); 4830 Builder.setFastMathFlags(Cmp->getFastMathFlags()); 4831 C = Builder.CreateFCmp(Cmp->getPredicate(), A, B); 4832 } else { 4833 C = Builder.CreateICmp(Cmp->getPredicate(), A, B); 4834 } 4835 State.set(Def, &I, C, Part); 4836 addMetadata(C, &I); 4837 } 4838 4839 break; 4840 } 4841 4842 case Instruction::ZExt: 4843 case Instruction::SExt: 4844 case Instruction::FPToUI: 4845 case Instruction::FPToSI: 4846 case Instruction::FPExt: 4847 case Instruction::PtrToInt: 4848 case Instruction::IntToPtr: 4849 case Instruction::SIToFP: 4850 case Instruction::UIToFP: 4851 case Instruction::Trunc: 4852 case Instruction::FPTrunc: 4853 case Instruction::BitCast: { 4854 auto *CI = cast<CastInst>(&I); 4855 setDebugLocFromInst(Builder, CI); 4856 4857 /// Vectorize casts. 4858 Type *DestTy = 4859 (VF.isScalar()) ? CI->getType() : VectorType::get(CI->getType(), VF); 4860 4861 for (unsigned Part = 0; Part < UF; ++Part) { 4862 Value *A = State.get(User.getOperand(0), Part); 4863 Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy); 4864 State.set(Def, &I, Cast, Part); 4865 addMetadata(Cast, &I); 4866 } 4867 break; 4868 } 4869 default: 4870 // This instruction is not vectorized by simple widening. 4871 LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I); 4872 llvm_unreachable("Unhandled instruction!"); 4873 } // end of switch. 4874 } 4875 4876 void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def, 4877 VPUser &ArgOperands, 4878 VPTransformState &State) { 4879 assert(!isa<DbgInfoIntrinsic>(I) && 4880 "DbgInfoIntrinsic should have been dropped during VPlan construction"); 4881 setDebugLocFromInst(Builder, &I); 4882 4883 Module *M = I.getParent()->getParent()->getParent(); 4884 auto *CI = cast<CallInst>(&I); 4885 4886 SmallVector<Type *, 4> Tys; 4887 for (Value *ArgOperand : CI->arg_operands()) 4888 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF.getKnownMinValue())); 4889 4890 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 4891 4892 // The flag shows whether we use Intrinsic or a usual Call for vectorized 4893 // version of the instruction. 4894 // Is it beneficial to perform intrinsic call compared to lib call? 4895 bool NeedToScalarize = false; 4896 InstructionCost CallCost = Cost->getVectorCallCost(CI, VF, NeedToScalarize); 4897 InstructionCost IntrinsicCost = ID ? Cost->getVectorIntrinsicCost(CI, VF) : 0; 4898 bool UseVectorIntrinsic = ID && IntrinsicCost <= CallCost; 4899 assert((UseVectorIntrinsic || !NeedToScalarize) && 4900 "Instruction should be scalarized elsewhere."); 4901 assert(IntrinsicCost.isValid() && CallCost.isValid() && 4902 "Cannot have invalid costs while widening"); 4903 4904 for (unsigned Part = 0; Part < UF; ++Part) { 4905 SmallVector<Value *, 4> Args; 4906 for (auto &I : enumerate(ArgOperands.operands())) { 4907 // Some intrinsics have a scalar argument - don't replace it with a 4908 // vector. 4909 Value *Arg; 4910 if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, I.index())) 4911 Arg = State.get(I.value(), Part); 4912 else 4913 Arg = State.get(I.value(), {0, 0}); 4914 Args.push_back(Arg); 4915 } 4916 4917 Function *VectorF; 4918 if (UseVectorIntrinsic) { 4919 // Use vector version of the intrinsic. 4920 Type *TysForDecl[] = {CI->getType()}; 4921 if (VF.isVector()) { 4922 assert(!VF.isScalable() && "VF is assumed to be non scalable."); 4923 TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF); 4924 } 4925 VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl); 4926 assert(VectorF && "Can't retrieve vector intrinsic."); 4927 } else { 4928 // Use vector version of the function call. 4929 const VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/); 4930 #ifndef NDEBUG 4931 assert(VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr && 4932 "Can't create vector function."); 4933 #endif 4934 VectorF = VFDatabase(*CI).getVectorizedFunction(Shape); 4935 } 4936 SmallVector<OperandBundleDef, 1> OpBundles; 4937 CI->getOperandBundlesAsDefs(OpBundles); 4938 CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles); 4939 4940 if (isa<FPMathOperator>(V)) 4941 V->copyFastMathFlags(CI); 4942 4943 State.set(Def, &I, V, Part); 4944 addMetadata(V, &I); 4945 } 4946 } 4947 4948 void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, VPValue *VPDef, 4949 VPUser &Operands, 4950 bool InvariantCond, 4951 VPTransformState &State) { 4952 setDebugLocFromInst(Builder, &I); 4953 4954 // The condition can be loop invariant but still defined inside the 4955 // loop. This means that we can't just use the original 'cond' value. 4956 // We have to take the 'vectorized' value and pick the first lane. 4957 // Instcombine will make this a no-op. 4958 auto *InvarCond = 4959 InvariantCond ? State.get(Operands.getOperand(0), {0, 0}) : nullptr; 4960 4961 for (unsigned Part = 0; Part < UF; ++Part) { 4962 Value *Cond = 4963 InvarCond ? InvarCond : State.get(Operands.getOperand(0), Part); 4964 Value *Op0 = State.get(Operands.getOperand(1), Part); 4965 Value *Op1 = State.get(Operands.getOperand(2), Part); 4966 Value *Sel = Builder.CreateSelect(Cond, Op0, Op1); 4967 State.set(VPDef, &I, Sel, Part); 4968 addMetadata(Sel, &I); 4969 } 4970 } 4971 4972 void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) { 4973 // We should not collect Scalars more than once per VF. Right now, this 4974 // function is called from collectUniformsAndScalars(), which already does 4975 // this check. Collecting Scalars for VF=1 does not make any sense. 4976 assert(VF.isVector() && Scalars.find(VF) == Scalars.end() && 4977 "This function should not be visited twice for the same VF"); 4978 4979 SmallSetVector<Instruction *, 8> Worklist; 4980 4981 // These sets are used to seed the analysis with pointers used by memory 4982 // accesses that will remain scalar. 4983 SmallSetVector<Instruction *, 8> ScalarPtrs; 4984 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs; 4985 auto *Latch = TheLoop->getLoopLatch(); 4986 4987 // A helper that returns true if the use of Ptr by MemAccess will be scalar. 4988 // The pointer operands of loads and stores will be scalar as long as the 4989 // memory access is not a gather or scatter operation. The value operand of a 4990 // store will remain scalar if the store is scalarized. 4991 auto isScalarUse = [&](Instruction *MemAccess, Value *Ptr) { 4992 InstWidening WideningDecision = getWideningDecision(MemAccess, VF); 4993 assert(WideningDecision != CM_Unknown && 4994 "Widening decision should be ready at this moment"); 4995 if (auto *Store = dyn_cast<StoreInst>(MemAccess)) 4996 if (Ptr == Store->getValueOperand()) 4997 return WideningDecision == CM_Scalarize; 4998 assert(Ptr == getLoadStorePointerOperand(MemAccess) && 4999 "Ptr is neither a value or pointer operand"); 5000 return WideningDecision != CM_GatherScatter; 5001 }; 5002 5003 // A helper that returns true if the given value is a bitcast or 5004 // getelementptr instruction contained in the loop. 5005 auto isLoopVaryingBitCastOrGEP = [&](Value *V) { 5006 return ((isa<BitCastInst>(V) && V->getType()->isPointerTy()) || 5007 isa<GetElementPtrInst>(V)) && 5008 !TheLoop->isLoopInvariant(V); 5009 }; 5010 5011 auto isScalarPtrInduction = [&](Instruction *MemAccess, Value *Ptr) { 5012 if (!isa<PHINode>(Ptr) || 5013 !Legal->getInductionVars().count(cast<PHINode>(Ptr))) 5014 return false; 5015 auto &Induction = Legal->getInductionVars()[cast<PHINode>(Ptr)]; 5016 if (Induction.getKind() != InductionDescriptor::IK_PtrInduction) 5017 return false; 5018 return isScalarUse(MemAccess, Ptr); 5019 }; 5020 5021 // A helper that evaluates a memory access's use of a pointer. If the 5022 // pointer is actually the pointer induction of a loop, it is being 5023 // inserted into Worklist. If the use will be a scalar use, and the 5024 // pointer is only used by memory accesses, we place the pointer in 5025 // ScalarPtrs. Otherwise, the pointer is placed in PossibleNonScalarPtrs. 5026 auto evaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) { 5027 if (isScalarPtrInduction(MemAccess, Ptr)) { 5028 Worklist.insert(cast<Instruction>(Ptr)); 5029 Instruction *Update = cast<Instruction>( 5030 cast<PHINode>(Ptr)->getIncomingValueForBlock(Latch)); 5031 Worklist.insert(Update); 5032 LLVM_DEBUG(dbgs() << "LV: Found new scalar instruction: " << *Ptr 5033 << "\n"); 5034 LLVM_DEBUG(dbgs() << "LV: Found new scalar instruction: " << *Update 5035 << "\n"); 5036 return; 5037 } 5038 // We only care about bitcast and getelementptr instructions contained in 5039 // the loop. 5040 if (!isLoopVaryingBitCastOrGEP(Ptr)) 5041 return; 5042 5043 // If the pointer has already been identified as scalar (e.g., if it was 5044 // also identified as uniform), there's nothing to do. 5045 auto *I = cast<Instruction>(Ptr); 5046 if (Worklist.count(I)) 5047 return; 5048 5049 // If the use of the pointer will be a scalar use, and all users of the 5050 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise, 5051 // place the pointer in PossibleNonScalarPtrs. 5052 if (isScalarUse(MemAccess, Ptr) && llvm::all_of(I->users(), [&](User *U) { 5053 return isa<LoadInst>(U) || isa<StoreInst>(U); 5054 })) 5055 ScalarPtrs.insert(I); 5056 else 5057 PossibleNonScalarPtrs.insert(I); 5058 }; 5059 5060 // We seed the scalars analysis with three classes of instructions: (1) 5061 // instructions marked uniform-after-vectorization and (2) bitcast, 5062 // getelementptr and (pointer) phi instructions used by memory accesses 5063 // requiring a scalar use. 5064 // 5065 // (1) Add to the worklist all instructions that have been identified as 5066 // uniform-after-vectorization. 5067 Worklist.insert(Uniforms[VF].begin(), Uniforms[VF].end()); 5068 5069 // (2) Add to the worklist all bitcast and getelementptr instructions used by 5070 // memory accesses requiring a scalar use. The pointer operands of loads and 5071 // stores will be scalar as long as the memory accesses is not a gather or 5072 // scatter operation. The value operand of a store will remain scalar if the 5073 // store is scalarized. 5074 for (auto *BB : TheLoop->blocks()) 5075 for (auto &I : *BB) { 5076 if (auto *Load = dyn_cast<LoadInst>(&I)) { 5077 evaluatePtrUse(Load, Load->getPointerOperand()); 5078 } else if (auto *Store = dyn_cast<StoreInst>(&I)) { 5079 evaluatePtrUse(Store, Store->getPointerOperand()); 5080 evaluatePtrUse(Store, Store->getValueOperand()); 5081 } 5082 } 5083 for (auto *I : ScalarPtrs) 5084 if (!PossibleNonScalarPtrs.count(I)) { 5085 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n"); 5086 Worklist.insert(I); 5087 } 5088 5089 // Insert the forced scalars. 5090 // FIXME: Currently widenPHIInstruction() often creates a dead vector 5091 // induction variable when the PHI user is scalarized. 5092 auto ForcedScalar = ForcedScalars.find(VF); 5093 if (ForcedScalar != ForcedScalars.end()) 5094 for (auto *I : ForcedScalar->second) 5095 Worklist.insert(I); 5096 5097 // Expand the worklist by looking through any bitcasts and getelementptr 5098 // instructions we've already identified as scalar. This is similar to the 5099 // expansion step in collectLoopUniforms(); however, here we're only 5100 // expanding to include additional bitcasts and getelementptr instructions. 5101 unsigned Idx = 0; 5102 while (Idx != Worklist.size()) { 5103 Instruction *Dst = Worklist[Idx++]; 5104 if (!isLoopVaryingBitCastOrGEP(Dst->getOperand(0))) 5105 continue; 5106 auto *Src = cast<Instruction>(Dst->getOperand(0)); 5107 if (llvm::all_of(Src->users(), [&](User *U) -> bool { 5108 auto *J = cast<Instruction>(U); 5109 return !TheLoop->contains(J) || Worklist.count(J) || 5110 ((isa<LoadInst>(J) || isa<StoreInst>(J)) && 5111 isScalarUse(J, Src)); 5112 })) { 5113 Worklist.insert(Src); 5114 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n"); 5115 } 5116 } 5117 5118 // An induction variable will remain scalar if all users of the induction 5119 // variable and induction variable update remain scalar. 5120 for (auto &Induction : Legal->getInductionVars()) { 5121 auto *Ind = Induction.first; 5122 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch)); 5123 5124 // If tail-folding is applied, the primary induction variable will be used 5125 // to feed a vector compare. 5126 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking()) 5127 continue; 5128 5129 // Determine if all users of the induction variable are scalar after 5130 // vectorization. 5131 auto ScalarInd = llvm::all_of(Ind->users(), [&](User *U) -> bool { 5132 auto *I = cast<Instruction>(U); 5133 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I); 5134 }); 5135 if (!ScalarInd) 5136 continue; 5137 5138 // Determine if all users of the induction variable update instruction are 5139 // scalar after vectorization. 5140 auto ScalarIndUpdate = 5141 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool { 5142 auto *I = cast<Instruction>(U); 5143 return I == Ind || !TheLoop->contains(I) || Worklist.count(I); 5144 }); 5145 if (!ScalarIndUpdate) 5146 continue; 5147 5148 // The induction variable and its update instruction will remain scalar. 5149 Worklist.insert(Ind); 5150 Worklist.insert(IndUpdate); 5151 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n"); 5152 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate 5153 << "\n"); 5154 } 5155 5156 Scalars[VF].insert(Worklist.begin(), Worklist.end()); 5157 } 5158 5159 bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I, 5160 ElementCount VF) { 5161 if (!blockNeedsPredication(I->getParent())) 5162 return false; 5163 switch(I->getOpcode()) { 5164 default: 5165 break; 5166 case Instruction::Load: 5167 case Instruction::Store: { 5168 if (!Legal->isMaskRequired(I)) 5169 return false; 5170 auto *Ptr = getLoadStorePointerOperand(I); 5171 auto *Ty = getMemInstValueType(I); 5172 // We have already decided how to vectorize this instruction, get that 5173 // result. 5174 if (VF.isVector()) { 5175 InstWidening WideningDecision = getWideningDecision(I, VF); 5176 assert(WideningDecision != CM_Unknown && 5177 "Widening decision should be ready at this moment"); 5178 return WideningDecision == CM_Scalarize; 5179 } 5180 const Align Alignment = getLoadStoreAlignment(I); 5181 return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) || 5182 isLegalMaskedGather(Ty, Alignment)) 5183 : !(isLegalMaskedStore(Ty, Ptr, Alignment) || 5184 isLegalMaskedScatter(Ty, Alignment)); 5185 } 5186 case Instruction::UDiv: 5187 case Instruction::SDiv: 5188 case Instruction::SRem: 5189 case Instruction::URem: 5190 return mayDivideByZero(*I); 5191 } 5192 return false; 5193 } 5194 5195 bool LoopVectorizationCostModel::interleavedAccessCanBeWidened( 5196 Instruction *I, ElementCount VF) { 5197 assert(isAccessInterleaved(I) && "Expecting interleaved access."); 5198 assert(getWideningDecision(I, VF) == CM_Unknown && 5199 "Decision should not be set yet."); 5200 auto *Group = getInterleavedAccessGroup(I); 5201 assert(Group && "Must have a group."); 5202 5203 // If the instruction's allocated size doesn't equal it's type size, it 5204 // requires padding and will be scalarized. 5205 auto &DL = I->getModule()->getDataLayout(); 5206 auto *ScalarTy = getMemInstValueType(I); 5207 if (hasIrregularType(ScalarTy, DL)) 5208 return false; 5209 5210 // Check if masking is required. 5211 // A Group may need masking for one of two reasons: it resides in a block that 5212 // needs predication, or it was decided to use masking to deal with gaps. 5213 bool PredicatedAccessRequiresMasking = 5214 Legal->blockNeedsPredication(I->getParent()) && Legal->isMaskRequired(I); 5215 bool AccessWithGapsRequiresMasking = 5216 Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed(); 5217 if (!PredicatedAccessRequiresMasking && !AccessWithGapsRequiresMasking) 5218 return true; 5219 5220 // If masked interleaving is required, we expect that the user/target had 5221 // enabled it, because otherwise it either wouldn't have been created or 5222 // it should have been invalidated by the CostModel. 5223 assert(useMaskedInterleavedAccesses(TTI) && 5224 "Masked interleave-groups for predicated accesses are not enabled."); 5225 5226 auto *Ty = getMemInstValueType(I); 5227 const Align Alignment = getLoadStoreAlignment(I); 5228 return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment) 5229 : TTI.isLegalMaskedStore(Ty, Alignment); 5230 } 5231 5232 bool LoopVectorizationCostModel::memoryInstructionCanBeWidened( 5233 Instruction *I, ElementCount VF) { 5234 // Get and ensure we have a valid memory instruction. 5235 LoadInst *LI = dyn_cast<LoadInst>(I); 5236 StoreInst *SI = dyn_cast<StoreInst>(I); 5237 assert((LI || SI) && "Invalid memory instruction"); 5238 5239 auto *Ptr = getLoadStorePointerOperand(I); 5240 5241 // In order to be widened, the pointer should be consecutive, first of all. 5242 if (!Legal->isConsecutivePtr(Ptr)) 5243 return false; 5244 5245 // If the instruction is a store located in a predicated block, it will be 5246 // scalarized. 5247 if (isScalarWithPredication(I)) 5248 return false; 5249 5250 // If the instruction's allocated size doesn't equal it's type size, it 5251 // requires padding and will be scalarized. 5252 auto &DL = I->getModule()->getDataLayout(); 5253 auto *ScalarTy = LI ? LI->getType() : SI->getValueOperand()->getType(); 5254 if (hasIrregularType(ScalarTy, DL)) 5255 return false; 5256 5257 return true; 5258 } 5259 5260 void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) { 5261 // We should not collect Uniforms more than once per VF. Right now, 5262 // this function is called from collectUniformsAndScalars(), which 5263 // already does this check. Collecting Uniforms for VF=1 does not make any 5264 // sense. 5265 5266 assert(VF.isVector() && Uniforms.find(VF) == Uniforms.end() && 5267 "This function should not be visited twice for the same VF"); 5268 5269 // Visit the list of Uniforms. If we'll not find any uniform value, we'll 5270 // not analyze again. Uniforms.count(VF) will return 1. 5271 Uniforms[VF].clear(); 5272 5273 // We now know that the loop is vectorizable! 5274 // Collect instructions inside the loop that will remain uniform after 5275 // vectorization. 5276 5277 // Global values, params and instructions outside of current loop are out of 5278 // scope. 5279 auto isOutOfScope = [&](Value *V) -> bool { 5280 Instruction *I = dyn_cast<Instruction>(V); 5281 return (!I || !TheLoop->contains(I)); 5282 }; 5283 5284 SetVector<Instruction *> Worklist; 5285 BasicBlock *Latch = TheLoop->getLoopLatch(); 5286 5287 // Instructions that are scalar with predication must not be considered 5288 // uniform after vectorization, because that would create an erroneous 5289 // replicating region where only a single instance out of VF should be formed. 5290 // TODO: optimize such seldom cases if found important, see PR40816. 5291 auto addToWorklistIfAllowed = [&](Instruction *I) -> void { 5292 if (isOutOfScope(I)) { 5293 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: " 5294 << *I << "\n"); 5295 return; 5296 } 5297 if (isScalarWithPredication(I, VF)) { 5298 LLVM_DEBUG(dbgs() << "LV: Found not uniform being ScalarWithPredication: " 5299 << *I << "\n"); 5300 return; 5301 } 5302 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n"); 5303 Worklist.insert(I); 5304 }; 5305 5306 // Start with the conditional branch. If the branch condition is an 5307 // instruction contained in the loop that is only used by the branch, it is 5308 // uniform. 5309 auto *Cmp = dyn_cast<Instruction>(Latch->getTerminator()->getOperand(0)); 5310 if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse()) 5311 addToWorklistIfAllowed(Cmp); 5312 5313 auto isUniformDecision = [&](Instruction *I, ElementCount VF) { 5314 InstWidening WideningDecision = getWideningDecision(I, VF); 5315 assert(WideningDecision != CM_Unknown && 5316 "Widening decision should be ready at this moment"); 5317 5318 // A uniform memory op is itself uniform. We exclude uniform stores 5319 // here as they demand the last lane, not the first one. 5320 if (isa<LoadInst>(I) && Legal->isUniformMemOp(*I)) { 5321 assert(WideningDecision == CM_Scalarize); 5322 return true; 5323 } 5324 5325 return (WideningDecision == CM_Widen || 5326 WideningDecision == CM_Widen_Reverse || 5327 WideningDecision == CM_Interleave); 5328 }; 5329 5330 5331 // Returns true if Ptr is the pointer operand of a memory access instruction 5332 // I, and I is known to not require scalarization. 5333 auto isVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool { 5334 return getLoadStorePointerOperand(I) == Ptr && isUniformDecision(I, VF); 5335 }; 5336 5337 // Holds a list of values which are known to have at least one uniform use. 5338 // Note that there may be other uses which aren't uniform. A "uniform use" 5339 // here is something which only demands lane 0 of the unrolled iterations; 5340 // it does not imply that all lanes produce the same value (e.g. this is not 5341 // the usual meaning of uniform) 5342 SmallPtrSet<Value *, 8> HasUniformUse; 5343 5344 // Scan the loop for instructions which are either a) known to have only 5345 // lane 0 demanded or b) are uses which demand only lane 0 of their operand. 5346 for (auto *BB : TheLoop->blocks()) 5347 for (auto &I : *BB) { 5348 // If there's no pointer operand, there's nothing to do. 5349 auto *Ptr = getLoadStorePointerOperand(&I); 5350 if (!Ptr) 5351 continue; 5352 5353 // A uniform memory op is itself uniform. We exclude uniform stores 5354 // here as they demand the last lane, not the first one. 5355 if (isa<LoadInst>(I) && Legal->isUniformMemOp(I)) 5356 addToWorklistIfAllowed(&I); 5357 5358 if (isUniformDecision(&I, VF)) { 5359 assert(isVectorizedMemAccessUse(&I, Ptr) && "consistency check"); 5360 HasUniformUse.insert(Ptr); 5361 } 5362 } 5363 5364 // Add to the worklist any operands which have *only* uniform (e.g. lane 0 5365 // demanding) users. Since loops are assumed to be in LCSSA form, this 5366 // disallows uses outside the loop as well. 5367 for (auto *V : HasUniformUse) { 5368 if (isOutOfScope(V)) 5369 continue; 5370 auto *I = cast<Instruction>(V); 5371 auto UsersAreMemAccesses = 5372 llvm::all_of(I->users(), [&](User *U) -> bool { 5373 return isVectorizedMemAccessUse(cast<Instruction>(U), V); 5374 }); 5375 if (UsersAreMemAccesses) 5376 addToWorklistIfAllowed(I); 5377 } 5378 5379 // Expand Worklist in topological order: whenever a new instruction 5380 // is added , its users should be already inside Worklist. It ensures 5381 // a uniform instruction will only be used by uniform instructions. 5382 unsigned idx = 0; 5383 while (idx != Worklist.size()) { 5384 Instruction *I = Worklist[idx++]; 5385 5386 for (auto OV : I->operand_values()) { 5387 // isOutOfScope operands cannot be uniform instructions. 5388 if (isOutOfScope(OV)) 5389 continue; 5390 // First order recurrence Phi's should typically be considered 5391 // non-uniform. 5392 auto *OP = dyn_cast<PHINode>(OV); 5393 if (OP && Legal->isFirstOrderRecurrence(OP)) 5394 continue; 5395 // If all the users of the operand are uniform, then add the 5396 // operand into the uniform worklist. 5397 auto *OI = cast<Instruction>(OV); 5398 if (llvm::all_of(OI->users(), [&](User *U) -> bool { 5399 auto *J = cast<Instruction>(U); 5400 return Worklist.count(J) || isVectorizedMemAccessUse(J, OI); 5401 })) 5402 addToWorklistIfAllowed(OI); 5403 } 5404 } 5405 5406 // For an instruction to be added into Worklist above, all its users inside 5407 // the loop should also be in Worklist. However, this condition cannot be 5408 // true for phi nodes that form a cyclic dependence. We must process phi 5409 // nodes separately. An induction variable will remain uniform if all users 5410 // of the induction variable and induction variable update remain uniform. 5411 // The code below handles both pointer and non-pointer induction variables. 5412 for (auto &Induction : Legal->getInductionVars()) { 5413 auto *Ind = Induction.first; 5414 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch)); 5415 5416 // Determine if all users of the induction variable are uniform after 5417 // vectorization. 5418 auto UniformInd = llvm::all_of(Ind->users(), [&](User *U) -> bool { 5419 auto *I = cast<Instruction>(U); 5420 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) || 5421 isVectorizedMemAccessUse(I, Ind); 5422 }); 5423 if (!UniformInd) 5424 continue; 5425 5426 // Determine if all users of the induction variable update instruction are 5427 // uniform after vectorization. 5428 auto UniformIndUpdate = 5429 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool { 5430 auto *I = cast<Instruction>(U); 5431 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) || 5432 isVectorizedMemAccessUse(I, IndUpdate); 5433 }); 5434 if (!UniformIndUpdate) 5435 continue; 5436 5437 // The induction variable and its update instruction will remain uniform. 5438 addToWorklistIfAllowed(Ind); 5439 addToWorklistIfAllowed(IndUpdate); 5440 } 5441 5442 Uniforms[VF].insert(Worklist.begin(), Worklist.end()); 5443 } 5444 5445 bool LoopVectorizationCostModel::runtimeChecksRequired() { 5446 LLVM_DEBUG(dbgs() << "LV: Performing code size checks.\n"); 5447 5448 if (Legal->getRuntimePointerChecking()->Need) { 5449 reportVectorizationFailure("Runtime ptr check is required with -Os/-Oz", 5450 "runtime pointer checks needed. Enable vectorization of this " 5451 "loop with '#pragma clang loop vectorize(enable)' when " 5452 "compiling with -Os/-Oz", 5453 "CantVersionLoopWithOptForSize", ORE, TheLoop); 5454 return true; 5455 } 5456 5457 if (!PSE.getUnionPredicate().getPredicates().empty()) { 5458 reportVectorizationFailure("Runtime SCEV check is required with -Os/-Oz", 5459 "runtime SCEV checks needed. Enable vectorization of this " 5460 "loop with '#pragma clang loop vectorize(enable)' when " 5461 "compiling with -Os/-Oz", 5462 "CantVersionLoopWithOptForSize", ORE, TheLoop); 5463 return true; 5464 } 5465 5466 // FIXME: Avoid specializing for stride==1 instead of bailing out. 5467 if (!Legal->getLAI()->getSymbolicStrides().empty()) { 5468 reportVectorizationFailure("Runtime stride check for small trip count", 5469 "runtime stride == 1 checks needed. Enable vectorization of " 5470 "this loop without such check by compiling with -Os/-Oz", 5471 "CantVersionLoopWithOptForSize", ORE, TheLoop); 5472 return true; 5473 } 5474 5475 return false; 5476 } 5477 5478 Optional<ElementCount> 5479 LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) { 5480 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) { 5481 // TODO: It may by useful to do since it's still likely to be dynamically 5482 // uniform if the target can skip. 5483 reportVectorizationFailure( 5484 "Not inserting runtime ptr check for divergent target", 5485 "runtime pointer checks needed. Not enabled for divergent target", 5486 "CantVersionLoopWithDivergentTarget", ORE, TheLoop); 5487 return None; 5488 } 5489 5490 unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop); 5491 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n'); 5492 if (TC == 1) { 5493 reportVectorizationFailure("Single iteration (non) loop", 5494 "loop trip count is one, irrelevant for vectorization", 5495 "SingleIterationLoop", ORE, TheLoop); 5496 return None; 5497 } 5498 5499 switch (ScalarEpilogueStatus) { 5500 case CM_ScalarEpilogueAllowed: 5501 return computeFeasibleMaxVF(TC, UserVF); 5502 case CM_ScalarEpilogueNotAllowedUsePredicate: 5503 LLVM_FALLTHROUGH; 5504 case CM_ScalarEpilogueNotNeededUsePredicate: 5505 LLVM_DEBUG( 5506 dbgs() << "LV: vector predicate hint/switch found.\n" 5507 << "LV: Not allowing scalar epilogue, creating predicated " 5508 << "vector loop.\n"); 5509 break; 5510 case CM_ScalarEpilogueNotAllowedLowTripLoop: 5511 // fallthrough as a special case of OptForSize 5512 case CM_ScalarEpilogueNotAllowedOptSize: 5513 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedOptSize) 5514 LLVM_DEBUG( 5515 dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n"); 5516 else 5517 LLVM_DEBUG(dbgs() << "LV: Not allowing scalar epilogue due to low trip " 5518 << "count.\n"); 5519 5520 // Bail if runtime checks are required, which are not good when optimising 5521 // for size. 5522 if (runtimeChecksRequired()) 5523 return None; 5524 5525 break; 5526 } 5527 5528 // The only loops we can vectorize without a scalar epilogue, are loops with 5529 // a bottom-test and a single exiting block. We'd have to handle the fact 5530 // that not every instruction executes on the last iteration. This will 5531 // require a lane mask which varies through the vector loop body. (TODO) 5532 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) { 5533 // If there was a tail-folding hint/switch, but we can't fold the tail by 5534 // masking, fallback to a vectorization with a scalar epilogue. 5535 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) { 5536 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a " 5537 "scalar epilogue instead.\n"); 5538 ScalarEpilogueStatus = CM_ScalarEpilogueAllowed; 5539 return computeFeasibleMaxVF(TC, UserVF); 5540 } 5541 return None; 5542 } 5543 5544 // Now try the tail folding 5545 5546 // Invalidate interleave groups that require an epilogue if we can't mask 5547 // the interleave-group. 5548 if (!useMaskedInterleavedAccesses(TTI)) { 5549 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() && 5550 "No decisions should have been taken at this point"); 5551 // Note: There is no need to invalidate any cost modeling decisions here, as 5552 // non where taken so far. 5553 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue(); 5554 } 5555 5556 ElementCount MaxVF = computeFeasibleMaxVF(TC, UserVF); 5557 assert(!MaxVF.isScalable() && 5558 "Scalable vectors do not yet support tail folding"); 5559 assert((UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())) && 5560 "MaxVF must be a power of 2"); 5561 unsigned MaxVFtimesIC = 5562 UserIC ? MaxVF.getFixedValue() * UserIC : MaxVF.getFixedValue(); 5563 // Avoid tail folding if the trip count is known to be a multiple of any VF we 5564 // chose. 5565 ScalarEvolution *SE = PSE.getSE(); 5566 const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount(); 5567 const SCEV *ExitCount = SE->getAddExpr( 5568 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType())); 5569 const SCEV *Rem = SE->getURemExpr( 5570 ExitCount, SE->getConstant(BackedgeTakenCount->getType(), MaxVFtimesIC)); 5571 if (Rem->isZero()) { 5572 // Accept MaxVF if we do not have a tail. 5573 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n"); 5574 return MaxVF; 5575 } 5576 5577 // If we don't know the precise trip count, or if the trip count that we 5578 // found modulo the vectorization factor is not zero, try to fold the tail 5579 // by masking. 5580 // FIXME: look for a smaller MaxVF that does divide TC rather than masking. 5581 if (Legal->prepareToFoldTailByMasking()) { 5582 FoldTailByMasking = true; 5583 return MaxVF; 5584 } 5585 5586 // If there was a tail-folding hint/switch, but we can't fold the tail by 5587 // masking, fallback to a vectorization with a scalar epilogue. 5588 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) { 5589 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a " 5590 "scalar epilogue instead.\n"); 5591 ScalarEpilogueStatus = CM_ScalarEpilogueAllowed; 5592 return MaxVF; 5593 } 5594 5595 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedUsePredicate) { 5596 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n"); 5597 return None; 5598 } 5599 5600 if (TC == 0) { 5601 reportVectorizationFailure( 5602 "Unable to calculate the loop count due to complex control flow", 5603 "unable to calculate the loop count due to complex control flow", 5604 "UnknownLoopCountComplexCFG", ORE, TheLoop); 5605 return None; 5606 } 5607 5608 reportVectorizationFailure( 5609 "Cannot optimize for size and vectorize at the same time.", 5610 "cannot optimize for size and vectorize at the same time. " 5611 "Enable vectorization of this loop with '#pragma clang loop " 5612 "vectorize(enable)' when compiling with -Os/-Oz", 5613 "NoTailLoopWithOptForSize", ORE, TheLoop); 5614 return None; 5615 } 5616 5617 ElementCount 5618 LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount, 5619 ElementCount UserVF) { 5620 bool IgnoreScalableUserVF = UserVF.isScalable() && 5621 !TTI.supportsScalableVectors() && 5622 !ForceTargetSupportsScalableVectors; 5623 if (IgnoreScalableUserVF) { 5624 LLVM_DEBUG( 5625 dbgs() << "LV: Ignoring VF=" << UserVF 5626 << " because target does not support scalable vectors.\n"); 5627 ORE->emit([&]() { 5628 return OptimizationRemarkAnalysis(DEBUG_TYPE, "IgnoreScalableUserVF", 5629 TheLoop->getStartLoc(), 5630 TheLoop->getHeader()) 5631 << "Ignoring VF=" << ore::NV("UserVF", UserVF) 5632 << " because target does not support scalable vectors."; 5633 }); 5634 } 5635 5636 // Beyond this point two scenarios are handled. If UserVF isn't specified 5637 // then a suitable VF is chosen. If UserVF is specified and there are 5638 // dependencies, check if it's legal. However, if a UserVF is specified and 5639 // there are no dependencies, then there's nothing to do. 5640 if (UserVF.isNonZero() && !IgnoreScalableUserVF && 5641 Legal->isSafeForAnyVectorWidth()) 5642 return UserVF; 5643 5644 MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI); 5645 unsigned SmallestType, WidestType; 5646 std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes(); 5647 unsigned WidestRegister = TTI.getRegisterBitWidth(true); 5648 5649 // Get the maximum safe dependence distance in bits computed by LAA. 5650 // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from 5651 // the memory accesses that is most restrictive (involved in the smallest 5652 // dependence distance). 5653 unsigned MaxSafeVectorWidthInBits = Legal->getMaxSafeVectorWidthInBits(); 5654 5655 // If the user vectorization factor is legally unsafe, clamp it to a safe 5656 // value. Otherwise, return as is. 5657 if (UserVF.isNonZero() && !IgnoreScalableUserVF) { 5658 unsigned MaxSafeElements = 5659 PowerOf2Floor(MaxSafeVectorWidthInBits / WidestType); 5660 ElementCount MaxSafeVF = ElementCount::getFixed(MaxSafeElements); 5661 5662 if (UserVF.isScalable()) { 5663 Optional<unsigned> MaxVScale = TTI.getMaxVScale(); 5664 5665 // Scale VF by vscale before checking if it's safe. 5666 MaxSafeVF = ElementCount::getScalable( 5667 MaxVScale ? (MaxSafeElements / MaxVScale.getValue()) : 0); 5668 5669 if (MaxSafeVF.isZero()) { 5670 // The dependence distance is too small to use scalable vectors, 5671 // fallback on fixed. 5672 LLVM_DEBUG( 5673 dbgs() 5674 << "LV: Max legal vector width too small, scalable vectorization " 5675 "unfeasible. Using fixed-width vectorization instead.\n"); 5676 ORE->emit([&]() { 5677 return OptimizationRemarkAnalysis(DEBUG_TYPE, "ScalableVFUnfeasible", 5678 TheLoop->getStartLoc(), 5679 TheLoop->getHeader()) 5680 << "Max legal vector width too small, scalable vectorization " 5681 << "unfeasible. Using fixed-width vectorization instead."; 5682 }); 5683 return computeFeasibleMaxVF( 5684 ConstTripCount, ElementCount::getFixed(UserVF.getKnownMinValue())); 5685 } 5686 } 5687 5688 LLVM_DEBUG(dbgs() << "LV: The max safe VF is: " << MaxSafeVF << ".\n"); 5689 5690 if (ElementCount::isKnownLE(UserVF, MaxSafeVF)) 5691 return UserVF; 5692 5693 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF 5694 << " is unsafe, clamping to max safe VF=" << MaxSafeVF 5695 << ".\n"); 5696 ORE->emit([&]() { 5697 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor", 5698 TheLoop->getStartLoc(), 5699 TheLoop->getHeader()) 5700 << "User-specified vectorization factor " 5701 << ore::NV("UserVectorizationFactor", UserVF) 5702 << " is unsafe, clamping to maximum safe vectorization factor " 5703 << ore::NV("VectorizationFactor", MaxSafeVF); 5704 }); 5705 return MaxSafeVF; 5706 } 5707 5708 WidestRegister = std::min(WidestRegister, MaxSafeVectorWidthInBits); 5709 5710 // Ensure MaxVF is a power of 2; the dependence distance bound may not be. 5711 // Note that both WidestRegister and WidestType may not be a powers of 2. 5712 unsigned MaxVectorSize = PowerOf2Floor(WidestRegister / WidestType); 5713 5714 LLVM_DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestType 5715 << " / " << WidestType << " bits.\n"); 5716 LLVM_DEBUG(dbgs() << "LV: The Widest register safe to use is: " 5717 << WidestRegister << " bits.\n"); 5718 5719 assert(MaxVectorSize <= WidestRegister && 5720 "Did not expect to pack so many elements" 5721 " into one vector!"); 5722 if (MaxVectorSize == 0) { 5723 LLVM_DEBUG(dbgs() << "LV: The target has no vector registers.\n"); 5724 MaxVectorSize = 1; 5725 return ElementCount::getFixed(MaxVectorSize); 5726 } else if (ConstTripCount && ConstTripCount < MaxVectorSize && 5727 isPowerOf2_32(ConstTripCount)) { 5728 // We need to clamp the VF to be the ConstTripCount. There is no point in 5729 // choosing a higher viable VF as done in the loop below. 5730 LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to the constant trip count: " 5731 << ConstTripCount << "\n"); 5732 MaxVectorSize = ConstTripCount; 5733 return ElementCount::getFixed(MaxVectorSize); 5734 } 5735 5736 unsigned MaxVF = MaxVectorSize; 5737 if (TTI.shouldMaximizeVectorBandwidth(!isScalarEpilogueAllowed()) || 5738 (MaximizeBandwidth && isScalarEpilogueAllowed())) { 5739 // Collect all viable vectorization factors larger than the default MaxVF 5740 // (i.e. MaxVectorSize). 5741 SmallVector<ElementCount, 8> VFs; 5742 unsigned NewMaxVectorSize = WidestRegister / SmallestType; 5743 for (unsigned VS = MaxVectorSize * 2; VS <= NewMaxVectorSize; VS *= 2) 5744 VFs.push_back(ElementCount::getFixed(VS)); 5745 5746 // For each VF calculate its register usage. 5747 auto RUs = calculateRegisterUsage(VFs); 5748 5749 // Select the largest VF which doesn't require more registers than existing 5750 // ones. 5751 for (int i = RUs.size() - 1; i >= 0; --i) { 5752 bool Selected = true; 5753 for (auto& pair : RUs[i].MaxLocalUsers) { 5754 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(pair.first); 5755 if (pair.second > TargetNumRegisters) 5756 Selected = false; 5757 } 5758 if (Selected) { 5759 MaxVF = VFs[i].getKnownMinValue(); 5760 break; 5761 } 5762 } 5763 if (unsigned MinVF = TTI.getMinimumVF(SmallestType)) { 5764 if (MaxVF < MinVF) { 5765 LLVM_DEBUG(dbgs() << "LV: Overriding calculated MaxVF(" << MaxVF 5766 << ") with target's minimum: " << MinVF << '\n'); 5767 MaxVF = MinVF; 5768 } 5769 } 5770 } 5771 return ElementCount::getFixed(MaxVF); 5772 } 5773 5774 VectorizationFactor 5775 LoopVectorizationCostModel::selectVectorizationFactor(ElementCount MaxVF) { 5776 // FIXME: This can be fixed for scalable vectors later, because at this stage 5777 // the LoopVectorizer will only consider vectorizing a loop with scalable 5778 // vectors when the loop has a hint to enable vectorization for a given VF. 5779 assert(!MaxVF.isScalable() && "scalable vectors not yet supported"); 5780 5781 InstructionCost ExpectedCost = expectedCost(ElementCount::getFixed(1)).first; 5782 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ExpectedCost << ".\n"); 5783 assert(ExpectedCost.isValid() && "Unexpected invalid cost for scalar loop"); 5784 5785 unsigned Width = 1; 5786 const float ScalarCost = *ExpectedCost.getValue(); 5787 float Cost = ScalarCost; 5788 5789 bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled; 5790 if (ForceVectorization && MaxVF.isVector()) { 5791 // Ignore scalar width, because the user explicitly wants vectorization. 5792 // Initialize cost to max so that VF = 2 is, at least, chosen during cost 5793 // evaluation. 5794 Cost = std::numeric_limits<float>::max(); 5795 } 5796 5797 for (unsigned i = 2; i <= MaxVF.getFixedValue(); i *= 2) { 5798 // Notice that the vector loop needs to be executed less times, so 5799 // we need to divide the cost of the vector loops by the width of 5800 // the vector elements. 5801 VectorizationCostTy C = expectedCost(ElementCount::getFixed(i)); 5802 assert(C.first.isValid() && "Unexpected invalid cost for vector loop"); 5803 float VectorCost = *C.first.getValue() / (float)i; 5804 LLVM_DEBUG(dbgs() << "LV: Vector loop of width " << i 5805 << " costs: " << (int)VectorCost << ".\n"); 5806 if (!C.second && !ForceVectorization) { 5807 LLVM_DEBUG( 5808 dbgs() << "LV: Not considering vector loop of width " << i 5809 << " because it will not generate any vector instructions.\n"); 5810 continue; 5811 } 5812 5813 // If profitable add it to ProfitableVF list. 5814 if (VectorCost < ScalarCost) { 5815 ProfitableVFs.push_back(VectorizationFactor( 5816 {ElementCount::getFixed(i), (unsigned)VectorCost})); 5817 } 5818 5819 if (VectorCost < Cost) { 5820 Cost = VectorCost; 5821 Width = i; 5822 } 5823 } 5824 5825 if (!EnableCondStoresVectorization && NumPredStores) { 5826 reportVectorizationFailure("There are conditional stores.", 5827 "store that is conditionally executed prevents vectorization", 5828 "ConditionalStore", ORE, TheLoop); 5829 Width = 1; 5830 Cost = ScalarCost; 5831 } 5832 5833 LLVM_DEBUG(if (ForceVectorization && Width > 1 && Cost >= ScalarCost) dbgs() 5834 << "LV: Vectorization seems to be not beneficial, " 5835 << "but was forced by a user.\n"); 5836 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << Width << ".\n"); 5837 VectorizationFactor Factor = {ElementCount::getFixed(Width), 5838 (unsigned)(Width * Cost)}; 5839 return Factor; 5840 } 5841 5842 bool LoopVectorizationCostModel::isCandidateForEpilogueVectorization( 5843 const Loop &L, ElementCount VF) const { 5844 // Cross iteration phis such as reductions need special handling and are 5845 // currently unsupported. 5846 if (any_of(L.getHeader()->phis(), [&](PHINode &Phi) { 5847 return Legal->isFirstOrderRecurrence(&Phi) || 5848 Legal->isReductionVariable(&Phi); 5849 })) 5850 return false; 5851 5852 // Phis with uses outside of the loop require special handling and are 5853 // currently unsupported. 5854 for (auto &Entry : Legal->getInductionVars()) { 5855 // Look for uses of the value of the induction at the last iteration. 5856 Value *PostInc = Entry.first->getIncomingValueForBlock(L.getLoopLatch()); 5857 for (User *U : PostInc->users()) 5858 if (!L.contains(cast<Instruction>(U))) 5859 return false; 5860 // Look for uses of penultimate value of the induction. 5861 for (User *U : Entry.first->users()) 5862 if (!L.contains(cast<Instruction>(U))) 5863 return false; 5864 } 5865 5866 // Induction variables that are widened require special handling that is 5867 // currently not supported. 5868 if (any_of(Legal->getInductionVars(), [&](auto &Entry) { 5869 return !(this->isScalarAfterVectorization(Entry.first, VF) || 5870 this->isProfitableToScalarize(Entry.first, VF)); 5871 })) 5872 return false; 5873 5874 return true; 5875 } 5876 5877 bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable( 5878 const ElementCount VF) const { 5879 // FIXME: We need a much better cost-model to take different parameters such 5880 // as register pressure, code size increase and cost of extra branches into 5881 // account. For now we apply a very crude heuristic and only consider loops 5882 // with vectorization factors larger than a certain value. 5883 // We also consider epilogue vectorization unprofitable for targets that don't 5884 // consider interleaving beneficial (eg. MVE). 5885 if (TTI.getMaxInterleaveFactor(VF.getKnownMinValue()) <= 1) 5886 return false; 5887 if (VF.getFixedValue() >= EpilogueVectorizationMinVF) 5888 return true; 5889 return false; 5890 } 5891 5892 VectorizationFactor 5893 LoopVectorizationCostModel::selectEpilogueVectorizationFactor( 5894 const ElementCount MainLoopVF, const LoopVectorizationPlanner &LVP) { 5895 VectorizationFactor Result = VectorizationFactor::Disabled(); 5896 if (!EnableEpilogueVectorization) { 5897 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n";); 5898 return Result; 5899 } 5900 5901 if (!isScalarEpilogueAllowed()) { 5902 LLVM_DEBUG( 5903 dbgs() << "LEV: Unable to vectorize epilogue because no epilogue is " 5904 "allowed.\n";); 5905 return Result; 5906 } 5907 5908 // FIXME: This can be fixed for scalable vectors later, because at this stage 5909 // the LoopVectorizer will only consider vectorizing a loop with scalable 5910 // vectors when the loop has a hint to enable vectorization for a given VF. 5911 if (MainLoopVF.isScalable()) { 5912 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization for scalable vectors not " 5913 "yet supported.\n"); 5914 return Result; 5915 } 5916 5917 // Not really a cost consideration, but check for unsupported cases here to 5918 // simplify the logic. 5919 if (!isCandidateForEpilogueVectorization(*TheLoop, MainLoopVF)) { 5920 LLVM_DEBUG( 5921 dbgs() << "LEV: Unable to vectorize epilogue because the loop is " 5922 "not a supported candidate.\n";); 5923 return Result; 5924 } 5925 5926 if (EpilogueVectorizationForceVF > 1) { 5927 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n";); 5928 if (LVP.hasPlanWithVFs( 5929 {MainLoopVF, ElementCount::getFixed(EpilogueVectorizationForceVF)})) 5930 return {ElementCount::getFixed(EpilogueVectorizationForceVF), 0}; 5931 else { 5932 LLVM_DEBUG( 5933 dbgs() 5934 << "LEV: Epilogue vectorization forced factor is not viable.\n";); 5935 return Result; 5936 } 5937 } 5938 5939 if (TheLoop->getHeader()->getParent()->hasOptSize() || 5940 TheLoop->getHeader()->getParent()->hasMinSize()) { 5941 LLVM_DEBUG( 5942 dbgs() 5943 << "LEV: Epilogue vectorization skipped due to opt for size.\n";); 5944 return Result; 5945 } 5946 5947 if (!isEpilogueVectorizationProfitable(MainLoopVF)) 5948 return Result; 5949 5950 for (auto &NextVF : ProfitableVFs) 5951 if (ElementCount::isKnownLT(NextVF.Width, MainLoopVF) && 5952 (Result.Width.getFixedValue() == 1 || NextVF.Cost < Result.Cost) && 5953 LVP.hasPlanWithVFs({MainLoopVF, NextVF.Width})) 5954 Result = NextVF; 5955 5956 if (Result != VectorizationFactor::Disabled()) 5957 LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = " 5958 << Result.Width.getFixedValue() << "\n";); 5959 return Result; 5960 } 5961 5962 std::pair<unsigned, unsigned> 5963 LoopVectorizationCostModel::getSmallestAndWidestTypes() { 5964 unsigned MinWidth = -1U; 5965 unsigned MaxWidth = 8; 5966 const DataLayout &DL = TheFunction->getParent()->getDataLayout(); 5967 5968 // For each block. 5969 for (BasicBlock *BB : TheLoop->blocks()) { 5970 // For each instruction in the loop. 5971 for (Instruction &I : BB->instructionsWithoutDebug()) { 5972 Type *T = I.getType(); 5973 5974 // Skip ignored values. 5975 if (ValuesToIgnore.count(&I)) 5976 continue; 5977 5978 // Only examine Loads, Stores and PHINodes. 5979 if (!isa<LoadInst>(I) && !isa<StoreInst>(I) && !isa<PHINode>(I)) 5980 continue; 5981 5982 // Examine PHI nodes that are reduction variables. Update the type to 5983 // account for the recurrence type. 5984 if (auto *PN = dyn_cast<PHINode>(&I)) { 5985 if (!Legal->isReductionVariable(PN)) 5986 continue; 5987 RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[PN]; 5988 if (PreferInLoopReductions || 5989 TTI.preferInLoopReduction(RdxDesc.getOpcode(), 5990 RdxDesc.getRecurrenceType(), 5991 TargetTransformInfo::ReductionFlags())) 5992 continue; 5993 T = RdxDesc.getRecurrenceType(); 5994 } 5995 5996 // Examine the stored values. 5997 if (auto *ST = dyn_cast<StoreInst>(&I)) 5998 T = ST->getValueOperand()->getType(); 5999 6000 // Ignore loaded pointer types and stored pointer types that are not 6001 // vectorizable. 6002 // 6003 // FIXME: The check here attempts to predict whether a load or store will 6004 // be vectorized. We only know this for certain after a VF has 6005 // been selected. Here, we assume that if an access can be 6006 // vectorized, it will be. We should also look at extending this 6007 // optimization to non-pointer types. 6008 // 6009 if (T->isPointerTy() && !isConsecutiveLoadOrStore(&I) && 6010 !isAccessInterleaved(&I) && !isLegalGatherOrScatter(&I)) 6011 continue; 6012 6013 MinWidth = std::min(MinWidth, 6014 (unsigned)DL.getTypeSizeInBits(T->getScalarType())); 6015 MaxWidth = std::max(MaxWidth, 6016 (unsigned)DL.getTypeSizeInBits(T->getScalarType())); 6017 } 6018 } 6019 6020 return {MinWidth, MaxWidth}; 6021 } 6022 6023 unsigned LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF, 6024 unsigned LoopCost) { 6025 // -- The interleave heuristics -- 6026 // We interleave the loop in order to expose ILP and reduce the loop overhead. 6027 // There are many micro-architectural considerations that we can't predict 6028 // at this level. For example, frontend pressure (on decode or fetch) due to 6029 // code size, or the number and capabilities of the execution ports. 6030 // 6031 // We use the following heuristics to select the interleave count: 6032 // 1. If the code has reductions, then we interleave to break the cross 6033 // iteration dependency. 6034 // 2. If the loop is really small, then we interleave to reduce the loop 6035 // overhead. 6036 // 3. We don't interleave if we think that we will spill registers to memory 6037 // due to the increased register pressure. 6038 6039 if (!isScalarEpilogueAllowed()) 6040 return 1; 6041 6042 // We used the distance for the interleave count. 6043 if (Legal->getMaxSafeDepDistBytes() != -1U) 6044 return 1; 6045 6046 auto BestKnownTC = getSmallBestKnownTC(*PSE.getSE(), TheLoop); 6047 const bool HasReductions = !Legal->getReductionVars().empty(); 6048 // Do not interleave loops with a relatively small known or estimated trip 6049 // count. But we will interleave when InterleaveSmallLoopScalarReduction is 6050 // enabled, and the code has scalar reductions(HasReductions && VF = 1), 6051 // because with the above conditions interleaving can expose ILP and break 6052 // cross iteration dependences for reductions. 6053 if (BestKnownTC && (*BestKnownTC < TinyTripCountInterleaveThreshold) && 6054 !(InterleaveSmallLoopScalarReduction && HasReductions && VF.isScalar())) 6055 return 1; 6056 6057 RegisterUsage R = calculateRegisterUsage({VF})[0]; 6058 // We divide by these constants so assume that we have at least one 6059 // instruction that uses at least one register. 6060 for (auto& pair : R.MaxLocalUsers) { 6061 pair.second = std::max(pair.second, 1U); 6062 } 6063 6064 // We calculate the interleave count using the following formula. 6065 // Subtract the number of loop invariants from the number of available 6066 // registers. These registers are used by all of the interleaved instances. 6067 // Next, divide the remaining registers by the number of registers that is 6068 // required by the loop, in order to estimate how many parallel instances 6069 // fit without causing spills. All of this is rounded down if necessary to be 6070 // a power of two. We want power of two interleave count to simplify any 6071 // addressing operations or alignment considerations. 6072 // We also want power of two interleave counts to ensure that the induction 6073 // variable of the vector loop wraps to zero, when tail is folded by masking; 6074 // this currently happens when OptForSize, in which case IC is set to 1 above. 6075 unsigned IC = UINT_MAX; 6076 6077 for (auto& pair : R.MaxLocalUsers) { 6078 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(pair.first); 6079 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters 6080 << " registers of " 6081 << TTI.getRegisterClassName(pair.first) << " register class\n"); 6082 if (VF.isScalar()) { 6083 if (ForceTargetNumScalarRegs.getNumOccurrences() > 0) 6084 TargetNumRegisters = ForceTargetNumScalarRegs; 6085 } else { 6086 if (ForceTargetNumVectorRegs.getNumOccurrences() > 0) 6087 TargetNumRegisters = ForceTargetNumVectorRegs; 6088 } 6089 unsigned MaxLocalUsers = pair.second; 6090 unsigned LoopInvariantRegs = 0; 6091 if (R.LoopInvariantRegs.find(pair.first) != R.LoopInvariantRegs.end()) 6092 LoopInvariantRegs = R.LoopInvariantRegs[pair.first]; 6093 6094 unsigned TmpIC = PowerOf2Floor((TargetNumRegisters - LoopInvariantRegs) / MaxLocalUsers); 6095 // Don't count the induction variable as interleaved. 6096 if (EnableIndVarRegisterHeur) { 6097 TmpIC = 6098 PowerOf2Floor((TargetNumRegisters - LoopInvariantRegs - 1) / 6099 std::max(1U, (MaxLocalUsers - 1))); 6100 } 6101 6102 IC = std::min(IC, TmpIC); 6103 } 6104 6105 // Clamp the interleave ranges to reasonable counts. 6106 unsigned MaxInterleaveCount = 6107 TTI.getMaxInterleaveFactor(VF.getKnownMinValue()); 6108 6109 // Check if the user has overridden the max. 6110 if (VF.isScalar()) { 6111 if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0) 6112 MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor; 6113 } else { 6114 if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0) 6115 MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor; 6116 } 6117 6118 // If trip count is known or estimated compile time constant, limit the 6119 // interleave count to be less than the trip count divided by VF, provided it 6120 // is at least 1. 6121 // 6122 // For scalable vectors we can't know if interleaving is beneficial. It may 6123 // not be beneficial for small loops if none of the lanes in the second vector 6124 // iterations is enabled. However, for larger loops, there is likely to be a 6125 // similar benefit as for fixed-width vectors. For now, we choose to leave 6126 // the InterleaveCount as if vscale is '1', although if some information about 6127 // the vector is known (e.g. min vector size), we can make a better decision. 6128 if (BestKnownTC) { 6129 MaxInterleaveCount = 6130 std::min(*BestKnownTC / VF.getKnownMinValue(), MaxInterleaveCount); 6131 // Make sure MaxInterleaveCount is greater than 0. 6132 MaxInterleaveCount = std::max(1u, MaxInterleaveCount); 6133 } 6134 6135 assert(MaxInterleaveCount > 0 && 6136 "Maximum interleave count must be greater than 0"); 6137 6138 // Clamp the calculated IC to be between the 1 and the max interleave count 6139 // that the target and trip count allows. 6140 if (IC > MaxInterleaveCount) 6141 IC = MaxInterleaveCount; 6142 else 6143 // Make sure IC is greater than 0. 6144 IC = std::max(1u, IC); 6145 6146 assert(IC > 0 && "Interleave count must be greater than 0."); 6147 6148 // If we did not calculate the cost for VF (because the user selected the VF) 6149 // then we calculate the cost of VF here. 6150 if (LoopCost == 0) { 6151 assert(expectedCost(VF).first.isValid() && "Expected a valid cost"); 6152 LoopCost = *expectedCost(VF).first.getValue(); 6153 } 6154 6155 assert(LoopCost && "Non-zero loop cost expected"); 6156 6157 // Interleave if we vectorized this loop and there is a reduction that could 6158 // benefit from interleaving. 6159 if (VF.isVector() && HasReductions) { 6160 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n"); 6161 return IC; 6162 } 6163 6164 // Note that if we've already vectorized the loop we will have done the 6165 // runtime check and so interleaving won't require further checks. 6166 bool InterleavingRequiresRuntimePointerCheck = 6167 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need); 6168 6169 // We want to interleave small loops in order to reduce the loop overhead and 6170 // potentially expose ILP opportunities. 6171 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n' 6172 << "LV: IC is " << IC << '\n' 6173 << "LV: VF is " << VF << '\n'); 6174 const bool AggressivelyInterleaveReductions = 6175 TTI.enableAggressiveInterleaving(HasReductions); 6176 if (!InterleavingRequiresRuntimePointerCheck && LoopCost < SmallLoopCost) { 6177 // We assume that the cost overhead is 1 and we use the cost model 6178 // to estimate the cost of the loop and interleave until the cost of the 6179 // loop overhead is about 5% of the cost of the loop. 6180 unsigned SmallIC = 6181 std::min(IC, (unsigned)PowerOf2Floor(SmallLoopCost / LoopCost)); 6182 6183 // Interleave until store/load ports (estimated by max interleave count) are 6184 // saturated. 6185 unsigned NumStores = Legal->getNumStores(); 6186 unsigned NumLoads = Legal->getNumLoads(); 6187 unsigned StoresIC = IC / (NumStores ? NumStores : 1); 6188 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1); 6189 6190 // If we have a scalar reduction (vector reductions are already dealt with 6191 // by this point), we can increase the critical path length if the loop 6192 // we're interleaving is inside another loop. Limit, by default to 2, so the 6193 // critical path only gets increased by one reduction operation. 6194 if (HasReductions && TheLoop->getLoopDepth() > 1) { 6195 unsigned F = static_cast<unsigned>(MaxNestedScalarReductionIC); 6196 SmallIC = std::min(SmallIC, F); 6197 StoresIC = std::min(StoresIC, F); 6198 LoadsIC = std::min(LoadsIC, F); 6199 } 6200 6201 if (EnableLoadStoreRuntimeInterleave && 6202 std::max(StoresIC, LoadsIC) > SmallIC) { 6203 LLVM_DEBUG( 6204 dbgs() << "LV: Interleaving to saturate store or load ports.\n"); 6205 return std::max(StoresIC, LoadsIC); 6206 } 6207 6208 // If there are scalar reductions and TTI has enabled aggressive 6209 // interleaving for reductions, we will interleave to expose ILP. 6210 if (InterleaveSmallLoopScalarReduction && VF.isScalar() && 6211 AggressivelyInterleaveReductions) { 6212 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n"); 6213 // Interleave no less than SmallIC but not as aggressive as the normal IC 6214 // to satisfy the rare situation when resources are too limited. 6215 return std::max(IC / 2, SmallIC); 6216 } else { 6217 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n"); 6218 return SmallIC; 6219 } 6220 } 6221 6222 // Interleave if this is a large loop (small loops are already dealt with by 6223 // this point) that could benefit from interleaving. 6224 if (AggressivelyInterleaveReductions) { 6225 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n"); 6226 return IC; 6227 } 6228 6229 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n"); 6230 return 1; 6231 } 6232 6233 SmallVector<LoopVectorizationCostModel::RegisterUsage, 8> 6234 LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<ElementCount> VFs) { 6235 // This function calculates the register usage by measuring the highest number 6236 // of values that are alive at a single location. Obviously, this is a very 6237 // rough estimation. We scan the loop in a topological order in order and 6238 // assign a number to each instruction. We use RPO to ensure that defs are 6239 // met before their users. We assume that each instruction that has in-loop 6240 // users starts an interval. We record every time that an in-loop value is 6241 // used, so we have a list of the first and last occurrences of each 6242 // instruction. Next, we transpose this data structure into a multi map that 6243 // holds the list of intervals that *end* at a specific location. This multi 6244 // map allows us to perform a linear search. We scan the instructions linearly 6245 // and record each time that a new interval starts, by placing it in a set. 6246 // If we find this value in the multi-map then we remove it from the set. 6247 // The max register usage is the maximum size of the set. 6248 // We also search for instructions that are defined outside the loop, but are 6249 // used inside the loop. We need this number separately from the max-interval 6250 // usage number because when we unroll, loop-invariant values do not take 6251 // more register. 6252 LoopBlocksDFS DFS(TheLoop); 6253 DFS.perform(LI); 6254 6255 RegisterUsage RU; 6256 6257 // Each 'key' in the map opens a new interval. The values 6258 // of the map are the index of the 'last seen' usage of the 6259 // instruction that is the key. 6260 using IntervalMap = DenseMap<Instruction *, unsigned>; 6261 6262 // Maps instruction to its index. 6263 SmallVector<Instruction *, 64> IdxToInstr; 6264 // Marks the end of each interval. 6265 IntervalMap EndPoint; 6266 // Saves the list of instruction indices that are used in the loop. 6267 SmallPtrSet<Instruction *, 8> Ends; 6268 // Saves the list of values that are used in the loop but are 6269 // defined outside the loop, such as arguments and constants. 6270 SmallPtrSet<Value *, 8> LoopInvariants; 6271 6272 for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) { 6273 for (Instruction &I : BB->instructionsWithoutDebug()) { 6274 IdxToInstr.push_back(&I); 6275 6276 // Save the end location of each USE. 6277 for (Value *U : I.operands()) { 6278 auto *Instr = dyn_cast<Instruction>(U); 6279 6280 // Ignore non-instruction values such as arguments, constants, etc. 6281 if (!Instr) 6282 continue; 6283 6284 // If this instruction is outside the loop then record it and continue. 6285 if (!TheLoop->contains(Instr)) { 6286 LoopInvariants.insert(Instr); 6287 continue; 6288 } 6289 6290 // Overwrite previous end points. 6291 EndPoint[Instr] = IdxToInstr.size(); 6292 Ends.insert(Instr); 6293 } 6294 } 6295 } 6296 6297 // Saves the list of intervals that end with the index in 'key'. 6298 using InstrList = SmallVector<Instruction *, 2>; 6299 DenseMap<unsigned, InstrList> TransposeEnds; 6300 6301 // Transpose the EndPoints to a list of values that end at each index. 6302 for (auto &Interval : EndPoint) 6303 TransposeEnds[Interval.second].push_back(Interval.first); 6304 6305 SmallPtrSet<Instruction *, 8> OpenIntervals; 6306 SmallVector<RegisterUsage, 8> RUs(VFs.size()); 6307 SmallVector<SmallMapVector<unsigned, unsigned, 4>, 8> MaxUsages(VFs.size()); 6308 6309 LLVM_DEBUG(dbgs() << "LV(REG): Calculating max register usage:\n"); 6310 6311 // A lambda that gets the register usage for the given type and VF. 6312 const auto &TTICapture = TTI; 6313 auto GetRegUsage = [&TTICapture](Type *Ty, ElementCount VF) { 6314 if (Ty->isTokenTy() || !VectorType::isValidElementType(Ty)) 6315 return 0U; 6316 return TTICapture.getRegUsageForType(VectorType::get(Ty, VF)); 6317 }; 6318 6319 for (unsigned int i = 0, s = IdxToInstr.size(); i < s; ++i) { 6320 Instruction *I = IdxToInstr[i]; 6321 6322 // Remove all of the instructions that end at this location. 6323 InstrList &List = TransposeEnds[i]; 6324 for (Instruction *ToRemove : List) 6325 OpenIntervals.erase(ToRemove); 6326 6327 // Ignore instructions that are never used within the loop. 6328 if (!Ends.count(I)) 6329 continue; 6330 6331 // Skip ignored values. 6332 if (ValuesToIgnore.count(I)) 6333 continue; 6334 6335 // For each VF find the maximum usage of registers. 6336 for (unsigned j = 0, e = VFs.size(); j < e; ++j) { 6337 // Count the number of live intervals. 6338 SmallMapVector<unsigned, unsigned, 4> RegUsage; 6339 6340 if (VFs[j].isScalar()) { 6341 for (auto Inst : OpenIntervals) { 6342 unsigned ClassID = TTI.getRegisterClassForType(false, Inst->getType()); 6343 if (RegUsage.find(ClassID) == RegUsage.end()) 6344 RegUsage[ClassID] = 1; 6345 else 6346 RegUsage[ClassID] += 1; 6347 } 6348 } else { 6349 collectUniformsAndScalars(VFs[j]); 6350 for (auto Inst : OpenIntervals) { 6351 // Skip ignored values for VF > 1. 6352 if (VecValuesToIgnore.count(Inst)) 6353 continue; 6354 if (isScalarAfterVectorization(Inst, VFs[j])) { 6355 unsigned ClassID = TTI.getRegisterClassForType(false, Inst->getType()); 6356 if (RegUsage.find(ClassID) == RegUsage.end()) 6357 RegUsage[ClassID] = 1; 6358 else 6359 RegUsage[ClassID] += 1; 6360 } else { 6361 unsigned ClassID = TTI.getRegisterClassForType(true, Inst->getType()); 6362 if (RegUsage.find(ClassID) == RegUsage.end()) 6363 RegUsage[ClassID] = GetRegUsage(Inst->getType(), VFs[j]); 6364 else 6365 RegUsage[ClassID] += GetRegUsage(Inst->getType(), VFs[j]); 6366 } 6367 } 6368 } 6369 6370 for (auto& pair : RegUsage) { 6371 if (MaxUsages[j].find(pair.first) != MaxUsages[j].end()) 6372 MaxUsages[j][pair.first] = std::max(MaxUsages[j][pair.first], pair.second); 6373 else 6374 MaxUsages[j][pair.first] = pair.second; 6375 } 6376 } 6377 6378 LLVM_DEBUG(dbgs() << "LV(REG): At #" << i << " Interval # " 6379 << OpenIntervals.size() << '\n'); 6380 6381 // Add the current instruction to the list of open intervals. 6382 OpenIntervals.insert(I); 6383 } 6384 6385 for (unsigned i = 0, e = VFs.size(); i < e; ++i) { 6386 SmallMapVector<unsigned, unsigned, 4> Invariant; 6387 6388 for (auto Inst : LoopInvariants) { 6389 unsigned Usage = 6390 VFs[i].isScalar() ? 1 : GetRegUsage(Inst->getType(), VFs[i]); 6391 unsigned ClassID = 6392 TTI.getRegisterClassForType(VFs[i].isVector(), Inst->getType()); 6393 if (Invariant.find(ClassID) == Invariant.end()) 6394 Invariant[ClassID] = Usage; 6395 else 6396 Invariant[ClassID] += Usage; 6397 } 6398 6399 LLVM_DEBUG({ 6400 dbgs() << "LV(REG): VF = " << VFs[i] << '\n'; 6401 dbgs() << "LV(REG): Found max usage: " << MaxUsages[i].size() 6402 << " item\n"; 6403 for (const auto &pair : MaxUsages[i]) { 6404 dbgs() << "LV(REG): RegisterClass: " 6405 << TTI.getRegisterClassName(pair.first) << ", " << pair.second 6406 << " registers\n"; 6407 } 6408 dbgs() << "LV(REG): Found invariant usage: " << Invariant.size() 6409 << " item\n"; 6410 for (const auto &pair : Invariant) { 6411 dbgs() << "LV(REG): RegisterClass: " 6412 << TTI.getRegisterClassName(pair.first) << ", " << pair.second 6413 << " registers\n"; 6414 } 6415 }); 6416 6417 RU.LoopInvariantRegs = Invariant; 6418 RU.MaxLocalUsers = MaxUsages[i]; 6419 RUs[i] = RU; 6420 } 6421 6422 return RUs; 6423 } 6424 6425 bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I){ 6426 // TODO: Cost model for emulated masked load/store is completely 6427 // broken. This hack guides the cost model to use an artificially 6428 // high enough value to practically disable vectorization with such 6429 // operations, except where previously deployed legality hack allowed 6430 // using very low cost values. This is to avoid regressions coming simply 6431 // from moving "masked load/store" check from legality to cost model. 6432 // Masked Load/Gather emulation was previously never allowed. 6433 // Limited number of Masked Store/Scatter emulation was allowed. 6434 assert(isPredicatedInst(I) && "Expecting a scalar emulated instruction"); 6435 return isa<LoadInst>(I) || 6436 (isa<StoreInst>(I) && 6437 NumPredStores > NumberOfStoresToPredicate); 6438 } 6439 6440 void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) { 6441 // If we aren't vectorizing the loop, or if we've already collected the 6442 // instructions to scalarize, there's nothing to do. Collection may already 6443 // have occurred if we have a user-selected VF and are now computing the 6444 // expected cost for interleaving. 6445 if (VF.isScalar() || VF.isZero() || 6446 InstsToScalarize.find(VF) != InstsToScalarize.end()) 6447 return; 6448 6449 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's 6450 // not profitable to scalarize any instructions, the presence of VF in the 6451 // map will indicate that we've analyzed it already. 6452 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF]; 6453 6454 // Find all the instructions that are scalar with predication in the loop and 6455 // determine if it would be better to not if-convert the blocks they are in. 6456 // If so, we also record the instructions to scalarize. 6457 for (BasicBlock *BB : TheLoop->blocks()) { 6458 if (!blockNeedsPredication(BB)) 6459 continue; 6460 for (Instruction &I : *BB) 6461 if (isScalarWithPredication(&I)) { 6462 ScalarCostsTy ScalarCosts; 6463 // Do not apply discount logic if hacked cost is needed 6464 // for emulated masked memrefs. 6465 if (!useEmulatedMaskMemRefHack(&I) && 6466 computePredInstDiscount(&I, ScalarCosts, VF) >= 0) 6467 ScalarCostsVF.insert(ScalarCosts.begin(), ScalarCosts.end()); 6468 // Remember that BB will remain after vectorization. 6469 PredicatedBBsAfterVectorization.insert(BB); 6470 } 6471 } 6472 } 6473 6474 int LoopVectorizationCostModel::computePredInstDiscount( 6475 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) { 6476 assert(!isUniformAfterVectorization(PredInst, VF) && 6477 "Instruction marked uniform-after-vectorization will be predicated"); 6478 6479 // Initialize the discount to zero, meaning that the scalar version and the 6480 // vector version cost the same. 6481 InstructionCost Discount = 0; 6482 6483 // Holds instructions to analyze. The instructions we visit are mapped in 6484 // ScalarCosts. Those instructions are the ones that would be scalarized if 6485 // we find that the scalar version costs less. 6486 SmallVector<Instruction *, 8> Worklist; 6487 6488 // Returns true if the given instruction can be scalarized. 6489 auto canBeScalarized = [&](Instruction *I) -> bool { 6490 // We only attempt to scalarize instructions forming a single-use chain 6491 // from the original predicated block that would otherwise be vectorized. 6492 // Although not strictly necessary, we give up on instructions we know will 6493 // already be scalar to avoid traversing chains that are unlikely to be 6494 // beneficial. 6495 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() || 6496 isScalarAfterVectorization(I, VF)) 6497 return false; 6498 6499 // If the instruction is scalar with predication, it will be analyzed 6500 // separately. We ignore it within the context of PredInst. 6501 if (isScalarWithPredication(I)) 6502 return false; 6503 6504 // If any of the instruction's operands are uniform after vectorization, 6505 // the instruction cannot be scalarized. This prevents, for example, a 6506 // masked load from being scalarized. 6507 // 6508 // We assume we will only emit a value for lane zero of an instruction 6509 // marked uniform after vectorization, rather than VF identical values. 6510 // Thus, if we scalarize an instruction that uses a uniform, we would 6511 // create uses of values corresponding to the lanes we aren't emitting code 6512 // for. This behavior can be changed by allowing getScalarValue to clone 6513 // the lane zero values for uniforms rather than asserting. 6514 for (Use &U : I->operands()) 6515 if (auto *J = dyn_cast<Instruction>(U.get())) 6516 if (isUniformAfterVectorization(J, VF)) 6517 return false; 6518 6519 // Otherwise, we can scalarize the instruction. 6520 return true; 6521 }; 6522 6523 // Compute the expected cost discount from scalarizing the entire expression 6524 // feeding the predicated instruction. We currently only consider expressions 6525 // that are single-use instruction chains. 6526 Worklist.push_back(PredInst); 6527 while (!Worklist.empty()) { 6528 Instruction *I = Worklist.pop_back_val(); 6529 6530 // If we've already analyzed the instruction, there's nothing to do. 6531 if (ScalarCosts.find(I) != ScalarCosts.end()) 6532 continue; 6533 6534 // Compute the cost of the vector instruction. Note that this cost already 6535 // includes the scalarization overhead of the predicated instruction. 6536 InstructionCost VectorCost = getInstructionCost(I, VF).first; 6537 6538 // Compute the cost of the scalarized instruction. This cost is the cost of 6539 // the instruction as if it wasn't if-converted and instead remained in the 6540 // predicated block. We will scale this cost by block probability after 6541 // computing the scalarization overhead. 6542 assert(!VF.isScalable() && "scalable vectors not yet supported."); 6543 InstructionCost ScalarCost = 6544 VF.getKnownMinValue() * 6545 getInstructionCost(I, ElementCount::getFixed(1)).first; 6546 6547 // Compute the scalarization overhead of needed insertelement instructions 6548 // and phi nodes. 6549 if (isScalarWithPredication(I) && !I->getType()->isVoidTy()) { 6550 ScalarCost += TTI.getScalarizationOverhead( 6551 cast<VectorType>(ToVectorTy(I->getType(), VF)), 6552 APInt::getAllOnesValue(VF.getKnownMinValue()), true, false); 6553 assert(!VF.isScalable() && "scalable vectors not yet supported."); 6554 ScalarCost += 6555 VF.getKnownMinValue() * 6556 TTI.getCFInstrCost(Instruction::PHI, TTI::TCK_RecipThroughput); 6557 } 6558 6559 // Compute the scalarization overhead of needed extractelement 6560 // instructions. For each of the instruction's operands, if the operand can 6561 // be scalarized, add it to the worklist; otherwise, account for the 6562 // overhead. 6563 for (Use &U : I->operands()) 6564 if (auto *J = dyn_cast<Instruction>(U.get())) { 6565 assert(VectorType::isValidElementType(J->getType()) && 6566 "Instruction has non-scalar type"); 6567 if (canBeScalarized(J)) 6568 Worklist.push_back(J); 6569 else if (needsExtract(J, VF)) { 6570 assert(!VF.isScalable() && "scalable vectors not yet supported."); 6571 ScalarCost += TTI.getScalarizationOverhead( 6572 cast<VectorType>(ToVectorTy(J->getType(), VF)), 6573 APInt::getAllOnesValue(VF.getKnownMinValue()), false, true); 6574 } 6575 } 6576 6577 // Scale the total scalar cost by block probability. 6578 ScalarCost /= getReciprocalPredBlockProb(); 6579 6580 // Compute the discount. A non-negative discount means the vector version 6581 // of the instruction costs more, and scalarizing would be beneficial. 6582 Discount += VectorCost - ScalarCost; 6583 ScalarCosts[I] = ScalarCost; 6584 } 6585 6586 return *Discount.getValue(); 6587 } 6588 6589 LoopVectorizationCostModel::VectorizationCostTy 6590 LoopVectorizationCostModel::expectedCost(ElementCount VF) { 6591 VectorizationCostTy Cost; 6592 6593 // For each block. 6594 for (BasicBlock *BB : TheLoop->blocks()) { 6595 VectorizationCostTy BlockCost; 6596 6597 // For each instruction in the old loop. 6598 for (Instruction &I : BB->instructionsWithoutDebug()) { 6599 // Skip ignored values. 6600 if (ValuesToIgnore.count(&I) || 6601 (VF.isVector() && VecValuesToIgnore.count(&I))) 6602 continue; 6603 6604 VectorizationCostTy C = getInstructionCost(&I, VF); 6605 6606 // Check if we should override the cost. 6607 if (ForceTargetInstructionCost.getNumOccurrences() > 0) 6608 C.first = InstructionCost(ForceTargetInstructionCost); 6609 6610 BlockCost.first += C.first; 6611 BlockCost.second |= C.second; 6612 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C.first 6613 << " for VF " << VF << " For instruction: " << I 6614 << '\n'); 6615 } 6616 6617 // If we are vectorizing a predicated block, it will have been 6618 // if-converted. This means that the block's instructions (aside from 6619 // stores and instructions that may divide by zero) will now be 6620 // unconditionally executed. For the scalar case, we may not always execute 6621 // the predicated block, if it is an if-else block. Thus, scale the block's 6622 // cost by the probability of executing it. blockNeedsPredication from 6623 // Legal is used so as to not include all blocks in tail folded loops. 6624 if (VF.isScalar() && Legal->blockNeedsPredication(BB)) 6625 BlockCost.first /= getReciprocalPredBlockProb(); 6626 6627 Cost.first += BlockCost.first; 6628 Cost.second |= BlockCost.second; 6629 } 6630 6631 return Cost; 6632 } 6633 6634 /// Gets Address Access SCEV after verifying that the access pattern 6635 /// is loop invariant except the induction variable dependence. 6636 /// 6637 /// This SCEV can be sent to the Target in order to estimate the address 6638 /// calculation cost. 6639 static const SCEV *getAddressAccessSCEV( 6640 Value *Ptr, 6641 LoopVectorizationLegality *Legal, 6642 PredicatedScalarEvolution &PSE, 6643 const Loop *TheLoop) { 6644 6645 auto *Gep = dyn_cast<GetElementPtrInst>(Ptr); 6646 if (!Gep) 6647 return nullptr; 6648 6649 // We are looking for a gep with all loop invariant indices except for one 6650 // which should be an induction variable. 6651 auto SE = PSE.getSE(); 6652 unsigned NumOperands = Gep->getNumOperands(); 6653 for (unsigned i = 1; i < NumOperands; ++i) { 6654 Value *Opd = Gep->getOperand(i); 6655 if (!SE->isLoopInvariant(SE->getSCEV(Opd), TheLoop) && 6656 !Legal->isInductionVariable(Opd)) 6657 return nullptr; 6658 } 6659 6660 // Now we know we have a GEP ptr, %inv, %ind, %inv. return the Ptr SCEV. 6661 return PSE.getSCEV(Ptr); 6662 } 6663 6664 static bool isStrideMul(Instruction *I, LoopVectorizationLegality *Legal) { 6665 return Legal->hasStride(I->getOperand(0)) || 6666 Legal->hasStride(I->getOperand(1)); 6667 } 6668 6669 InstructionCost 6670 LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I, 6671 ElementCount VF) { 6672 assert(VF.isVector() && 6673 "Scalarization cost of instruction implies vectorization."); 6674 assert(!VF.isScalable() && "scalable vectors not yet supported."); 6675 Type *ValTy = getMemInstValueType(I); 6676 auto SE = PSE.getSE(); 6677 6678 unsigned AS = getLoadStoreAddressSpace(I); 6679 Value *Ptr = getLoadStorePointerOperand(I); 6680 Type *PtrTy = ToVectorTy(Ptr->getType(), VF); 6681 6682 // Figure out whether the access is strided and get the stride value 6683 // if it's known in compile time 6684 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, Legal, PSE, TheLoop); 6685 6686 // Get the cost of the scalar memory instruction and address computation. 6687 InstructionCost Cost = 6688 VF.getKnownMinValue() * TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV); 6689 6690 // Don't pass *I here, since it is scalar but will actually be part of a 6691 // vectorized loop where the user of it is a vectorized instruction. 6692 const Align Alignment = getLoadStoreAlignment(I); 6693 Cost += VF.getKnownMinValue() * 6694 TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(), Alignment, 6695 AS, TTI::TCK_RecipThroughput); 6696 6697 // Get the overhead of the extractelement and insertelement instructions 6698 // we might create due to scalarization. 6699 Cost += getScalarizationOverhead(I, VF); 6700 6701 // If we have a predicated store, it may not be executed for each vector 6702 // lane. Scale the cost by the probability of executing the predicated 6703 // block. 6704 if (isPredicatedInst(I)) { 6705 Cost /= getReciprocalPredBlockProb(); 6706 6707 if (useEmulatedMaskMemRefHack(I)) 6708 // Artificially setting to a high enough value to practically disable 6709 // vectorization with such operations. 6710 Cost = 3000000; 6711 } 6712 6713 return Cost; 6714 } 6715 6716 InstructionCost 6717 LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I, 6718 ElementCount VF) { 6719 Type *ValTy = getMemInstValueType(I); 6720 auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF)); 6721 Value *Ptr = getLoadStorePointerOperand(I); 6722 unsigned AS = getLoadStoreAddressSpace(I); 6723 int ConsecutiveStride = Legal->isConsecutivePtr(Ptr); 6724 enum TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; 6725 6726 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) && 6727 "Stride should be 1 or -1 for consecutive memory access"); 6728 const Align Alignment = getLoadStoreAlignment(I); 6729 InstructionCost Cost = 0; 6730 if (Legal->isMaskRequired(I)) 6731 Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS, 6732 CostKind); 6733 else 6734 Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS, 6735 CostKind, I); 6736 6737 bool Reverse = ConsecutiveStride < 0; 6738 if (Reverse) 6739 Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, 0); 6740 return Cost; 6741 } 6742 6743 InstructionCost 6744 LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I, 6745 ElementCount VF) { 6746 assert(Legal->isUniformMemOp(*I)); 6747 6748 Type *ValTy = getMemInstValueType(I); 6749 auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF)); 6750 const Align Alignment = getLoadStoreAlignment(I); 6751 unsigned AS = getLoadStoreAddressSpace(I); 6752 enum TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; 6753 if (isa<LoadInst>(I)) { 6754 return TTI.getAddressComputationCost(ValTy) + 6755 TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS, 6756 CostKind) + 6757 TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy); 6758 } 6759 StoreInst *SI = cast<StoreInst>(I); 6760 6761 bool isLoopInvariantStoreValue = Legal->isUniform(SI->getValueOperand()); 6762 return TTI.getAddressComputationCost(ValTy) + 6763 TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS, 6764 CostKind) + 6765 (isLoopInvariantStoreValue 6766 ? 0 6767 : TTI.getVectorInstrCost(Instruction::ExtractElement, VectorTy, 6768 VF.getKnownMinValue() - 1)); 6769 } 6770 6771 InstructionCost 6772 LoopVectorizationCostModel::getGatherScatterCost(Instruction *I, 6773 ElementCount VF) { 6774 Type *ValTy = getMemInstValueType(I); 6775 auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF)); 6776 const Align Alignment = getLoadStoreAlignment(I); 6777 const Value *Ptr = getLoadStorePointerOperand(I); 6778 6779 return TTI.getAddressComputationCost(VectorTy) + 6780 TTI.getGatherScatterOpCost( 6781 I->getOpcode(), VectorTy, Ptr, Legal->isMaskRequired(I), Alignment, 6782 TargetTransformInfo::TCK_RecipThroughput, I); 6783 } 6784 6785 InstructionCost 6786 LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I, 6787 ElementCount VF) { 6788 Type *ValTy = getMemInstValueType(I); 6789 auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF)); 6790 unsigned AS = getLoadStoreAddressSpace(I); 6791 6792 auto Group = getInterleavedAccessGroup(I); 6793 assert(Group && "Fail to get an interleaved access group."); 6794 6795 unsigned InterleaveFactor = Group->getFactor(); 6796 assert(!VF.isScalable() && "scalable vectors not yet supported."); 6797 auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor); 6798 6799 // Holds the indices of existing members in an interleaved load group. 6800 // An interleaved store group doesn't need this as it doesn't allow gaps. 6801 SmallVector<unsigned, 4> Indices; 6802 if (isa<LoadInst>(I)) { 6803 for (unsigned i = 0; i < InterleaveFactor; i++) 6804 if (Group->getMember(i)) 6805 Indices.push_back(i); 6806 } 6807 6808 // Calculate the cost of the whole interleaved group. 6809 bool UseMaskForGaps = 6810 Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed(); 6811 InstructionCost Cost = TTI.getInterleavedMemoryOpCost( 6812 I->getOpcode(), WideVecTy, Group->getFactor(), Indices, Group->getAlign(), 6813 AS, TTI::TCK_RecipThroughput, Legal->isMaskRequired(I), UseMaskForGaps); 6814 6815 if (Group->isReverse()) { 6816 // TODO: Add support for reversed masked interleaved access. 6817 assert(!Legal->isMaskRequired(I) && 6818 "Reverse masked interleaved access not supported."); 6819 Cost += Group->getNumMembers() * 6820 TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, 0); 6821 } 6822 return Cost; 6823 } 6824 6825 InstructionCost LoopVectorizationCostModel::getReductionPatternCost( 6826 Instruction *I, ElementCount VF, Type *Ty, TTI::TargetCostKind CostKind) { 6827 // Early exit for no inloop reductions 6828 if (InLoopReductionChains.empty() || VF.isScalar() || !isa<VectorType>(Ty)) 6829 return InstructionCost::getInvalid(); 6830 auto *VectorTy = cast<VectorType>(Ty); 6831 6832 // We are looking for a pattern of, and finding the minimal acceptable cost: 6833 // reduce(mul(ext(A), ext(B))) or 6834 // reduce(mul(A, B)) or 6835 // reduce(ext(A)) or 6836 // reduce(A). 6837 // The basic idea is that we walk down the tree to do that, finding the root 6838 // reduction instruction in InLoopReductionImmediateChains. From there we find 6839 // the pattern of mul/ext and test the cost of the entire pattern vs the cost 6840 // of the components. If the reduction cost is lower then we return it for the 6841 // reduction instruction and 0 for the other instructions in the pattern. If 6842 // it is not we return an invalid cost specifying the orignal cost method 6843 // should be used. 6844 Instruction *RetI = I; 6845 if ((RetI->getOpcode() == Instruction::SExt || 6846 RetI->getOpcode() == Instruction::ZExt)) { 6847 if (!RetI->hasOneUser()) 6848 return InstructionCost::getInvalid(); 6849 RetI = RetI->user_back(); 6850 } 6851 if (RetI->getOpcode() == Instruction::Mul && 6852 RetI->user_back()->getOpcode() == Instruction::Add) { 6853 if (!RetI->hasOneUser()) 6854 return InstructionCost::getInvalid(); 6855 RetI = RetI->user_back(); 6856 } 6857 6858 // Test if the found instruction is a reduction, and if not return an invalid 6859 // cost specifying the parent to use the original cost modelling. 6860 if (!InLoopReductionImmediateChains.count(RetI)) 6861 return InstructionCost::getInvalid(); 6862 6863 // Find the reduction this chain is a part of and calculate the basic cost of 6864 // the reduction on its own. 6865 Instruction *LastChain = InLoopReductionImmediateChains[RetI]; 6866 Instruction *ReductionPhi = LastChain; 6867 while (!isa<PHINode>(ReductionPhi)) 6868 ReductionPhi = InLoopReductionImmediateChains[ReductionPhi]; 6869 6870 RecurrenceDescriptor RdxDesc = 6871 Legal->getReductionVars()[cast<PHINode>(ReductionPhi)]; 6872 unsigned BaseCost = TTI.getArithmeticReductionCost(RdxDesc.getOpcode(), 6873 VectorTy, false, CostKind); 6874 6875 // Get the operand that was not the reduction chain and match it to one of the 6876 // patterns, returning the better cost if it is found. 6877 Instruction *RedOp = RetI->getOperand(1) == LastChain 6878 ? dyn_cast<Instruction>(RetI->getOperand(0)) 6879 : dyn_cast<Instruction>(RetI->getOperand(1)); 6880 6881 VectorTy = VectorType::get(I->getOperand(0)->getType(), VectorTy); 6882 6883 if (RedOp && (isa<SExtInst>(RedOp) || isa<ZExtInst>(RedOp)) && 6884 !TheLoop->isLoopInvariant(RedOp)) { 6885 bool IsUnsigned = isa<ZExtInst>(RedOp); 6886 auto *ExtType = VectorType::get(RedOp->getOperand(0)->getType(), VectorTy); 6887 InstructionCost RedCost = TTI.getExtendedAddReductionCost( 6888 /*IsMLA=*/false, IsUnsigned, RdxDesc.getRecurrenceType(), ExtType, 6889 CostKind); 6890 6891 unsigned ExtCost = 6892 TTI.getCastInstrCost(RedOp->getOpcode(), VectorTy, ExtType, 6893 TTI::CastContextHint::None, CostKind, RedOp); 6894 if (RedCost.isValid() && RedCost < BaseCost + ExtCost) 6895 return I == RetI ? *RedCost.getValue() : 0; 6896 } else if (RedOp && RedOp->getOpcode() == Instruction::Mul) { 6897 Instruction *Mul = RedOp; 6898 Instruction *Op0 = dyn_cast<Instruction>(Mul->getOperand(0)); 6899 Instruction *Op1 = dyn_cast<Instruction>(Mul->getOperand(1)); 6900 if (Op0 && Op1 && (isa<SExtInst>(Op0) || isa<ZExtInst>(Op0)) && 6901 Op0->getOpcode() == Op1->getOpcode() && 6902 Op0->getOperand(0)->getType() == Op1->getOperand(0)->getType() && 6903 !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1)) { 6904 bool IsUnsigned = isa<ZExtInst>(Op0); 6905 auto *ExtType = VectorType::get(Op0->getOperand(0)->getType(), VectorTy); 6906 // reduce(mul(ext, ext)) 6907 unsigned ExtCost = 6908 TTI.getCastInstrCost(Op0->getOpcode(), VectorTy, ExtType, 6909 TTI::CastContextHint::None, CostKind, Op0); 6910 unsigned MulCost = 6911 TTI.getArithmeticInstrCost(Mul->getOpcode(), VectorTy, CostKind); 6912 6913 InstructionCost RedCost = TTI.getExtendedAddReductionCost( 6914 /*IsMLA=*/true, IsUnsigned, RdxDesc.getRecurrenceType(), ExtType, 6915 CostKind); 6916 6917 if (RedCost.isValid() && RedCost < ExtCost * 2 + MulCost + BaseCost) 6918 return I == RetI ? *RedCost.getValue() : 0; 6919 } else { 6920 unsigned MulCost = 6921 TTI.getArithmeticInstrCost(Mul->getOpcode(), VectorTy, CostKind); 6922 6923 InstructionCost RedCost = TTI.getExtendedAddReductionCost( 6924 /*IsMLA=*/true, true, RdxDesc.getRecurrenceType(), VectorTy, 6925 CostKind); 6926 6927 if (RedCost.isValid() && RedCost < MulCost + BaseCost) 6928 return I == RetI ? *RedCost.getValue() : 0; 6929 } 6930 } 6931 6932 return I == RetI ? BaseCost : InstructionCost::getInvalid(); 6933 } 6934 6935 InstructionCost 6936 LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I, 6937 ElementCount VF) { 6938 // Calculate scalar cost only. Vectorization cost should be ready at this 6939 // moment. 6940 if (VF.isScalar()) { 6941 Type *ValTy = getMemInstValueType(I); 6942 const Align Alignment = getLoadStoreAlignment(I); 6943 unsigned AS = getLoadStoreAddressSpace(I); 6944 6945 return TTI.getAddressComputationCost(ValTy) + 6946 TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS, 6947 TTI::TCK_RecipThroughput, I); 6948 } 6949 return getWideningCost(I, VF); 6950 } 6951 6952 LoopVectorizationCostModel::VectorizationCostTy 6953 LoopVectorizationCostModel::getInstructionCost(Instruction *I, 6954 ElementCount VF) { 6955 // If we know that this instruction will remain uniform, check the cost of 6956 // the scalar version. 6957 if (isUniformAfterVectorization(I, VF)) 6958 VF = ElementCount::getFixed(1); 6959 6960 if (VF.isVector() && isProfitableToScalarize(I, VF)) 6961 return VectorizationCostTy(InstsToScalarize[VF][I], false); 6962 6963 // Forced scalars do not have any scalarization overhead. 6964 auto ForcedScalar = ForcedScalars.find(VF); 6965 if (VF.isVector() && ForcedScalar != ForcedScalars.end()) { 6966 auto InstSet = ForcedScalar->second; 6967 if (InstSet.count(I)) 6968 return VectorizationCostTy( 6969 (getInstructionCost(I, ElementCount::getFixed(1)).first * 6970 VF.getKnownMinValue()), 6971 false); 6972 } 6973 6974 Type *VectorTy; 6975 InstructionCost C = getInstructionCost(I, VF, VectorTy); 6976 6977 bool TypeNotScalarized = 6978 VF.isVector() && VectorTy->isVectorTy() && 6979 TTI.getNumberOfParts(VectorTy) < VF.getKnownMinValue(); 6980 return VectorizationCostTy(C, TypeNotScalarized); 6981 } 6982 6983 InstructionCost 6984 LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I, 6985 ElementCount VF) { 6986 6987 assert(!VF.isScalable() && 6988 "cannot compute scalarization overhead for scalable vectorization"); 6989 if (VF.isScalar()) 6990 return 0; 6991 6992 InstructionCost Cost = 0; 6993 Type *RetTy = ToVectorTy(I->getType(), VF); 6994 if (!RetTy->isVoidTy() && 6995 (!isa<LoadInst>(I) || !TTI.supportsEfficientVectorElementLoadStore())) 6996 Cost += TTI.getScalarizationOverhead( 6997 cast<VectorType>(RetTy), APInt::getAllOnesValue(VF.getKnownMinValue()), 6998 true, false); 6999 7000 // Some targets keep addresses scalar. 7001 if (isa<LoadInst>(I) && !TTI.prefersVectorizedAddressing()) 7002 return Cost; 7003 7004 // Some targets support efficient element stores. 7005 if (isa<StoreInst>(I) && TTI.supportsEfficientVectorElementLoadStore()) 7006 return Cost; 7007 7008 // Collect operands to consider. 7009 CallInst *CI = dyn_cast<CallInst>(I); 7010 Instruction::op_range Ops = CI ? CI->arg_operands() : I->operands(); 7011 7012 // Skip operands that do not require extraction/scalarization and do not incur 7013 // any overhead. 7014 return Cost + TTI.getOperandsScalarizationOverhead( 7015 filterExtractingOperands(Ops, VF), VF.getKnownMinValue()); 7016 } 7017 7018 void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) { 7019 if (VF.isScalar()) 7020 return; 7021 NumPredStores = 0; 7022 for (BasicBlock *BB : TheLoop->blocks()) { 7023 // For each instruction in the old loop. 7024 for (Instruction &I : *BB) { 7025 Value *Ptr = getLoadStorePointerOperand(&I); 7026 if (!Ptr) 7027 continue; 7028 7029 // TODO: We should generate better code and update the cost model for 7030 // predicated uniform stores. Today they are treated as any other 7031 // predicated store (see added test cases in 7032 // invariant-store-vectorization.ll). 7033 if (isa<StoreInst>(&I) && isScalarWithPredication(&I)) 7034 NumPredStores++; 7035 7036 if (Legal->isUniformMemOp(I)) { 7037 // TODO: Avoid replicating loads and stores instead of 7038 // relying on instcombine to remove them. 7039 // Load: Scalar load + broadcast 7040 // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract 7041 InstructionCost Cost = getUniformMemOpCost(&I, VF); 7042 setWideningDecision(&I, VF, CM_Scalarize, Cost); 7043 continue; 7044 } 7045 7046 // We assume that widening is the best solution when possible. 7047 if (memoryInstructionCanBeWidened(&I, VF)) { 7048 InstructionCost Cost = getConsecutiveMemOpCost(&I, VF); 7049 int ConsecutiveStride = 7050 Legal->isConsecutivePtr(getLoadStorePointerOperand(&I)); 7051 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) && 7052 "Expected consecutive stride."); 7053 InstWidening Decision = 7054 ConsecutiveStride == 1 ? CM_Widen : CM_Widen_Reverse; 7055 setWideningDecision(&I, VF, Decision, Cost); 7056 continue; 7057 } 7058 7059 // Choose between Interleaving, Gather/Scatter or Scalarization. 7060 InstructionCost InterleaveCost = std::numeric_limits<int>::max(); 7061 unsigned NumAccesses = 1; 7062 if (isAccessInterleaved(&I)) { 7063 auto Group = getInterleavedAccessGroup(&I); 7064 assert(Group && "Fail to get an interleaved access group."); 7065 7066 // Make one decision for the whole group. 7067 if (getWideningDecision(&I, VF) != CM_Unknown) 7068 continue; 7069 7070 NumAccesses = Group->getNumMembers(); 7071 if (interleavedAccessCanBeWidened(&I, VF)) 7072 InterleaveCost = getInterleaveGroupCost(&I, VF); 7073 } 7074 7075 InstructionCost GatherScatterCost = 7076 isLegalGatherOrScatter(&I) 7077 ? getGatherScatterCost(&I, VF) * NumAccesses 7078 : std::numeric_limits<int>::max(); 7079 7080 InstructionCost ScalarizationCost = 7081 getMemInstScalarizationCost(&I, VF) * NumAccesses; 7082 7083 // Choose better solution for the current VF, 7084 // write down this decision and use it during vectorization. 7085 InstructionCost Cost; 7086 InstWidening Decision; 7087 if (InterleaveCost <= GatherScatterCost && 7088 InterleaveCost < ScalarizationCost) { 7089 Decision = CM_Interleave; 7090 Cost = InterleaveCost; 7091 } else if (GatherScatterCost < ScalarizationCost) { 7092 Decision = CM_GatherScatter; 7093 Cost = GatherScatterCost; 7094 } else { 7095 Decision = CM_Scalarize; 7096 Cost = ScalarizationCost; 7097 } 7098 // If the instructions belongs to an interleave group, the whole group 7099 // receives the same decision. The whole group receives the cost, but 7100 // the cost will actually be assigned to one instruction. 7101 if (auto Group = getInterleavedAccessGroup(&I)) 7102 setWideningDecision(Group, VF, Decision, Cost); 7103 else 7104 setWideningDecision(&I, VF, Decision, Cost); 7105 } 7106 } 7107 7108 // Make sure that any load of address and any other address computation 7109 // remains scalar unless there is gather/scatter support. This avoids 7110 // inevitable extracts into address registers, and also has the benefit of 7111 // activating LSR more, since that pass can't optimize vectorized 7112 // addresses. 7113 if (TTI.prefersVectorizedAddressing()) 7114 return; 7115 7116 // Start with all scalar pointer uses. 7117 SmallPtrSet<Instruction *, 8> AddrDefs; 7118 for (BasicBlock *BB : TheLoop->blocks()) 7119 for (Instruction &I : *BB) { 7120 Instruction *PtrDef = 7121 dyn_cast_or_null<Instruction>(getLoadStorePointerOperand(&I)); 7122 if (PtrDef && TheLoop->contains(PtrDef) && 7123 getWideningDecision(&I, VF) != CM_GatherScatter) 7124 AddrDefs.insert(PtrDef); 7125 } 7126 7127 // Add all instructions used to generate the addresses. 7128 SmallVector<Instruction *, 4> Worklist; 7129 append_range(Worklist, AddrDefs); 7130 while (!Worklist.empty()) { 7131 Instruction *I = Worklist.pop_back_val(); 7132 for (auto &Op : I->operands()) 7133 if (auto *InstOp = dyn_cast<Instruction>(Op)) 7134 if ((InstOp->getParent() == I->getParent()) && !isa<PHINode>(InstOp) && 7135 AddrDefs.insert(InstOp).second) 7136 Worklist.push_back(InstOp); 7137 } 7138 7139 for (auto *I : AddrDefs) { 7140 if (isa<LoadInst>(I)) { 7141 // Setting the desired widening decision should ideally be handled in 7142 // by cost functions, but since this involves the task of finding out 7143 // if the loaded register is involved in an address computation, it is 7144 // instead changed here when we know this is the case. 7145 InstWidening Decision = getWideningDecision(I, VF); 7146 if (Decision == CM_Widen || Decision == CM_Widen_Reverse) 7147 // Scalarize a widened load of address. 7148 setWideningDecision( 7149 I, VF, CM_Scalarize, 7150 (VF.getKnownMinValue() * 7151 getMemoryInstructionCost(I, ElementCount::getFixed(1)))); 7152 else if (auto Group = getInterleavedAccessGroup(I)) { 7153 // Scalarize an interleave group of address loads. 7154 for (unsigned I = 0; I < Group->getFactor(); ++I) { 7155 if (Instruction *Member = Group->getMember(I)) 7156 setWideningDecision( 7157 Member, VF, CM_Scalarize, 7158 (VF.getKnownMinValue() * 7159 getMemoryInstructionCost(Member, ElementCount::getFixed(1)))); 7160 } 7161 } 7162 } else 7163 // Make sure I gets scalarized and a cost estimate without 7164 // scalarization overhead. 7165 ForcedScalars[VF].insert(I); 7166 } 7167 } 7168 7169 InstructionCost 7170 LoopVectorizationCostModel::getInstructionCost(Instruction *I, ElementCount VF, 7171 Type *&VectorTy) { 7172 Type *RetTy = I->getType(); 7173 if (canTruncateToMinimalBitwidth(I, VF)) 7174 RetTy = IntegerType::get(RetTy->getContext(), MinBWs[I]); 7175 VectorTy = isScalarAfterVectorization(I, VF) ? RetTy : ToVectorTy(RetTy, VF); 7176 auto SE = PSE.getSE(); 7177 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; 7178 7179 // TODO: We need to estimate the cost of intrinsic calls. 7180 switch (I->getOpcode()) { 7181 case Instruction::GetElementPtr: 7182 // We mark this instruction as zero-cost because the cost of GEPs in 7183 // vectorized code depends on whether the corresponding memory instruction 7184 // is scalarized or not. Therefore, we handle GEPs with the memory 7185 // instruction cost. 7186 return 0; 7187 case Instruction::Br: { 7188 // In cases of scalarized and predicated instructions, there will be VF 7189 // predicated blocks in the vectorized loop. Each branch around these 7190 // blocks requires also an extract of its vector compare i1 element. 7191 bool ScalarPredicatedBB = false; 7192 BranchInst *BI = cast<BranchInst>(I); 7193 if (VF.isVector() && BI->isConditional() && 7194 (PredicatedBBsAfterVectorization.count(BI->getSuccessor(0)) || 7195 PredicatedBBsAfterVectorization.count(BI->getSuccessor(1)))) 7196 ScalarPredicatedBB = true; 7197 7198 if (ScalarPredicatedBB) { 7199 // Return cost for branches around scalarized and predicated blocks. 7200 assert(!VF.isScalable() && "scalable vectors not yet supported."); 7201 auto *Vec_i1Ty = 7202 VectorType::get(IntegerType::getInt1Ty(RetTy->getContext()), VF); 7203 return (TTI.getScalarizationOverhead( 7204 Vec_i1Ty, APInt::getAllOnesValue(VF.getKnownMinValue()), 7205 false, true) + 7206 (TTI.getCFInstrCost(Instruction::Br, CostKind) * 7207 VF.getKnownMinValue())); 7208 } else if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar()) 7209 // The back-edge branch will remain, as will all scalar branches. 7210 return TTI.getCFInstrCost(Instruction::Br, CostKind); 7211 else 7212 // This branch will be eliminated by if-conversion. 7213 return 0; 7214 // Note: We currently assume zero cost for an unconditional branch inside 7215 // a predicated block since it will become a fall-through, although we 7216 // may decide in the future to call TTI for all branches. 7217 } 7218 case Instruction::PHI: { 7219 auto *Phi = cast<PHINode>(I); 7220 7221 // First-order recurrences are replaced by vector shuffles inside the loop. 7222 // NOTE: Don't use ToVectorTy as SK_ExtractSubvector expects a vector type. 7223 if (VF.isVector() && Legal->isFirstOrderRecurrence(Phi)) 7224 return TTI.getShuffleCost( 7225 TargetTransformInfo::SK_ExtractSubvector, cast<VectorType>(VectorTy), 7226 VF.getKnownMinValue() - 1, FixedVectorType::get(RetTy, 1)); 7227 7228 // Phi nodes in non-header blocks (not inductions, reductions, etc.) are 7229 // converted into select instructions. We require N - 1 selects per phi 7230 // node, where N is the number of incoming values. 7231 if (VF.isVector() && Phi->getParent() != TheLoop->getHeader()) 7232 return (Phi->getNumIncomingValues() - 1) * 7233 TTI.getCmpSelInstrCost( 7234 Instruction::Select, ToVectorTy(Phi->getType(), VF), 7235 ToVectorTy(Type::getInt1Ty(Phi->getContext()), VF), 7236 CmpInst::BAD_ICMP_PREDICATE, CostKind); 7237 7238 return TTI.getCFInstrCost(Instruction::PHI, CostKind); 7239 } 7240 case Instruction::UDiv: 7241 case Instruction::SDiv: 7242 case Instruction::URem: 7243 case Instruction::SRem: 7244 // If we have a predicated instruction, it may not be executed for each 7245 // vector lane. Get the scalarization cost and scale this amount by the 7246 // probability of executing the predicated block. If the instruction is not 7247 // predicated, we fall through to the next case. 7248 if (VF.isVector() && isScalarWithPredication(I)) { 7249 InstructionCost Cost = 0; 7250 7251 // These instructions have a non-void type, so account for the phi nodes 7252 // that we will create. This cost is likely to be zero. The phi node 7253 // cost, if any, should be scaled by the block probability because it 7254 // models a copy at the end of each predicated block. 7255 Cost += VF.getKnownMinValue() * 7256 TTI.getCFInstrCost(Instruction::PHI, CostKind); 7257 7258 // The cost of the non-predicated instruction. 7259 Cost += VF.getKnownMinValue() * 7260 TTI.getArithmeticInstrCost(I->getOpcode(), RetTy, CostKind); 7261 7262 // The cost of insertelement and extractelement instructions needed for 7263 // scalarization. 7264 Cost += getScalarizationOverhead(I, VF); 7265 7266 // Scale the cost by the probability of executing the predicated blocks. 7267 // This assumes the predicated block for each vector lane is equally 7268 // likely. 7269 return Cost / getReciprocalPredBlockProb(); 7270 } 7271 LLVM_FALLTHROUGH; 7272 case Instruction::Add: 7273 case Instruction::FAdd: 7274 case Instruction::Sub: 7275 case Instruction::FSub: 7276 case Instruction::Mul: 7277 case Instruction::FMul: 7278 case Instruction::FDiv: 7279 case Instruction::FRem: 7280 case Instruction::Shl: 7281 case Instruction::LShr: 7282 case Instruction::AShr: 7283 case Instruction::And: 7284 case Instruction::Or: 7285 case Instruction::Xor: { 7286 // Since we will replace the stride by 1 the multiplication should go away. 7287 if (I->getOpcode() == Instruction::Mul && isStrideMul(I, Legal)) 7288 return 0; 7289 7290 // Detect reduction patterns 7291 InstructionCost RedCost; 7292 if ((RedCost = getReductionPatternCost(I, VF, VectorTy, CostKind)) 7293 .isValid()) 7294 return RedCost; 7295 7296 // Certain instructions can be cheaper to vectorize if they have a constant 7297 // second vector operand. One example of this are shifts on x86. 7298 Value *Op2 = I->getOperand(1); 7299 TargetTransformInfo::OperandValueProperties Op2VP; 7300 TargetTransformInfo::OperandValueKind Op2VK = 7301 TTI.getOperandInfo(Op2, Op2VP); 7302 if (Op2VK == TargetTransformInfo::OK_AnyValue && Legal->isUniform(Op2)) 7303 Op2VK = TargetTransformInfo::OK_UniformValue; 7304 7305 SmallVector<const Value *, 4> Operands(I->operand_values()); 7306 unsigned N = isScalarAfterVectorization(I, VF) ? VF.getKnownMinValue() : 1; 7307 return N * TTI.getArithmeticInstrCost( 7308 I->getOpcode(), VectorTy, CostKind, 7309 TargetTransformInfo::OK_AnyValue, 7310 Op2VK, TargetTransformInfo::OP_None, Op2VP, Operands, I); 7311 } 7312 case Instruction::FNeg: { 7313 assert(!VF.isScalable() && "VF is assumed to be non scalable."); 7314 unsigned N = isScalarAfterVectorization(I, VF) ? VF.getKnownMinValue() : 1; 7315 return N * TTI.getArithmeticInstrCost( 7316 I->getOpcode(), VectorTy, CostKind, 7317 TargetTransformInfo::OK_AnyValue, 7318 TargetTransformInfo::OK_AnyValue, 7319 TargetTransformInfo::OP_None, TargetTransformInfo::OP_None, 7320 I->getOperand(0), I); 7321 } 7322 case Instruction::Select: { 7323 SelectInst *SI = cast<SelectInst>(I); 7324 const SCEV *CondSCEV = SE->getSCEV(SI->getCondition()); 7325 bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop)); 7326 Type *CondTy = SI->getCondition()->getType(); 7327 if (!ScalarCond) 7328 CondTy = VectorType::get(CondTy, VF); 7329 return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy, 7330 CmpInst::BAD_ICMP_PREDICATE, CostKind, I); 7331 } 7332 case Instruction::ICmp: 7333 case Instruction::FCmp: { 7334 Type *ValTy = I->getOperand(0)->getType(); 7335 Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0)); 7336 if (canTruncateToMinimalBitwidth(Op0AsInstruction, VF)) 7337 ValTy = IntegerType::get(ValTy->getContext(), MinBWs[Op0AsInstruction]); 7338 VectorTy = ToVectorTy(ValTy, VF); 7339 return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, nullptr, 7340 CmpInst::BAD_ICMP_PREDICATE, CostKind, I); 7341 } 7342 case Instruction::Store: 7343 case Instruction::Load: { 7344 ElementCount Width = VF; 7345 if (Width.isVector()) { 7346 InstWidening Decision = getWideningDecision(I, Width); 7347 assert(Decision != CM_Unknown && 7348 "CM decision should be taken at this point"); 7349 if (Decision == CM_Scalarize) 7350 Width = ElementCount::getFixed(1); 7351 } 7352 VectorTy = ToVectorTy(getMemInstValueType(I), Width); 7353 return getMemoryInstructionCost(I, VF); 7354 } 7355 case Instruction::ZExt: 7356 case Instruction::SExt: 7357 case Instruction::FPToUI: 7358 case Instruction::FPToSI: 7359 case Instruction::FPExt: 7360 case Instruction::PtrToInt: 7361 case Instruction::IntToPtr: 7362 case Instruction::SIToFP: 7363 case Instruction::UIToFP: 7364 case Instruction::Trunc: 7365 case Instruction::FPTrunc: 7366 case Instruction::BitCast: { 7367 // Computes the CastContextHint from a Load/Store instruction. 7368 auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint { 7369 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && 7370 "Expected a load or a store!"); 7371 7372 if (VF.isScalar() || !TheLoop->contains(I)) 7373 return TTI::CastContextHint::Normal; 7374 7375 switch (getWideningDecision(I, VF)) { 7376 case LoopVectorizationCostModel::CM_GatherScatter: 7377 return TTI::CastContextHint::GatherScatter; 7378 case LoopVectorizationCostModel::CM_Interleave: 7379 return TTI::CastContextHint::Interleave; 7380 case LoopVectorizationCostModel::CM_Scalarize: 7381 case LoopVectorizationCostModel::CM_Widen: 7382 return Legal->isMaskRequired(I) ? TTI::CastContextHint::Masked 7383 : TTI::CastContextHint::Normal; 7384 case LoopVectorizationCostModel::CM_Widen_Reverse: 7385 return TTI::CastContextHint::Reversed; 7386 case LoopVectorizationCostModel::CM_Unknown: 7387 llvm_unreachable("Instr did not go through cost modelling?"); 7388 } 7389 7390 llvm_unreachable("Unhandled case!"); 7391 }; 7392 7393 unsigned Opcode = I->getOpcode(); 7394 TTI::CastContextHint CCH = TTI::CastContextHint::None; 7395 // For Trunc, the context is the only user, which must be a StoreInst. 7396 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) { 7397 if (I->hasOneUse()) 7398 if (StoreInst *Store = dyn_cast<StoreInst>(*I->user_begin())) 7399 CCH = ComputeCCH(Store); 7400 } 7401 // For Z/Sext, the context is the operand, which must be a LoadInst. 7402 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt || 7403 Opcode == Instruction::FPExt) { 7404 if (LoadInst *Load = dyn_cast<LoadInst>(I->getOperand(0))) 7405 CCH = ComputeCCH(Load); 7406 } 7407 7408 // We optimize the truncation of induction variables having constant 7409 // integer steps. The cost of these truncations is the same as the scalar 7410 // operation. 7411 if (isOptimizableIVTruncate(I, VF)) { 7412 auto *Trunc = cast<TruncInst>(I); 7413 return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(), 7414 Trunc->getSrcTy(), CCH, CostKind, Trunc); 7415 } 7416 7417 // Detect reduction patterns 7418 InstructionCost RedCost; 7419 if ((RedCost = getReductionPatternCost(I, VF, VectorTy, CostKind)) 7420 .isValid()) 7421 return RedCost; 7422 7423 Type *SrcScalarTy = I->getOperand(0)->getType(); 7424 Type *SrcVecTy = 7425 VectorTy->isVectorTy() ? ToVectorTy(SrcScalarTy, VF) : SrcScalarTy; 7426 if (canTruncateToMinimalBitwidth(I, VF)) { 7427 // This cast is going to be shrunk. This may remove the cast or it might 7428 // turn it into slightly different cast. For example, if MinBW == 16, 7429 // "zext i8 %1 to i32" becomes "zext i8 %1 to i16". 7430 // 7431 // Calculate the modified src and dest types. 7432 Type *MinVecTy = VectorTy; 7433 if (Opcode == Instruction::Trunc) { 7434 SrcVecTy = smallestIntegerVectorType(SrcVecTy, MinVecTy); 7435 VectorTy = 7436 largestIntegerVectorType(ToVectorTy(I->getType(), VF), MinVecTy); 7437 } else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt) { 7438 SrcVecTy = largestIntegerVectorType(SrcVecTy, MinVecTy); 7439 VectorTy = 7440 smallestIntegerVectorType(ToVectorTy(I->getType(), VF), MinVecTy); 7441 } 7442 } 7443 7444 assert(!VF.isScalable() && "VF is assumed to be non scalable"); 7445 unsigned N = isScalarAfterVectorization(I, VF) ? VF.getKnownMinValue() : 1; 7446 return N * 7447 TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH, CostKind, I); 7448 } 7449 case Instruction::Call: { 7450 bool NeedToScalarize; 7451 CallInst *CI = cast<CallInst>(I); 7452 InstructionCost CallCost = getVectorCallCost(CI, VF, NeedToScalarize); 7453 if (getVectorIntrinsicIDForCall(CI, TLI)) { 7454 InstructionCost IntrinsicCost = getVectorIntrinsicCost(CI, VF); 7455 return std::min(CallCost, IntrinsicCost); 7456 } 7457 return CallCost; 7458 } 7459 case Instruction::ExtractValue: 7460 return TTI.getInstructionCost(I, TTI::TCK_RecipThroughput); 7461 default: 7462 // The cost of executing VF copies of the scalar instruction. This opcode 7463 // is unknown. Assume that it is the same as 'mul'. 7464 return VF.getKnownMinValue() * TTI.getArithmeticInstrCost( 7465 Instruction::Mul, VectorTy, CostKind) + 7466 getScalarizationOverhead(I, VF); 7467 } // end of switch. 7468 } 7469 7470 char LoopVectorize::ID = 0; 7471 7472 static const char lv_name[] = "Loop Vectorization"; 7473 7474 INITIALIZE_PASS_BEGIN(LoopVectorize, LV_NAME, lv_name, false, false) 7475 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) 7476 INITIALIZE_PASS_DEPENDENCY(BasicAAWrapperPass) 7477 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 7478 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass) 7479 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 7480 INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) 7481 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 7482 INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass) 7483 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) 7484 INITIALIZE_PASS_DEPENDENCY(LoopAccessLegacyAnalysis) 7485 INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass) 7486 INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass) 7487 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) 7488 INITIALIZE_PASS_DEPENDENCY(InjectTLIMappingsLegacy) 7489 INITIALIZE_PASS_END(LoopVectorize, LV_NAME, lv_name, false, false) 7490 7491 namespace llvm { 7492 7493 Pass *createLoopVectorizePass() { return new LoopVectorize(); } 7494 7495 Pass *createLoopVectorizePass(bool InterleaveOnlyWhenForced, 7496 bool VectorizeOnlyWhenForced) { 7497 return new LoopVectorize(InterleaveOnlyWhenForced, VectorizeOnlyWhenForced); 7498 } 7499 7500 } // end namespace llvm 7501 7502 bool LoopVectorizationCostModel::isConsecutiveLoadOrStore(Instruction *Inst) { 7503 // Check if the pointer operand of a load or store instruction is 7504 // consecutive. 7505 if (auto *Ptr = getLoadStorePointerOperand(Inst)) 7506 return Legal->isConsecutivePtr(Ptr); 7507 return false; 7508 } 7509 7510 void LoopVectorizationCostModel::collectValuesToIgnore() { 7511 // Ignore ephemeral values. 7512 CodeMetrics::collectEphemeralValues(TheLoop, AC, ValuesToIgnore); 7513 7514 // Ignore type-promoting instructions we identified during reduction 7515 // detection. 7516 for (auto &Reduction : Legal->getReductionVars()) { 7517 RecurrenceDescriptor &RedDes = Reduction.second; 7518 const SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts(); 7519 VecValuesToIgnore.insert(Casts.begin(), Casts.end()); 7520 } 7521 // Ignore type-casting instructions we identified during induction 7522 // detection. 7523 for (auto &Induction : Legal->getInductionVars()) { 7524 InductionDescriptor &IndDes = Induction.second; 7525 const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts(); 7526 VecValuesToIgnore.insert(Casts.begin(), Casts.end()); 7527 } 7528 } 7529 7530 void LoopVectorizationCostModel::collectInLoopReductions() { 7531 for (auto &Reduction : Legal->getReductionVars()) { 7532 PHINode *Phi = Reduction.first; 7533 RecurrenceDescriptor &RdxDesc = Reduction.second; 7534 7535 // We don't collect reductions that are type promoted (yet). 7536 if (RdxDesc.getRecurrenceType() != Phi->getType()) 7537 continue; 7538 7539 // If the target would prefer this reduction to happen "in-loop", then we 7540 // want to record it as such. 7541 unsigned Opcode = RdxDesc.getOpcode(); 7542 if (!PreferInLoopReductions && 7543 !TTI.preferInLoopReduction(Opcode, Phi->getType(), 7544 TargetTransformInfo::ReductionFlags())) 7545 continue; 7546 7547 // Check that we can correctly put the reductions into the loop, by 7548 // finding the chain of operations that leads from the phi to the loop 7549 // exit value. 7550 SmallVector<Instruction *, 4> ReductionOperations = 7551 RdxDesc.getReductionOpChain(Phi, TheLoop); 7552 bool InLoop = !ReductionOperations.empty(); 7553 if (InLoop) { 7554 InLoopReductionChains[Phi] = ReductionOperations; 7555 // Add the elements to InLoopReductionImmediateChains for cost modelling. 7556 Instruction *LastChain = Phi; 7557 for (auto *I : ReductionOperations) { 7558 InLoopReductionImmediateChains[I] = LastChain; 7559 LastChain = I; 7560 } 7561 } 7562 LLVM_DEBUG(dbgs() << "LV: Using " << (InLoop ? "inloop" : "out of loop") 7563 << " reduction for phi: " << *Phi << "\n"); 7564 } 7565 } 7566 7567 // TODO: we could return a pair of values that specify the max VF and 7568 // min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of 7569 // `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment 7570 // doesn't have a cost model that can choose which plan to execute if 7571 // more than one is generated. 7572 static unsigned determineVPlanVF(const unsigned WidestVectorRegBits, 7573 LoopVectorizationCostModel &CM) { 7574 unsigned WidestType; 7575 std::tie(std::ignore, WidestType) = CM.getSmallestAndWidestTypes(); 7576 return WidestVectorRegBits / WidestType; 7577 } 7578 7579 VectorizationFactor 7580 LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) { 7581 assert(!UserVF.isScalable() && "scalable vectors not yet supported"); 7582 ElementCount VF = UserVF; 7583 // Outer loop handling: They may require CFG and instruction level 7584 // transformations before even evaluating whether vectorization is profitable. 7585 // Since we cannot modify the incoming IR, we need to build VPlan upfront in 7586 // the vectorization pipeline. 7587 if (!OrigLoop->isInnermost()) { 7588 // If the user doesn't provide a vectorization factor, determine a 7589 // reasonable one. 7590 if (UserVF.isZero()) { 7591 VF = ElementCount::getFixed( 7592 determineVPlanVF(TTI->getRegisterBitWidth(true /* Vector*/), CM)); 7593 LLVM_DEBUG(dbgs() << "LV: VPlan computed VF " << VF << ".\n"); 7594 7595 // Make sure we have a VF > 1 for stress testing. 7596 if (VPlanBuildStressTest && (VF.isScalar() || VF.isZero())) { 7597 LLVM_DEBUG(dbgs() << "LV: VPlan stress testing: " 7598 << "overriding computed VF.\n"); 7599 VF = ElementCount::getFixed(4); 7600 } 7601 } 7602 assert(EnableVPlanNativePath && "VPlan-native path is not enabled."); 7603 assert(isPowerOf2_32(VF.getKnownMinValue()) && 7604 "VF needs to be a power of two"); 7605 LLVM_DEBUG(dbgs() << "LV: Using " << (!UserVF.isZero() ? "user " : "") 7606 << "VF " << VF << " to build VPlans.\n"); 7607 buildVPlans(VF, VF); 7608 7609 // For VPlan build stress testing, we bail out after VPlan construction. 7610 if (VPlanBuildStressTest) 7611 return VectorizationFactor::Disabled(); 7612 7613 return {VF, 0 /*Cost*/}; 7614 } 7615 7616 LLVM_DEBUG( 7617 dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the " 7618 "VPlan-native path.\n"); 7619 return VectorizationFactor::Disabled(); 7620 } 7621 7622 Optional<VectorizationFactor> 7623 LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) { 7624 assert(OrigLoop->isInnermost() && "Inner loop expected."); 7625 Optional<ElementCount> MaybeMaxVF = CM.computeMaxVF(UserVF, UserIC); 7626 if (!MaybeMaxVF) // Cases that should not to be vectorized nor interleaved. 7627 return None; 7628 7629 // Invalidate interleave groups if all blocks of loop will be predicated. 7630 if (CM.blockNeedsPredication(OrigLoop->getHeader()) && 7631 !useMaskedInterleavedAccesses(*TTI)) { 7632 LLVM_DEBUG( 7633 dbgs() 7634 << "LV: Invalidate all interleaved groups due to fold-tail by masking " 7635 "which requires masked-interleaved support.\n"); 7636 if (CM.InterleaveInfo.invalidateGroups()) 7637 // Invalidating interleave groups also requires invalidating all decisions 7638 // based on them, which includes widening decisions and uniform and scalar 7639 // values. 7640 CM.invalidateCostModelingDecisions(); 7641 } 7642 7643 ElementCount MaxVF = MaybeMaxVF.getValue(); 7644 assert(MaxVF.isNonZero() && "MaxVF is zero."); 7645 7646 bool UserVFIsLegal = ElementCount::isKnownLE(UserVF, MaxVF); 7647 if (!UserVF.isZero() && 7648 (UserVFIsLegal || (UserVF.isScalable() && MaxVF.isScalable()))) { 7649 // FIXME: MaxVF is temporarily used inplace of UserVF for illegal scalable 7650 // VFs here, this should be reverted to only use legal UserVFs once the 7651 // loop below supports scalable VFs. 7652 ElementCount VF = UserVFIsLegal ? UserVF : MaxVF; 7653 LLVM_DEBUG(dbgs() << "LV: Using " << (UserVFIsLegal ? "user" : "max") 7654 << " VF " << VF << ".\n"); 7655 assert(isPowerOf2_32(VF.getKnownMinValue()) && 7656 "VF needs to be a power of two"); 7657 // Collect the instructions (and their associated costs) that will be more 7658 // profitable to scalarize. 7659 CM.selectUserVectorizationFactor(VF); 7660 CM.collectInLoopReductions(); 7661 buildVPlansWithVPRecipes(VF, VF); 7662 LLVM_DEBUG(printPlans(dbgs())); 7663 return {{VF, 0}}; 7664 } 7665 7666 assert(!MaxVF.isScalable() && 7667 "Scalable vectors not yet supported beyond this point"); 7668 7669 for (ElementCount VF = ElementCount::getFixed(1); 7670 ElementCount::isKnownLE(VF, MaxVF); VF *= 2) { 7671 // Collect Uniform and Scalar instructions after vectorization with VF. 7672 CM.collectUniformsAndScalars(VF); 7673 7674 // Collect the instructions (and their associated costs) that will be more 7675 // profitable to scalarize. 7676 if (VF.isVector()) 7677 CM.collectInstsToScalarize(VF); 7678 } 7679 7680 CM.collectInLoopReductions(); 7681 7682 buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxVF); 7683 LLVM_DEBUG(printPlans(dbgs())); 7684 if (MaxVF.isScalar()) 7685 return VectorizationFactor::Disabled(); 7686 7687 // Select the optimal vectorization factor. 7688 return CM.selectVectorizationFactor(MaxVF); 7689 } 7690 7691 void LoopVectorizationPlanner::setBestPlan(ElementCount VF, unsigned UF) { 7692 LLVM_DEBUG(dbgs() << "Setting best plan to VF=" << VF << ", UF=" << UF 7693 << '\n'); 7694 BestVF = VF; 7695 BestUF = UF; 7696 7697 erase_if(VPlans, [VF](const VPlanPtr &Plan) { 7698 return !Plan->hasVF(VF); 7699 }); 7700 assert(VPlans.size() == 1 && "Best VF has not a single VPlan."); 7701 } 7702 7703 void LoopVectorizationPlanner::executePlan(InnerLoopVectorizer &ILV, 7704 DominatorTree *DT) { 7705 // Perform the actual loop transformation. 7706 7707 // 1. Create a new empty loop. Unlink the old loop and connect the new one. 7708 VPCallbackILV CallbackILV(ILV); 7709 7710 assert(BestVF.hasValue() && "Vectorization Factor is missing"); 7711 7712 VPTransformState State{*BestVF, 7713 BestUF, 7714 OrigLoop, 7715 LI, 7716 DT, 7717 ILV.Builder, 7718 ILV.VectorLoopValueMap, 7719 &ILV, 7720 CallbackILV}; 7721 State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton(); 7722 State.TripCount = ILV.getOrCreateTripCount(nullptr); 7723 State.CanonicalIV = ILV.Induction; 7724 7725 ILV.printDebugTracesAtStart(); 7726 7727 //===------------------------------------------------===// 7728 // 7729 // Notice: any optimization or new instruction that go 7730 // into the code below should also be implemented in 7731 // the cost-model. 7732 // 7733 //===------------------------------------------------===// 7734 7735 // 2. Copy and widen instructions from the old loop into the new loop. 7736 assert(VPlans.size() == 1 && "Not a single VPlan to execute."); 7737 VPlans.front()->execute(&State); 7738 7739 // 3. Fix the vectorized code: take care of header phi's, live-outs, 7740 // predication, updating analyses. 7741 ILV.fixVectorizedLoop(); 7742 7743 ILV.printDebugTracesAtEnd(); 7744 } 7745 7746 void LoopVectorizationPlanner::collectTriviallyDeadInstructions( 7747 SmallPtrSetImpl<Instruction *> &DeadInstructions) { 7748 7749 // We create new control-flow for the vectorized loop, so the original exit 7750 // conditions will be dead after vectorization if it's only used by the 7751 // terminator 7752 SmallVector<BasicBlock*> ExitingBlocks; 7753 OrigLoop->getExitingBlocks(ExitingBlocks); 7754 for (auto *BB : ExitingBlocks) { 7755 auto *Cmp = dyn_cast<Instruction>(BB->getTerminator()->getOperand(0)); 7756 if (!Cmp || !Cmp->hasOneUse()) 7757 continue; 7758 7759 // TODO: we should introduce a getUniqueExitingBlocks on Loop 7760 if (!DeadInstructions.insert(Cmp).second) 7761 continue; 7762 7763 // The operands of the icmp is often a dead trunc, used by IndUpdate. 7764 // TODO: can recurse through operands in general 7765 for (Value *Op : Cmp->operands()) { 7766 if (isa<TruncInst>(Op) && Op->hasOneUse()) 7767 DeadInstructions.insert(cast<Instruction>(Op)); 7768 } 7769 } 7770 7771 // We create new "steps" for induction variable updates to which the original 7772 // induction variables map. An original update instruction will be dead if 7773 // all its users except the induction variable are dead. 7774 auto *Latch = OrigLoop->getLoopLatch(); 7775 for (auto &Induction : Legal->getInductionVars()) { 7776 PHINode *Ind = Induction.first; 7777 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch)); 7778 7779 // If the tail is to be folded by masking, the primary induction variable, 7780 // if exists, isn't dead: it will be used for masking. Don't kill it. 7781 if (CM.foldTailByMasking() && IndUpdate == Legal->getPrimaryInduction()) 7782 continue; 7783 7784 if (llvm::all_of(IndUpdate->users(), [&](User *U) -> bool { 7785 return U == Ind || DeadInstructions.count(cast<Instruction>(U)); 7786 })) 7787 DeadInstructions.insert(IndUpdate); 7788 7789 // We record as "Dead" also the type-casting instructions we had identified 7790 // during induction analysis. We don't need any handling for them in the 7791 // vectorized loop because we have proven that, under a proper runtime 7792 // test guarding the vectorized loop, the value of the phi, and the casted 7793 // value of the phi, are the same. The last instruction in this casting chain 7794 // will get its scalar/vector/widened def from the scalar/vector/widened def 7795 // of the respective phi node. Any other casts in the induction def-use chain 7796 // have no other uses outside the phi update chain, and will be ignored. 7797 InductionDescriptor &IndDes = Induction.second; 7798 const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts(); 7799 DeadInstructions.insert(Casts.begin(), Casts.end()); 7800 } 7801 } 7802 7803 Value *InnerLoopUnroller::reverseVector(Value *Vec) { return Vec; } 7804 7805 Value *InnerLoopUnroller::getBroadcastInstrs(Value *V) { return V; } 7806 7807 Value *InnerLoopUnroller::getStepVector(Value *Val, int StartIdx, Value *Step, 7808 Instruction::BinaryOps BinOp) { 7809 // When unrolling and the VF is 1, we only need to add a simple scalar. 7810 Type *Ty = Val->getType(); 7811 assert(!Ty->isVectorTy() && "Val must be a scalar"); 7812 7813 if (Ty->isFloatingPointTy()) { 7814 Constant *C = ConstantFP::get(Ty, (double)StartIdx); 7815 7816 // Floating point operations had to be 'fast' to enable the unrolling. 7817 Value *MulOp = addFastMathFlag(Builder.CreateFMul(C, Step)); 7818 return addFastMathFlag(Builder.CreateBinOp(BinOp, Val, MulOp)); 7819 } 7820 Constant *C = ConstantInt::get(Ty, StartIdx); 7821 return Builder.CreateAdd(Val, Builder.CreateMul(C, Step), "induction"); 7822 } 7823 7824 static void AddRuntimeUnrollDisableMetaData(Loop *L) { 7825 SmallVector<Metadata *, 4> MDs; 7826 // Reserve first location for self reference to the LoopID metadata node. 7827 MDs.push_back(nullptr); 7828 bool IsUnrollMetadata = false; 7829 MDNode *LoopID = L->getLoopID(); 7830 if (LoopID) { 7831 // First find existing loop unrolling disable metadata. 7832 for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) { 7833 auto *MD = dyn_cast<MDNode>(LoopID->getOperand(i)); 7834 if (MD) { 7835 const auto *S = dyn_cast<MDString>(MD->getOperand(0)); 7836 IsUnrollMetadata = 7837 S && S->getString().startswith("llvm.loop.unroll.disable"); 7838 } 7839 MDs.push_back(LoopID->getOperand(i)); 7840 } 7841 } 7842 7843 if (!IsUnrollMetadata) { 7844 // Add runtime unroll disable metadata. 7845 LLVMContext &Context = L->getHeader()->getContext(); 7846 SmallVector<Metadata *, 1> DisableOperands; 7847 DisableOperands.push_back( 7848 MDString::get(Context, "llvm.loop.unroll.runtime.disable")); 7849 MDNode *DisableNode = MDNode::get(Context, DisableOperands); 7850 MDs.push_back(DisableNode); 7851 MDNode *NewLoopID = MDNode::get(Context, MDs); 7852 // Set operand 0 to refer to the loop id itself. 7853 NewLoopID->replaceOperandWith(0, NewLoopID); 7854 L->setLoopID(NewLoopID); 7855 } 7856 } 7857 7858 //===--------------------------------------------------------------------===// 7859 // EpilogueVectorizerMainLoop 7860 //===--------------------------------------------------------------------===// 7861 7862 /// This function is partially responsible for generating the control flow 7863 /// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization. 7864 BasicBlock *EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton() { 7865 MDNode *OrigLoopID = OrigLoop->getLoopID(); 7866 Loop *Lp = createVectorLoopSkeleton(""); 7867 7868 // Generate the code to check the minimum iteration count of the vector 7869 // epilogue (see below). 7870 EPI.EpilogueIterationCountCheck = 7871 emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader, true); 7872 EPI.EpilogueIterationCountCheck->setName("iter.check"); 7873 7874 // Generate the code to check any assumptions that we've made for SCEV 7875 // expressions. 7876 BasicBlock *SavedPreHeader = LoopVectorPreHeader; 7877 emitSCEVChecks(Lp, LoopScalarPreHeader); 7878 7879 // If a safety check was generated save it. 7880 if (SavedPreHeader != LoopVectorPreHeader) 7881 EPI.SCEVSafetyCheck = SavedPreHeader; 7882 7883 // Generate the code that checks at runtime if arrays overlap. We put the 7884 // checks into a separate block to make the more common case of few elements 7885 // faster. 7886 SavedPreHeader = LoopVectorPreHeader; 7887 emitMemRuntimeChecks(Lp, LoopScalarPreHeader); 7888 7889 // If a safety check was generated save/overwite it. 7890 if (SavedPreHeader != LoopVectorPreHeader) 7891 EPI.MemSafetyCheck = SavedPreHeader; 7892 7893 // Generate the iteration count check for the main loop, *after* the check 7894 // for the epilogue loop, so that the path-length is shorter for the case 7895 // that goes directly through the vector epilogue. The longer-path length for 7896 // the main loop is compensated for, by the gain from vectorizing the larger 7897 // trip count. Note: the branch will get updated later on when we vectorize 7898 // the epilogue. 7899 EPI.MainLoopIterationCountCheck = 7900 emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader, false); 7901 7902 // Generate the induction variable. 7903 OldInduction = Legal->getPrimaryInduction(); 7904 Type *IdxTy = Legal->getWidestInductionType(); 7905 Value *StartIdx = ConstantInt::get(IdxTy, 0); 7906 Constant *Step = ConstantInt::get(IdxTy, VF.getKnownMinValue() * UF); 7907 Value *CountRoundDown = getOrCreateVectorTripCount(Lp); 7908 EPI.VectorTripCount = CountRoundDown; 7909 Induction = 7910 createInductionVariable(Lp, StartIdx, CountRoundDown, Step, 7911 getDebugLocFromInstOrOperands(OldInduction)); 7912 7913 // Skip induction resume value creation here because they will be created in 7914 // the second pass. If we created them here, they wouldn't be used anyway, 7915 // because the vplan in the second pass still contains the inductions from the 7916 // original loop. 7917 7918 return completeLoopSkeleton(Lp, OrigLoopID); 7919 } 7920 7921 void EpilogueVectorizerMainLoop::printDebugTracesAtStart() { 7922 LLVM_DEBUG({ 7923 dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n" 7924 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue() 7925 << ", Main Loop UF:" << EPI.MainLoopUF 7926 << ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue() 7927 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n"; 7928 }); 7929 } 7930 7931 void EpilogueVectorizerMainLoop::printDebugTracesAtEnd() { 7932 DEBUG_WITH_TYPE(VerboseDebug, { 7933 dbgs() << "intermediate fn:\n" << *Induction->getFunction() << "\n"; 7934 }); 7935 } 7936 7937 BasicBlock *EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck( 7938 Loop *L, BasicBlock *Bypass, bool ForEpilogue) { 7939 assert(L && "Expected valid Loop."); 7940 assert(Bypass && "Expected valid bypass basic block."); 7941 unsigned VFactor = 7942 ForEpilogue ? EPI.EpilogueVF.getKnownMinValue() : VF.getKnownMinValue(); 7943 unsigned UFactor = ForEpilogue ? EPI.EpilogueUF : UF; 7944 Value *Count = getOrCreateTripCount(L); 7945 // Reuse existing vector loop preheader for TC checks. 7946 // Note that new preheader block is generated for vector loop. 7947 BasicBlock *const TCCheckBlock = LoopVectorPreHeader; 7948 IRBuilder<> Builder(TCCheckBlock->getTerminator()); 7949 7950 // Generate code to check if the loop's trip count is less than VF * UF of the 7951 // main vector loop. 7952 auto P = 7953 Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_ULT; 7954 7955 Value *CheckMinIters = Builder.CreateICmp( 7956 P, Count, ConstantInt::get(Count->getType(), VFactor * UFactor), 7957 "min.iters.check"); 7958 7959 if (!ForEpilogue) 7960 TCCheckBlock->setName("vector.main.loop.iter.check"); 7961 7962 // Create new preheader for vector loop. 7963 LoopVectorPreHeader = SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(), 7964 DT, LI, nullptr, "vector.ph"); 7965 7966 if (ForEpilogue) { 7967 assert(DT->properlyDominates(DT->getNode(TCCheckBlock), 7968 DT->getNode(Bypass)->getIDom()) && 7969 "TC check is expected to dominate Bypass"); 7970 7971 // Update dominator for Bypass & LoopExit. 7972 DT->changeImmediateDominator(Bypass, TCCheckBlock); 7973 DT->changeImmediateDominator(LoopExitBlock, TCCheckBlock); 7974 7975 LoopBypassBlocks.push_back(TCCheckBlock); 7976 7977 // Save the trip count so we don't have to regenerate it in the 7978 // vec.epilog.iter.check. This is safe to do because the trip count 7979 // generated here dominates the vector epilog iter check. 7980 EPI.TripCount = Count; 7981 } 7982 7983 ReplaceInstWithInst( 7984 TCCheckBlock->getTerminator(), 7985 BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters)); 7986 7987 return TCCheckBlock; 7988 } 7989 7990 //===--------------------------------------------------------------------===// 7991 // EpilogueVectorizerEpilogueLoop 7992 //===--------------------------------------------------------------------===// 7993 7994 /// This function is partially responsible for generating the control flow 7995 /// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization. 7996 BasicBlock * 7997 EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() { 7998 MDNode *OrigLoopID = OrigLoop->getLoopID(); 7999 Loop *Lp = createVectorLoopSkeleton("vec.epilog."); 8000 8001 // Now, compare the remaining count and if there aren't enough iterations to 8002 // execute the vectorized epilogue skip to the scalar part. 8003 BasicBlock *VecEpilogueIterationCountCheck = LoopVectorPreHeader; 8004 VecEpilogueIterationCountCheck->setName("vec.epilog.iter.check"); 8005 LoopVectorPreHeader = 8006 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT, 8007 LI, nullptr, "vec.epilog.ph"); 8008 emitMinimumVectorEpilogueIterCountCheck(Lp, LoopScalarPreHeader, 8009 VecEpilogueIterationCountCheck); 8010 8011 // Adjust the control flow taking the state info from the main loop 8012 // vectorization into account. 8013 assert(EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck && 8014 "expected this to be saved from the previous pass."); 8015 EPI.MainLoopIterationCountCheck->getTerminator()->replaceUsesOfWith( 8016 VecEpilogueIterationCountCheck, LoopVectorPreHeader); 8017 8018 DT->changeImmediateDominator(LoopVectorPreHeader, 8019 EPI.MainLoopIterationCountCheck); 8020 8021 EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith( 8022 VecEpilogueIterationCountCheck, LoopScalarPreHeader); 8023 8024 if (EPI.SCEVSafetyCheck) 8025 EPI.SCEVSafetyCheck->getTerminator()->replaceUsesOfWith( 8026 VecEpilogueIterationCountCheck, LoopScalarPreHeader); 8027 if (EPI.MemSafetyCheck) 8028 EPI.MemSafetyCheck->getTerminator()->replaceUsesOfWith( 8029 VecEpilogueIterationCountCheck, LoopScalarPreHeader); 8030 8031 DT->changeImmediateDominator( 8032 VecEpilogueIterationCountCheck, 8033 VecEpilogueIterationCountCheck->getSinglePredecessor()); 8034 8035 DT->changeImmediateDominator(LoopScalarPreHeader, 8036 EPI.EpilogueIterationCountCheck); 8037 DT->changeImmediateDominator(LoopExitBlock, EPI.EpilogueIterationCountCheck); 8038 8039 // Keep track of bypass blocks, as they feed start values to the induction 8040 // phis in the scalar loop preheader. 8041 if (EPI.SCEVSafetyCheck) 8042 LoopBypassBlocks.push_back(EPI.SCEVSafetyCheck); 8043 if (EPI.MemSafetyCheck) 8044 LoopBypassBlocks.push_back(EPI.MemSafetyCheck); 8045 LoopBypassBlocks.push_back(EPI.EpilogueIterationCountCheck); 8046 8047 // Generate a resume induction for the vector epilogue and put it in the 8048 // vector epilogue preheader 8049 Type *IdxTy = Legal->getWidestInductionType(); 8050 PHINode *EPResumeVal = PHINode::Create(IdxTy, 2, "vec.epilog.resume.val", 8051 LoopVectorPreHeader->getFirstNonPHI()); 8052 EPResumeVal->addIncoming(EPI.VectorTripCount, VecEpilogueIterationCountCheck); 8053 EPResumeVal->addIncoming(ConstantInt::get(IdxTy, 0), 8054 EPI.MainLoopIterationCountCheck); 8055 8056 // Generate the induction variable. 8057 OldInduction = Legal->getPrimaryInduction(); 8058 Value *CountRoundDown = getOrCreateVectorTripCount(Lp); 8059 Constant *Step = ConstantInt::get(IdxTy, VF.getKnownMinValue() * UF); 8060 Value *StartIdx = EPResumeVal; 8061 Induction = 8062 createInductionVariable(Lp, StartIdx, CountRoundDown, Step, 8063 getDebugLocFromInstOrOperands(OldInduction)); 8064 8065 // Generate induction resume values. These variables save the new starting 8066 // indexes for the scalar loop. They are used to test if there are any tail 8067 // iterations left once the vector loop has completed. 8068 // Note that when the vectorized epilogue is skipped due to iteration count 8069 // check, then the resume value for the induction variable comes from 8070 // the trip count of the main vector loop, hence passing the AdditionalBypass 8071 // argument. 8072 createInductionResumeValues(Lp, CountRoundDown, 8073 {VecEpilogueIterationCountCheck, 8074 EPI.VectorTripCount} /* AdditionalBypass */); 8075 8076 AddRuntimeUnrollDisableMetaData(Lp); 8077 return completeLoopSkeleton(Lp, OrigLoopID); 8078 } 8079 8080 BasicBlock * 8081 EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck( 8082 Loop *L, BasicBlock *Bypass, BasicBlock *Insert) { 8083 8084 assert(EPI.TripCount && 8085 "Expected trip count to have been safed in the first pass."); 8086 assert( 8087 (!isa<Instruction>(EPI.TripCount) || 8088 DT->dominates(cast<Instruction>(EPI.TripCount)->getParent(), Insert)) && 8089 "saved trip count does not dominate insertion point."); 8090 Value *TC = EPI.TripCount; 8091 IRBuilder<> Builder(Insert->getTerminator()); 8092 Value *Count = Builder.CreateSub(TC, EPI.VectorTripCount, "n.vec.remaining"); 8093 8094 // Generate code to check if the loop's trip count is less than VF * UF of the 8095 // vector epilogue loop. 8096 auto P = 8097 Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_ULT; 8098 8099 Value *CheckMinIters = Builder.CreateICmp( 8100 P, Count, 8101 ConstantInt::get(Count->getType(), 8102 EPI.EpilogueVF.getKnownMinValue() * EPI.EpilogueUF), 8103 "min.epilog.iters.check"); 8104 8105 ReplaceInstWithInst( 8106 Insert->getTerminator(), 8107 BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters)); 8108 8109 LoopBypassBlocks.push_back(Insert); 8110 return Insert; 8111 } 8112 8113 void EpilogueVectorizerEpilogueLoop::printDebugTracesAtStart() { 8114 LLVM_DEBUG({ 8115 dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n" 8116 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue() 8117 << ", Main Loop UF:" << EPI.MainLoopUF 8118 << ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue() 8119 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n"; 8120 }); 8121 } 8122 8123 void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() { 8124 DEBUG_WITH_TYPE(VerboseDebug, { 8125 dbgs() << "final fn:\n" << *Induction->getFunction() << "\n"; 8126 }); 8127 } 8128 8129 bool LoopVectorizationPlanner::getDecisionAndClampRange( 8130 const std::function<bool(ElementCount)> &Predicate, VFRange &Range) { 8131 assert(!Range.isEmpty() && "Trying to test an empty VF range."); 8132 bool PredicateAtRangeStart = Predicate(Range.Start); 8133 8134 for (ElementCount TmpVF = Range.Start * 2; 8135 ElementCount::isKnownLT(TmpVF, Range.End); TmpVF *= 2) 8136 if (Predicate(TmpVF) != PredicateAtRangeStart) { 8137 Range.End = TmpVF; 8138 break; 8139 } 8140 8141 return PredicateAtRangeStart; 8142 } 8143 8144 /// Build VPlans for the full range of feasible VF's = {\p MinVF, 2 * \p MinVF, 8145 /// 4 * \p MinVF, ..., \p MaxVF} by repeatedly building a VPlan for a sub-range 8146 /// of VF's starting at a given VF and extending it as much as possible. Each 8147 /// vectorization decision can potentially shorten this sub-range during 8148 /// buildVPlan(). 8149 void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF, 8150 ElementCount MaxVF) { 8151 auto MaxVFPlusOne = MaxVF.getWithIncrement(1); 8152 for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) { 8153 VFRange SubRange = {VF, MaxVFPlusOne}; 8154 VPlans.push_back(buildVPlan(SubRange)); 8155 VF = SubRange.End; 8156 } 8157 } 8158 8159 VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst, 8160 VPlanPtr &Plan) { 8161 assert(is_contained(predecessors(Dst), Src) && "Invalid edge"); 8162 8163 // Look for cached value. 8164 std::pair<BasicBlock *, BasicBlock *> Edge(Src, Dst); 8165 EdgeMaskCacheTy::iterator ECEntryIt = EdgeMaskCache.find(Edge); 8166 if (ECEntryIt != EdgeMaskCache.end()) 8167 return ECEntryIt->second; 8168 8169 VPValue *SrcMask = createBlockInMask(Src, Plan); 8170 8171 // The terminator has to be a branch inst! 8172 BranchInst *BI = dyn_cast<BranchInst>(Src->getTerminator()); 8173 assert(BI && "Unexpected terminator found"); 8174 8175 if (!BI->isConditional() || BI->getSuccessor(0) == BI->getSuccessor(1)) 8176 return EdgeMaskCache[Edge] = SrcMask; 8177 8178 // If source is an exiting block, we know the exit edge is dynamically dead 8179 // in the vector loop, and thus we don't need to restrict the mask. Avoid 8180 // adding uses of an otherwise potentially dead instruction. 8181 if (OrigLoop->isLoopExiting(Src)) 8182 return EdgeMaskCache[Edge] = SrcMask; 8183 8184 VPValue *EdgeMask = Plan->getOrAddVPValue(BI->getCondition()); 8185 assert(EdgeMask && "No Edge Mask found for condition"); 8186 8187 if (BI->getSuccessor(0) != Dst) 8188 EdgeMask = Builder.createNot(EdgeMask); 8189 8190 if (SrcMask) { // Otherwise block in-mask is all-one, no need to AND. 8191 // The condition is 'SrcMask && EdgeMask', which is equivalent to 8192 // 'select i1 SrcMask, i1 EdgeMask, i1 false'. 8193 // The select version does not introduce new UB if SrcMask is false and 8194 // EdgeMask is poison. Using 'and' here introduces undefined behavior. 8195 VPValue *False = Plan->getOrAddVPValue( 8196 ConstantInt::getFalse(BI->getCondition()->getType())); 8197 EdgeMask = Builder.createSelect(SrcMask, EdgeMask, False); 8198 } 8199 8200 return EdgeMaskCache[Edge] = EdgeMask; 8201 } 8202 8203 VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlanPtr &Plan) { 8204 assert(OrigLoop->contains(BB) && "Block is not a part of a loop"); 8205 8206 // Look for cached value. 8207 BlockMaskCacheTy::iterator BCEntryIt = BlockMaskCache.find(BB); 8208 if (BCEntryIt != BlockMaskCache.end()) 8209 return BCEntryIt->second; 8210 8211 // All-one mask is modelled as no-mask following the convention for masked 8212 // load/store/gather/scatter. Initialize BlockMask to no-mask. 8213 VPValue *BlockMask = nullptr; 8214 8215 if (OrigLoop->getHeader() == BB) { 8216 if (!CM.blockNeedsPredication(BB)) 8217 return BlockMaskCache[BB] = BlockMask; // Loop incoming mask is all-one. 8218 8219 // Create the block in mask as the first non-phi instruction in the block. 8220 VPBuilder::InsertPointGuard Guard(Builder); 8221 auto NewInsertionPoint = Builder.getInsertBlock()->getFirstNonPhi(); 8222 Builder.setInsertPoint(Builder.getInsertBlock(), NewInsertionPoint); 8223 8224 // Introduce the early-exit compare IV <= BTC to form header block mask. 8225 // This is used instead of IV < TC because TC may wrap, unlike BTC. 8226 // Start by constructing the desired canonical IV. 8227 VPValue *IV = nullptr; 8228 if (Legal->getPrimaryInduction()) 8229 IV = Plan->getOrAddVPValue(Legal->getPrimaryInduction()); 8230 else { 8231 auto IVRecipe = new VPWidenCanonicalIVRecipe(); 8232 Builder.getInsertBlock()->insert(IVRecipe, NewInsertionPoint); 8233 IV = IVRecipe->getVPValue(); 8234 } 8235 VPValue *BTC = Plan->getOrCreateBackedgeTakenCount(); 8236 bool TailFolded = !CM.isScalarEpilogueAllowed(); 8237 8238 if (TailFolded && CM.TTI.emitGetActiveLaneMask()) { 8239 // While ActiveLaneMask is a binary op that consumes the loop tripcount 8240 // as a second argument, we only pass the IV here and extract the 8241 // tripcount from the transform state where codegen of the VP instructions 8242 // happen. 8243 BlockMask = Builder.createNaryOp(VPInstruction::ActiveLaneMask, {IV}); 8244 } else { 8245 BlockMask = Builder.createNaryOp(VPInstruction::ICmpULE, {IV, BTC}); 8246 } 8247 return BlockMaskCache[BB] = BlockMask; 8248 } 8249 8250 // This is the block mask. We OR all incoming edges. 8251 for (auto *Predecessor : predecessors(BB)) { 8252 VPValue *EdgeMask = createEdgeMask(Predecessor, BB, Plan); 8253 if (!EdgeMask) // Mask of predecessor is all-one so mask of block is too. 8254 return BlockMaskCache[BB] = EdgeMask; 8255 8256 if (!BlockMask) { // BlockMask has its initialized nullptr value. 8257 BlockMask = EdgeMask; 8258 continue; 8259 } 8260 8261 BlockMask = Builder.createOr(BlockMask, EdgeMask); 8262 } 8263 8264 return BlockMaskCache[BB] = BlockMask; 8265 } 8266 8267 VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(Instruction *I, VFRange &Range, 8268 VPlanPtr &Plan) { 8269 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && 8270 "Must be called with either a load or store"); 8271 8272 auto willWiden = [&](ElementCount VF) -> bool { 8273 if (VF.isScalar()) 8274 return false; 8275 LoopVectorizationCostModel::InstWidening Decision = 8276 CM.getWideningDecision(I, VF); 8277 assert(Decision != LoopVectorizationCostModel::CM_Unknown && 8278 "CM decision should be taken at this point."); 8279 if (Decision == LoopVectorizationCostModel::CM_Interleave) 8280 return true; 8281 if (CM.isScalarAfterVectorization(I, VF) || 8282 CM.isProfitableToScalarize(I, VF)) 8283 return false; 8284 return Decision != LoopVectorizationCostModel::CM_Scalarize; 8285 }; 8286 8287 if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range)) 8288 return nullptr; 8289 8290 VPValue *Mask = nullptr; 8291 if (Legal->isMaskRequired(I)) 8292 Mask = createBlockInMask(I->getParent(), Plan); 8293 8294 VPValue *Addr = Plan->getOrAddVPValue(getLoadStorePointerOperand(I)); 8295 if (LoadInst *Load = dyn_cast<LoadInst>(I)) 8296 return new VPWidenMemoryInstructionRecipe(*Load, Addr, Mask); 8297 8298 StoreInst *Store = cast<StoreInst>(I); 8299 VPValue *StoredValue = Plan->getOrAddVPValue(Store->getValueOperand()); 8300 return new VPWidenMemoryInstructionRecipe(*Store, Addr, StoredValue, Mask); 8301 } 8302 8303 VPWidenIntOrFpInductionRecipe * 8304 VPRecipeBuilder::tryToOptimizeInductionPHI(PHINode *Phi, VPlan &Plan) const { 8305 // Check if this is an integer or fp induction. If so, build the recipe that 8306 // produces its scalar and vector values. 8307 InductionDescriptor II = Legal->getInductionVars().lookup(Phi); 8308 if (II.getKind() == InductionDescriptor::IK_IntInduction || 8309 II.getKind() == InductionDescriptor::IK_FpInduction) { 8310 VPValue *Start = Plan.getOrAddVPValue(II.getStartValue()); 8311 return new VPWidenIntOrFpInductionRecipe(Phi, Start); 8312 } 8313 8314 return nullptr; 8315 } 8316 8317 VPWidenIntOrFpInductionRecipe * 8318 VPRecipeBuilder::tryToOptimizeInductionTruncate(TruncInst *I, VFRange &Range, 8319 VPlan &Plan) const { 8320 // Optimize the special case where the source is a constant integer 8321 // induction variable. Notice that we can only optimize the 'trunc' case 8322 // because (a) FP conversions lose precision, (b) sext/zext may wrap, and 8323 // (c) other casts depend on pointer size. 8324 8325 // Determine whether \p K is a truncation based on an induction variable that 8326 // can be optimized. 8327 auto isOptimizableIVTruncate = 8328 [&](Instruction *K) -> std::function<bool(ElementCount)> { 8329 return [=](ElementCount VF) -> bool { 8330 return CM.isOptimizableIVTruncate(K, VF); 8331 }; 8332 }; 8333 8334 if (LoopVectorizationPlanner::getDecisionAndClampRange( 8335 isOptimizableIVTruncate(I), Range)) { 8336 8337 InductionDescriptor II = 8338 Legal->getInductionVars().lookup(cast<PHINode>(I->getOperand(0))); 8339 VPValue *Start = Plan.getOrAddVPValue(II.getStartValue()); 8340 return new VPWidenIntOrFpInductionRecipe(cast<PHINode>(I->getOperand(0)), 8341 Start, I); 8342 } 8343 return nullptr; 8344 } 8345 8346 VPBlendRecipe *VPRecipeBuilder::tryToBlend(PHINode *Phi, VPlanPtr &Plan) { 8347 // We know that all PHIs in non-header blocks are converted into selects, so 8348 // we don't have to worry about the insertion order and we can just use the 8349 // builder. At this point we generate the predication tree. There may be 8350 // duplications since this is a simple recursive scan, but future 8351 // optimizations will clean it up. 8352 8353 SmallVector<VPValue *, 2> Operands; 8354 unsigned NumIncoming = Phi->getNumIncomingValues(); 8355 for (unsigned In = 0; In < NumIncoming; In++) { 8356 VPValue *EdgeMask = 8357 createEdgeMask(Phi->getIncomingBlock(In), Phi->getParent(), Plan); 8358 assert((EdgeMask || NumIncoming == 1) && 8359 "Multiple predecessors with one having a full mask"); 8360 Operands.push_back(Plan->getOrAddVPValue(Phi->getIncomingValue(In))); 8361 if (EdgeMask) 8362 Operands.push_back(EdgeMask); 8363 } 8364 return new VPBlendRecipe(Phi, Operands); 8365 } 8366 8367 VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, VFRange &Range, 8368 VPlan &Plan) const { 8369 8370 bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange( 8371 [this, CI](ElementCount VF) { 8372 return CM.isScalarWithPredication(CI, VF); 8373 }, 8374 Range); 8375 8376 if (IsPredicated) 8377 return nullptr; 8378 8379 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 8380 if (ID && (ID == Intrinsic::assume || ID == Intrinsic::lifetime_end || 8381 ID == Intrinsic::lifetime_start || ID == Intrinsic::sideeffect || 8382 ID == Intrinsic::pseudoprobe || 8383 ID == Intrinsic::experimental_noalias_scope_decl)) 8384 return nullptr; 8385 8386 auto willWiden = [&](ElementCount VF) -> bool { 8387 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 8388 // The following case may be scalarized depending on the VF. 8389 // The flag shows whether we use Intrinsic or a usual Call for vectorized 8390 // version of the instruction. 8391 // Is it beneficial to perform intrinsic call compared to lib call? 8392 bool NeedToScalarize = false; 8393 InstructionCost CallCost = CM.getVectorCallCost(CI, VF, NeedToScalarize); 8394 InstructionCost IntrinsicCost = ID ? CM.getVectorIntrinsicCost(CI, VF) : 0; 8395 bool UseVectorIntrinsic = ID && IntrinsicCost <= CallCost; 8396 assert(IntrinsicCost.isValid() && CallCost.isValid() && 8397 "Cannot have invalid costs while widening"); 8398 return UseVectorIntrinsic || !NeedToScalarize; 8399 }; 8400 8401 if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range)) 8402 return nullptr; 8403 8404 return new VPWidenCallRecipe(*CI, Plan.mapToVPValues(CI->arg_operands())); 8405 } 8406 8407 bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const { 8408 assert(!isa<BranchInst>(I) && !isa<PHINode>(I) && !isa<LoadInst>(I) && 8409 !isa<StoreInst>(I) && "Instruction should have been handled earlier"); 8410 // Instruction should be widened, unless it is scalar after vectorization, 8411 // scalarization is profitable or it is predicated. 8412 auto WillScalarize = [this, I](ElementCount VF) -> bool { 8413 return CM.isScalarAfterVectorization(I, VF) || 8414 CM.isProfitableToScalarize(I, VF) || 8415 CM.isScalarWithPredication(I, VF); 8416 }; 8417 return !LoopVectorizationPlanner::getDecisionAndClampRange(WillScalarize, 8418 Range); 8419 } 8420 8421 VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VPlan &Plan) const { 8422 auto IsVectorizableOpcode = [](unsigned Opcode) { 8423 switch (Opcode) { 8424 case Instruction::Add: 8425 case Instruction::And: 8426 case Instruction::AShr: 8427 case Instruction::BitCast: 8428 case Instruction::FAdd: 8429 case Instruction::FCmp: 8430 case Instruction::FDiv: 8431 case Instruction::FMul: 8432 case Instruction::FNeg: 8433 case Instruction::FPExt: 8434 case Instruction::FPToSI: 8435 case Instruction::FPToUI: 8436 case Instruction::FPTrunc: 8437 case Instruction::FRem: 8438 case Instruction::FSub: 8439 case Instruction::ICmp: 8440 case Instruction::IntToPtr: 8441 case Instruction::LShr: 8442 case Instruction::Mul: 8443 case Instruction::Or: 8444 case Instruction::PtrToInt: 8445 case Instruction::SDiv: 8446 case Instruction::Select: 8447 case Instruction::SExt: 8448 case Instruction::Shl: 8449 case Instruction::SIToFP: 8450 case Instruction::SRem: 8451 case Instruction::Sub: 8452 case Instruction::Trunc: 8453 case Instruction::UDiv: 8454 case Instruction::UIToFP: 8455 case Instruction::URem: 8456 case Instruction::Xor: 8457 case Instruction::ZExt: 8458 return true; 8459 } 8460 return false; 8461 }; 8462 8463 if (!IsVectorizableOpcode(I->getOpcode())) 8464 return nullptr; 8465 8466 // Success: widen this instruction. 8467 return new VPWidenRecipe(*I, Plan.mapToVPValues(I->operands())); 8468 } 8469 8470 VPBasicBlock *VPRecipeBuilder::handleReplication( 8471 Instruction *I, VFRange &Range, VPBasicBlock *VPBB, 8472 DenseMap<Instruction *, VPReplicateRecipe *> &PredInst2Recipe, 8473 VPlanPtr &Plan) { 8474 bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange( 8475 [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); }, 8476 Range); 8477 8478 bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange( 8479 [&](ElementCount VF) { return CM.isScalarWithPredication(I, VF); }, 8480 Range); 8481 8482 auto *Recipe = new VPReplicateRecipe(I, Plan->mapToVPValues(I->operands()), 8483 IsUniform, IsPredicated); 8484 setRecipe(I, Recipe); 8485 Plan->addVPValue(I, Recipe); 8486 8487 // Find if I uses a predicated instruction. If so, it will use its scalar 8488 // value. Avoid hoisting the insert-element which packs the scalar value into 8489 // a vector value, as that happens iff all users use the vector value. 8490 for (auto &Op : I->operands()) 8491 if (auto *PredInst = dyn_cast<Instruction>(Op)) 8492 if (PredInst2Recipe.find(PredInst) != PredInst2Recipe.end()) 8493 PredInst2Recipe[PredInst]->setAlsoPack(false); 8494 8495 // Finalize the recipe for Instr, first if it is not predicated. 8496 if (!IsPredicated) { 8497 LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n"); 8498 VPBB->appendRecipe(Recipe); 8499 return VPBB; 8500 } 8501 LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n"); 8502 assert(VPBB->getSuccessors().empty() && 8503 "VPBB has successors when handling predicated replication."); 8504 // Record predicated instructions for above packing optimizations. 8505 PredInst2Recipe[I] = Recipe; 8506 VPBlockBase *Region = createReplicateRegion(I, Recipe, Plan); 8507 VPBlockUtils::insertBlockAfter(Region, VPBB); 8508 auto *RegSucc = new VPBasicBlock(); 8509 VPBlockUtils::insertBlockAfter(RegSucc, Region); 8510 return RegSucc; 8511 } 8512 8513 VPRegionBlock *VPRecipeBuilder::createReplicateRegion(Instruction *Instr, 8514 VPRecipeBase *PredRecipe, 8515 VPlanPtr &Plan) { 8516 // Instructions marked for predication are replicated and placed under an 8517 // if-then construct to prevent side-effects. 8518 8519 // Generate recipes to compute the block mask for this region. 8520 VPValue *BlockInMask = createBlockInMask(Instr->getParent(), Plan); 8521 8522 // Build the triangular if-then region. 8523 std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str(); 8524 assert(Instr->getParent() && "Predicated instruction not in any basic block"); 8525 auto *BOMRecipe = new VPBranchOnMaskRecipe(BlockInMask); 8526 auto *Entry = new VPBasicBlock(Twine(RegionName) + ".entry", BOMRecipe); 8527 auto *PHIRecipe = Instr->getType()->isVoidTy() 8528 ? nullptr 8529 : new VPPredInstPHIRecipe(Plan->getOrAddVPValue(Instr)); 8530 auto *Exit = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe); 8531 auto *Pred = new VPBasicBlock(Twine(RegionName) + ".if", PredRecipe); 8532 VPRegionBlock *Region = new VPRegionBlock(Entry, Exit, RegionName, true); 8533 8534 // Note: first set Entry as region entry and then connect successors starting 8535 // from it in order, to propagate the "parent" of each VPBasicBlock. 8536 VPBlockUtils::insertTwoBlocksAfter(Pred, Exit, BlockInMask, Entry); 8537 VPBlockUtils::connectBlocks(Pred, Exit); 8538 8539 return Region; 8540 } 8541 8542 VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr, 8543 VFRange &Range, 8544 VPlanPtr &Plan) { 8545 // First, check for specific widening recipes that deal with calls, memory 8546 // operations, inductions and Phi nodes. 8547 if (auto *CI = dyn_cast<CallInst>(Instr)) 8548 return tryToWidenCall(CI, Range, *Plan); 8549 8550 if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr)) 8551 return tryToWidenMemory(Instr, Range, Plan); 8552 8553 VPRecipeBase *Recipe; 8554 if (auto Phi = dyn_cast<PHINode>(Instr)) { 8555 if (Phi->getParent() != OrigLoop->getHeader()) 8556 return tryToBlend(Phi, Plan); 8557 if ((Recipe = tryToOptimizeInductionPHI(Phi, *Plan))) 8558 return Recipe; 8559 8560 if (Legal->isReductionVariable(Phi)) { 8561 RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi]; 8562 VPValue *StartV = 8563 Plan->getOrAddVPValue(RdxDesc.getRecurrenceStartValue()); 8564 return new VPWidenPHIRecipe(Phi, RdxDesc, *StartV); 8565 } 8566 8567 return new VPWidenPHIRecipe(Phi); 8568 } 8569 8570 if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate( 8571 cast<TruncInst>(Instr), Range, *Plan))) 8572 return Recipe; 8573 8574 if (!shouldWiden(Instr, Range)) 8575 return nullptr; 8576 8577 if (auto GEP = dyn_cast<GetElementPtrInst>(Instr)) 8578 return new VPWidenGEPRecipe(GEP, Plan->mapToVPValues(GEP->operands()), 8579 OrigLoop); 8580 8581 if (auto *SI = dyn_cast<SelectInst>(Instr)) { 8582 bool InvariantCond = 8583 PSE.getSE()->isLoopInvariant(PSE.getSCEV(SI->getOperand(0)), OrigLoop); 8584 return new VPWidenSelectRecipe(*SI, Plan->mapToVPValues(SI->operands()), 8585 InvariantCond); 8586 } 8587 8588 return tryToWiden(Instr, *Plan); 8589 } 8590 8591 void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, 8592 ElementCount MaxVF) { 8593 assert(OrigLoop->isInnermost() && "Inner loop expected."); 8594 8595 // Collect instructions from the original loop that will become trivially dead 8596 // in the vectorized loop. We don't need to vectorize these instructions. For 8597 // example, original induction update instructions can become dead because we 8598 // separately emit induction "steps" when generating code for the new loop. 8599 // Similarly, we create a new latch condition when setting up the structure 8600 // of the new loop, so the old one can become dead. 8601 SmallPtrSet<Instruction *, 4> DeadInstructions; 8602 collectTriviallyDeadInstructions(DeadInstructions); 8603 8604 // Add assume instructions we need to drop to DeadInstructions, to prevent 8605 // them from being added to the VPlan. 8606 // TODO: We only need to drop assumes in blocks that get flattend. If the 8607 // control flow is preserved, we should keep them. 8608 auto &ConditionalAssumes = Legal->getConditionalAssumes(); 8609 DeadInstructions.insert(ConditionalAssumes.begin(), ConditionalAssumes.end()); 8610 8611 DenseMap<Instruction *, Instruction *> &SinkAfter = Legal->getSinkAfter(); 8612 // Dead instructions do not need sinking. Remove them from SinkAfter. 8613 for (Instruction *I : DeadInstructions) 8614 SinkAfter.erase(I); 8615 8616 auto MaxVFPlusOne = MaxVF.getWithIncrement(1); 8617 for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) { 8618 VFRange SubRange = {VF, MaxVFPlusOne}; 8619 VPlans.push_back( 8620 buildVPlanWithVPRecipes(SubRange, DeadInstructions, SinkAfter)); 8621 VF = SubRange.End; 8622 } 8623 } 8624 8625 VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes( 8626 VFRange &Range, SmallPtrSetImpl<Instruction *> &DeadInstructions, 8627 const DenseMap<Instruction *, Instruction *> &SinkAfter) { 8628 8629 // Hold a mapping from predicated instructions to their recipes, in order to 8630 // fix their AlsoPack behavior if a user is determined to replicate and use a 8631 // scalar instead of vector value. 8632 DenseMap<Instruction *, VPReplicateRecipe *> PredInst2Recipe; 8633 8634 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups; 8635 8636 VPRecipeBuilder RecipeBuilder(OrigLoop, TLI, Legal, CM, PSE, Builder); 8637 8638 // --------------------------------------------------------------------------- 8639 // Pre-construction: record ingredients whose recipes we'll need to further 8640 // process after constructing the initial VPlan. 8641 // --------------------------------------------------------------------------- 8642 8643 // Mark instructions we'll need to sink later and their targets as 8644 // ingredients whose recipe we'll need to record. 8645 for (auto &Entry : SinkAfter) { 8646 RecipeBuilder.recordRecipeOf(Entry.first); 8647 RecipeBuilder.recordRecipeOf(Entry.second); 8648 } 8649 for (auto &Reduction : CM.getInLoopReductionChains()) { 8650 PHINode *Phi = Reduction.first; 8651 RecurKind Kind = Legal->getReductionVars()[Phi].getRecurrenceKind(); 8652 const SmallVector<Instruction *, 4> &ReductionOperations = Reduction.second; 8653 8654 RecipeBuilder.recordRecipeOf(Phi); 8655 for (auto &R : ReductionOperations) { 8656 RecipeBuilder.recordRecipeOf(R); 8657 // For min/max reducitons, where we have a pair of icmp/select, we also 8658 // need to record the ICmp recipe, so it can be removed later. 8659 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) 8660 RecipeBuilder.recordRecipeOf(cast<Instruction>(R->getOperand(0))); 8661 } 8662 } 8663 8664 // For each interleave group which is relevant for this (possibly trimmed) 8665 // Range, add it to the set of groups to be later applied to the VPlan and add 8666 // placeholders for its members' Recipes which we'll be replacing with a 8667 // single VPInterleaveRecipe. 8668 for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) { 8669 auto applyIG = [IG, this](ElementCount VF) -> bool { 8670 return (VF.isVector() && // Query is illegal for VF == 1 8671 CM.getWideningDecision(IG->getInsertPos(), VF) == 8672 LoopVectorizationCostModel::CM_Interleave); 8673 }; 8674 if (!getDecisionAndClampRange(applyIG, Range)) 8675 continue; 8676 InterleaveGroups.insert(IG); 8677 for (unsigned i = 0; i < IG->getFactor(); i++) 8678 if (Instruction *Member = IG->getMember(i)) 8679 RecipeBuilder.recordRecipeOf(Member); 8680 }; 8681 8682 // --------------------------------------------------------------------------- 8683 // Build initial VPlan: Scan the body of the loop in a topological order to 8684 // visit each basic block after having visited its predecessor basic blocks. 8685 // --------------------------------------------------------------------------- 8686 8687 // Create a dummy pre-entry VPBasicBlock to start building the VPlan. 8688 auto Plan = std::make_unique<VPlan>(); 8689 VPBasicBlock *VPBB = new VPBasicBlock("Pre-Entry"); 8690 Plan->setEntry(VPBB); 8691 8692 // Scan the body of the loop in a topological order to visit each basic block 8693 // after having visited its predecessor basic blocks. 8694 LoopBlocksDFS DFS(OrigLoop); 8695 DFS.perform(LI); 8696 8697 for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) { 8698 // Relevant instructions from basic block BB will be grouped into VPRecipe 8699 // ingredients and fill a new VPBasicBlock. 8700 unsigned VPBBsForBB = 0; 8701 auto *FirstVPBBForBB = new VPBasicBlock(BB->getName()); 8702 VPBlockUtils::insertBlockAfter(FirstVPBBForBB, VPBB); 8703 VPBB = FirstVPBBForBB; 8704 Builder.setInsertPoint(VPBB); 8705 8706 // Introduce each ingredient into VPlan. 8707 // TODO: Model and preserve debug instrinsics in VPlan. 8708 for (Instruction &I : BB->instructionsWithoutDebug()) { 8709 Instruction *Instr = &I; 8710 8711 // First filter out irrelevant instructions, to ensure no recipes are 8712 // built for them. 8713 if (isa<BranchInst>(Instr) || DeadInstructions.count(Instr)) 8714 continue; 8715 8716 if (auto Recipe = 8717 RecipeBuilder.tryToCreateWidenRecipe(Instr, Range, Plan)) { 8718 for (auto *Def : Recipe->definedValues()) { 8719 auto *UV = Def->getUnderlyingValue(); 8720 Plan->addVPValue(UV, Def); 8721 } 8722 8723 RecipeBuilder.setRecipe(Instr, Recipe); 8724 VPBB->appendRecipe(Recipe); 8725 continue; 8726 } 8727 8728 // Otherwise, if all widening options failed, Instruction is to be 8729 // replicated. This may create a successor for VPBB. 8730 VPBasicBlock *NextVPBB = RecipeBuilder.handleReplication( 8731 Instr, Range, VPBB, PredInst2Recipe, Plan); 8732 if (NextVPBB != VPBB) { 8733 VPBB = NextVPBB; 8734 VPBB->setName(BB->hasName() ? BB->getName() + "." + Twine(VPBBsForBB++) 8735 : ""); 8736 } 8737 } 8738 } 8739 8740 // Discard empty dummy pre-entry VPBasicBlock. Note that other VPBasicBlocks 8741 // may also be empty, such as the last one VPBB, reflecting original 8742 // basic-blocks with no recipes. 8743 VPBasicBlock *PreEntry = cast<VPBasicBlock>(Plan->getEntry()); 8744 assert(PreEntry->empty() && "Expecting empty pre-entry block."); 8745 VPBlockBase *Entry = Plan->setEntry(PreEntry->getSingleSuccessor()); 8746 VPBlockUtils::disconnectBlocks(PreEntry, Entry); 8747 delete PreEntry; 8748 8749 // --------------------------------------------------------------------------- 8750 // Transform initial VPlan: Apply previously taken decisions, in order, to 8751 // bring the VPlan to its final state. 8752 // --------------------------------------------------------------------------- 8753 8754 // Apply Sink-After legal constraints. 8755 for (auto &Entry : SinkAfter) { 8756 VPRecipeBase *Sink = RecipeBuilder.getRecipe(Entry.first); 8757 VPRecipeBase *Target = RecipeBuilder.getRecipe(Entry.second); 8758 // If the target is in a replication region, make sure to move Sink to the 8759 // block after it, not into the replication region itself. 8760 if (auto *Region = 8761 dyn_cast_or_null<VPRegionBlock>(Target->getParent()->getParent())) { 8762 if (Region->isReplicator()) { 8763 assert(Region->getNumSuccessors() == 1 && "Expected SESE region!"); 8764 VPBasicBlock *NextBlock = 8765 cast<VPBasicBlock>(Region->getSuccessors().front()); 8766 Sink->moveBefore(*NextBlock, NextBlock->getFirstNonPhi()); 8767 continue; 8768 } 8769 } 8770 Sink->moveAfter(Target); 8771 } 8772 8773 // Interleave memory: for each Interleave Group we marked earlier as relevant 8774 // for this VPlan, replace the Recipes widening its memory instructions with a 8775 // single VPInterleaveRecipe at its insertion point. 8776 for (auto IG : InterleaveGroups) { 8777 auto *Recipe = cast<VPWidenMemoryInstructionRecipe>( 8778 RecipeBuilder.getRecipe(IG->getInsertPos())); 8779 SmallVector<VPValue *, 4> StoredValues; 8780 for (unsigned i = 0; i < IG->getFactor(); ++i) 8781 if (auto *SI = dyn_cast_or_null<StoreInst>(IG->getMember(i))) 8782 StoredValues.push_back(Plan->getOrAddVPValue(SI->getOperand(0))); 8783 8784 auto *VPIG = new VPInterleaveRecipe(IG, Recipe->getAddr(), StoredValues, 8785 Recipe->getMask()); 8786 VPIG->insertBefore(Recipe); 8787 unsigned J = 0; 8788 for (unsigned i = 0; i < IG->getFactor(); ++i) 8789 if (Instruction *Member = IG->getMember(i)) { 8790 if (!Member->getType()->isVoidTy()) { 8791 VPValue *OriginalV = Plan->getVPValue(Member); 8792 Plan->removeVPValueFor(Member); 8793 Plan->addVPValue(Member, VPIG->getVPValue(J)); 8794 OriginalV->replaceAllUsesWith(VPIG->getVPValue(J)); 8795 J++; 8796 } 8797 RecipeBuilder.getRecipe(Member)->eraseFromParent(); 8798 } 8799 } 8800 8801 // Adjust the recipes for any inloop reductions. 8802 if (Range.Start.isVector()) 8803 adjustRecipesForInLoopReductions(Plan, RecipeBuilder); 8804 8805 // Finally, if tail is folded by masking, introduce selects between the phi 8806 // and the live-out instruction of each reduction, at the end of the latch. 8807 if (CM.foldTailByMasking() && !Legal->getReductionVars().empty()) { 8808 Builder.setInsertPoint(VPBB); 8809 auto *Cond = RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), Plan); 8810 for (auto &Reduction : Legal->getReductionVars()) { 8811 if (CM.isInLoopReduction(Reduction.first)) 8812 continue; 8813 VPValue *Phi = Plan->getOrAddVPValue(Reduction.first); 8814 VPValue *Red = Plan->getOrAddVPValue(Reduction.second.getLoopExitInstr()); 8815 Builder.createNaryOp(Instruction::Select, {Cond, Red, Phi}); 8816 } 8817 } 8818 8819 std::string PlanName; 8820 raw_string_ostream RSO(PlanName); 8821 ElementCount VF = Range.Start; 8822 Plan->addVF(VF); 8823 RSO << "Initial VPlan for VF={" << VF; 8824 for (VF *= 2; ElementCount::isKnownLT(VF, Range.End); VF *= 2) { 8825 Plan->addVF(VF); 8826 RSO << "," << VF; 8827 } 8828 RSO << "},UF>=1"; 8829 RSO.flush(); 8830 Plan->setName(PlanName); 8831 8832 return Plan; 8833 } 8834 8835 VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) { 8836 // Outer loop handling: They may require CFG and instruction level 8837 // transformations before even evaluating whether vectorization is profitable. 8838 // Since we cannot modify the incoming IR, we need to build VPlan upfront in 8839 // the vectorization pipeline. 8840 assert(!OrigLoop->isInnermost()); 8841 assert(EnableVPlanNativePath && "VPlan-native path is not enabled."); 8842 8843 // Create new empty VPlan 8844 auto Plan = std::make_unique<VPlan>(); 8845 8846 // Build hierarchical CFG 8847 VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan); 8848 HCFGBuilder.buildHierarchicalCFG(); 8849 8850 for (ElementCount VF = Range.Start; ElementCount::isKnownLT(VF, Range.End); 8851 VF *= 2) 8852 Plan->addVF(VF); 8853 8854 if (EnableVPlanPredication) { 8855 VPlanPredicator VPP(*Plan); 8856 VPP.predicate(); 8857 8858 // Avoid running transformation to recipes until masked code generation in 8859 // VPlan-native path is in place. 8860 return Plan; 8861 } 8862 8863 SmallPtrSet<Instruction *, 1> DeadInstructions; 8864 VPlanTransforms::VPInstructionsToVPRecipes( 8865 OrigLoop, Plan, Legal->getInductionVars(), DeadInstructions); 8866 return Plan; 8867 } 8868 8869 // Adjust the recipes for any inloop reductions. The chain of instructions 8870 // leading from the loop exit instr to the phi need to be converted to 8871 // reductions, with one operand being vector and the other being the scalar 8872 // reduction chain. 8873 void LoopVectorizationPlanner::adjustRecipesForInLoopReductions( 8874 VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder) { 8875 for (auto &Reduction : CM.getInLoopReductionChains()) { 8876 PHINode *Phi = Reduction.first; 8877 RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi]; 8878 const SmallVector<Instruction *, 4> &ReductionOperations = Reduction.second; 8879 8880 // ReductionOperations are orders top-down from the phi's use to the 8881 // LoopExitValue. We keep a track of the previous item (the Chain) to tell 8882 // which of the two operands will remain scalar and which will be reduced. 8883 // For minmax the chain will be the select instructions. 8884 Instruction *Chain = Phi; 8885 for (Instruction *R : ReductionOperations) { 8886 VPRecipeBase *WidenRecipe = RecipeBuilder.getRecipe(R); 8887 RecurKind Kind = RdxDesc.getRecurrenceKind(); 8888 8889 VPValue *ChainOp = Plan->getVPValue(Chain); 8890 unsigned FirstOpId; 8891 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) { 8892 assert(isa<VPWidenSelectRecipe>(WidenRecipe) && 8893 "Expected to replace a VPWidenSelectSC"); 8894 FirstOpId = 1; 8895 } else { 8896 assert(isa<VPWidenRecipe>(WidenRecipe) && 8897 "Expected to replace a VPWidenSC"); 8898 FirstOpId = 0; 8899 } 8900 unsigned VecOpId = 8901 R->getOperand(FirstOpId) == Chain ? FirstOpId + 1 : FirstOpId; 8902 VPValue *VecOp = Plan->getVPValue(R->getOperand(VecOpId)); 8903 8904 auto *CondOp = CM.foldTailByMasking() 8905 ? RecipeBuilder.createBlockInMask(R->getParent(), Plan) 8906 : nullptr; 8907 VPReductionRecipe *RedRecipe = new VPReductionRecipe( 8908 &RdxDesc, R, ChainOp, VecOp, CondOp, Legal->hasFunNoNaNAttr(), TTI); 8909 WidenRecipe->getVPValue()->replaceAllUsesWith(RedRecipe); 8910 Plan->removeVPValueFor(R); 8911 Plan->addVPValue(R, RedRecipe); 8912 WidenRecipe->getParent()->insert(RedRecipe, WidenRecipe->getIterator()); 8913 WidenRecipe->getVPValue()->replaceAllUsesWith(RedRecipe); 8914 WidenRecipe->eraseFromParent(); 8915 8916 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) { 8917 VPRecipeBase *CompareRecipe = 8918 RecipeBuilder.getRecipe(cast<Instruction>(R->getOperand(0))); 8919 assert(isa<VPWidenRecipe>(CompareRecipe) && 8920 "Expected to replace a VPWidenSC"); 8921 assert(cast<VPWidenRecipe>(CompareRecipe)->getNumUsers() == 0 && 8922 "Expected no remaining users"); 8923 CompareRecipe->eraseFromParent(); 8924 } 8925 Chain = R; 8926 } 8927 } 8928 } 8929 8930 Value* LoopVectorizationPlanner::VPCallbackILV:: 8931 getOrCreateVectorValues(Value *V, unsigned Part) { 8932 return ILV.getOrCreateVectorValue(V, Part); 8933 } 8934 8935 Value *LoopVectorizationPlanner::VPCallbackILV::getOrCreateScalarValue( 8936 Value *V, const VPIteration &Instance) { 8937 return ILV.getOrCreateScalarValue(V, Instance); 8938 } 8939 8940 void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent, 8941 VPSlotTracker &SlotTracker) const { 8942 O << "\"INTERLEAVE-GROUP with factor " << IG->getFactor() << " at "; 8943 IG->getInsertPos()->printAsOperand(O, false); 8944 O << ", "; 8945 getAddr()->printAsOperand(O, SlotTracker); 8946 VPValue *Mask = getMask(); 8947 if (Mask) { 8948 O << ", "; 8949 Mask->printAsOperand(O, SlotTracker); 8950 } 8951 for (unsigned i = 0; i < IG->getFactor(); ++i) 8952 if (Instruction *I = IG->getMember(i)) 8953 O << "\\l\" +\n" << Indent << "\" " << VPlanIngredient(I) << " " << i; 8954 } 8955 8956 void VPWidenCallRecipe::execute(VPTransformState &State) { 8957 State.ILV->widenCallInstruction(*cast<CallInst>(getUnderlyingInstr()), this, 8958 *this, State); 8959 } 8960 8961 void VPWidenSelectRecipe::execute(VPTransformState &State) { 8962 State.ILV->widenSelectInstruction(*cast<SelectInst>(getUnderlyingInstr()), 8963 this, *this, InvariantCond, State); 8964 } 8965 8966 void VPWidenRecipe::execute(VPTransformState &State) { 8967 State.ILV->widenInstruction(*getUnderlyingInstr(), this, *this, State); 8968 } 8969 8970 void VPWidenGEPRecipe::execute(VPTransformState &State) { 8971 State.ILV->widenGEP(cast<GetElementPtrInst>(getUnderlyingInstr()), this, 8972 *this, State.UF, State.VF, IsPtrLoopInvariant, 8973 IsIndexLoopInvariant, State); 8974 } 8975 8976 void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { 8977 assert(!State.Instance && "Int or FP induction being replicated."); 8978 State.ILV->widenIntOrFpInduction(IV, getStartValue()->getLiveInIRValue(), 8979 Trunc); 8980 } 8981 8982 void VPWidenPHIRecipe::execute(VPTransformState &State) { 8983 Value *StartV = 8984 getStartValue() ? getStartValue()->getLiveInIRValue() : nullptr; 8985 State.ILV->widenPHIInstruction(Phi, RdxDesc, StartV, State.UF, State.VF); 8986 } 8987 8988 void VPBlendRecipe::execute(VPTransformState &State) { 8989 State.ILV->setDebugLocFromInst(State.Builder, Phi); 8990 // We know that all PHIs in non-header blocks are converted into 8991 // selects, so we don't have to worry about the insertion order and we 8992 // can just use the builder. 8993 // At this point we generate the predication tree. There may be 8994 // duplications since this is a simple recursive scan, but future 8995 // optimizations will clean it up. 8996 8997 unsigned NumIncoming = getNumIncomingValues(); 8998 8999 // Generate a sequence of selects of the form: 9000 // SELECT(Mask3, In3, 9001 // SELECT(Mask2, In2, 9002 // SELECT(Mask1, In1, 9003 // In0))) 9004 // Note that Mask0 is never used: lanes for which no path reaches this phi and 9005 // are essentially undef are taken from In0. 9006 InnerLoopVectorizer::VectorParts Entry(State.UF); 9007 for (unsigned In = 0; In < NumIncoming; ++In) { 9008 for (unsigned Part = 0; Part < State.UF; ++Part) { 9009 // We might have single edge PHIs (blocks) - use an identity 9010 // 'select' for the first PHI operand. 9011 Value *In0 = State.get(getIncomingValue(In), Part); 9012 if (In == 0) 9013 Entry[Part] = In0; // Initialize with the first incoming value. 9014 else { 9015 // Select between the current value and the previous incoming edge 9016 // based on the incoming mask. 9017 Value *Cond = State.get(getMask(In), Part); 9018 Entry[Part] = 9019 State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi"); 9020 } 9021 } 9022 } 9023 for (unsigned Part = 0; Part < State.UF; ++Part) 9024 State.ValueMap.setVectorValue(Phi, Part, Entry[Part]); 9025 } 9026 9027 void VPInterleaveRecipe::execute(VPTransformState &State) { 9028 assert(!State.Instance && "Interleave group being replicated."); 9029 State.ILV->vectorizeInterleaveGroup(IG, definedValues(), State, getAddr(), 9030 getStoredValues(), getMask()); 9031 } 9032 9033 void VPReductionRecipe::execute(VPTransformState &State) { 9034 assert(!State.Instance && "Reduction being replicated."); 9035 for (unsigned Part = 0; Part < State.UF; ++Part) { 9036 RecurKind Kind = RdxDesc->getRecurrenceKind(); 9037 Value *NewVecOp = State.get(getVecOp(), Part); 9038 if (VPValue *Cond = getCondOp()) { 9039 Value *NewCond = State.get(Cond, Part); 9040 VectorType *VecTy = cast<VectorType>(NewVecOp->getType()); 9041 Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity( 9042 Kind, VecTy->getElementType()); 9043 Constant *IdenVec = 9044 ConstantVector::getSplat(VecTy->getElementCount(), Iden); 9045 Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, IdenVec); 9046 NewVecOp = Select; 9047 } 9048 Value *NewRed = 9049 createTargetReduction(State.Builder, TTI, *RdxDesc, NewVecOp); 9050 Value *PrevInChain = State.get(getChainOp(), Part); 9051 Value *NextInChain; 9052 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) { 9053 NextInChain = 9054 createMinMaxOp(State.Builder, RdxDesc->getRecurrenceKind(), 9055 NewRed, PrevInChain); 9056 } else { 9057 NextInChain = State.Builder.CreateBinOp( 9058 (Instruction::BinaryOps)getUnderlyingInstr()->getOpcode(), NewRed, 9059 PrevInChain); 9060 } 9061 State.set(this, getUnderlyingInstr(), NextInChain, Part); 9062 } 9063 } 9064 9065 void VPReplicateRecipe::execute(VPTransformState &State) { 9066 if (State.Instance) { // Generate a single instance. 9067 assert(!State.VF.isScalable() && "Can't scalarize a scalable vector"); 9068 State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this, 9069 *State.Instance, IsPredicated, State); 9070 // Insert scalar instance packing it into a vector. 9071 if (AlsoPack && State.VF.isVector()) { 9072 // If we're constructing lane 0, initialize to start from poison. 9073 if (State.Instance->Lane == 0) { 9074 assert(!State.VF.isScalable() && "VF is assumed to be non scalable."); 9075 Value *Poison = PoisonValue::get( 9076 VectorType::get(getUnderlyingValue()->getType(), State.VF)); 9077 State.ValueMap.setVectorValue(getUnderlyingInstr(), 9078 State.Instance->Part, Poison); 9079 } 9080 State.ILV->packScalarIntoVectorValue(getUnderlyingInstr(), 9081 *State.Instance); 9082 } 9083 return; 9084 } 9085 9086 // Generate scalar instances for all VF lanes of all UF parts, unless the 9087 // instruction is uniform inwhich case generate only the first lane for each 9088 // of the UF parts. 9089 unsigned EndLane = IsUniform ? 1 : State.VF.getKnownMinValue(); 9090 assert((!State.VF.isScalable() || IsUniform) && 9091 "Can't scalarize a scalable vector"); 9092 for (unsigned Part = 0; Part < State.UF; ++Part) 9093 for (unsigned Lane = 0; Lane < EndLane; ++Lane) 9094 State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this, {Part, Lane}, 9095 IsPredicated, State); 9096 } 9097 9098 void VPBranchOnMaskRecipe::execute(VPTransformState &State) { 9099 assert(State.Instance && "Branch on Mask works only on single instance."); 9100 9101 unsigned Part = State.Instance->Part; 9102 unsigned Lane = State.Instance->Lane; 9103 9104 Value *ConditionBit = nullptr; 9105 VPValue *BlockInMask = getMask(); 9106 if (BlockInMask) { 9107 ConditionBit = State.get(BlockInMask, Part); 9108 if (ConditionBit->getType()->isVectorTy()) 9109 ConditionBit = State.Builder.CreateExtractElement( 9110 ConditionBit, State.Builder.getInt32(Lane)); 9111 } else // Block in mask is all-one. 9112 ConditionBit = State.Builder.getTrue(); 9113 9114 // Replace the temporary unreachable terminator with a new conditional branch, 9115 // whose two destinations will be set later when they are created. 9116 auto *CurrentTerminator = State.CFG.PrevBB->getTerminator(); 9117 assert(isa<UnreachableInst>(CurrentTerminator) && 9118 "Expected to replace unreachable terminator with conditional branch."); 9119 auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit); 9120 CondBr->setSuccessor(0, nullptr); 9121 ReplaceInstWithInst(CurrentTerminator, CondBr); 9122 } 9123 9124 void VPPredInstPHIRecipe::execute(VPTransformState &State) { 9125 assert(State.Instance && "Predicated instruction PHI works per instance."); 9126 Instruction *ScalarPredInst = 9127 cast<Instruction>(State.get(getOperand(0), *State.Instance)); 9128 BasicBlock *PredicatedBB = ScalarPredInst->getParent(); 9129 BasicBlock *PredicatingBB = PredicatedBB->getSinglePredecessor(); 9130 assert(PredicatingBB && "Predicated block has no single predecessor."); 9131 9132 // By current pack/unpack logic we need to generate only a single phi node: if 9133 // a vector value for the predicated instruction exists at this point it means 9134 // the instruction has vector users only, and a phi for the vector value is 9135 // needed. In this case the recipe of the predicated instruction is marked to 9136 // also do that packing, thereby "hoisting" the insert-element sequence. 9137 // Otherwise, a phi node for the scalar value is needed. 9138 unsigned Part = State.Instance->Part; 9139 Instruction *PredInst = 9140 cast<Instruction>(getOperand(0)->getUnderlyingValue()); 9141 if (State.ValueMap.hasVectorValue(PredInst, Part)) { 9142 Value *VectorValue = State.ValueMap.getVectorValue(PredInst, Part); 9143 InsertElementInst *IEI = cast<InsertElementInst>(VectorValue); 9144 PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2); 9145 VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector. 9146 VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element. 9147 State.ValueMap.resetVectorValue(PredInst, Part, VPhi); // Update cache. 9148 } else { 9149 Type *PredInstType = PredInst->getType(); 9150 PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2); 9151 Phi->addIncoming(PoisonValue::get(ScalarPredInst->getType()), PredicatingBB); 9152 Phi->addIncoming(ScalarPredInst, PredicatedBB); 9153 State.ValueMap.resetScalarValue(PredInst, *State.Instance, Phi); 9154 } 9155 } 9156 9157 void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) { 9158 VPValue *StoredValue = isStore() ? getStoredValue() : nullptr; 9159 State.ILV->vectorizeMemoryInstruction(&Ingredient, State, 9160 StoredValue ? nullptr : getVPValue(), 9161 getAddr(), StoredValue, getMask()); 9162 } 9163 9164 // Determine how to lower the scalar epilogue, which depends on 1) optimising 9165 // for minimum code-size, 2) predicate compiler options, 3) loop hints forcing 9166 // predication, and 4) a TTI hook that analyses whether the loop is suitable 9167 // for predication. 9168 static ScalarEpilogueLowering getScalarEpilogueLowering( 9169 Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI, 9170 BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, 9171 AssumptionCache *AC, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, 9172 LoopVectorizationLegality &LVL) { 9173 // 1) OptSize takes precedence over all other options, i.e. if this is set, 9174 // don't look at hints or options, and don't request a scalar epilogue. 9175 // (For PGSO, as shouldOptimizeForSize isn't currently accessible from 9176 // LoopAccessInfo (due to code dependency and not being able to reliably get 9177 // PSI/BFI from a loop analysis under NPM), we cannot suppress the collection 9178 // of strides in LoopAccessInfo::analyzeLoop() and vectorize without 9179 // versioning when the vectorization is forced, unlike hasOptSize. So revert 9180 // back to the old way and vectorize with versioning when forced. See D81345.) 9181 if (F->hasOptSize() || (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI, 9182 PGSOQueryType::IRPass) && 9183 Hints.getForce() != LoopVectorizeHints::FK_Enabled)) 9184 return CM_ScalarEpilogueNotAllowedOptSize; 9185 9186 // 2) If set, obey the directives 9187 if (PreferPredicateOverEpilogue.getNumOccurrences()) { 9188 switch (PreferPredicateOverEpilogue) { 9189 case PreferPredicateTy::ScalarEpilogue: 9190 return CM_ScalarEpilogueAllowed; 9191 case PreferPredicateTy::PredicateElseScalarEpilogue: 9192 return CM_ScalarEpilogueNotNeededUsePredicate; 9193 case PreferPredicateTy::PredicateOrDontVectorize: 9194 return CM_ScalarEpilogueNotAllowedUsePredicate; 9195 }; 9196 } 9197 9198 // 3) If set, obey the hints 9199 switch (Hints.getPredicate()) { 9200 case LoopVectorizeHints::FK_Enabled: 9201 return CM_ScalarEpilogueNotNeededUsePredicate; 9202 case LoopVectorizeHints::FK_Disabled: 9203 return CM_ScalarEpilogueAllowed; 9204 }; 9205 9206 // 4) if the TTI hook indicates this is profitable, request predication. 9207 if (TTI->preferPredicateOverEpilogue(L, LI, *SE, *AC, TLI, DT, 9208 LVL.getLAI())) 9209 return CM_ScalarEpilogueNotNeededUsePredicate; 9210 9211 return CM_ScalarEpilogueAllowed; 9212 } 9213 9214 void VPTransformState::set(VPValue *Def, Value *IRDef, Value *V, 9215 unsigned Part) { 9216 set(Def, V, Part); 9217 ILV->setVectorValue(IRDef, Part, V); 9218 } 9219 9220 // Process the loop in the VPlan-native vectorization path. This path builds 9221 // VPlan upfront in the vectorization pipeline, which allows to apply 9222 // VPlan-to-VPlan transformations from the very beginning without modifying the 9223 // input LLVM IR. 9224 static bool processLoopInVPlanNativePath( 9225 Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, 9226 LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, 9227 TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, 9228 OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI, 9229 ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints) { 9230 9231 if (isa<SCEVCouldNotCompute>(PSE.getBackedgeTakenCount())) { 9232 LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n"); 9233 return false; 9234 } 9235 assert(EnableVPlanNativePath && "VPlan-native path is disabled."); 9236 Function *F = L->getHeader()->getParent(); 9237 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL->getLAI()); 9238 9239 ScalarEpilogueLowering SEL = getScalarEpilogueLowering( 9240 F, L, Hints, PSI, BFI, TTI, TLI, AC, LI, PSE.getSE(), DT, *LVL); 9241 9242 LoopVectorizationCostModel CM(SEL, L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE, F, 9243 &Hints, IAI); 9244 // Use the planner for outer loop vectorization. 9245 // TODO: CM is not used at this point inside the planner. Turn CM into an 9246 // optional argument if we don't need it in the future. 9247 LoopVectorizationPlanner LVP(L, LI, TLI, TTI, LVL, CM, IAI, PSE); 9248 9249 // Get user vectorization factor. 9250 ElementCount UserVF = Hints.getWidth(); 9251 9252 // Plan how to best vectorize, return the best VF and its cost. 9253 const VectorizationFactor VF = LVP.planInVPlanNativePath(UserVF); 9254 9255 // If we are stress testing VPlan builds, do not attempt to generate vector 9256 // code. Masked vector code generation support will follow soon. 9257 // Also, do not attempt to vectorize if no vector code will be produced. 9258 if (VPlanBuildStressTest || EnableVPlanPredication || 9259 VectorizationFactor::Disabled() == VF) 9260 return false; 9261 9262 LVP.setBestPlan(VF.Width, 1); 9263 9264 InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width, 1, LVL, 9265 &CM, BFI, PSI); 9266 LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" 9267 << L->getHeader()->getParent()->getName() << "\"\n"); 9268 LVP.executePlan(LB, DT); 9269 9270 // Mark the loop as already vectorized to avoid vectorizing again. 9271 Hints.setAlreadyVectorized(); 9272 9273 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs())); 9274 return true; 9275 } 9276 9277 LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts) 9278 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced || 9279 !EnableLoopInterleaving), 9280 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced || 9281 !EnableLoopVectorization) {} 9282 9283 bool LoopVectorizePass::processLoop(Loop *L) { 9284 assert((EnableVPlanNativePath || L->isInnermost()) && 9285 "VPlan-native path is not enabled. Only process inner loops."); 9286 9287 #ifndef NDEBUG 9288 const std::string DebugLocStr = getDebugLocString(L); 9289 #endif /* NDEBUG */ 9290 9291 LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in \"" 9292 << L->getHeader()->getParent()->getName() << "\" from " 9293 << DebugLocStr << "\n"); 9294 9295 LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE); 9296 9297 LLVM_DEBUG( 9298 dbgs() << "LV: Loop hints:" 9299 << " force=" 9300 << (Hints.getForce() == LoopVectorizeHints::FK_Disabled 9301 ? "disabled" 9302 : (Hints.getForce() == LoopVectorizeHints::FK_Enabled 9303 ? "enabled" 9304 : "?")) 9305 << " width=" << Hints.getWidth() 9306 << " unroll=" << Hints.getInterleave() << "\n"); 9307 9308 // Function containing loop 9309 Function *F = L->getHeader()->getParent(); 9310 9311 // Looking at the diagnostic output is the only way to determine if a loop 9312 // was vectorized (other than looking at the IR or machine code), so it 9313 // is important to generate an optimization remark for each loop. Most of 9314 // these messages are generated as OptimizationRemarkAnalysis. Remarks 9315 // generated as OptimizationRemark and OptimizationRemarkMissed are 9316 // less verbose reporting vectorized loops and unvectorized loops that may 9317 // benefit from vectorization, respectively. 9318 9319 if (!Hints.allowVectorization(F, L, VectorizeOnlyWhenForced)) { 9320 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n"); 9321 return false; 9322 } 9323 9324 PredicatedScalarEvolution PSE(*SE, *L); 9325 9326 // Check if it is legal to vectorize the loop. 9327 LoopVectorizationRequirements Requirements(*ORE); 9328 LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, AA, F, GetLAA, LI, ORE, 9329 &Requirements, &Hints, DB, AC, BFI, PSI); 9330 if (!LVL.canVectorize(EnableVPlanNativePath)) { 9331 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n"); 9332 Hints.emitRemarkWithHints(); 9333 return false; 9334 } 9335 9336 // Check the function attributes and profiles to find out if this function 9337 // should be optimized for size. 9338 ScalarEpilogueLowering SEL = getScalarEpilogueLowering( 9339 F, L, Hints, PSI, BFI, TTI, TLI, AC, LI, PSE.getSE(), DT, LVL); 9340 9341 // Entrance to the VPlan-native vectorization path. Outer loops are processed 9342 // here. They may require CFG and instruction level transformations before 9343 // even evaluating whether vectorization is profitable. Since we cannot modify 9344 // the incoming IR, we need to build VPlan upfront in the vectorization 9345 // pipeline. 9346 if (!L->isInnermost()) 9347 return processLoopInVPlanNativePath(L, PSE, LI, DT, &LVL, TTI, TLI, DB, AC, 9348 ORE, BFI, PSI, Hints); 9349 9350 assert(L->isInnermost() && "Inner loop expected."); 9351 9352 // Check the loop for a trip count threshold: vectorize loops with a tiny trip 9353 // count by optimizing for size, to minimize overheads. 9354 auto ExpectedTC = getSmallBestKnownTC(*SE, L); 9355 if (ExpectedTC && *ExpectedTC < TinyTripCountVectorThreshold) { 9356 LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. " 9357 << "This loop is worth vectorizing only if no scalar " 9358 << "iteration overheads are incurred."); 9359 if (Hints.getForce() == LoopVectorizeHints::FK_Enabled) 9360 LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n"); 9361 else { 9362 LLVM_DEBUG(dbgs() << "\n"); 9363 SEL = CM_ScalarEpilogueNotAllowedLowTripLoop; 9364 } 9365 } 9366 9367 // Check the function attributes to see if implicit floats are allowed. 9368 // FIXME: This check doesn't seem possibly correct -- what if the loop is 9369 // an integer loop and the vector instructions selected are purely integer 9370 // vector instructions? 9371 if (F->hasFnAttribute(Attribute::NoImplicitFloat)) { 9372 reportVectorizationFailure( 9373 "Can't vectorize when the NoImplicitFloat attribute is used", 9374 "loop not vectorized due to NoImplicitFloat attribute", 9375 "NoImplicitFloat", ORE, L); 9376 Hints.emitRemarkWithHints(); 9377 return false; 9378 } 9379 9380 // Check if the target supports potentially unsafe FP vectorization. 9381 // FIXME: Add a check for the type of safety issue (denormal, signaling) 9382 // for the target we're vectorizing for, to make sure none of the 9383 // additional fp-math flags can help. 9384 if (Hints.isPotentiallyUnsafe() && 9385 TTI->isFPVectorizationPotentiallyUnsafe()) { 9386 reportVectorizationFailure( 9387 "Potentially unsafe FP op prevents vectorization", 9388 "loop not vectorized due to unsafe FP support.", 9389 "UnsafeFP", ORE, L); 9390 Hints.emitRemarkWithHints(); 9391 return false; 9392 } 9393 9394 bool UseInterleaved = TTI->enableInterleavedAccessVectorization(); 9395 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI()); 9396 9397 // If an override option has been passed in for interleaved accesses, use it. 9398 if (EnableInterleavedMemAccesses.getNumOccurrences() > 0) 9399 UseInterleaved = EnableInterleavedMemAccesses; 9400 9401 // Analyze interleaved memory accesses. 9402 if (UseInterleaved) { 9403 IAI.analyzeInterleaving(useMaskedInterleavedAccesses(*TTI)); 9404 } 9405 9406 // Use the cost model. 9407 LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, DB, AC, ORE, 9408 F, &Hints, IAI); 9409 CM.collectValuesToIgnore(); 9410 9411 // Use the planner for vectorization. 9412 LoopVectorizationPlanner LVP(L, LI, TLI, TTI, &LVL, CM, IAI, PSE); 9413 9414 // Get user vectorization factor and interleave count. 9415 ElementCount UserVF = Hints.getWidth(); 9416 unsigned UserIC = Hints.getInterleave(); 9417 9418 // Plan how to best vectorize, return the best VF and its cost. 9419 Optional<VectorizationFactor> MaybeVF = LVP.plan(UserVF, UserIC); 9420 9421 VectorizationFactor VF = VectorizationFactor::Disabled(); 9422 unsigned IC = 1; 9423 9424 if (MaybeVF) { 9425 VF = *MaybeVF; 9426 // Select the interleave count. 9427 IC = CM.selectInterleaveCount(VF.Width, VF.Cost); 9428 } 9429 9430 // Identify the diagnostic messages that should be produced. 9431 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg; 9432 bool VectorizeLoop = true, InterleaveLoop = true; 9433 if (Requirements.doesNotMeet(F, L, Hints)) { 9434 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: loop did not meet vectorization " 9435 "requirements.\n"); 9436 Hints.emitRemarkWithHints(); 9437 return false; 9438 } 9439 9440 if (VF.Width.isScalar()) { 9441 LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n"); 9442 VecDiagMsg = std::make_pair( 9443 "VectorizationNotBeneficial", 9444 "the cost-model indicates that vectorization is not beneficial"); 9445 VectorizeLoop = false; 9446 } 9447 9448 if (!MaybeVF && UserIC > 1) { 9449 // Tell the user interleaving was avoided up-front, despite being explicitly 9450 // requested. 9451 LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and " 9452 "interleaving should be avoided up front\n"); 9453 IntDiagMsg = std::make_pair( 9454 "InterleavingAvoided", 9455 "Ignoring UserIC, because interleaving was avoided up front"); 9456 InterleaveLoop = false; 9457 } else if (IC == 1 && UserIC <= 1) { 9458 // Tell the user interleaving is not beneficial. 9459 LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n"); 9460 IntDiagMsg = std::make_pair( 9461 "InterleavingNotBeneficial", 9462 "the cost-model indicates that interleaving is not beneficial"); 9463 InterleaveLoop = false; 9464 if (UserIC == 1) { 9465 IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled"; 9466 IntDiagMsg.second += 9467 " and is explicitly disabled or interleave count is set to 1"; 9468 } 9469 } else if (IC > 1 && UserIC == 1) { 9470 // Tell the user interleaving is beneficial, but it explicitly disabled. 9471 LLVM_DEBUG( 9472 dbgs() << "LV: Interleaving is beneficial but is explicitly disabled."); 9473 IntDiagMsg = std::make_pair( 9474 "InterleavingBeneficialButDisabled", 9475 "the cost-model indicates that interleaving is beneficial " 9476 "but is explicitly disabled or interleave count is set to 1"); 9477 InterleaveLoop = false; 9478 } 9479 9480 // Override IC if user provided an interleave count. 9481 IC = UserIC > 0 ? UserIC : IC; 9482 9483 // Emit diagnostic messages, if any. 9484 const char *VAPassName = Hints.vectorizeAnalysisPassName(); 9485 if (!VectorizeLoop && !InterleaveLoop) { 9486 // Do not vectorize or interleaving the loop. 9487 ORE->emit([&]() { 9488 return OptimizationRemarkMissed(VAPassName, VecDiagMsg.first, 9489 L->getStartLoc(), L->getHeader()) 9490 << VecDiagMsg.second; 9491 }); 9492 ORE->emit([&]() { 9493 return OptimizationRemarkMissed(LV_NAME, IntDiagMsg.first, 9494 L->getStartLoc(), L->getHeader()) 9495 << IntDiagMsg.second; 9496 }); 9497 return false; 9498 } else if (!VectorizeLoop && InterleaveLoop) { 9499 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n'); 9500 ORE->emit([&]() { 9501 return OptimizationRemarkAnalysis(VAPassName, VecDiagMsg.first, 9502 L->getStartLoc(), L->getHeader()) 9503 << VecDiagMsg.second; 9504 }); 9505 } else if (VectorizeLoop && !InterleaveLoop) { 9506 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width 9507 << ") in " << DebugLocStr << '\n'); 9508 ORE->emit([&]() { 9509 return OptimizationRemarkAnalysis(LV_NAME, IntDiagMsg.first, 9510 L->getStartLoc(), L->getHeader()) 9511 << IntDiagMsg.second; 9512 }); 9513 } else if (VectorizeLoop && InterleaveLoop) { 9514 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width 9515 << ") in " << DebugLocStr << '\n'); 9516 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n'); 9517 } 9518 9519 LVP.setBestPlan(VF.Width, IC); 9520 9521 using namespace ore; 9522 bool DisableRuntimeUnroll = false; 9523 MDNode *OrigLoopID = L->getLoopID(); 9524 9525 if (!VectorizeLoop) { 9526 assert(IC > 1 && "interleave count should not be 1 or 0"); 9527 // If we decided that it is not legal to vectorize the loop, then 9528 // interleave it. 9529 InnerLoopUnroller Unroller(L, PSE, LI, DT, TLI, TTI, AC, ORE, IC, &LVL, &CM, 9530 BFI, PSI); 9531 LVP.executePlan(Unroller, DT); 9532 9533 ORE->emit([&]() { 9534 return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(), 9535 L->getHeader()) 9536 << "interleaved loop (interleaved count: " 9537 << NV("InterleaveCount", IC) << ")"; 9538 }); 9539 } else { 9540 // If we decided that it is *legal* to vectorize the loop, then do it. 9541 9542 // Consider vectorizing the epilogue too if it's profitable. 9543 VectorizationFactor EpilogueVF = 9544 CM.selectEpilogueVectorizationFactor(VF.Width, LVP); 9545 if (EpilogueVF.Width.isVector()) { 9546 9547 // The first pass vectorizes the main loop and creates a scalar epilogue 9548 // to be vectorized by executing the plan (potentially with a different 9549 // factor) again shortly afterwards. 9550 EpilogueLoopVectorizationInfo EPI(VF.Width.getKnownMinValue(), IC, 9551 EpilogueVF.Width.getKnownMinValue(), 1); 9552 EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TLI, TTI, AC, ORE, EPI, 9553 &LVL, &CM, BFI, PSI); 9554 9555 LVP.setBestPlan(EPI.MainLoopVF, EPI.MainLoopUF); 9556 LVP.executePlan(MainILV, DT); 9557 ++LoopsVectorized; 9558 9559 simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */); 9560 formLCSSARecursively(*L, *DT, LI, SE); 9561 9562 // Second pass vectorizes the epilogue and adjusts the control flow 9563 // edges from the first pass. 9564 LVP.setBestPlan(EPI.EpilogueVF, EPI.EpilogueUF); 9565 EPI.MainLoopVF = EPI.EpilogueVF; 9566 EPI.MainLoopUF = EPI.EpilogueUF; 9567 EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TLI, TTI, AC, 9568 ORE, EPI, &LVL, &CM, BFI, PSI); 9569 LVP.executePlan(EpilogILV, DT); 9570 ++LoopsEpilogueVectorized; 9571 9572 if (!MainILV.areSafetyChecksAdded()) 9573 DisableRuntimeUnroll = true; 9574 } else { 9575 InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width, IC, 9576 &LVL, &CM, BFI, PSI); 9577 LVP.executePlan(LB, DT); 9578 ++LoopsVectorized; 9579 9580 // Add metadata to disable runtime unrolling a scalar loop when there are 9581 // no runtime checks about strides and memory. A scalar loop that is 9582 // rarely used is not worth unrolling. 9583 if (!LB.areSafetyChecksAdded()) 9584 DisableRuntimeUnroll = true; 9585 } 9586 9587 // Report the vectorization decision. 9588 ORE->emit([&]() { 9589 return OptimizationRemark(LV_NAME, "Vectorized", L->getStartLoc(), 9590 L->getHeader()) 9591 << "vectorized loop (vectorization width: " 9592 << NV("VectorizationFactor", VF.Width) 9593 << ", interleaved count: " << NV("InterleaveCount", IC) << ")"; 9594 }); 9595 } 9596 9597 Optional<MDNode *> RemainderLoopID = 9598 makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll, 9599 LLVMLoopVectorizeFollowupEpilogue}); 9600 if (RemainderLoopID.hasValue()) { 9601 L->setLoopID(RemainderLoopID.getValue()); 9602 } else { 9603 if (DisableRuntimeUnroll) 9604 AddRuntimeUnrollDisableMetaData(L); 9605 9606 // Mark the loop as already vectorized to avoid vectorizing again. 9607 Hints.setAlreadyVectorized(); 9608 } 9609 9610 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs())); 9611 return true; 9612 } 9613 9614 LoopVectorizeResult LoopVectorizePass::runImpl( 9615 Function &F, ScalarEvolution &SE_, LoopInfo &LI_, TargetTransformInfo &TTI_, 9616 DominatorTree &DT_, BlockFrequencyInfo &BFI_, TargetLibraryInfo *TLI_, 9617 DemandedBits &DB_, AAResults &AA_, AssumptionCache &AC_, 9618 std::function<const LoopAccessInfo &(Loop &)> &GetLAA_, 9619 OptimizationRemarkEmitter &ORE_, ProfileSummaryInfo *PSI_) { 9620 SE = &SE_; 9621 LI = &LI_; 9622 TTI = &TTI_; 9623 DT = &DT_; 9624 BFI = &BFI_; 9625 TLI = TLI_; 9626 AA = &AA_; 9627 AC = &AC_; 9628 GetLAA = &GetLAA_; 9629 DB = &DB_; 9630 ORE = &ORE_; 9631 PSI = PSI_; 9632 9633 // Don't attempt if 9634 // 1. the target claims to have no vector registers, and 9635 // 2. interleaving won't help ILP. 9636 // 9637 // The second condition is necessary because, even if the target has no 9638 // vector registers, loop vectorization may still enable scalar 9639 // interleaving. 9640 if (!TTI->getNumberOfRegisters(TTI->getRegisterClassForType(true)) && 9641 TTI->getMaxInterleaveFactor(1) < 2) 9642 return LoopVectorizeResult(false, false); 9643 9644 bool Changed = false, CFGChanged = false; 9645 9646 // The vectorizer requires loops to be in simplified form. 9647 // Since simplification may add new inner loops, it has to run before the 9648 // legality and profitability checks. This means running the loop vectorizer 9649 // will simplify all loops, regardless of whether anything end up being 9650 // vectorized. 9651 for (auto &L : *LI) 9652 Changed |= CFGChanged |= 9653 simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */); 9654 9655 // Build up a worklist of inner-loops to vectorize. This is necessary as 9656 // the act of vectorizing or partially unrolling a loop creates new loops 9657 // and can invalidate iterators across the loops. 9658 SmallVector<Loop *, 8> Worklist; 9659 9660 for (Loop *L : *LI) 9661 collectSupportedLoops(*L, LI, ORE, Worklist); 9662 9663 LoopsAnalyzed += Worklist.size(); 9664 9665 // Now walk the identified inner loops. 9666 while (!Worklist.empty()) { 9667 Loop *L = Worklist.pop_back_val(); 9668 9669 // For the inner loops we actually process, form LCSSA to simplify the 9670 // transform. 9671 Changed |= formLCSSARecursively(*L, *DT, LI, SE); 9672 9673 Changed |= CFGChanged |= processLoop(L); 9674 } 9675 9676 // Process each loop nest in the function. 9677 return LoopVectorizeResult(Changed, CFGChanged); 9678 } 9679 9680 PreservedAnalyses LoopVectorizePass::run(Function &F, 9681 FunctionAnalysisManager &AM) { 9682 auto &SE = AM.getResult<ScalarEvolutionAnalysis>(F); 9683 auto &LI = AM.getResult<LoopAnalysis>(F); 9684 auto &TTI = AM.getResult<TargetIRAnalysis>(F); 9685 auto &DT = AM.getResult<DominatorTreeAnalysis>(F); 9686 auto &BFI = AM.getResult<BlockFrequencyAnalysis>(F); 9687 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F); 9688 auto &AA = AM.getResult<AAManager>(F); 9689 auto &AC = AM.getResult<AssumptionAnalysis>(F); 9690 auto &DB = AM.getResult<DemandedBitsAnalysis>(F); 9691 auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F); 9692 MemorySSA *MSSA = EnableMSSALoopDependency 9693 ? &AM.getResult<MemorySSAAnalysis>(F).getMSSA() 9694 : nullptr; 9695 9696 auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager(); 9697 std::function<const LoopAccessInfo &(Loop &)> GetLAA = 9698 [&](Loop &L) -> const LoopAccessInfo & { 9699 LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, 9700 TLI, TTI, nullptr, MSSA}; 9701 return LAM.getResult<LoopAccessAnalysis>(L, AR); 9702 }; 9703 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F); 9704 ProfileSummaryInfo *PSI = 9705 MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent()); 9706 LoopVectorizeResult Result = 9707 runImpl(F, SE, LI, TTI, DT, BFI, &TLI, DB, AA, AC, GetLAA, ORE, PSI); 9708 if (!Result.MadeAnyChange) 9709 return PreservedAnalyses::all(); 9710 PreservedAnalyses PA; 9711 9712 // We currently do not preserve loopinfo/dominator analyses with outer loop 9713 // vectorization. Until this is addressed, mark these analyses as preserved 9714 // only for non-VPlan-native path. 9715 // TODO: Preserve Loop and Dominator analyses for VPlan-native path. 9716 if (!EnableVPlanNativePath) { 9717 PA.preserve<LoopAnalysis>(); 9718 PA.preserve<DominatorTreeAnalysis>(); 9719 } 9720 PA.preserve<BasicAA>(); 9721 PA.preserve<GlobalsAA>(); 9722 if (!Result.MadeCFGChange) 9723 PA.preserveSet<CFGAnalyses>(); 9724 return PA; 9725 } 9726