1 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive 10 // stores that can be put together into vector-stores. Next, it attempts to 11 // construct vectorizable tree using the use-def chains. If a profitable tree 12 // was found, the SLP vectorizer performs vectorization on the tree. 13 // 14 // The pass is inspired by the work described in the paper: 15 // "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks. 16 // 17 //===----------------------------------------------------------------------===// 18 19 #include "llvm/Transforms/Vectorize/SLPVectorizer.h" 20 #include "llvm/ADT/ArrayRef.h" 21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/ADT/DenseSet.h" 23 #include "llvm/ADT/MapVector.h" 24 #include "llvm/ADT/None.h" 25 #include "llvm/ADT/Optional.h" 26 #include "llvm/ADT/PostOrderIterator.h" 27 #include "llvm/ADT/STLExtras.h" 28 #include "llvm/ADT/SetVector.h" 29 #include "llvm/ADT/SmallBitVector.h" 30 #include "llvm/ADT/SmallPtrSet.h" 31 #include "llvm/ADT/SmallSet.h" 32 #include "llvm/ADT/SmallVector.h" 33 #include "llvm/ADT/Statistic.h" 34 #include "llvm/ADT/iterator.h" 35 #include "llvm/ADT/iterator_range.h" 36 #include "llvm/Analysis/AliasAnalysis.h" 37 #include "llvm/Analysis/CodeMetrics.h" 38 #include "llvm/Analysis/DemandedBits.h" 39 #include "llvm/Analysis/GlobalsModRef.h" 40 #include "llvm/Analysis/LoopAccessAnalysis.h" 41 #include "llvm/Analysis/LoopInfo.h" 42 #include "llvm/Analysis/MemoryLocation.h" 43 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 44 #include "llvm/Analysis/ScalarEvolution.h" 45 #include "llvm/Analysis/ScalarEvolutionExpressions.h" 46 #include "llvm/Analysis/TargetLibraryInfo.h" 47 #include "llvm/Analysis/TargetTransformInfo.h" 48 #include "llvm/Analysis/ValueTracking.h" 49 #include "llvm/Analysis/VectorUtils.h" 50 #include "llvm/IR/Attributes.h" 51 #include "llvm/IR/BasicBlock.h" 52 #include "llvm/IR/Constant.h" 53 #include "llvm/IR/Constants.h" 54 #include "llvm/IR/DataLayout.h" 55 #include "llvm/IR/DebugLoc.h" 56 #include "llvm/IR/DerivedTypes.h" 57 #include "llvm/IR/Dominators.h" 58 #include "llvm/IR/Function.h" 59 #include "llvm/IR/IRBuilder.h" 60 #include "llvm/IR/InstrTypes.h" 61 #include "llvm/IR/Instruction.h" 62 #include "llvm/IR/Instructions.h" 63 #include "llvm/IR/IntrinsicInst.h" 64 #include "llvm/IR/Intrinsics.h" 65 #include "llvm/IR/Module.h" 66 #include "llvm/IR/NoFolder.h" 67 #include "llvm/IR/Operator.h" 68 #include "llvm/IR/PassManager.h" 69 #include "llvm/IR/PatternMatch.h" 70 #include "llvm/IR/Type.h" 71 #include "llvm/IR/Use.h" 72 #include "llvm/IR/User.h" 73 #include "llvm/IR/Value.h" 74 #include "llvm/IR/ValueHandle.h" 75 #include "llvm/IR/Verifier.h" 76 #include "llvm/InitializePasses.h" 77 #include "llvm/Pass.h" 78 #include "llvm/Support/Casting.h" 79 #include "llvm/Support/CommandLine.h" 80 #include "llvm/Support/Compiler.h" 81 #include "llvm/Support/DOTGraphTraits.h" 82 #include "llvm/Support/Debug.h" 83 #include "llvm/Support/ErrorHandling.h" 84 #include "llvm/Support/GraphWriter.h" 85 #include "llvm/Support/KnownBits.h" 86 #include "llvm/Support/MathExtras.h" 87 #include "llvm/Support/raw_ostream.h" 88 #include "llvm/Transforms/Utils/LoopUtils.h" 89 #include "llvm/Transforms/Vectorize.h" 90 #include <algorithm> 91 #include <cassert> 92 #include <cstdint> 93 #include <iterator> 94 #include <memory> 95 #include <set> 96 #include <string> 97 #include <tuple> 98 #include <utility> 99 #include <vector> 100 101 using namespace llvm; 102 using namespace llvm::PatternMatch; 103 using namespace slpvectorizer; 104 105 #define SV_NAME "slp-vectorizer" 106 #define DEBUG_TYPE "SLP" 107 108 STATISTIC(NumVectorInstructions, "Number of vector instructions generated"); 109 110 cl::opt<bool> 111 llvm::RunSLPVectorization("vectorize-slp", cl::init(false), cl::Hidden, 112 cl::desc("Run the SLP vectorization passes")); 113 114 static cl::opt<int> 115 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden, 116 cl::desc("Only vectorize if you gain more than this " 117 "number ")); 118 119 static cl::opt<bool> 120 ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden, 121 cl::desc("Attempt to vectorize horizontal reductions")); 122 123 static cl::opt<bool> ShouldStartVectorizeHorAtStore( 124 "slp-vectorize-hor-store", cl::init(false), cl::Hidden, 125 cl::desc( 126 "Attempt to vectorize horizontal reductions feeding into a store")); 127 128 static cl::opt<int> 129 MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden, 130 cl::desc("Attempt to vectorize for this register size in bits")); 131 132 static cl::opt<int> 133 MaxStoreLookup("slp-max-store-lookup", cl::init(32), cl::Hidden, 134 cl::desc("Maximum depth of the lookup for consecutive stores.")); 135 136 /// Limits the size of scheduling regions in a block. 137 /// It avoid long compile times for _very_ large blocks where vector 138 /// instructions are spread over a wide range. 139 /// This limit is way higher than needed by real-world functions. 140 static cl::opt<int> 141 ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden, 142 cl::desc("Limit the size of the SLP scheduling region per block")); 143 144 static cl::opt<int> MinVectorRegSizeOption( 145 "slp-min-reg-size", cl::init(128), cl::Hidden, 146 cl::desc("Attempt to vectorize for this register size in bits")); 147 148 static cl::opt<unsigned> RecursionMaxDepth( 149 "slp-recursion-max-depth", cl::init(12), cl::Hidden, 150 cl::desc("Limit the recursion depth when building a vectorizable tree")); 151 152 static cl::opt<unsigned> MinTreeSize( 153 "slp-min-tree-size", cl::init(3), cl::Hidden, 154 cl::desc("Only vectorize small trees if they are fully vectorizable")); 155 156 // The maximum depth that the look-ahead score heuristic will explore. 157 // The higher this value, the higher the compilation time overhead. 158 static cl::opt<int> LookAheadMaxDepth( 159 "slp-max-look-ahead-depth", cl::init(2), cl::Hidden, 160 cl::desc("The maximum look-ahead depth for operand reordering scores")); 161 162 // The Look-ahead heuristic goes through the users of the bundle to calculate 163 // the users cost in getExternalUsesCost(). To avoid compilation time increase 164 // we limit the number of users visited to this value. 165 static cl::opt<unsigned> LookAheadUsersBudget( 166 "slp-look-ahead-users-budget", cl::init(2), cl::Hidden, 167 cl::desc("The maximum number of users to visit while visiting the " 168 "predecessors. This prevents compilation time increase.")); 169 170 static cl::opt<bool> 171 ViewSLPTree("view-slp-tree", cl::Hidden, 172 cl::desc("Display the SLP trees with Graphviz")); 173 174 // Limit the number of alias checks. The limit is chosen so that 175 // it has no negative effect on the llvm benchmarks. 176 static const unsigned AliasedCheckLimit = 10; 177 178 // Another limit for the alias checks: The maximum distance between load/store 179 // instructions where alias checks are done. 180 // This limit is useful for very large basic blocks. 181 static const unsigned MaxMemDepDistance = 160; 182 183 /// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling 184 /// regions to be handled. 185 static const int MinScheduleRegionSize = 16; 186 187 /// Predicate for the element types that the SLP vectorizer supports. 188 /// 189 /// The most important thing to filter here are types which are invalid in LLVM 190 /// vectors. We also filter target specific types which have absolutely no 191 /// meaningful vectorization path such as x86_fp80 and ppc_f128. This just 192 /// avoids spending time checking the cost model and realizing that they will 193 /// be inevitably scalarized. 194 static bool isValidElementType(Type *Ty) { 195 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() && 196 !Ty->isPPC_FP128Ty(); 197 } 198 199 /// \returns true if all of the instructions in \p VL are in the same block or 200 /// false otherwise. 201 static bool allSameBlock(ArrayRef<Value *> VL) { 202 Instruction *I0 = dyn_cast<Instruction>(VL[0]); 203 if (!I0) 204 return false; 205 BasicBlock *BB = I0->getParent(); 206 for (int i = 1, e = VL.size(); i < e; i++) { 207 Instruction *I = dyn_cast<Instruction>(VL[i]); 208 if (!I) 209 return false; 210 211 if (BB != I->getParent()) 212 return false; 213 } 214 return true; 215 } 216 217 /// \returns True if all of the values in \p VL are constants (but not 218 /// globals/constant expressions). 219 static bool allConstant(ArrayRef<Value *> VL) { 220 // Constant expressions and globals can't be vectorized like normal integer/FP 221 // constants. 222 for (Value *i : VL) 223 if (!isa<Constant>(i) || isa<ConstantExpr>(i) || isa<GlobalValue>(i)) 224 return false; 225 return true; 226 } 227 228 /// \returns True if all of the values in \p VL are identical. 229 static bool isSplat(ArrayRef<Value *> VL) { 230 for (unsigned i = 1, e = VL.size(); i < e; ++i) 231 if (VL[i] != VL[0]) 232 return false; 233 return true; 234 } 235 236 /// \returns True if \p I is commutative, handles CmpInst as well as Instruction. 237 static bool isCommutative(Instruction *I) { 238 if (auto *IC = dyn_cast<CmpInst>(I)) 239 return IC->isCommutative(); 240 return I->isCommutative(); 241 } 242 243 /// Checks if the vector of instructions can be represented as a shuffle, like: 244 /// %x0 = extractelement <4 x i8> %x, i32 0 245 /// %x3 = extractelement <4 x i8> %x, i32 3 246 /// %y1 = extractelement <4 x i8> %y, i32 1 247 /// %y2 = extractelement <4 x i8> %y, i32 2 248 /// %x0x0 = mul i8 %x0, %x0 249 /// %x3x3 = mul i8 %x3, %x3 250 /// %y1y1 = mul i8 %y1, %y1 251 /// %y2y2 = mul i8 %y2, %y2 252 /// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0 253 /// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1 254 /// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2 255 /// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3 256 /// ret <4 x i8> %ins4 257 /// can be transformed into: 258 /// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5, 259 /// i32 6> 260 /// %2 = mul <4 x i8> %1, %1 261 /// ret <4 x i8> %2 262 /// We convert this initially to something like: 263 /// %x0 = extractelement <4 x i8> %x, i32 0 264 /// %x3 = extractelement <4 x i8> %x, i32 3 265 /// %y1 = extractelement <4 x i8> %y, i32 1 266 /// %y2 = extractelement <4 x i8> %y, i32 2 267 /// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0 268 /// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1 269 /// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2 270 /// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3 271 /// %5 = mul <4 x i8> %4, %4 272 /// %6 = extractelement <4 x i8> %5, i32 0 273 /// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0 274 /// %7 = extractelement <4 x i8> %5, i32 1 275 /// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1 276 /// %8 = extractelement <4 x i8> %5, i32 2 277 /// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2 278 /// %9 = extractelement <4 x i8> %5, i32 3 279 /// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3 280 /// ret <4 x i8> %ins4 281 /// InstCombiner transforms this into a shuffle and vector mul 282 /// TODO: Can we split off and reuse the shuffle mask detection from 283 /// TargetTransformInfo::getInstructionThroughput? 284 static Optional<TargetTransformInfo::ShuffleKind> 285 isShuffle(ArrayRef<Value *> VL) { 286 auto *EI0 = cast<ExtractElementInst>(VL[0]); 287 unsigned Size = EI0->getVectorOperandType()->getVectorNumElements(); 288 Value *Vec1 = nullptr; 289 Value *Vec2 = nullptr; 290 enum ShuffleMode { Unknown, Select, Permute }; 291 ShuffleMode CommonShuffleMode = Unknown; 292 for (unsigned I = 0, E = VL.size(); I < E; ++I) { 293 auto *EI = cast<ExtractElementInst>(VL[I]); 294 auto *Vec = EI->getVectorOperand(); 295 // All vector operands must have the same number of vector elements. 296 if (Vec->getType()->getVectorNumElements() != Size) 297 return None; 298 auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand()); 299 if (!Idx) 300 return None; 301 // Undefined behavior if Idx is negative or >= Size. 302 if (Idx->getValue().uge(Size)) 303 continue; 304 unsigned IntIdx = Idx->getValue().getZExtValue(); 305 // We can extractelement from undef vector. 306 if (isa<UndefValue>(Vec)) 307 continue; 308 // For correct shuffling we have to have at most 2 different vector operands 309 // in all extractelement instructions. 310 if (!Vec1 || Vec1 == Vec) 311 Vec1 = Vec; 312 else if (!Vec2 || Vec2 == Vec) 313 Vec2 = Vec; 314 else 315 return None; 316 if (CommonShuffleMode == Permute) 317 continue; 318 // If the extract index is not the same as the operation number, it is a 319 // permutation. 320 if (IntIdx != I) { 321 CommonShuffleMode = Permute; 322 continue; 323 } 324 CommonShuffleMode = Select; 325 } 326 // If we're not crossing lanes in different vectors, consider it as blending. 327 if (CommonShuffleMode == Select && Vec2) 328 return TargetTransformInfo::SK_Select; 329 // If Vec2 was never used, we have a permutation of a single vector, otherwise 330 // we have permutation of 2 vectors. 331 return Vec2 ? TargetTransformInfo::SK_PermuteTwoSrc 332 : TargetTransformInfo::SK_PermuteSingleSrc; 333 } 334 335 namespace { 336 337 /// Main data required for vectorization of instructions. 338 struct InstructionsState { 339 /// The very first instruction in the list with the main opcode. 340 Value *OpValue = nullptr; 341 342 /// The main/alternate instruction. 343 Instruction *MainOp = nullptr; 344 Instruction *AltOp = nullptr; 345 346 /// The main/alternate opcodes for the list of instructions. 347 unsigned getOpcode() const { 348 return MainOp ? MainOp->getOpcode() : 0; 349 } 350 351 unsigned getAltOpcode() const { 352 return AltOp ? AltOp->getOpcode() : 0; 353 } 354 355 /// Some of the instructions in the list have alternate opcodes. 356 bool isAltShuffle() const { return getOpcode() != getAltOpcode(); } 357 358 bool isOpcodeOrAlt(Instruction *I) const { 359 unsigned CheckedOpcode = I->getOpcode(); 360 return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode; 361 } 362 363 InstructionsState() = delete; 364 InstructionsState(Value *OpValue, Instruction *MainOp, Instruction *AltOp) 365 : OpValue(OpValue), MainOp(MainOp), AltOp(AltOp) {} 366 }; 367 368 } // end anonymous namespace 369 370 /// Chooses the correct key for scheduling data. If \p Op has the same (or 371 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p 372 /// OpValue. 373 static Value *isOneOf(const InstructionsState &S, Value *Op) { 374 auto *I = dyn_cast<Instruction>(Op); 375 if (I && S.isOpcodeOrAlt(I)) 376 return Op; 377 return S.OpValue; 378 } 379 380 /// \returns true if \p Opcode is allowed as part of of the main/alternate 381 /// instruction for SLP vectorization. 382 /// 383 /// Example of unsupported opcode is SDIV that can potentially cause UB if the 384 /// "shuffled out" lane would result in division by zero. 385 static bool isValidForAlternation(unsigned Opcode) { 386 if (Instruction::isIntDivRem(Opcode)) 387 return false; 388 389 return true; 390 } 391 392 /// \returns analysis of the Instructions in \p VL described in 393 /// InstructionsState, the Opcode that we suppose the whole list 394 /// could be vectorized even if its structure is diverse. 395 static InstructionsState getSameOpcode(ArrayRef<Value *> VL, 396 unsigned BaseIndex = 0) { 397 // Make sure these are all Instructions. 398 if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); })) 399 return InstructionsState(VL[BaseIndex], nullptr, nullptr); 400 401 bool IsCastOp = isa<CastInst>(VL[BaseIndex]); 402 bool IsBinOp = isa<BinaryOperator>(VL[BaseIndex]); 403 unsigned Opcode = cast<Instruction>(VL[BaseIndex])->getOpcode(); 404 unsigned AltOpcode = Opcode; 405 unsigned AltIndex = BaseIndex; 406 407 // Check for one alternate opcode from another BinaryOperator. 408 // TODO - generalize to support all operators (types, calls etc.). 409 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) { 410 unsigned InstOpcode = cast<Instruction>(VL[Cnt])->getOpcode(); 411 if (IsBinOp && isa<BinaryOperator>(VL[Cnt])) { 412 if (InstOpcode == Opcode || InstOpcode == AltOpcode) 413 continue; 414 if (Opcode == AltOpcode && isValidForAlternation(InstOpcode) && 415 isValidForAlternation(Opcode)) { 416 AltOpcode = InstOpcode; 417 AltIndex = Cnt; 418 continue; 419 } 420 } else if (IsCastOp && isa<CastInst>(VL[Cnt])) { 421 Type *Ty0 = cast<Instruction>(VL[BaseIndex])->getOperand(0)->getType(); 422 Type *Ty1 = cast<Instruction>(VL[Cnt])->getOperand(0)->getType(); 423 if (Ty0 == Ty1) { 424 if (InstOpcode == Opcode || InstOpcode == AltOpcode) 425 continue; 426 if (Opcode == AltOpcode) { 427 assert(isValidForAlternation(Opcode) && 428 isValidForAlternation(InstOpcode) && 429 "Cast isn't safe for alternation, logic needs to be updated!"); 430 AltOpcode = InstOpcode; 431 AltIndex = Cnt; 432 continue; 433 } 434 } 435 } else if (InstOpcode == Opcode || InstOpcode == AltOpcode) 436 continue; 437 return InstructionsState(VL[BaseIndex], nullptr, nullptr); 438 } 439 440 return InstructionsState(VL[BaseIndex], cast<Instruction>(VL[BaseIndex]), 441 cast<Instruction>(VL[AltIndex])); 442 } 443 444 /// \returns true if all of the values in \p VL have the same type or false 445 /// otherwise. 446 static bool allSameType(ArrayRef<Value *> VL) { 447 Type *Ty = VL[0]->getType(); 448 for (int i = 1, e = VL.size(); i < e; i++) 449 if (VL[i]->getType() != Ty) 450 return false; 451 452 return true; 453 } 454 455 /// \returns True if Extract{Value,Element} instruction extracts element Idx. 456 static Optional<unsigned> getExtractIndex(Instruction *E) { 457 unsigned Opcode = E->getOpcode(); 458 assert((Opcode == Instruction::ExtractElement || 459 Opcode == Instruction::ExtractValue) && 460 "Expected extractelement or extractvalue instruction."); 461 if (Opcode == Instruction::ExtractElement) { 462 auto *CI = dyn_cast<ConstantInt>(E->getOperand(1)); 463 if (!CI) 464 return None; 465 return CI->getZExtValue(); 466 } 467 ExtractValueInst *EI = cast<ExtractValueInst>(E); 468 if (EI->getNumIndices() != 1) 469 return None; 470 return *EI->idx_begin(); 471 } 472 473 /// \returns True if in-tree use also needs extract. This refers to 474 /// possible scalar operand in vectorized instruction. 475 static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst, 476 TargetLibraryInfo *TLI) { 477 unsigned Opcode = UserInst->getOpcode(); 478 switch (Opcode) { 479 case Instruction::Load: { 480 LoadInst *LI = cast<LoadInst>(UserInst); 481 return (LI->getPointerOperand() == Scalar); 482 } 483 case Instruction::Store: { 484 StoreInst *SI = cast<StoreInst>(UserInst); 485 return (SI->getPointerOperand() == Scalar); 486 } 487 case Instruction::Call: { 488 CallInst *CI = cast<CallInst>(UserInst); 489 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 490 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) { 491 if (hasVectorInstrinsicScalarOpd(ID, i)) 492 return (CI->getArgOperand(i) == Scalar); 493 } 494 LLVM_FALLTHROUGH; 495 } 496 default: 497 return false; 498 } 499 } 500 501 /// \returns the AA location that is being access by the instruction. 502 static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) { 503 if (StoreInst *SI = dyn_cast<StoreInst>(I)) 504 return MemoryLocation::get(SI); 505 if (LoadInst *LI = dyn_cast<LoadInst>(I)) 506 return MemoryLocation::get(LI); 507 return MemoryLocation(); 508 } 509 510 /// \returns True if the instruction is not a volatile or atomic load/store. 511 static bool isSimple(Instruction *I) { 512 if (LoadInst *LI = dyn_cast<LoadInst>(I)) 513 return LI->isSimple(); 514 if (StoreInst *SI = dyn_cast<StoreInst>(I)) 515 return SI->isSimple(); 516 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I)) 517 return !MI->isVolatile(); 518 return true; 519 } 520 521 namespace llvm { 522 523 namespace slpvectorizer { 524 525 /// Bottom Up SLP Vectorizer. 526 class BoUpSLP { 527 struct TreeEntry; 528 struct ScheduleData; 529 530 public: 531 using ValueList = SmallVector<Value *, 8>; 532 using InstrList = SmallVector<Instruction *, 16>; 533 using ValueSet = SmallPtrSet<Value *, 16>; 534 using StoreList = SmallVector<StoreInst *, 8>; 535 using ExtraValueToDebugLocsMap = 536 MapVector<Value *, SmallVector<Instruction *, 2>>; 537 538 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti, 539 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li, 540 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB, 541 const DataLayout *DL, OptimizationRemarkEmitter *ORE) 542 : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC), 543 DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) { 544 CodeMetrics::collectEphemeralValues(F, AC, EphValues); 545 // Use the vector register size specified by the target unless overridden 546 // by a command-line option. 547 // TODO: It would be better to limit the vectorization factor based on 548 // data type rather than just register size. For example, x86 AVX has 549 // 256-bit registers, but it does not support integer operations 550 // at that width (that requires AVX2). 551 if (MaxVectorRegSizeOption.getNumOccurrences()) 552 MaxVecRegSize = MaxVectorRegSizeOption; 553 else 554 MaxVecRegSize = TTI->getRegisterBitWidth(true); 555 556 if (MinVectorRegSizeOption.getNumOccurrences()) 557 MinVecRegSize = MinVectorRegSizeOption; 558 else 559 MinVecRegSize = TTI->getMinVectorRegisterBitWidth(); 560 } 561 562 /// Vectorize the tree that starts with the elements in \p VL. 563 /// Returns the vectorized root. 564 Value *vectorizeTree(); 565 566 /// Vectorize the tree but with the list of externally used values \p 567 /// ExternallyUsedValues. Values in this MapVector can be replaced but the 568 /// generated extractvalue instructions. 569 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues); 570 571 /// \returns the cost incurred by unwanted spills and fills, caused by 572 /// holding live values over call sites. 573 int getSpillCost() const; 574 575 /// \returns the vectorization cost of the subtree that starts at \p VL. 576 /// A negative number means that this is profitable. 577 int getTreeCost(); 578 579 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for 580 /// the purpose of scheduling and extraction in the \p UserIgnoreLst. 581 void buildTree(ArrayRef<Value *> Roots, 582 ArrayRef<Value *> UserIgnoreLst = None); 583 584 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for 585 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking 586 /// into account (and updating it, if required) list of externally used 587 /// values stored in \p ExternallyUsedValues. 588 void buildTree(ArrayRef<Value *> Roots, 589 ExtraValueToDebugLocsMap &ExternallyUsedValues, 590 ArrayRef<Value *> UserIgnoreLst = None); 591 592 /// Clear the internal data structures that are created by 'buildTree'. 593 void deleteTree() { 594 VectorizableTree.clear(); 595 ScalarToTreeEntry.clear(); 596 MustGather.clear(); 597 ExternalUses.clear(); 598 NumOpsWantToKeepOrder.clear(); 599 NumOpsWantToKeepOriginalOrder = 0; 600 for (auto &Iter : BlocksSchedules) { 601 BlockScheduling *BS = Iter.second.get(); 602 BS->clear(); 603 } 604 MinBWs.clear(); 605 } 606 607 unsigned getTreeSize() const { return VectorizableTree.size(); } 608 609 /// Perform LICM and CSE on the newly generated gather sequences. 610 void optimizeGatherSequence(); 611 612 /// \returns The best order of instructions for vectorization. 613 Optional<ArrayRef<unsigned>> bestOrder() const { 614 auto I = std::max_element( 615 NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(), 616 [](const decltype(NumOpsWantToKeepOrder)::value_type &D1, 617 const decltype(NumOpsWantToKeepOrder)::value_type &D2) { 618 return D1.second < D2.second; 619 }); 620 if (I == NumOpsWantToKeepOrder.end() || 621 I->getSecond() <= NumOpsWantToKeepOriginalOrder) 622 return None; 623 624 return makeArrayRef(I->getFirst()); 625 } 626 627 /// \return The vector element size in bits to use when vectorizing the 628 /// expression tree ending at \p V. If V is a store, the size is the width of 629 /// the stored value. Otherwise, the size is the width of the largest loaded 630 /// value reaching V. This method is used by the vectorizer to calculate 631 /// vectorization factors. 632 unsigned getVectorElementSize(Value *V) const; 633 634 /// Compute the minimum type sizes required to represent the entries in a 635 /// vectorizable tree. 636 void computeMinimumValueSizes(); 637 638 // \returns maximum vector register size as set by TTI or overridden by cl::opt. 639 unsigned getMaxVecRegSize() const { 640 return MaxVecRegSize; 641 } 642 643 // \returns minimum vector register size as set by cl::opt. 644 unsigned getMinVecRegSize() const { 645 return MinVecRegSize; 646 } 647 648 /// Check if homogeneous aggregate is isomorphic to some VectorType. 649 /// Accepts homogeneous multidimensional aggregate of scalars/vectors like 650 /// {[4 x i16], [4 x i16]}, { <2 x float>, <2 x float> }, 651 /// {{{i16, i16}, {i16, i16}}, {{i16, i16}, {i16, i16}}} and so on. 652 /// 653 /// \returns number of elements in vector if isomorphism exists, 0 otherwise. 654 unsigned canMapToVector(Type *T, const DataLayout &DL) const; 655 656 /// \returns True if the VectorizableTree is both tiny and not fully 657 /// vectorizable. We do not vectorize such trees. 658 bool isTreeTinyAndNotFullyVectorizable() const; 659 660 /// Assume that a legal-sized 'or'-reduction of shifted/zexted loaded values 661 /// can be load combined in the backend. Load combining may not be allowed in 662 /// the IR optimizer, so we do not want to alter the pattern. For example, 663 /// partially transforming a scalar bswap() pattern into vector code is 664 /// effectively impossible for the backend to undo. 665 /// TODO: If load combining is allowed in the IR optimizer, this analysis 666 /// may not be necessary. 667 bool isLoadCombineReductionCandidate(unsigned ReductionOpcode) const; 668 669 OptimizationRemarkEmitter *getORE() { return ORE; } 670 671 /// This structure holds any data we need about the edges being traversed 672 /// during buildTree_rec(). We keep track of: 673 /// (i) the user TreeEntry index, and 674 /// (ii) the index of the edge. 675 struct EdgeInfo { 676 EdgeInfo() = default; 677 EdgeInfo(TreeEntry *UserTE, unsigned EdgeIdx) 678 : UserTE(UserTE), EdgeIdx(EdgeIdx) {} 679 /// The user TreeEntry. 680 TreeEntry *UserTE = nullptr; 681 /// The operand index of the use. 682 unsigned EdgeIdx = UINT_MAX; 683 #ifndef NDEBUG 684 friend inline raw_ostream &operator<<(raw_ostream &OS, 685 const BoUpSLP::EdgeInfo &EI) { 686 EI.dump(OS); 687 return OS; 688 } 689 /// Debug print. 690 void dump(raw_ostream &OS) const { 691 OS << "{User:" << (UserTE ? std::to_string(UserTE->Idx) : "null") 692 << " EdgeIdx:" << EdgeIdx << "}"; 693 } 694 LLVM_DUMP_METHOD void dump() const { dump(dbgs()); } 695 #endif 696 }; 697 698 /// A helper data structure to hold the operands of a vector of instructions. 699 /// This supports a fixed vector length for all operand vectors. 700 class VLOperands { 701 /// For each operand we need (i) the value, and (ii) the opcode that it 702 /// would be attached to if the expression was in a left-linearized form. 703 /// This is required to avoid illegal operand reordering. 704 /// For example: 705 /// \verbatim 706 /// 0 Op1 707 /// |/ 708 /// Op1 Op2 Linearized + Op2 709 /// \ / ----------> |/ 710 /// - - 711 /// 712 /// Op1 - Op2 (0 + Op1) - Op2 713 /// \endverbatim 714 /// 715 /// Value Op1 is attached to a '+' operation, and Op2 to a '-'. 716 /// 717 /// Another way to think of this is to track all the operations across the 718 /// path from the operand all the way to the root of the tree and to 719 /// calculate the operation that corresponds to this path. For example, the 720 /// path from Op2 to the root crosses the RHS of the '-', therefore the 721 /// corresponding operation is a '-' (which matches the one in the 722 /// linearized tree, as shown above). 723 /// 724 /// For lack of a better term, we refer to this operation as Accumulated 725 /// Path Operation (APO). 726 struct OperandData { 727 OperandData() = default; 728 OperandData(Value *V, bool APO, bool IsUsed) 729 : V(V), APO(APO), IsUsed(IsUsed) {} 730 /// The operand value. 731 Value *V = nullptr; 732 /// TreeEntries only allow a single opcode, or an alternate sequence of 733 /// them (e.g, +, -). Therefore, we can safely use a boolean value for the 734 /// APO. It is set to 'true' if 'V' is attached to an inverse operation 735 /// in the left-linearized form (e.g., Sub/Div), and 'false' otherwise 736 /// (e.g., Add/Mul) 737 bool APO = false; 738 /// Helper data for the reordering function. 739 bool IsUsed = false; 740 }; 741 742 /// During operand reordering, we are trying to select the operand at lane 743 /// that matches best with the operand at the neighboring lane. Our 744 /// selection is based on the type of value we are looking for. For example, 745 /// if the neighboring lane has a load, we need to look for a load that is 746 /// accessing a consecutive address. These strategies are summarized in the 747 /// 'ReorderingMode' enumerator. 748 enum class ReorderingMode { 749 Load, ///< Matching loads to consecutive memory addresses 750 Opcode, ///< Matching instructions based on opcode (same or alternate) 751 Constant, ///< Matching constants 752 Splat, ///< Matching the same instruction multiple times (broadcast) 753 Failed, ///< We failed to create a vectorizable group 754 }; 755 756 using OperandDataVec = SmallVector<OperandData, 2>; 757 758 /// A vector of operand vectors. 759 SmallVector<OperandDataVec, 4> OpsVec; 760 761 const DataLayout &DL; 762 ScalarEvolution &SE; 763 const BoUpSLP &R; 764 765 /// \returns the operand data at \p OpIdx and \p Lane. 766 OperandData &getData(unsigned OpIdx, unsigned Lane) { 767 return OpsVec[OpIdx][Lane]; 768 } 769 770 /// \returns the operand data at \p OpIdx and \p Lane. Const version. 771 const OperandData &getData(unsigned OpIdx, unsigned Lane) const { 772 return OpsVec[OpIdx][Lane]; 773 } 774 775 /// Clears the used flag for all entries. 776 void clearUsed() { 777 for (unsigned OpIdx = 0, NumOperands = getNumOperands(); 778 OpIdx != NumOperands; ++OpIdx) 779 for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes; 780 ++Lane) 781 OpsVec[OpIdx][Lane].IsUsed = false; 782 } 783 784 /// Swap the operand at \p OpIdx1 with that one at \p OpIdx2. 785 void swap(unsigned OpIdx1, unsigned OpIdx2, unsigned Lane) { 786 std::swap(OpsVec[OpIdx1][Lane], OpsVec[OpIdx2][Lane]); 787 } 788 789 // The hard-coded scores listed here are not very important. When computing 790 // the scores of matching one sub-tree with another, we are basically 791 // counting the number of values that are matching. So even if all scores 792 // are set to 1, we would still get a decent matching result. 793 // However, sometimes we have to break ties. For example we may have to 794 // choose between matching loads vs matching opcodes. This is what these 795 // scores are helping us with: they provide the order of preference. 796 797 /// Loads from consecutive memory addresses, e.g. load(A[i]), load(A[i+1]). 798 static const int ScoreConsecutiveLoads = 3; 799 /// ExtractElementInst from same vector and consecutive indexes. 800 static const int ScoreConsecutiveExtracts = 3; 801 /// Constants. 802 static const int ScoreConstants = 2; 803 /// Instructions with the same opcode. 804 static const int ScoreSameOpcode = 2; 805 /// Instructions with alt opcodes (e.g, add + sub). 806 static const int ScoreAltOpcodes = 1; 807 /// Identical instructions (a.k.a. splat or broadcast). 808 static const int ScoreSplat = 1; 809 /// Matching with an undef is preferable to failing. 810 static const int ScoreUndef = 1; 811 /// Score for failing to find a decent match. 812 static const int ScoreFail = 0; 813 /// User exteranl to the vectorized code. 814 static const int ExternalUseCost = 1; 815 /// The user is internal but in a different lane. 816 static const int UserInDiffLaneCost = ExternalUseCost; 817 818 /// \returns the score of placing \p V1 and \p V2 in consecutive lanes. 819 static int getShallowScore(Value *V1, Value *V2, const DataLayout &DL, 820 ScalarEvolution &SE) { 821 auto *LI1 = dyn_cast<LoadInst>(V1); 822 auto *LI2 = dyn_cast<LoadInst>(V2); 823 if (LI1 && LI2) 824 return isConsecutiveAccess(LI1, LI2, DL, SE) 825 ? VLOperands::ScoreConsecutiveLoads 826 : VLOperands::ScoreFail; 827 828 auto *C1 = dyn_cast<Constant>(V1); 829 auto *C2 = dyn_cast<Constant>(V2); 830 if (C1 && C2) 831 return VLOperands::ScoreConstants; 832 833 // Extracts from consecutive indexes of the same vector better score as 834 // the extracts could be optimized away. 835 Value *EV; 836 ConstantInt *Ex1Idx, *Ex2Idx; 837 if (match(V1, m_ExtractElement(m_Value(EV), m_ConstantInt(Ex1Idx))) && 838 match(V2, m_ExtractElement(m_Deferred(EV), m_ConstantInt(Ex2Idx))) && 839 Ex1Idx->getZExtValue() + 1 == Ex2Idx->getZExtValue()) 840 return VLOperands::ScoreConsecutiveExtracts; 841 842 auto *I1 = dyn_cast<Instruction>(V1); 843 auto *I2 = dyn_cast<Instruction>(V2); 844 if (I1 && I2) { 845 if (I1 == I2) 846 return VLOperands::ScoreSplat; 847 InstructionsState S = getSameOpcode({I1, I2}); 848 // Note: Only consider instructions with <= 2 operands to avoid 849 // complexity explosion. 850 if (S.getOpcode() && S.MainOp->getNumOperands() <= 2) 851 return S.isAltShuffle() ? VLOperands::ScoreAltOpcodes 852 : VLOperands::ScoreSameOpcode; 853 } 854 855 if (isa<UndefValue>(V2)) 856 return VLOperands::ScoreUndef; 857 858 return VLOperands::ScoreFail; 859 } 860 861 /// Holds the values and their lane that are taking part in the look-ahead 862 /// score calculation. This is used in the external uses cost calculation. 863 SmallDenseMap<Value *, int> InLookAheadValues; 864 865 /// \Returns the additinal cost due to uses of \p LHS and \p RHS that are 866 /// either external to the vectorized code, or require shuffling. 867 int getExternalUsesCost(const std::pair<Value *, int> &LHS, 868 const std::pair<Value *, int> &RHS) { 869 int Cost = 0; 870 SmallVector<std::pair<Value *, int>, 2> Values = {LHS, RHS}; 871 for (int Idx = 0, IdxE = Values.size(); Idx != IdxE; ++Idx) { 872 Value *V = Values[Idx].first; 873 // Calculate the absolute lane, using the minimum relative lane of LHS 874 // and RHS as base and Idx as the offset. 875 int Ln = std::min(LHS.second, RHS.second) + Idx; 876 assert(Ln >= 0 && "Bad lane calculation"); 877 unsigned UsersBudget = LookAheadUsersBudget; 878 for (User *U : V->users()) { 879 if (const TreeEntry *UserTE = R.getTreeEntry(U)) { 880 // The user is in the VectorizableTree. Check if we need to insert. 881 auto It = llvm::find(UserTE->Scalars, U); 882 assert(It != UserTE->Scalars.end() && "U is in UserTE"); 883 int UserLn = std::distance(UserTE->Scalars.begin(), It); 884 assert(UserLn >= 0 && "Bad lane"); 885 if (UserLn != Ln) 886 Cost += UserInDiffLaneCost; 887 } else { 888 // Check if the user is in the look-ahead code. 889 auto It2 = InLookAheadValues.find(U); 890 if (It2 != InLookAheadValues.end()) { 891 // The user is in the look-ahead code. Check the lane. 892 if (It2->second != Ln) 893 Cost += UserInDiffLaneCost; 894 } else { 895 // The user is neither in SLP tree nor in the look-ahead code. 896 Cost += ExternalUseCost; 897 } 898 } 899 // Limit the number of visited uses to cap compilation time. 900 if (--UsersBudget == 0) 901 break; 902 } 903 } 904 return Cost; 905 } 906 907 /// Go through the operands of \p LHS and \p RHS recursively until \p 908 /// MaxLevel, and return the cummulative score. For example: 909 /// \verbatim 910 /// A[0] B[0] A[1] B[1] C[0] D[0] B[1] A[1] 911 /// \ / \ / \ / \ / 912 /// + + + + 913 /// G1 G2 G3 G4 914 /// \endverbatim 915 /// The getScoreAtLevelRec(G1, G2) function will try to match the nodes at 916 /// each level recursively, accumulating the score. It starts from matching 917 /// the additions at level 0, then moves on to the loads (level 1). The 918 /// score of G1 and G2 is higher than G1 and G3, because {A[0],A[1]} and 919 /// {B[0],B[1]} match with VLOperands::ScoreConsecutiveLoads, while 920 /// {A[0],C[0]} has a score of VLOperands::ScoreFail. 921 /// Please note that the order of the operands does not matter, as we 922 /// evaluate the score of all profitable combinations of operands. In 923 /// other words the score of G1 and G4 is the same as G1 and G2. This 924 /// heuristic is based on ideas described in: 925 /// Look-ahead SLP: Auto-vectorization in the presence of commutative 926 /// operations, CGO 2018 by Vasileios Porpodas, Rodrigo C. O. Rocha, 927 /// Luís F. W. Góes 928 int getScoreAtLevelRec(const std::pair<Value *, int> &LHS, 929 const std::pair<Value *, int> &RHS, int CurrLevel, 930 int MaxLevel) { 931 932 Value *V1 = LHS.first; 933 Value *V2 = RHS.first; 934 // Get the shallow score of V1 and V2. 935 int ShallowScoreAtThisLevel = 936 std::max((int)ScoreFail, getShallowScore(V1, V2, DL, SE) - 937 getExternalUsesCost(LHS, RHS)); 938 int Lane1 = LHS.second; 939 int Lane2 = RHS.second; 940 941 // If reached MaxLevel, 942 // or if V1 and V2 are not instructions, 943 // or if they are SPLAT, 944 // or if they are not consecutive, early return the current cost. 945 auto *I1 = dyn_cast<Instruction>(V1); 946 auto *I2 = dyn_cast<Instruction>(V2); 947 if (CurrLevel == MaxLevel || !(I1 && I2) || I1 == I2 || 948 ShallowScoreAtThisLevel == VLOperands::ScoreFail || 949 (isa<LoadInst>(I1) && isa<LoadInst>(I2) && ShallowScoreAtThisLevel)) 950 return ShallowScoreAtThisLevel; 951 assert(I1 && I2 && "Should have early exited."); 952 953 // Keep track of in-tree values for determining the external-use cost. 954 InLookAheadValues[V1] = Lane1; 955 InLookAheadValues[V2] = Lane2; 956 957 // Contains the I2 operand indexes that got matched with I1 operands. 958 SmallSet<unsigned, 4> Op2Used; 959 960 // Recursion towards the operands of I1 and I2. We are trying all possbile 961 // operand pairs, and keeping track of the best score. 962 for (unsigned OpIdx1 = 0, NumOperands1 = I1->getNumOperands(); 963 OpIdx1 != NumOperands1; ++OpIdx1) { 964 // Try to pair op1I with the best operand of I2. 965 int MaxTmpScore = 0; 966 unsigned MaxOpIdx2 = 0; 967 bool FoundBest = false; 968 // If I2 is commutative try all combinations. 969 unsigned FromIdx = isCommutative(I2) ? 0 : OpIdx1; 970 unsigned ToIdx = isCommutative(I2) 971 ? I2->getNumOperands() 972 : std::min(I2->getNumOperands(), OpIdx1 + 1); 973 assert(FromIdx <= ToIdx && "Bad index"); 974 for (unsigned OpIdx2 = FromIdx; OpIdx2 != ToIdx; ++OpIdx2) { 975 // Skip operands already paired with OpIdx1. 976 if (Op2Used.count(OpIdx2)) 977 continue; 978 // Recursively calculate the cost at each level 979 int TmpScore = getScoreAtLevelRec({I1->getOperand(OpIdx1), Lane1}, 980 {I2->getOperand(OpIdx2), Lane2}, 981 CurrLevel + 1, MaxLevel); 982 // Look for the best score. 983 if (TmpScore > VLOperands::ScoreFail && TmpScore > MaxTmpScore) { 984 MaxTmpScore = TmpScore; 985 MaxOpIdx2 = OpIdx2; 986 FoundBest = true; 987 } 988 } 989 if (FoundBest) { 990 // Pair {OpIdx1, MaxOpIdx2} was found to be best. Never revisit it. 991 Op2Used.insert(MaxOpIdx2); 992 ShallowScoreAtThisLevel += MaxTmpScore; 993 } 994 } 995 return ShallowScoreAtThisLevel; 996 } 997 998 /// \Returns the look-ahead score, which tells us how much the sub-trees 999 /// rooted at \p LHS and \p RHS match, the more they match the higher the 1000 /// score. This helps break ties in an informed way when we cannot decide on 1001 /// the order of the operands by just considering the immediate 1002 /// predecessors. 1003 int getLookAheadScore(const std::pair<Value *, int> &LHS, 1004 const std::pair<Value *, int> &RHS) { 1005 InLookAheadValues.clear(); 1006 return getScoreAtLevelRec(LHS, RHS, 1, LookAheadMaxDepth); 1007 } 1008 1009 // Search all operands in Ops[*][Lane] for the one that matches best 1010 // Ops[OpIdx][LastLane] and return its opreand index. 1011 // If no good match can be found, return None. 1012 Optional<unsigned> 1013 getBestOperand(unsigned OpIdx, int Lane, int LastLane, 1014 ArrayRef<ReorderingMode> ReorderingModes) { 1015 unsigned NumOperands = getNumOperands(); 1016 1017 // The operand of the previous lane at OpIdx. 1018 Value *OpLastLane = getData(OpIdx, LastLane).V; 1019 1020 // Our strategy mode for OpIdx. 1021 ReorderingMode RMode = ReorderingModes[OpIdx]; 1022 1023 // The linearized opcode of the operand at OpIdx, Lane. 1024 bool OpIdxAPO = getData(OpIdx, Lane).APO; 1025 1026 // The best operand index and its score. 1027 // Sometimes we have more than one option (e.g., Opcode and Undefs), so we 1028 // are using the score to differentiate between the two. 1029 struct BestOpData { 1030 Optional<unsigned> Idx = None; 1031 unsigned Score = 0; 1032 } BestOp; 1033 1034 // Iterate through all unused operands and look for the best. 1035 for (unsigned Idx = 0; Idx != NumOperands; ++Idx) { 1036 // Get the operand at Idx and Lane. 1037 OperandData &OpData = getData(Idx, Lane); 1038 Value *Op = OpData.V; 1039 bool OpAPO = OpData.APO; 1040 1041 // Skip already selected operands. 1042 if (OpData.IsUsed) 1043 continue; 1044 1045 // Skip if we are trying to move the operand to a position with a 1046 // different opcode in the linearized tree form. This would break the 1047 // semantics. 1048 if (OpAPO != OpIdxAPO) 1049 continue; 1050 1051 // Look for an operand that matches the current mode. 1052 switch (RMode) { 1053 case ReorderingMode::Load: 1054 case ReorderingMode::Constant: 1055 case ReorderingMode::Opcode: { 1056 bool LeftToRight = Lane > LastLane; 1057 Value *OpLeft = (LeftToRight) ? OpLastLane : Op; 1058 Value *OpRight = (LeftToRight) ? Op : OpLastLane; 1059 unsigned Score = 1060 getLookAheadScore({OpLeft, LastLane}, {OpRight, Lane}); 1061 if (Score > BestOp.Score) { 1062 BestOp.Idx = Idx; 1063 BestOp.Score = Score; 1064 } 1065 break; 1066 } 1067 case ReorderingMode::Splat: 1068 if (Op == OpLastLane) 1069 BestOp.Idx = Idx; 1070 break; 1071 case ReorderingMode::Failed: 1072 return None; 1073 } 1074 } 1075 1076 if (BestOp.Idx) { 1077 getData(BestOp.Idx.getValue(), Lane).IsUsed = true; 1078 return BestOp.Idx; 1079 } 1080 // If we could not find a good match return None. 1081 return None; 1082 } 1083 1084 /// Helper for reorderOperandVecs. \Returns the lane that we should start 1085 /// reordering from. This is the one which has the least number of operands 1086 /// that can freely move about. 1087 unsigned getBestLaneToStartReordering() const { 1088 unsigned BestLane = 0; 1089 unsigned Min = UINT_MAX; 1090 for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes; 1091 ++Lane) { 1092 unsigned NumFreeOps = getMaxNumOperandsThatCanBeReordered(Lane); 1093 if (NumFreeOps < Min) { 1094 Min = NumFreeOps; 1095 BestLane = Lane; 1096 } 1097 } 1098 return BestLane; 1099 } 1100 1101 /// \Returns the maximum number of operands that are allowed to be reordered 1102 /// for \p Lane. This is used as a heuristic for selecting the first lane to 1103 /// start operand reordering. 1104 unsigned getMaxNumOperandsThatCanBeReordered(unsigned Lane) const { 1105 unsigned CntTrue = 0; 1106 unsigned NumOperands = getNumOperands(); 1107 // Operands with the same APO can be reordered. We therefore need to count 1108 // how many of them we have for each APO, like this: Cnt[APO] = x. 1109 // Since we only have two APOs, namely true and false, we can avoid using 1110 // a map. Instead we can simply count the number of operands that 1111 // correspond to one of them (in this case the 'true' APO), and calculate 1112 // the other by subtracting it from the total number of operands. 1113 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) 1114 if (getData(OpIdx, Lane).APO) 1115 ++CntTrue; 1116 unsigned CntFalse = NumOperands - CntTrue; 1117 return std::max(CntTrue, CntFalse); 1118 } 1119 1120 /// Go through the instructions in VL and append their operands. 1121 void appendOperandsOfVL(ArrayRef<Value *> VL) { 1122 assert(!VL.empty() && "Bad VL"); 1123 assert((empty() || VL.size() == getNumLanes()) && 1124 "Expected same number of lanes"); 1125 assert(isa<Instruction>(VL[0]) && "Expected instruction"); 1126 unsigned NumOperands = cast<Instruction>(VL[0])->getNumOperands(); 1127 OpsVec.resize(NumOperands); 1128 unsigned NumLanes = VL.size(); 1129 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) { 1130 OpsVec[OpIdx].resize(NumLanes); 1131 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) { 1132 assert(isa<Instruction>(VL[Lane]) && "Expected instruction"); 1133 // Our tree has just 3 nodes: the root and two operands. 1134 // It is therefore trivial to get the APO. We only need to check the 1135 // opcode of VL[Lane] and whether the operand at OpIdx is the LHS or 1136 // RHS operand. The LHS operand of both add and sub is never attached 1137 // to an inversese operation in the linearized form, therefore its APO 1138 // is false. The RHS is true only if VL[Lane] is an inverse operation. 1139 1140 // Since operand reordering is performed on groups of commutative 1141 // operations or alternating sequences (e.g., +, -), we can safely 1142 // tell the inverse operations by checking commutativity. 1143 bool IsInverseOperation = !isCommutative(cast<Instruction>(VL[Lane])); 1144 bool APO = (OpIdx == 0) ? false : IsInverseOperation; 1145 OpsVec[OpIdx][Lane] = {cast<Instruction>(VL[Lane])->getOperand(OpIdx), 1146 APO, false}; 1147 } 1148 } 1149 } 1150 1151 /// \returns the number of operands. 1152 unsigned getNumOperands() const { return OpsVec.size(); } 1153 1154 /// \returns the number of lanes. 1155 unsigned getNumLanes() const { return OpsVec[0].size(); } 1156 1157 /// \returns the operand value at \p OpIdx and \p Lane. 1158 Value *getValue(unsigned OpIdx, unsigned Lane) const { 1159 return getData(OpIdx, Lane).V; 1160 } 1161 1162 /// \returns true if the data structure is empty. 1163 bool empty() const { return OpsVec.empty(); } 1164 1165 /// Clears the data. 1166 void clear() { OpsVec.clear(); } 1167 1168 /// \Returns true if there are enough operands identical to \p Op to fill 1169 /// the whole vector. 1170 /// Note: This modifies the 'IsUsed' flag, so a cleanUsed() must follow. 1171 bool shouldBroadcast(Value *Op, unsigned OpIdx, unsigned Lane) { 1172 bool OpAPO = getData(OpIdx, Lane).APO; 1173 for (unsigned Ln = 0, Lns = getNumLanes(); Ln != Lns; ++Ln) { 1174 if (Ln == Lane) 1175 continue; 1176 // This is set to true if we found a candidate for broadcast at Lane. 1177 bool FoundCandidate = false; 1178 for (unsigned OpI = 0, OpE = getNumOperands(); OpI != OpE; ++OpI) { 1179 OperandData &Data = getData(OpI, Ln); 1180 if (Data.APO != OpAPO || Data.IsUsed) 1181 continue; 1182 if (Data.V == Op) { 1183 FoundCandidate = true; 1184 Data.IsUsed = true; 1185 break; 1186 } 1187 } 1188 if (!FoundCandidate) 1189 return false; 1190 } 1191 return true; 1192 } 1193 1194 public: 1195 /// Initialize with all the operands of the instruction vector \p RootVL. 1196 VLOperands(ArrayRef<Value *> RootVL, const DataLayout &DL, 1197 ScalarEvolution &SE, const BoUpSLP &R) 1198 : DL(DL), SE(SE), R(R) { 1199 // Append all the operands of RootVL. 1200 appendOperandsOfVL(RootVL); 1201 } 1202 1203 /// \Returns a value vector with the operands across all lanes for the 1204 /// opearnd at \p OpIdx. 1205 ValueList getVL(unsigned OpIdx) const { 1206 ValueList OpVL(OpsVec[OpIdx].size()); 1207 assert(OpsVec[OpIdx].size() == getNumLanes() && 1208 "Expected same num of lanes across all operands"); 1209 for (unsigned Lane = 0, Lanes = getNumLanes(); Lane != Lanes; ++Lane) 1210 OpVL[Lane] = OpsVec[OpIdx][Lane].V; 1211 return OpVL; 1212 } 1213 1214 // Performs operand reordering for 2 or more operands. 1215 // The original operands are in OrigOps[OpIdx][Lane]. 1216 // The reordered operands are returned in 'SortedOps[OpIdx][Lane]'. 1217 void reorder() { 1218 unsigned NumOperands = getNumOperands(); 1219 unsigned NumLanes = getNumLanes(); 1220 // Each operand has its own mode. We are using this mode to help us select 1221 // the instructions for each lane, so that they match best with the ones 1222 // we have selected so far. 1223 SmallVector<ReorderingMode, 2> ReorderingModes(NumOperands); 1224 1225 // This is a greedy single-pass algorithm. We are going over each lane 1226 // once and deciding on the best order right away with no back-tracking. 1227 // However, in order to increase its effectiveness, we start with the lane 1228 // that has operands that can move the least. For example, given the 1229 // following lanes: 1230 // Lane 0 : A[0] = B[0] + C[0] // Visited 3rd 1231 // Lane 1 : A[1] = C[1] - B[1] // Visited 1st 1232 // Lane 2 : A[2] = B[2] + C[2] // Visited 2nd 1233 // Lane 3 : A[3] = C[3] - B[3] // Visited 4th 1234 // we will start at Lane 1, since the operands of the subtraction cannot 1235 // be reordered. Then we will visit the rest of the lanes in a circular 1236 // fashion. That is, Lanes 2, then Lane 0, and finally Lane 3. 1237 1238 // Find the first lane that we will start our search from. 1239 unsigned FirstLane = getBestLaneToStartReordering(); 1240 1241 // Initialize the modes. 1242 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) { 1243 Value *OpLane0 = getValue(OpIdx, FirstLane); 1244 // Keep track if we have instructions with all the same opcode on one 1245 // side. 1246 if (isa<LoadInst>(OpLane0)) 1247 ReorderingModes[OpIdx] = ReorderingMode::Load; 1248 else if (isa<Instruction>(OpLane0)) { 1249 // Check if OpLane0 should be broadcast. 1250 if (shouldBroadcast(OpLane0, OpIdx, FirstLane)) 1251 ReorderingModes[OpIdx] = ReorderingMode::Splat; 1252 else 1253 ReorderingModes[OpIdx] = ReorderingMode::Opcode; 1254 } 1255 else if (isa<Constant>(OpLane0)) 1256 ReorderingModes[OpIdx] = ReorderingMode::Constant; 1257 else if (isa<Argument>(OpLane0)) 1258 // Our best hope is a Splat. It may save some cost in some cases. 1259 ReorderingModes[OpIdx] = ReorderingMode::Splat; 1260 else 1261 // NOTE: This should be unreachable. 1262 ReorderingModes[OpIdx] = ReorderingMode::Failed; 1263 } 1264 1265 // If the initial strategy fails for any of the operand indexes, then we 1266 // perform reordering again in a second pass. This helps avoid assigning 1267 // high priority to the failed strategy, and should improve reordering for 1268 // the non-failed operand indexes. 1269 for (int Pass = 0; Pass != 2; ++Pass) { 1270 // Skip the second pass if the first pass did not fail. 1271 bool StrategyFailed = false; 1272 // Mark all operand data as free to use. 1273 clearUsed(); 1274 // We keep the original operand order for the FirstLane, so reorder the 1275 // rest of the lanes. We are visiting the nodes in a circular fashion, 1276 // using FirstLane as the center point and increasing the radius 1277 // distance. 1278 for (unsigned Distance = 1; Distance != NumLanes; ++Distance) { 1279 // Visit the lane on the right and then the lane on the left. 1280 for (int Direction : {+1, -1}) { 1281 int Lane = FirstLane + Direction * Distance; 1282 if (Lane < 0 || Lane >= (int)NumLanes) 1283 continue; 1284 int LastLane = Lane - Direction; 1285 assert(LastLane >= 0 && LastLane < (int)NumLanes && 1286 "Out of bounds"); 1287 // Look for a good match for each operand. 1288 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) { 1289 // Search for the operand that matches SortedOps[OpIdx][Lane-1]. 1290 Optional<unsigned> BestIdx = 1291 getBestOperand(OpIdx, Lane, LastLane, ReorderingModes); 1292 // By not selecting a value, we allow the operands that follow to 1293 // select a better matching value. We will get a non-null value in 1294 // the next run of getBestOperand(). 1295 if (BestIdx) { 1296 // Swap the current operand with the one returned by 1297 // getBestOperand(). 1298 swap(OpIdx, BestIdx.getValue(), Lane); 1299 } else { 1300 // We failed to find a best operand, set mode to 'Failed'. 1301 ReorderingModes[OpIdx] = ReorderingMode::Failed; 1302 // Enable the second pass. 1303 StrategyFailed = true; 1304 } 1305 } 1306 } 1307 } 1308 // Skip second pass if the strategy did not fail. 1309 if (!StrategyFailed) 1310 break; 1311 } 1312 } 1313 1314 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 1315 LLVM_DUMP_METHOD static StringRef getModeStr(ReorderingMode RMode) { 1316 switch (RMode) { 1317 case ReorderingMode::Load: 1318 return "Load"; 1319 case ReorderingMode::Opcode: 1320 return "Opcode"; 1321 case ReorderingMode::Constant: 1322 return "Constant"; 1323 case ReorderingMode::Splat: 1324 return "Splat"; 1325 case ReorderingMode::Failed: 1326 return "Failed"; 1327 } 1328 llvm_unreachable("Unimplemented Reordering Type"); 1329 } 1330 1331 LLVM_DUMP_METHOD static raw_ostream &printMode(ReorderingMode RMode, 1332 raw_ostream &OS) { 1333 return OS << getModeStr(RMode); 1334 } 1335 1336 /// Debug print. 1337 LLVM_DUMP_METHOD static void dumpMode(ReorderingMode RMode) { 1338 printMode(RMode, dbgs()); 1339 } 1340 1341 friend raw_ostream &operator<<(raw_ostream &OS, ReorderingMode RMode) { 1342 return printMode(RMode, OS); 1343 } 1344 1345 LLVM_DUMP_METHOD raw_ostream &print(raw_ostream &OS) const { 1346 const unsigned Indent = 2; 1347 unsigned Cnt = 0; 1348 for (const OperandDataVec &OpDataVec : OpsVec) { 1349 OS << "Operand " << Cnt++ << "\n"; 1350 for (const OperandData &OpData : OpDataVec) { 1351 OS.indent(Indent) << "{"; 1352 if (Value *V = OpData.V) 1353 OS << *V; 1354 else 1355 OS << "null"; 1356 OS << ", APO:" << OpData.APO << "}\n"; 1357 } 1358 OS << "\n"; 1359 } 1360 return OS; 1361 } 1362 1363 /// Debug print. 1364 LLVM_DUMP_METHOD void dump() const { print(dbgs()); } 1365 #endif 1366 }; 1367 1368 /// Checks if the instruction is marked for deletion. 1369 bool isDeleted(Instruction *I) const { return DeletedInstructions.count(I); } 1370 1371 /// Marks values operands for later deletion by replacing them with Undefs. 1372 void eraseInstructions(ArrayRef<Value *> AV); 1373 1374 ~BoUpSLP(); 1375 1376 private: 1377 /// Checks if all users of \p I are the part of the vectorization tree. 1378 bool areAllUsersVectorized(Instruction *I) const; 1379 1380 /// \returns the cost of the vectorizable entry. 1381 int getEntryCost(TreeEntry *E); 1382 1383 /// This is the recursive part of buildTree. 1384 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, 1385 const EdgeInfo &EI); 1386 1387 /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can 1388 /// be vectorized to use the original vector (or aggregate "bitcast" to a 1389 /// vector) and sets \p CurrentOrder to the identity permutation; otherwise 1390 /// returns false, setting \p CurrentOrder to either an empty vector or a 1391 /// non-identity permutation that allows to reuse extract instructions. 1392 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue, 1393 SmallVectorImpl<unsigned> &CurrentOrder) const; 1394 1395 /// Vectorize a single entry in the tree. 1396 Value *vectorizeTree(TreeEntry *E); 1397 1398 /// Vectorize a single entry in the tree, starting in \p VL. 1399 Value *vectorizeTree(ArrayRef<Value *> VL); 1400 1401 /// \returns the scalarization cost for this type. Scalarization in this 1402 /// context means the creation of vectors from a group of scalars. 1403 int getGatherCost(Type *Ty, const DenseSet<unsigned> &ShuffledIndices) const; 1404 1405 /// \returns the scalarization cost for this list of values. Assuming that 1406 /// this subtree gets vectorized, we may need to extract the values from the 1407 /// roots. This method calculates the cost of extracting the values. 1408 int getGatherCost(ArrayRef<Value *> VL) const; 1409 1410 /// Set the Builder insert point to one after the last instruction in 1411 /// the bundle 1412 void setInsertPointAfterBundle(TreeEntry *E); 1413 1414 /// \returns a vector from a collection of scalars in \p VL. 1415 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty); 1416 1417 /// \returns whether the VectorizableTree is fully vectorizable and will 1418 /// be beneficial even the tree height is tiny. 1419 bool isFullyVectorizableTinyTree() const; 1420 1421 /// Reorder commutative or alt operands to get better probability of 1422 /// generating vectorized code. 1423 static void reorderInputsAccordingToOpcode(ArrayRef<Value *> VL, 1424 SmallVectorImpl<Value *> &Left, 1425 SmallVectorImpl<Value *> &Right, 1426 const DataLayout &DL, 1427 ScalarEvolution &SE, 1428 const BoUpSLP &R); 1429 struct TreeEntry { 1430 using VecTreeTy = SmallVector<std::unique_ptr<TreeEntry>, 8>; 1431 TreeEntry(VecTreeTy &Container) : Container(Container) {} 1432 1433 /// \returns true if the scalars in VL are equal to this entry. 1434 bool isSame(ArrayRef<Value *> VL) const { 1435 if (VL.size() == Scalars.size()) 1436 return std::equal(VL.begin(), VL.end(), Scalars.begin()); 1437 return VL.size() == ReuseShuffleIndices.size() && 1438 std::equal( 1439 VL.begin(), VL.end(), ReuseShuffleIndices.begin(), 1440 [this](Value *V, unsigned Idx) { return V == Scalars[Idx]; }); 1441 } 1442 1443 /// A vector of scalars. 1444 ValueList Scalars; 1445 1446 /// The Scalars are vectorized into this value. It is initialized to Null. 1447 Value *VectorizedValue = nullptr; 1448 1449 /// Do we need to gather this sequence ? 1450 enum EntryState { Vectorize, NeedToGather }; 1451 EntryState State; 1452 1453 /// Does this sequence require some shuffling? 1454 SmallVector<unsigned, 4> ReuseShuffleIndices; 1455 1456 /// Does this entry require reordering? 1457 ArrayRef<unsigned> ReorderIndices; 1458 1459 /// Points back to the VectorizableTree. 1460 /// 1461 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has 1462 /// to be a pointer and needs to be able to initialize the child iterator. 1463 /// Thus we need a reference back to the container to translate the indices 1464 /// to entries. 1465 VecTreeTy &Container; 1466 1467 /// The TreeEntry index containing the user of this entry. We can actually 1468 /// have multiple users so the data structure is not truly a tree. 1469 SmallVector<EdgeInfo, 1> UserTreeIndices; 1470 1471 /// The index of this treeEntry in VectorizableTree. 1472 int Idx = -1; 1473 1474 private: 1475 /// The operands of each instruction in each lane Operands[op_index][lane]. 1476 /// Note: This helps avoid the replication of the code that performs the 1477 /// reordering of operands during buildTree_rec() and vectorizeTree(). 1478 SmallVector<ValueList, 2> Operands; 1479 1480 /// The main/alternate instruction. 1481 Instruction *MainOp = nullptr; 1482 Instruction *AltOp = nullptr; 1483 1484 public: 1485 /// Set this bundle's \p OpIdx'th operand to \p OpVL. 1486 void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) { 1487 if (Operands.size() < OpIdx + 1) 1488 Operands.resize(OpIdx + 1); 1489 assert(Operands[OpIdx].size() == 0 && "Already resized?"); 1490 Operands[OpIdx].resize(Scalars.size()); 1491 for (unsigned Lane = 0, E = Scalars.size(); Lane != E; ++Lane) 1492 Operands[OpIdx][Lane] = OpVL[Lane]; 1493 } 1494 1495 /// Set the operands of this bundle in their original order. 1496 void setOperandsInOrder() { 1497 assert(Operands.empty() && "Already initialized?"); 1498 auto *I0 = cast<Instruction>(Scalars[0]); 1499 Operands.resize(I0->getNumOperands()); 1500 unsigned NumLanes = Scalars.size(); 1501 for (unsigned OpIdx = 0, NumOperands = I0->getNumOperands(); 1502 OpIdx != NumOperands; ++OpIdx) { 1503 Operands[OpIdx].resize(NumLanes); 1504 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) { 1505 auto *I = cast<Instruction>(Scalars[Lane]); 1506 assert(I->getNumOperands() == NumOperands && 1507 "Expected same number of operands"); 1508 Operands[OpIdx][Lane] = I->getOperand(OpIdx); 1509 } 1510 } 1511 } 1512 1513 /// \returns the \p OpIdx operand of this TreeEntry. 1514 ValueList &getOperand(unsigned OpIdx) { 1515 assert(OpIdx < Operands.size() && "Off bounds"); 1516 return Operands[OpIdx]; 1517 } 1518 1519 /// \returns the number of operands. 1520 unsigned getNumOperands() const { return Operands.size(); } 1521 1522 /// \return the single \p OpIdx operand. 1523 Value *getSingleOperand(unsigned OpIdx) const { 1524 assert(OpIdx < Operands.size() && "Off bounds"); 1525 assert(!Operands[OpIdx].empty() && "No operand available"); 1526 return Operands[OpIdx][0]; 1527 } 1528 1529 /// Some of the instructions in the list have alternate opcodes. 1530 bool isAltShuffle() const { 1531 return getOpcode() != getAltOpcode(); 1532 } 1533 1534 bool isOpcodeOrAlt(Instruction *I) const { 1535 unsigned CheckedOpcode = I->getOpcode(); 1536 return (getOpcode() == CheckedOpcode || 1537 getAltOpcode() == CheckedOpcode); 1538 } 1539 1540 /// Chooses the correct key for scheduling data. If \p Op has the same (or 1541 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is 1542 /// \p OpValue. 1543 Value *isOneOf(Value *Op) const { 1544 auto *I = dyn_cast<Instruction>(Op); 1545 if (I && isOpcodeOrAlt(I)) 1546 return Op; 1547 return MainOp; 1548 } 1549 1550 void setOperations(const InstructionsState &S) { 1551 MainOp = S.MainOp; 1552 AltOp = S.AltOp; 1553 } 1554 1555 Instruction *getMainOp() const { 1556 return MainOp; 1557 } 1558 1559 Instruction *getAltOp() const { 1560 return AltOp; 1561 } 1562 1563 /// The main/alternate opcodes for the list of instructions. 1564 unsigned getOpcode() const { 1565 return MainOp ? MainOp->getOpcode() : 0; 1566 } 1567 1568 unsigned getAltOpcode() const { 1569 return AltOp ? AltOp->getOpcode() : 0; 1570 } 1571 1572 /// Update operations state of this entry if reorder occurred. 1573 bool updateStateIfReorder() { 1574 if (ReorderIndices.empty()) 1575 return false; 1576 InstructionsState S = getSameOpcode(Scalars, ReorderIndices.front()); 1577 setOperations(S); 1578 return true; 1579 } 1580 1581 #ifndef NDEBUG 1582 /// Debug printer. 1583 LLVM_DUMP_METHOD void dump() const { 1584 dbgs() << Idx << ".\n"; 1585 for (unsigned OpI = 0, OpE = Operands.size(); OpI != OpE; ++OpI) { 1586 dbgs() << "Operand " << OpI << ":\n"; 1587 for (const Value *V : Operands[OpI]) 1588 dbgs().indent(2) << *V << "\n"; 1589 } 1590 dbgs() << "Scalars: \n"; 1591 for (Value *V : Scalars) 1592 dbgs().indent(2) << *V << "\n"; 1593 dbgs() << "State: "; 1594 switch (State) { 1595 case Vectorize: 1596 dbgs() << "Vectorize\n"; 1597 break; 1598 case NeedToGather: 1599 dbgs() << "NeedToGather\n"; 1600 break; 1601 } 1602 dbgs() << "MainOp: "; 1603 if (MainOp) 1604 dbgs() << *MainOp << "\n"; 1605 else 1606 dbgs() << "NULL\n"; 1607 dbgs() << "AltOp: "; 1608 if (AltOp) 1609 dbgs() << *AltOp << "\n"; 1610 else 1611 dbgs() << "NULL\n"; 1612 dbgs() << "VectorizedValue: "; 1613 if (VectorizedValue) 1614 dbgs() << *VectorizedValue << "\n"; 1615 else 1616 dbgs() << "NULL\n"; 1617 dbgs() << "ReuseShuffleIndices: "; 1618 if (ReuseShuffleIndices.empty()) 1619 dbgs() << "Emtpy"; 1620 else 1621 for (unsigned ReuseIdx : ReuseShuffleIndices) 1622 dbgs() << ReuseIdx << ", "; 1623 dbgs() << "\n"; 1624 dbgs() << "ReorderIndices: "; 1625 for (unsigned ReorderIdx : ReorderIndices) 1626 dbgs() << ReorderIdx << ", "; 1627 dbgs() << "\n"; 1628 dbgs() << "UserTreeIndices: "; 1629 for (const auto &EInfo : UserTreeIndices) 1630 dbgs() << EInfo << ", "; 1631 dbgs() << "\n"; 1632 } 1633 #endif 1634 }; 1635 1636 /// Create a new VectorizableTree entry. 1637 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, Optional<ScheduleData *> Bundle, 1638 const InstructionsState &S, 1639 const EdgeInfo &UserTreeIdx, 1640 ArrayRef<unsigned> ReuseShuffleIndices = None, 1641 ArrayRef<unsigned> ReorderIndices = None) { 1642 bool Vectorized = (bool)Bundle; 1643 VectorizableTree.push_back(std::make_unique<TreeEntry>(VectorizableTree)); 1644 TreeEntry *Last = VectorizableTree.back().get(); 1645 Last->Idx = VectorizableTree.size() - 1; 1646 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end()); 1647 Last->State = Vectorized ? TreeEntry::Vectorize : TreeEntry::NeedToGather; 1648 Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(), 1649 ReuseShuffleIndices.end()); 1650 Last->ReorderIndices = ReorderIndices; 1651 Last->setOperations(S); 1652 if (Vectorized) { 1653 for (int i = 0, e = VL.size(); i != e; ++i) { 1654 assert(!getTreeEntry(VL[i]) && "Scalar already in tree!"); 1655 ScalarToTreeEntry[VL[i]] = Last; 1656 } 1657 // Update the scheduler bundle to point to this TreeEntry. 1658 unsigned Lane = 0; 1659 for (ScheduleData *BundleMember = Bundle.getValue(); BundleMember; 1660 BundleMember = BundleMember->NextInBundle) { 1661 BundleMember->TE = Last; 1662 BundleMember->Lane = Lane; 1663 ++Lane; 1664 } 1665 assert((!Bundle.getValue() || Lane == VL.size()) && 1666 "Bundle and VL out of sync"); 1667 } else { 1668 MustGather.insert(VL.begin(), VL.end()); 1669 } 1670 1671 if (UserTreeIdx.UserTE) 1672 Last->UserTreeIndices.push_back(UserTreeIdx); 1673 1674 return Last; 1675 } 1676 1677 /// -- Vectorization State -- 1678 /// Holds all of the tree entries. 1679 TreeEntry::VecTreeTy VectorizableTree; 1680 1681 #ifndef NDEBUG 1682 /// Debug printer. 1683 LLVM_DUMP_METHOD void dumpVectorizableTree() const { 1684 for (unsigned Id = 0, IdE = VectorizableTree.size(); Id != IdE; ++Id) { 1685 VectorizableTree[Id]->dump(); 1686 dbgs() << "\n"; 1687 } 1688 } 1689 #endif 1690 1691 TreeEntry *getTreeEntry(Value *V) { 1692 auto I = ScalarToTreeEntry.find(V); 1693 if (I != ScalarToTreeEntry.end()) 1694 return I->second; 1695 return nullptr; 1696 } 1697 1698 const TreeEntry *getTreeEntry(Value *V) const { 1699 auto I = ScalarToTreeEntry.find(V); 1700 if (I != ScalarToTreeEntry.end()) 1701 return I->second; 1702 return nullptr; 1703 } 1704 1705 /// Maps a specific scalar to its tree entry. 1706 SmallDenseMap<Value*, TreeEntry *> ScalarToTreeEntry; 1707 1708 /// A list of scalars that we found that we need to keep as scalars. 1709 ValueSet MustGather; 1710 1711 /// This POD struct describes one external user in the vectorized tree. 1712 struct ExternalUser { 1713 ExternalUser(Value *S, llvm::User *U, int L) 1714 : Scalar(S), User(U), Lane(L) {} 1715 1716 // Which scalar in our function. 1717 Value *Scalar; 1718 1719 // Which user that uses the scalar. 1720 llvm::User *User; 1721 1722 // Which lane does the scalar belong to. 1723 int Lane; 1724 }; 1725 using UserList = SmallVector<ExternalUser, 16>; 1726 1727 /// Checks if two instructions may access the same memory. 1728 /// 1729 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it 1730 /// is invariant in the calling loop. 1731 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1, 1732 Instruction *Inst2) { 1733 // First check if the result is already in the cache. 1734 AliasCacheKey key = std::make_pair(Inst1, Inst2); 1735 Optional<bool> &result = AliasCache[key]; 1736 if (result.hasValue()) { 1737 return result.getValue(); 1738 } 1739 MemoryLocation Loc2 = getLocation(Inst2, AA); 1740 bool aliased = true; 1741 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) { 1742 // Do the alias check. 1743 aliased = AA->alias(Loc1, Loc2); 1744 } 1745 // Store the result in the cache. 1746 result = aliased; 1747 return aliased; 1748 } 1749 1750 using AliasCacheKey = std::pair<Instruction *, Instruction *>; 1751 1752 /// Cache for alias results. 1753 /// TODO: consider moving this to the AliasAnalysis itself. 1754 DenseMap<AliasCacheKey, Optional<bool>> AliasCache; 1755 1756 /// Removes an instruction from its block and eventually deletes it. 1757 /// It's like Instruction::eraseFromParent() except that the actual deletion 1758 /// is delayed until BoUpSLP is destructed. 1759 /// This is required to ensure that there are no incorrect collisions in the 1760 /// AliasCache, which can happen if a new instruction is allocated at the 1761 /// same address as a previously deleted instruction. 1762 void eraseInstruction(Instruction *I, bool ReplaceOpsWithUndef = false) { 1763 auto It = DeletedInstructions.try_emplace(I, ReplaceOpsWithUndef).first; 1764 It->getSecond() = It->getSecond() && ReplaceOpsWithUndef; 1765 } 1766 1767 /// Temporary store for deleted instructions. Instructions will be deleted 1768 /// eventually when the BoUpSLP is destructed. 1769 DenseMap<Instruction *, bool> DeletedInstructions; 1770 1771 /// A list of values that need to extracted out of the tree. 1772 /// This list holds pairs of (Internal Scalar : External User). External User 1773 /// can be nullptr, it means that this Internal Scalar will be used later, 1774 /// after vectorization. 1775 UserList ExternalUses; 1776 1777 /// Values used only by @llvm.assume calls. 1778 SmallPtrSet<const Value *, 32> EphValues; 1779 1780 /// Holds all of the instructions that we gathered. 1781 SetVector<Instruction *> GatherSeq; 1782 1783 /// A list of blocks that we are going to CSE. 1784 SetVector<BasicBlock *> CSEBlocks; 1785 1786 /// Contains all scheduling relevant data for an instruction. 1787 /// A ScheduleData either represents a single instruction or a member of an 1788 /// instruction bundle (= a group of instructions which is combined into a 1789 /// vector instruction). 1790 struct ScheduleData { 1791 // The initial value for the dependency counters. It means that the 1792 // dependencies are not calculated yet. 1793 enum { InvalidDeps = -1 }; 1794 1795 ScheduleData() = default; 1796 1797 void init(int BlockSchedulingRegionID, Value *OpVal) { 1798 FirstInBundle = this; 1799 NextInBundle = nullptr; 1800 NextLoadStore = nullptr; 1801 IsScheduled = false; 1802 SchedulingRegionID = BlockSchedulingRegionID; 1803 UnscheduledDepsInBundle = UnscheduledDeps; 1804 clearDependencies(); 1805 OpValue = OpVal; 1806 TE = nullptr; 1807 Lane = -1; 1808 } 1809 1810 /// Returns true if the dependency information has been calculated. 1811 bool hasValidDependencies() const { return Dependencies != InvalidDeps; } 1812 1813 /// Returns true for single instructions and for bundle representatives 1814 /// (= the head of a bundle). 1815 bool isSchedulingEntity() const { return FirstInBundle == this; } 1816 1817 /// Returns true if it represents an instruction bundle and not only a 1818 /// single instruction. 1819 bool isPartOfBundle() const { 1820 return NextInBundle != nullptr || FirstInBundle != this; 1821 } 1822 1823 /// Returns true if it is ready for scheduling, i.e. it has no more 1824 /// unscheduled depending instructions/bundles. 1825 bool isReady() const { 1826 assert(isSchedulingEntity() && 1827 "can't consider non-scheduling entity for ready list"); 1828 return UnscheduledDepsInBundle == 0 && !IsScheduled; 1829 } 1830 1831 /// Modifies the number of unscheduled dependencies, also updating it for 1832 /// the whole bundle. 1833 int incrementUnscheduledDeps(int Incr) { 1834 UnscheduledDeps += Incr; 1835 return FirstInBundle->UnscheduledDepsInBundle += Incr; 1836 } 1837 1838 /// Sets the number of unscheduled dependencies to the number of 1839 /// dependencies. 1840 void resetUnscheduledDeps() { 1841 incrementUnscheduledDeps(Dependencies - UnscheduledDeps); 1842 } 1843 1844 /// Clears all dependency information. 1845 void clearDependencies() { 1846 Dependencies = InvalidDeps; 1847 resetUnscheduledDeps(); 1848 MemoryDependencies.clear(); 1849 } 1850 1851 void dump(raw_ostream &os) const { 1852 if (!isSchedulingEntity()) { 1853 os << "/ " << *Inst; 1854 } else if (NextInBundle) { 1855 os << '[' << *Inst; 1856 ScheduleData *SD = NextInBundle; 1857 while (SD) { 1858 os << ';' << *SD->Inst; 1859 SD = SD->NextInBundle; 1860 } 1861 os << ']'; 1862 } else { 1863 os << *Inst; 1864 } 1865 } 1866 1867 Instruction *Inst = nullptr; 1868 1869 /// Points to the head in an instruction bundle (and always to this for 1870 /// single instructions). 1871 ScheduleData *FirstInBundle = nullptr; 1872 1873 /// Single linked list of all instructions in a bundle. Null if it is a 1874 /// single instruction. 1875 ScheduleData *NextInBundle = nullptr; 1876 1877 /// Single linked list of all memory instructions (e.g. load, store, call) 1878 /// in the block - until the end of the scheduling region. 1879 ScheduleData *NextLoadStore = nullptr; 1880 1881 /// The dependent memory instructions. 1882 /// This list is derived on demand in calculateDependencies(). 1883 SmallVector<ScheduleData *, 4> MemoryDependencies; 1884 1885 /// This ScheduleData is in the current scheduling region if this matches 1886 /// the current SchedulingRegionID of BlockScheduling. 1887 int SchedulingRegionID = 0; 1888 1889 /// Used for getting a "good" final ordering of instructions. 1890 int SchedulingPriority = 0; 1891 1892 /// The number of dependencies. Constitutes of the number of users of the 1893 /// instruction plus the number of dependent memory instructions (if any). 1894 /// This value is calculated on demand. 1895 /// If InvalidDeps, the number of dependencies is not calculated yet. 1896 int Dependencies = InvalidDeps; 1897 1898 /// The number of dependencies minus the number of dependencies of scheduled 1899 /// instructions. As soon as this is zero, the instruction/bundle gets ready 1900 /// for scheduling. 1901 /// Note that this is negative as long as Dependencies is not calculated. 1902 int UnscheduledDeps = InvalidDeps; 1903 1904 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for 1905 /// single instructions. 1906 int UnscheduledDepsInBundle = InvalidDeps; 1907 1908 /// True if this instruction is scheduled (or considered as scheduled in the 1909 /// dry-run). 1910 bool IsScheduled = false; 1911 1912 /// Opcode of the current instruction in the schedule data. 1913 Value *OpValue = nullptr; 1914 1915 /// The TreeEntry that this instruction corresponds to. 1916 TreeEntry *TE = nullptr; 1917 1918 /// The lane of this node in the TreeEntry. 1919 int Lane = -1; 1920 }; 1921 1922 #ifndef NDEBUG 1923 friend inline raw_ostream &operator<<(raw_ostream &os, 1924 const BoUpSLP::ScheduleData &SD) { 1925 SD.dump(os); 1926 return os; 1927 } 1928 #endif 1929 1930 friend struct GraphTraits<BoUpSLP *>; 1931 friend struct DOTGraphTraits<BoUpSLP *>; 1932 1933 /// Contains all scheduling data for a basic block. 1934 struct BlockScheduling { 1935 BlockScheduling(BasicBlock *BB) 1936 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {} 1937 1938 void clear() { 1939 ReadyInsts.clear(); 1940 ScheduleStart = nullptr; 1941 ScheduleEnd = nullptr; 1942 FirstLoadStoreInRegion = nullptr; 1943 LastLoadStoreInRegion = nullptr; 1944 1945 // Reduce the maximum schedule region size by the size of the 1946 // previous scheduling run. 1947 ScheduleRegionSizeLimit -= ScheduleRegionSize; 1948 if (ScheduleRegionSizeLimit < MinScheduleRegionSize) 1949 ScheduleRegionSizeLimit = MinScheduleRegionSize; 1950 ScheduleRegionSize = 0; 1951 1952 // Make a new scheduling region, i.e. all existing ScheduleData is not 1953 // in the new region yet. 1954 ++SchedulingRegionID; 1955 } 1956 1957 ScheduleData *getScheduleData(Value *V) { 1958 ScheduleData *SD = ScheduleDataMap[V]; 1959 if (SD && SD->SchedulingRegionID == SchedulingRegionID) 1960 return SD; 1961 return nullptr; 1962 } 1963 1964 ScheduleData *getScheduleData(Value *V, Value *Key) { 1965 if (V == Key) 1966 return getScheduleData(V); 1967 auto I = ExtraScheduleDataMap.find(V); 1968 if (I != ExtraScheduleDataMap.end()) { 1969 ScheduleData *SD = I->second[Key]; 1970 if (SD && SD->SchedulingRegionID == SchedulingRegionID) 1971 return SD; 1972 } 1973 return nullptr; 1974 } 1975 1976 bool isInSchedulingRegion(ScheduleData *SD) const { 1977 return SD->SchedulingRegionID == SchedulingRegionID; 1978 } 1979 1980 /// Marks an instruction as scheduled and puts all dependent ready 1981 /// instructions into the ready-list. 1982 template <typename ReadyListType> 1983 void schedule(ScheduleData *SD, ReadyListType &ReadyList) { 1984 SD->IsScheduled = true; 1985 LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n"); 1986 1987 ScheduleData *BundleMember = SD; 1988 while (BundleMember) { 1989 if (BundleMember->Inst != BundleMember->OpValue) { 1990 BundleMember = BundleMember->NextInBundle; 1991 continue; 1992 } 1993 // Handle the def-use chain dependencies. 1994 1995 // Decrement the unscheduled counter and insert to ready list if ready. 1996 auto &&DecrUnsched = [this, &ReadyList](Instruction *I) { 1997 doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) { 1998 if (OpDef && OpDef->hasValidDependencies() && 1999 OpDef->incrementUnscheduledDeps(-1) == 0) { 2000 // There are no more unscheduled dependencies after 2001 // decrementing, so we can put the dependent instruction 2002 // into the ready list. 2003 ScheduleData *DepBundle = OpDef->FirstInBundle; 2004 assert(!DepBundle->IsScheduled && 2005 "already scheduled bundle gets ready"); 2006 ReadyList.insert(DepBundle); 2007 LLVM_DEBUG(dbgs() 2008 << "SLP: gets ready (def): " << *DepBundle << "\n"); 2009 } 2010 }); 2011 }; 2012 2013 // If BundleMember is a vector bundle, its operands may have been 2014 // reordered duiring buildTree(). We therefore need to get its operands 2015 // through the TreeEntry. 2016 if (TreeEntry *TE = BundleMember->TE) { 2017 int Lane = BundleMember->Lane; 2018 assert(Lane >= 0 && "Lane not set"); 2019 for (unsigned OpIdx = 0, NumOperands = TE->getNumOperands(); 2020 OpIdx != NumOperands; ++OpIdx) 2021 if (auto *I = dyn_cast<Instruction>(TE->getOperand(OpIdx)[Lane])) 2022 DecrUnsched(I); 2023 } else { 2024 // If BundleMember is a stand-alone instruction, no operand reordering 2025 // has taken place, so we directly access its operands. 2026 for (Use &U : BundleMember->Inst->operands()) 2027 if (auto *I = dyn_cast<Instruction>(U.get())) 2028 DecrUnsched(I); 2029 } 2030 // Handle the memory dependencies. 2031 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) { 2032 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) { 2033 // There are no more unscheduled dependencies after decrementing, 2034 // so we can put the dependent instruction into the ready list. 2035 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle; 2036 assert(!DepBundle->IsScheduled && 2037 "already scheduled bundle gets ready"); 2038 ReadyList.insert(DepBundle); 2039 LLVM_DEBUG(dbgs() 2040 << "SLP: gets ready (mem): " << *DepBundle << "\n"); 2041 } 2042 } 2043 BundleMember = BundleMember->NextInBundle; 2044 } 2045 } 2046 2047 void doForAllOpcodes(Value *V, 2048 function_ref<void(ScheduleData *SD)> Action) { 2049 if (ScheduleData *SD = getScheduleData(V)) 2050 Action(SD); 2051 auto I = ExtraScheduleDataMap.find(V); 2052 if (I != ExtraScheduleDataMap.end()) 2053 for (auto &P : I->second) 2054 if (P.second->SchedulingRegionID == SchedulingRegionID) 2055 Action(P.second); 2056 } 2057 2058 /// Put all instructions into the ReadyList which are ready for scheduling. 2059 template <typename ReadyListType> 2060 void initialFillReadyList(ReadyListType &ReadyList) { 2061 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) { 2062 doForAllOpcodes(I, [&](ScheduleData *SD) { 2063 if (SD->isSchedulingEntity() && SD->isReady()) { 2064 ReadyList.insert(SD); 2065 LLVM_DEBUG(dbgs() 2066 << "SLP: initially in ready list: " << *I << "\n"); 2067 } 2068 }); 2069 } 2070 } 2071 2072 /// Checks if a bundle of instructions can be scheduled, i.e. has no 2073 /// cyclic dependencies. This is only a dry-run, no instructions are 2074 /// actually moved at this stage. 2075 /// \returns the scheduling bundle. The returned Optional value is non-None 2076 /// if \p VL is allowed to be scheduled. 2077 Optional<ScheduleData *> 2078 tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, 2079 const InstructionsState &S); 2080 2081 /// Un-bundles a group of instructions. 2082 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue); 2083 2084 /// Allocates schedule data chunk. 2085 ScheduleData *allocateScheduleDataChunks(); 2086 2087 /// Extends the scheduling region so that V is inside the region. 2088 /// \returns true if the region size is within the limit. 2089 bool extendSchedulingRegion(Value *V, const InstructionsState &S); 2090 2091 /// Initialize the ScheduleData structures for new instructions in the 2092 /// scheduling region. 2093 void initScheduleData(Instruction *FromI, Instruction *ToI, 2094 ScheduleData *PrevLoadStore, 2095 ScheduleData *NextLoadStore); 2096 2097 /// Updates the dependency information of a bundle and of all instructions/ 2098 /// bundles which depend on the original bundle. 2099 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList, 2100 BoUpSLP *SLP); 2101 2102 /// Sets all instruction in the scheduling region to un-scheduled. 2103 void resetSchedule(); 2104 2105 BasicBlock *BB; 2106 2107 /// Simple memory allocation for ScheduleData. 2108 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks; 2109 2110 /// The size of a ScheduleData array in ScheduleDataChunks. 2111 int ChunkSize; 2112 2113 /// The allocator position in the current chunk, which is the last entry 2114 /// of ScheduleDataChunks. 2115 int ChunkPos; 2116 2117 /// Attaches ScheduleData to Instruction. 2118 /// Note that the mapping survives during all vectorization iterations, i.e. 2119 /// ScheduleData structures are recycled. 2120 DenseMap<Value *, ScheduleData *> ScheduleDataMap; 2121 2122 /// Attaches ScheduleData to Instruction with the leading key. 2123 DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>> 2124 ExtraScheduleDataMap; 2125 2126 struct ReadyList : SmallVector<ScheduleData *, 8> { 2127 void insert(ScheduleData *SD) { push_back(SD); } 2128 }; 2129 2130 /// The ready-list for scheduling (only used for the dry-run). 2131 ReadyList ReadyInsts; 2132 2133 /// The first instruction of the scheduling region. 2134 Instruction *ScheduleStart = nullptr; 2135 2136 /// The first instruction _after_ the scheduling region. 2137 Instruction *ScheduleEnd = nullptr; 2138 2139 /// The first memory accessing instruction in the scheduling region 2140 /// (can be null). 2141 ScheduleData *FirstLoadStoreInRegion = nullptr; 2142 2143 /// The last memory accessing instruction in the scheduling region 2144 /// (can be null). 2145 ScheduleData *LastLoadStoreInRegion = nullptr; 2146 2147 /// The current size of the scheduling region. 2148 int ScheduleRegionSize = 0; 2149 2150 /// The maximum size allowed for the scheduling region. 2151 int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget; 2152 2153 /// The ID of the scheduling region. For a new vectorization iteration this 2154 /// is incremented which "removes" all ScheduleData from the region. 2155 // Make sure that the initial SchedulingRegionID is greater than the 2156 // initial SchedulingRegionID in ScheduleData (which is 0). 2157 int SchedulingRegionID = 1; 2158 }; 2159 2160 /// Attaches the BlockScheduling structures to basic blocks. 2161 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules; 2162 2163 /// Performs the "real" scheduling. Done before vectorization is actually 2164 /// performed in a basic block. 2165 void scheduleBlock(BlockScheduling *BS); 2166 2167 /// List of users to ignore during scheduling and that don't need extracting. 2168 ArrayRef<Value *> UserIgnoreList; 2169 2170 using OrdersType = SmallVector<unsigned, 4>; 2171 /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of 2172 /// sorted SmallVectors of unsigned. 2173 struct OrdersTypeDenseMapInfo { 2174 static OrdersType getEmptyKey() { 2175 OrdersType V; 2176 V.push_back(~1U); 2177 return V; 2178 } 2179 2180 static OrdersType getTombstoneKey() { 2181 OrdersType V; 2182 V.push_back(~2U); 2183 return V; 2184 } 2185 2186 static unsigned getHashValue(const OrdersType &V) { 2187 return static_cast<unsigned>(hash_combine_range(V.begin(), V.end())); 2188 } 2189 2190 static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) { 2191 return LHS == RHS; 2192 } 2193 }; 2194 2195 /// Contains orders of operations along with the number of bundles that have 2196 /// operations in this order. It stores only those orders that require 2197 /// reordering, if reordering is not required it is counted using \a 2198 /// NumOpsWantToKeepOriginalOrder. 2199 DenseMap<OrdersType, unsigned, OrdersTypeDenseMapInfo> NumOpsWantToKeepOrder; 2200 /// Number of bundles that do not require reordering. 2201 unsigned NumOpsWantToKeepOriginalOrder = 0; 2202 2203 // Analysis and block reference. 2204 Function *F; 2205 ScalarEvolution *SE; 2206 TargetTransformInfo *TTI; 2207 TargetLibraryInfo *TLI; 2208 AliasAnalysis *AA; 2209 LoopInfo *LI; 2210 DominatorTree *DT; 2211 AssumptionCache *AC; 2212 DemandedBits *DB; 2213 const DataLayout *DL; 2214 OptimizationRemarkEmitter *ORE; 2215 2216 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt. 2217 unsigned MinVecRegSize; // Set by cl::opt (default: 128). 2218 2219 /// Instruction builder to construct the vectorized tree. 2220 IRBuilder<> Builder; 2221 2222 /// A map of scalar integer values to the smallest bit width with which they 2223 /// can legally be represented. The values map to (width, signed) pairs, 2224 /// where "width" indicates the minimum bit width and "signed" is True if the 2225 /// value must be signed-extended, rather than zero-extended, back to its 2226 /// original width. 2227 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs; 2228 }; 2229 2230 } // end namespace slpvectorizer 2231 2232 template <> struct GraphTraits<BoUpSLP *> { 2233 using TreeEntry = BoUpSLP::TreeEntry; 2234 2235 /// NodeRef has to be a pointer per the GraphWriter. 2236 using NodeRef = TreeEntry *; 2237 2238 using ContainerTy = BoUpSLP::TreeEntry::VecTreeTy; 2239 2240 /// Add the VectorizableTree to the index iterator to be able to return 2241 /// TreeEntry pointers. 2242 struct ChildIteratorType 2243 : public iterator_adaptor_base< 2244 ChildIteratorType, SmallVector<BoUpSLP::EdgeInfo, 1>::iterator> { 2245 ContainerTy &VectorizableTree; 2246 2247 ChildIteratorType(SmallVector<BoUpSLP::EdgeInfo, 1>::iterator W, 2248 ContainerTy &VT) 2249 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {} 2250 2251 NodeRef operator*() { return I->UserTE; } 2252 }; 2253 2254 static NodeRef getEntryNode(BoUpSLP &R) { 2255 return R.VectorizableTree[0].get(); 2256 } 2257 2258 static ChildIteratorType child_begin(NodeRef N) { 2259 return {N->UserTreeIndices.begin(), N->Container}; 2260 } 2261 2262 static ChildIteratorType child_end(NodeRef N) { 2263 return {N->UserTreeIndices.end(), N->Container}; 2264 } 2265 2266 /// For the node iterator we just need to turn the TreeEntry iterator into a 2267 /// TreeEntry* iterator so that it dereferences to NodeRef. 2268 class nodes_iterator { 2269 using ItTy = ContainerTy::iterator; 2270 ItTy It; 2271 2272 public: 2273 nodes_iterator(const ItTy &It2) : It(It2) {} 2274 NodeRef operator*() { return It->get(); } 2275 nodes_iterator operator++() { 2276 ++It; 2277 return *this; 2278 } 2279 bool operator!=(const nodes_iterator &N2) const { return N2.It != It; } 2280 }; 2281 2282 static nodes_iterator nodes_begin(BoUpSLP *R) { 2283 return nodes_iterator(R->VectorizableTree.begin()); 2284 } 2285 2286 static nodes_iterator nodes_end(BoUpSLP *R) { 2287 return nodes_iterator(R->VectorizableTree.end()); 2288 } 2289 2290 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); } 2291 }; 2292 2293 template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits { 2294 using TreeEntry = BoUpSLP::TreeEntry; 2295 2296 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} 2297 2298 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) { 2299 std::string Str; 2300 raw_string_ostream OS(Str); 2301 if (isSplat(Entry->Scalars)) { 2302 OS << "<splat> " << *Entry->Scalars[0]; 2303 return Str; 2304 } 2305 for (auto V : Entry->Scalars) { 2306 OS << *V; 2307 if (std::any_of( 2308 R->ExternalUses.begin(), R->ExternalUses.end(), 2309 [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; })) 2310 OS << " <extract>"; 2311 OS << "\n"; 2312 } 2313 return Str; 2314 } 2315 2316 static std::string getNodeAttributes(const TreeEntry *Entry, 2317 const BoUpSLP *) { 2318 if (Entry->State == TreeEntry::NeedToGather) 2319 return "color=red"; 2320 return ""; 2321 } 2322 }; 2323 2324 } // end namespace llvm 2325 2326 BoUpSLP::~BoUpSLP() { 2327 for (const auto &Pair : DeletedInstructions) { 2328 // Replace operands of ignored instructions with Undefs in case if they were 2329 // marked for deletion. 2330 if (Pair.getSecond()) { 2331 Value *Undef = UndefValue::get(Pair.getFirst()->getType()); 2332 Pair.getFirst()->replaceAllUsesWith(Undef); 2333 } 2334 Pair.getFirst()->dropAllReferences(); 2335 } 2336 for (const auto &Pair : DeletedInstructions) { 2337 assert(Pair.getFirst()->use_empty() && 2338 "trying to erase instruction with users."); 2339 Pair.getFirst()->eraseFromParent(); 2340 } 2341 } 2342 2343 void BoUpSLP::eraseInstructions(ArrayRef<Value *> AV) { 2344 for (auto *V : AV) { 2345 if (auto *I = dyn_cast<Instruction>(V)) 2346 eraseInstruction(I, /*ReplaceWithUndef=*/true); 2347 }; 2348 } 2349 2350 void BoUpSLP::buildTree(ArrayRef<Value *> Roots, 2351 ArrayRef<Value *> UserIgnoreLst) { 2352 ExtraValueToDebugLocsMap ExternallyUsedValues; 2353 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst); 2354 } 2355 2356 void BoUpSLP::buildTree(ArrayRef<Value *> Roots, 2357 ExtraValueToDebugLocsMap &ExternallyUsedValues, 2358 ArrayRef<Value *> UserIgnoreLst) { 2359 deleteTree(); 2360 UserIgnoreList = UserIgnoreLst; 2361 if (!allSameType(Roots)) 2362 return; 2363 buildTree_rec(Roots, 0, EdgeInfo()); 2364 2365 // Collect the values that we need to extract from the tree. 2366 for (auto &TEPtr : VectorizableTree) { 2367 TreeEntry *Entry = TEPtr.get(); 2368 2369 // No need to handle users of gathered values. 2370 if (Entry->State == TreeEntry::NeedToGather) 2371 continue; 2372 2373 // For each lane: 2374 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) { 2375 Value *Scalar = Entry->Scalars[Lane]; 2376 int FoundLane = Lane; 2377 if (!Entry->ReuseShuffleIndices.empty()) { 2378 FoundLane = 2379 std::distance(Entry->ReuseShuffleIndices.begin(), 2380 llvm::find(Entry->ReuseShuffleIndices, FoundLane)); 2381 } 2382 2383 // Check if the scalar is externally used as an extra arg. 2384 auto ExtI = ExternallyUsedValues.find(Scalar); 2385 if (ExtI != ExternallyUsedValues.end()) { 2386 LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " 2387 << Lane << " from " << *Scalar << ".\n"); 2388 ExternalUses.emplace_back(Scalar, nullptr, FoundLane); 2389 } 2390 for (User *U : Scalar->users()) { 2391 LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n"); 2392 2393 Instruction *UserInst = dyn_cast<Instruction>(U); 2394 if (!UserInst) 2395 continue; 2396 2397 // Skip in-tree scalars that become vectors 2398 if (TreeEntry *UseEntry = getTreeEntry(U)) { 2399 Value *UseScalar = UseEntry->Scalars[0]; 2400 // Some in-tree scalars will remain as scalar in vectorized 2401 // instructions. If that is the case, the one in Lane 0 will 2402 // be used. 2403 if (UseScalar != U || 2404 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) { 2405 LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U 2406 << ".\n"); 2407 assert(UseEntry->State != TreeEntry::NeedToGather && "Bad state"); 2408 continue; 2409 } 2410 } 2411 2412 // Ignore users in the user ignore list. 2413 if (is_contained(UserIgnoreList, UserInst)) 2414 continue; 2415 2416 LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " 2417 << Lane << " from " << *Scalar << ".\n"); 2418 ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane)); 2419 } 2420 } 2421 } 2422 } 2423 2424 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth, 2425 const EdgeInfo &UserTreeIdx) { 2426 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!"); 2427 2428 InstructionsState S = getSameOpcode(VL); 2429 if (Depth == RecursionMaxDepth) { 2430 LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n"); 2431 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2432 return; 2433 } 2434 2435 // Don't handle vectors. 2436 if (S.OpValue->getType()->isVectorTy()) { 2437 LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n"); 2438 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2439 return; 2440 } 2441 2442 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue)) 2443 if (SI->getValueOperand()->getType()->isVectorTy()) { 2444 LLVM_DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n"); 2445 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2446 return; 2447 } 2448 2449 // If all of the operands are identical or constant we have a simple solution. 2450 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.getOpcode()) { 2451 LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n"); 2452 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2453 return; 2454 } 2455 2456 // We now know that this is a vector of instructions of the same type from 2457 // the same block. 2458 2459 // Don't vectorize ephemeral values. 2460 for (Value *V : VL) { 2461 if (EphValues.count(V)) { 2462 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V 2463 << ") is ephemeral.\n"); 2464 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2465 return; 2466 } 2467 } 2468 2469 // Check if this is a duplicate of another entry. 2470 if (TreeEntry *E = getTreeEntry(S.OpValue)) { 2471 LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n"); 2472 if (!E->isSame(VL)) { 2473 LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n"); 2474 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2475 return; 2476 } 2477 // Record the reuse of the tree node. FIXME, currently this is only used to 2478 // properly draw the graph rather than for the actual vectorization. 2479 E->UserTreeIndices.push_back(UserTreeIdx); 2480 LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue 2481 << ".\n"); 2482 return; 2483 } 2484 2485 // Check that none of the instructions in the bundle are already in the tree. 2486 for (Value *V : VL) { 2487 auto *I = dyn_cast<Instruction>(V); 2488 if (!I) 2489 continue; 2490 if (getTreeEntry(I)) { 2491 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V 2492 << ") is already in tree.\n"); 2493 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2494 return; 2495 } 2496 } 2497 2498 // If any of the scalars is marked as a value that needs to stay scalar, then 2499 // we need to gather the scalars. 2500 // The reduction nodes (stored in UserIgnoreList) also should stay scalar. 2501 for (Value *V : VL) { 2502 if (MustGather.count(V) || is_contained(UserIgnoreList, V)) { 2503 LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n"); 2504 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2505 return; 2506 } 2507 } 2508 2509 // Check that all of the users of the scalars that we want to vectorize are 2510 // schedulable. 2511 auto *VL0 = cast<Instruction>(S.OpValue); 2512 BasicBlock *BB = VL0->getParent(); 2513 2514 if (!DT->isReachableFromEntry(BB)) { 2515 // Don't go into unreachable blocks. They may contain instructions with 2516 // dependency cycles which confuse the final scheduling. 2517 LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n"); 2518 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2519 return; 2520 } 2521 2522 // Check that every instruction appears once in this bundle. 2523 SmallVector<unsigned, 4> ReuseShuffleIndicies; 2524 SmallVector<Value *, 4> UniqueValues; 2525 DenseMap<Value *, unsigned> UniquePositions; 2526 for (Value *V : VL) { 2527 auto Res = UniquePositions.try_emplace(V, UniqueValues.size()); 2528 ReuseShuffleIndicies.emplace_back(Res.first->second); 2529 if (Res.second) 2530 UniqueValues.emplace_back(V); 2531 } 2532 size_t NumUniqueScalarValues = UniqueValues.size(); 2533 if (NumUniqueScalarValues == VL.size()) { 2534 ReuseShuffleIndicies.clear(); 2535 } else { 2536 LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n"); 2537 if (NumUniqueScalarValues <= 1 || 2538 !llvm::isPowerOf2_32(NumUniqueScalarValues)) { 2539 LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n"); 2540 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx); 2541 return; 2542 } 2543 VL = UniqueValues; 2544 } 2545 2546 auto &BSRef = BlocksSchedules[BB]; 2547 if (!BSRef) 2548 BSRef = std::make_unique<BlockScheduling>(BB); 2549 2550 BlockScheduling &BS = *BSRef.get(); 2551 2552 Optional<ScheduleData *> Bundle = BS.tryScheduleBundle(VL, this, S); 2553 if (!Bundle) { 2554 LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n"); 2555 assert((!BS.getScheduleData(VL0) || 2556 !BS.getScheduleData(VL0)->isPartOfBundle()) && 2557 "tryScheduleBundle should cancelScheduling on failure"); 2558 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2559 ReuseShuffleIndicies); 2560 return; 2561 } 2562 LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n"); 2563 2564 unsigned ShuffleOrOp = S.isAltShuffle() ? 2565 (unsigned) Instruction::ShuffleVector : S.getOpcode(); 2566 switch (ShuffleOrOp) { 2567 case Instruction::PHI: { 2568 auto *PH = cast<PHINode>(VL0); 2569 2570 // Check for terminator values (e.g. invoke). 2571 for (unsigned j = 0; j < VL.size(); ++j) 2572 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) { 2573 Instruction *Term = dyn_cast<Instruction>( 2574 cast<PHINode>(VL[j])->getIncomingValueForBlock( 2575 PH->getIncomingBlock(i))); 2576 if (Term && Term->isTerminator()) { 2577 LLVM_DEBUG(dbgs() 2578 << "SLP: Need to swizzle PHINodes (terminator use).\n"); 2579 BS.cancelScheduling(VL, VL0); 2580 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2581 ReuseShuffleIndicies); 2582 return; 2583 } 2584 } 2585 2586 TreeEntry *TE = 2587 newTreeEntry(VL, Bundle, S, UserTreeIdx, ReuseShuffleIndicies); 2588 LLVM_DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n"); 2589 2590 // Keeps the reordered operands to avoid code duplication. 2591 SmallVector<ValueList, 2> OperandsVec; 2592 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) { 2593 ValueList Operands; 2594 // Prepare the operand vector. 2595 for (Value *j : VL) 2596 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock( 2597 PH->getIncomingBlock(i))); 2598 TE->setOperand(i, Operands); 2599 OperandsVec.push_back(Operands); 2600 } 2601 for (unsigned OpIdx = 0, OpE = OperandsVec.size(); OpIdx != OpE; ++OpIdx) 2602 buildTree_rec(OperandsVec[OpIdx], Depth + 1, {TE, OpIdx}); 2603 return; 2604 } 2605 case Instruction::ExtractValue: 2606 case Instruction::ExtractElement: { 2607 OrdersType CurrentOrder; 2608 bool Reuse = canReuseExtract(VL, VL0, CurrentOrder); 2609 if (Reuse) { 2610 LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n"); 2611 ++NumOpsWantToKeepOriginalOrder; 2612 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2613 ReuseShuffleIndicies); 2614 // This is a special case, as it does not gather, but at the same time 2615 // we are not extending buildTree_rec() towards the operands. 2616 ValueList Op0; 2617 Op0.assign(VL.size(), VL0->getOperand(0)); 2618 VectorizableTree.back()->setOperand(0, Op0); 2619 return; 2620 } 2621 if (!CurrentOrder.empty()) { 2622 LLVM_DEBUG({ 2623 dbgs() << "SLP: Reusing or shuffling of reordered extract sequence " 2624 "with order"; 2625 for (unsigned Idx : CurrentOrder) 2626 dbgs() << " " << Idx; 2627 dbgs() << "\n"; 2628 }); 2629 // Insert new order with initial value 0, if it does not exist, 2630 // otherwise return the iterator to the existing one. 2631 auto StoredCurrentOrderAndNum = 2632 NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first; 2633 ++StoredCurrentOrderAndNum->getSecond(); 2634 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2635 ReuseShuffleIndicies, 2636 StoredCurrentOrderAndNum->getFirst()); 2637 // This is a special case, as it does not gather, but at the same time 2638 // we are not extending buildTree_rec() towards the operands. 2639 ValueList Op0; 2640 Op0.assign(VL.size(), VL0->getOperand(0)); 2641 VectorizableTree.back()->setOperand(0, Op0); 2642 return; 2643 } 2644 LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n"); 2645 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2646 ReuseShuffleIndicies); 2647 BS.cancelScheduling(VL, VL0); 2648 return; 2649 } 2650 case Instruction::Load: { 2651 // Check that a vectorized load would load the same memory as a scalar 2652 // load. For example, we don't want to vectorize loads that are smaller 2653 // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM 2654 // treats loading/storing it as an i8 struct. If we vectorize loads/stores 2655 // from such a struct, we read/write packed bits disagreeing with the 2656 // unvectorized version. 2657 Type *ScalarTy = VL0->getType(); 2658 2659 if (DL->getTypeSizeInBits(ScalarTy) != 2660 DL->getTypeAllocSizeInBits(ScalarTy)) { 2661 BS.cancelScheduling(VL, VL0); 2662 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2663 ReuseShuffleIndicies); 2664 LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n"); 2665 return; 2666 } 2667 2668 // Make sure all loads in the bundle are simple - we can't vectorize 2669 // atomic or volatile loads. 2670 SmallVector<Value *, 4> PointerOps(VL.size()); 2671 auto POIter = PointerOps.begin(); 2672 for (Value *V : VL) { 2673 auto *L = cast<LoadInst>(V); 2674 if (!L->isSimple()) { 2675 BS.cancelScheduling(VL, VL0); 2676 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2677 ReuseShuffleIndicies); 2678 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n"); 2679 return; 2680 } 2681 *POIter = L->getPointerOperand(); 2682 ++POIter; 2683 } 2684 2685 OrdersType CurrentOrder; 2686 // Check the order of pointer operands. 2687 if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) { 2688 Value *Ptr0; 2689 Value *PtrN; 2690 if (CurrentOrder.empty()) { 2691 Ptr0 = PointerOps.front(); 2692 PtrN = PointerOps.back(); 2693 } else { 2694 Ptr0 = PointerOps[CurrentOrder.front()]; 2695 PtrN = PointerOps[CurrentOrder.back()]; 2696 } 2697 const SCEV *Scev0 = SE->getSCEV(Ptr0); 2698 const SCEV *ScevN = SE->getSCEV(PtrN); 2699 const auto *Diff = 2700 dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0)); 2701 uint64_t Size = DL->getTypeAllocSize(ScalarTy); 2702 // Check that the sorted loads are consecutive. 2703 if (Diff && Diff->getAPInt() == (VL.size() - 1) * Size) { 2704 if (CurrentOrder.empty()) { 2705 // Original loads are consecutive and does not require reordering. 2706 ++NumOpsWantToKeepOriginalOrder; 2707 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, 2708 UserTreeIdx, ReuseShuffleIndicies); 2709 TE->setOperandsInOrder(); 2710 LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n"); 2711 } else { 2712 // Need to reorder. 2713 auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first; 2714 ++I->getSecond(); 2715 TreeEntry *TE = 2716 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2717 ReuseShuffleIndicies, I->getFirst()); 2718 TE->setOperandsInOrder(); 2719 LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n"); 2720 } 2721 return; 2722 } 2723 } 2724 2725 LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n"); 2726 BS.cancelScheduling(VL, VL0); 2727 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2728 ReuseShuffleIndicies); 2729 return; 2730 } 2731 case Instruction::ZExt: 2732 case Instruction::SExt: 2733 case Instruction::FPToUI: 2734 case Instruction::FPToSI: 2735 case Instruction::FPExt: 2736 case Instruction::PtrToInt: 2737 case Instruction::IntToPtr: 2738 case Instruction::SIToFP: 2739 case Instruction::UIToFP: 2740 case Instruction::Trunc: 2741 case Instruction::FPTrunc: 2742 case Instruction::BitCast: { 2743 Type *SrcTy = VL0->getOperand(0)->getType(); 2744 for (Value *V : VL) { 2745 Type *Ty = cast<Instruction>(V)->getOperand(0)->getType(); 2746 if (Ty != SrcTy || !isValidElementType(Ty)) { 2747 BS.cancelScheduling(VL, VL0); 2748 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2749 ReuseShuffleIndicies); 2750 LLVM_DEBUG(dbgs() 2751 << "SLP: Gathering casts with different src types.\n"); 2752 return; 2753 } 2754 } 2755 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2756 ReuseShuffleIndicies); 2757 LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n"); 2758 2759 TE->setOperandsInOrder(); 2760 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) { 2761 ValueList Operands; 2762 // Prepare the operand vector. 2763 for (Value *V : VL) 2764 Operands.push_back(cast<Instruction>(V)->getOperand(i)); 2765 2766 buildTree_rec(Operands, Depth + 1, {TE, i}); 2767 } 2768 return; 2769 } 2770 case Instruction::ICmp: 2771 case Instruction::FCmp: { 2772 // Check that all of the compares have the same predicate. 2773 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate(); 2774 CmpInst::Predicate SwapP0 = CmpInst::getSwappedPredicate(P0); 2775 Type *ComparedTy = VL0->getOperand(0)->getType(); 2776 for (Value *V : VL) { 2777 CmpInst *Cmp = cast<CmpInst>(V); 2778 if ((Cmp->getPredicate() != P0 && Cmp->getPredicate() != SwapP0) || 2779 Cmp->getOperand(0)->getType() != ComparedTy) { 2780 BS.cancelScheduling(VL, VL0); 2781 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2782 ReuseShuffleIndicies); 2783 LLVM_DEBUG(dbgs() 2784 << "SLP: Gathering cmp with different predicate.\n"); 2785 return; 2786 } 2787 } 2788 2789 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2790 ReuseShuffleIndicies); 2791 LLVM_DEBUG(dbgs() << "SLP: added a vector of compares.\n"); 2792 2793 ValueList Left, Right; 2794 if (cast<CmpInst>(VL0)->isCommutative()) { 2795 // Commutative predicate - collect + sort operands of the instructions 2796 // so that each side is more likely to have the same opcode. 2797 assert(P0 == SwapP0 && "Commutative Predicate mismatch"); 2798 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this); 2799 } else { 2800 // Collect operands - commute if it uses the swapped predicate. 2801 for (Value *V : VL) { 2802 auto *Cmp = cast<CmpInst>(V); 2803 Value *LHS = Cmp->getOperand(0); 2804 Value *RHS = Cmp->getOperand(1); 2805 if (Cmp->getPredicate() != P0) 2806 std::swap(LHS, RHS); 2807 Left.push_back(LHS); 2808 Right.push_back(RHS); 2809 } 2810 } 2811 TE->setOperand(0, Left); 2812 TE->setOperand(1, Right); 2813 buildTree_rec(Left, Depth + 1, {TE, 0}); 2814 buildTree_rec(Right, Depth + 1, {TE, 1}); 2815 return; 2816 } 2817 case Instruction::Select: 2818 case Instruction::FNeg: 2819 case Instruction::Add: 2820 case Instruction::FAdd: 2821 case Instruction::Sub: 2822 case Instruction::FSub: 2823 case Instruction::Mul: 2824 case Instruction::FMul: 2825 case Instruction::UDiv: 2826 case Instruction::SDiv: 2827 case Instruction::FDiv: 2828 case Instruction::URem: 2829 case Instruction::SRem: 2830 case Instruction::FRem: 2831 case Instruction::Shl: 2832 case Instruction::LShr: 2833 case Instruction::AShr: 2834 case Instruction::And: 2835 case Instruction::Or: 2836 case Instruction::Xor: { 2837 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2838 ReuseShuffleIndicies); 2839 LLVM_DEBUG(dbgs() << "SLP: added a vector of un/bin op.\n"); 2840 2841 // Sort operands of the instructions so that each side is more likely to 2842 // have the same opcode. 2843 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) { 2844 ValueList Left, Right; 2845 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this); 2846 TE->setOperand(0, Left); 2847 TE->setOperand(1, Right); 2848 buildTree_rec(Left, Depth + 1, {TE, 0}); 2849 buildTree_rec(Right, Depth + 1, {TE, 1}); 2850 return; 2851 } 2852 2853 TE->setOperandsInOrder(); 2854 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) { 2855 ValueList Operands; 2856 // Prepare the operand vector. 2857 for (Value *j : VL) 2858 Operands.push_back(cast<Instruction>(j)->getOperand(i)); 2859 2860 buildTree_rec(Operands, Depth + 1, {TE, i}); 2861 } 2862 return; 2863 } 2864 case Instruction::GetElementPtr: { 2865 // We don't combine GEPs with complicated (nested) indexing. 2866 for (Value *V : VL) { 2867 if (cast<Instruction>(V)->getNumOperands() != 2) { 2868 LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"); 2869 BS.cancelScheduling(VL, VL0); 2870 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2871 ReuseShuffleIndicies); 2872 return; 2873 } 2874 } 2875 2876 // We can't combine several GEPs into one vector if they operate on 2877 // different types. 2878 Type *Ty0 = VL0->getOperand(0)->getType(); 2879 for (Value *V : VL) { 2880 Type *CurTy = cast<Instruction>(V)->getOperand(0)->getType(); 2881 if (Ty0 != CurTy) { 2882 LLVM_DEBUG(dbgs() 2883 << "SLP: not-vectorizable GEP (different types).\n"); 2884 BS.cancelScheduling(VL, VL0); 2885 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2886 ReuseShuffleIndicies); 2887 return; 2888 } 2889 } 2890 2891 // We don't combine GEPs with non-constant indexes. 2892 Type *Ty1 = VL0->getOperand(1)->getType(); 2893 for (Value *V : VL) { 2894 auto Op = cast<Instruction>(V)->getOperand(1); 2895 if (!isa<ConstantInt>(Op) || 2896 (Op->getType() != Ty1 && 2897 Op->getType()->getScalarSizeInBits() > 2898 DL->getIndexSizeInBits( 2899 V->getType()->getPointerAddressSpace()))) { 2900 LLVM_DEBUG(dbgs() 2901 << "SLP: not-vectorizable GEP (non-constant indexes).\n"); 2902 BS.cancelScheduling(VL, VL0); 2903 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2904 ReuseShuffleIndicies); 2905 return; 2906 } 2907 } 2908 2909 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2910 ReuseShuffleIndicies); 2911 LLVM_DEBUG(dbgs() << "SLP: added a vector of GEPs.\n"); 2912 TE->setOperandsInOrder(); 2913 for (unsigned i = 0, e = 2; i < e; ++i) { 2914 ValueList Operands; 2915 // Prepare the operand vector. 2916 for (Value *V : VL) 2917 Operands.push_back(cast<Instruction>(V)->getOperand(i)); 2918 2919 buildTree_rec(Operands, Depth + 1, {TE, i}); 2920 } 2921 return; 2922 } 2923 case Instruction::Store: { 2924 // Check if the stores are consecutive or if we need to swizzle them. 2925 llvm::Type *ScalarTy = cast<StoreInst>(VL0)->getValueOperand()->getType(); 2926 // Make sure all stores in the bundle are simple - we can't vectorize 2927 // atomic or volatile stores. 2928 SmallVector<Value *, 4> PointerOps(VL.size()); 2929 ValueList Operands(VL.size()); 2930 auto POIter = PointerOps.begin(); 2931 auto OIter = Operands.begin(); 2932 for (Value *V : VL) { 2933 auto *SI = cast<StoreInst>(V); 2934 if (!SI->isSimple()) { 2935 BS.cancelScheduling(VL, VL0); 2936 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2937 ReuseShuffleIndicies); 2938 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple stores.\n"); 2939 return; 2940 } 2941 *POIter = SI->getPointerOperand(); 2942 *OIter = SI->getValueOperand(); 2943 ++POIter; 2944 ++OIter; 2945 } 2946 2947 OrdersType CurrentOrder; 2948 // Check the order of pointer operands. 2949 if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) { 2950 Value *Ptr0; 2951 Value *PtrN; 2952 if (CurrentOrder.empty()) { 2953 Ptr0 = PointerOps.front(); 2954 PtrN = PointerOps.back(); 2955 } else { 2956 Ptr0 = PointerOps[CurrentOrder.front()]; 2957 PtrN = PointerOps[CurrentOrder.back()]; 2958 } 2959 const SCEV *Scev0 = SE->getSCEV(Ptr0); 2960 const SCEV *ScevN = SE->getSCEV(PtrN); 2961 const auto *Diff = 2962 dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0)); 2963 uint64_t Size = DL->getTypeAllocSize(ScalarTy); 2964 // Check that the sorted pointer operands are consecutive. 2965 if (Diff && Diff->getAPInt() == (VL.size() - 1) * Size) { 2966 if (CurrentOrder.empty()) { 2967 // Original stores are consecutive and does not require reordering. 2968 ++NumOpsWantToKeepOriginalOrder; 2969 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, 2970 UserTreeIdx, ReuseShuffleIndicies); 2971 TE->setOperandsInOrder(); 2972 buildTree_rec(Operands, Depth + 1, {TE, 0}); 2973 LLVM_DEBUG(dbgs() << "SLP: added a vector of stores.\n"); 2974 } else { 2975 // Need to reorder. 2976 auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first; 2977 ++(I->getSecond()); 2978 TreeEntry *TE = 2979 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 2980 ReuseShuffleIndicies, I->getFirst()); 2981 TE->setOperandsInOrder(); 2982 buildTree_rec(Operands, Depth + 1, {TE, 0}); 2983 LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled stores.\n"); 2984 } 2985 return; 2986 } 2987 } 2988 2989 BS.cancelScheduling(VL, VL0); 2990 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 2991 ReuseShuffleIndicies); 2992 LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n"); 2993 return; 2994 } 2995 case Instruction::Call: { 2996 // Check if the calls are all to the same vectorizable intrinsic. 2997 CallInst *CI = cast<CallInst>(VL0); 2998 // Check if this is an Intrinsic call or something that can be 2999 // represented by an intrinsic call 3000 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 3001 if (!isTriviallyVectorizable(ID)) { 3002 BS.cancelScheduling(VL, VL0); 3003 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3004 ReuseShuffleIndicies); 3005 LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n"); 3006 return; 3007 } 3008 Function *Int = CI->getCalledFunction(); 3009 unsigned NumArgs = CI->getNumArgOperands(); 3010 SmallVector<Value*, 4> ScalarArgs(NumArgs, nullptr); 3011 for (unsigned j = 0; j != NumArgs; ++j) 3012 if (hasVectorInstrinsicScalarOpd(ID, j)) 3013 ScalarArgs[j] = CI->getArgOperand(j); 3014 for (Value *V : VL) { 3015 CallInst *CI2 = dyn_cast<CallInst>(V); 3016 if (!CI2 || CI2->getCalledFunction() != Int || 3017 getVectorIntrinsicIDForCall(CI2, TLI) != ID || 3018 !CI->hasIdenticalOperandBundleSchema(*CI2)) { 3019 BS.cancelScheduling(VL, VL0); 3020 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3021 ReuseShuffleIndicies); 3022 LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *V 3023 << "\n"); 3024 return; 3025 } 3026 // Some intrinsics have scalar arguments and should be same in order for 3027 // them to be vectorized. 3028 for (unsigned j = 0; j != NumArgs; ++j) { 3029 if (hasVectorInstrinsicScalarOpd(ID, j)) { 3030 Value *A1J = CI2->getArgOperand(j); 3031 if (ScalarArgs[j] != A1J) { 3032 BS.cancelScheduling(VL, VL0); 3033 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3034 ReuseShuffleIndicies); 3035 LLVM_DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI 3036 << " argument " << ScalarArgs[j] << "!=" << A1J 3037 << "\n"); 3038 return; 3039 } 3040 } 3041 } 3042 // Verify that the bundle operands are identical between the two calls. 3043 if (CI->hasOperandBundles() && 3044 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(), 3045 CI->op_begin() + CI->getBundleOperandsEndIndex(), 3046 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) { 3047 BS.cancelScheduling(VL, VL0); 3048 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3049 ReuseShuffleIndicies); 3050 LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" 3051 << *CI << "!=" << *V << '\n'); 3052 return; 3053 } 3054 } 3055 3056 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 3057 ReuseShuffleIndicies); 3058 TE->setOperandsInOrder(); 3059 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) { 3060 ValueList Operands; 3061 // Prepare the operand vector. 3062 for (Value *V : VL) { 3063 auto *CI2 = cast<CallInst>(V); 3064 Operands.push_back(CI2->getArgOperand(i)); 3065 } 3066 buildTree_rec(Operands, Depth + 1, {TE, i}); 3067 } 3068 return; 3069 } 3070 case Instruction::ShuffleVector: { 3071 // If this is not an alternate sequence of opcode like add-sub 3072 // then do not vectorize this instruction. 3073 if (!S.isAltShuffle()) { 3074 BS.cancelScheduling(VL, VL0); 3075 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3076 ReuseShuffleIndicies); 3077 LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n"); 3078 return; 3079 } 3080 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, 3081 ReuseShuffleIndicies); 3082 LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n"); 3083 3084 // Reorder operands if reordering would enable vectorization. 3085 if (isa<BinaryOperator>(VL0)) { 3086 ValueList Left, Right; 3087 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this); 3088 TE->setOperand(0, Left); 3089 TE->setOperand(1, Right); 3090 buildTree_rec(Left, Depth + 1, {TE, 0}); 3091 buildTree_rec(Right, Depth + 1, {TE, 1}); 3092 return; 3093 } 3094 3095 TE->setOperandsInOrder(); 3096 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) { 3097 ValueList Operands; 3098 // Prepare the operand vector. 3099 for (Value *V : VL) 3100 Operands.push_back(cast<Instruction>(V)->getOperand(i)); 3101 3102 buildTree_rec(Operands, Depth + 1, {TE, i}); 3103 } 3104 return; 3105 } 3106 default: 3107 BS.cancelScheduling(VL, VL0); 3108 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx, 3109 ReuseShuffleIndicies); 3110 LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n"); 3111 return; 3112 } 3113 } 3114 3115 unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const { 3116 unsigned N = 1; 3117 Type *EltTy = T; 3118 3119 while (isa<CompositeType>(EltTy)) { 3120 if (auto *ST = dyn_cast<StructType>(EltTy)) { 3121 // Check that struct is homogeneous. 3122 for (const auto *Ty : ST->elements()) 3123 if (Ty != *ST->element_begin()) 3124 return 0; 3125 N *= ST->getNumElements(); 3126 EltTy = *ST->element_begin(); 3127 } else { 3128 auto *SeqT = cast<SequentialType>(EltTy); 3129 N *= SeqT->getNumElements(); 3130 EltTy = SeqT->getElementType(); 3131 } 3132 } 3133 3134 if (!isValidElementType(EltTy)) 3135 return 0; 3136 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N)); 3137 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T)) 3138 return 0; 3139 return N; 3140 } 3141 3142 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue, 3143 SmallVectorImpl<unsigned> &CurrentOrder) const { 3144 Instruction *E0 = cast<Instruction>(OpValue); 3145 assert(E0->getOpcode() == Instruction::ExtractElement || 3146 E0->getOpcode() == Instruction::ExtractValue); 3147 assert(E0->getOpcode() == getSameOpcode(VL).getOpcode() && "Invalid opcode"); 3148 // Check if all of the extracts come from the same vector and from the 3149 // correct offset. 3150 Value *Vec = E0->getOperand(0); 3151 3152 CurrentOrder.clear(); 3153 3154 // We have to extract from a vector/aggregate with the same number of elements. 3155 unsigned NElts; 3156 if (E0->getOpcode() == Instruction::ExtractValue) { 3157 const DataLayout &DL = E0->getModule()->getDataLayout(); 3158 NElts = canMapToVector(Vec->getType(), DL); 3159 if (!NElts) 3160 return false; 3161 // Check if load can be rewritten as load of vector. 3162 LoadInst *LI = dyn_cast<LoadInst>(Vec); 3163 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size())) 3164 return false; 3165 } else { 3166 NElts = Vec->getType()->getVectorNumElements(); 3167 } 3168 3169 if (NElts != VL.size()) 3170 return false; 3171 3172 // Check that all of the indices extract from the correct offset. 3173 bool ShouldKeepOrder = true; 3174 unsigned E = VL.size(); 3175 // Assign to all items the initial value E + 1 so we can check if the extract 3176 // instruction index was used already. 3177 // Also, later we can check that all the indices are used and we have a 3178 // consecutive access in the extract instructions, by checking that no 3179 // element of CurrentOrder still has value E + 1. 3180 CurrentOrder.assign(E, E + 1); 3181 unsigned I = 0; 3182 for (; I < E; ++I) { 3183 auto *Inst = cast<Instruction>(VL[I]); 3184 if (Inst->getOperand(0) != Vec) 3185 break; 3186 Optional<unsigned> Idx = getExtractIndex(Inst); 3187 if (!Idx) 3188 break; 3189 const unsigned ExtIdx = *Idx; 3190 if (ExtIdx != I) { 3191 if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1) 3192 break; 3193 ShouldKeepOrder = false; 3194 CurrentOrder[ExtIdx] = I; 3195 } else { 3196 if (CurrentOrder[I] != E + 1) 3197 break; 3198 CurrentOrder[I] = I; 3199 } 3200 } 3201 if (I < E) { 3202 CurrentOrder.clear(); 3203 return false; 3204 } 3205 3206 return ShouldKeepOrder; 3207 } 3208 3209 bool BoUpSLP::areAllUsersVectorized(Instruction *I) const { 3210 return I->hasOneUse() || 3211 std::all_of(I->user_begin(), I->user_end(), [this](User *U) { 3212 return ScalarToTreeEntry.count(U) > 0; 3213 }); 3214 } 3215 3216 int BoUpSLP::getEntryCost(TreeEntry *E) { 3217 ArrayRef<Value*> VL = E->Scalars; 3218 3219 Type *ScalarTy = VL[0]->getType(); 3220 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0])) 3221 ScalarTy = SI->getValueOperand()->getType(); 3222 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0])) 3223 ScalarTy = CI->getOperand(0)->getType(); 3224 VectorType *VecTy = VectorType::get(ScalarTy, VL.size()); 3225 3226 // If we have computed a smaller type for the expression, update VecTy so 3227 // that the costs will be accurate. 3228 if (MinBWs.count(VL[0])) 3229 VecTy = VectorType::get( 3230 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size()); 3231 3232 unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size(); 3233 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty(); 3234 int ReuseShuffleCost = 0; 3235 if (NeedToShuffleReuses) { 3236 ReuseShuffleCost = 3237 TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy); 3238 } 3239 if (E->State == TreeEntry::NeedToGather) { 3240 if (allConstant(VL)) 3241 return 0; 3242 if (isSplat(VL)) { 3243 return ReuseShuffleCost + 3244 TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0); 3245 } 3246 if (E->getOpcode() == Instruction::ExtractElement && 3247 allSameType(VL) && allSameBlock(VL)) { 3248 Optional<TargetTransformInfo::ShuffleKind> ShuffleKind = isShuffle(VL); 3249 if (ShuffleKind.hasValue()) { 3250 int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy); 3251 for (auto *V : VL) { 3252 // If all users of instruction are going to be vectorized and this 3253 // instruction itself is not going to be vectorized, consider this 3254 // instruction as dead and remove its cost from the final cost of the 3255 // vectorized tree. 3256 if (areAllUsersVectorized(cast<Instruction>(V)) && 3257 !ScalarToTreeEntry.count(V)) { 3258 auto *IO = cast<ConstantInt>( 3259 cast<ExtractElementInst>(V)->getIndexOperand()); 3260 Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, 3261 IO->getZExtValue()); 3262 } 3263 } 3264 return ReuseShuffleCost + Cost; 3265 } 3266 } 3267 return ReuseShuffleCost + getGatherCost(VL); 3268 } 3269 assert(E->getOpcode() && allSameType(VL) && allSameBlock(VL) && "Invalid VL"); 3270 Instruction *VL0 = E->getMainOp(); 3271 unsigned ShuffleOrOp = 3272 E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode(); 3273 switch (ShuffleOrOp) { 3274 case Instruction::PHI: 3275 return 0; 3276 3277 case Instruction::ExtractValue: 3278 case Instruction::ExtractElement: 3279 if (NeedToShuffleReuses) { 3280 unsigned Idx = 0; 3281 for (unsigned I : E->ReuseShuffleIndices) { 3282 if (ShuffleOrOp == Instruction::ExtractElement) { 3283 auto *IO = cast<ConstantInt>( 3284 cast<ExtractElementInst>(VL[I])->getIndexOperand()); 3285 Idx = IO->getZExtValue(); 3286 ReuseShuffleCost -= TTI->getVectorInstrCost( 3287 Instruction::ExtractElement, VecTy, Idx); 3288 } else { 3289 ReuseShuffleCost -= TTI->getVectorInstrCost( 3290 Instruction::ExtractElement, VecTy, Idx); 3291 ++Idx; 3292 } 3293 } 3294 Idx = ReuseShuffleNumbers; 3295 for (Value *V : VL) { 3296 if (ShuffleOrOp == Instruction::ExtractElement) { 3297 auto *IO = cast<ConstantInt>( 3298 cast<ExtractElementInst>(V)->getIndexOperand()); 3299 Idx = IO->getZExtValue(); 3300 } else { 3301 --Idx; 3302 } 3303 ReuseShuffleCost += 3304 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx); 3305 } 3306 } 3307 if (E->State == TreeEntry::Vectorize) { 3308 int DeadCost = ReuseShuffleCost; 3309 if (!E->ReorderIndices.empty()) { 3310 // TODO: Merge this shuffle with the ReuseShuffleCost. 3311 DeadCost += TTI->getShuffleCost( 3312 TargetTransformInfo::SK_PermuteSingleSrc, VecTy); 3313 } 3314 for (unsigned i = 0, e = VL.size(); i < e; ++i) { 3315 Instruction *E = cast<Instruction>(VL[i]); 3316 // If all users are going to be vectorized, instruction can be 3317 // considered as dead. 3318 // The same, if have only one user, it will be vectorized for sure. 3319 if (areAllUsersVectorized(E)) { 3320 // Take credit for instruction that will become dead. 3321 if (E->hasOneUse()) { 3322 Instruction *Ext = E->user_back(); 3323 if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) && 3324 all_of(Ext->users(), 3325 [](User *U) { return isa<GetElementPtrInst>(U); })) { 3326 // Use getExtractWithExtendCost() to calculate the cost of 3327 // extractelement/ext pair. 3328 DeadCost -= TTI->getExtractWithExtendCost( 3329 Ext->getOpcode(), Ext->getType(), VecTy, i); 3330 // Add back the cost of s|zext which is subtracted separately. 3331 DeadCost += TTI->getCastInstrCost( 3332 Ext->getOpcode(), Ext->getType(), E->getType(), Ext); 3333 continue; 3334 } 3335 } 3336 DeadCost -= 3337 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i); 3338 } 3339 } 3340 return DeadCost; 3341 } 3342 return ReuseShuffleCost + getGatherCost(VL); 3343 3344 case Instruction::ZExt: 3345 case Instruction::SExt: 3346 case Instruction::FPToUI: 3347 case Instruction::FPToSI: 3348 case Instruction::FPExt: 3349 case Instruction::PtrToInt: 3350 case Instruction::IntToPtr: 3351 case Instruction::SIToFP: 3352 case Instruction::UIToFP: 3353 case Instruction::Trunc: 3354 case Instruction::FPTrunc: 3355 case Instruction::BitCast: { 3356 Type *SrcTy = VL0->getOperand(0)->getType(); 3357 int ScalarEltCost = 3358 TTI->getCastInstrCost(E->getOpcode(), ScalarTy, SrcTy, VL0); 3359 if (NeedToShuffleReuses) { 3360 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3361 } 3362 3363 // Calculate the cost of this instruction. 3364 int ScalarCost = VL.size() * ScalarEltCost; 3365 3366 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size()); 3367 int VecCost = 0; 3368 // Check if the values are candidates to demote. 3369 if (!MinBWs.count(VL0) || VecTy != SrcVecTy) { 3370 VecCost = ReuseShuffleCost + 3371 TTI->getCastInstrCost(E->getOpcode(), VecTy, SrcVecTy, VL0); 3372 } 3373 return VecCost - ScalarCost; 3374 } 3375 case Instruction::FCmp: 3376 case Instruction::ICmp: 3377 case Instruction::Select: { 3378 // Calculate the cost of this instruction. 3379 int ScalarEltCost = TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy, 3380 Builder.getInt1Ty(), VL0); 3381 if (NeedToShuffleReuses) { 3382 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3383 } 3384 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size()); 3385 int ScalarCost = VecTy->getNumElements() * ScalarEltCost; 3386 int VecCost = TTI->getCmpSelInstrCost(E->getOpcode(), VecTy, MaskTy, VL0); 3387 return ReuseShuffleCost + VecCost - ScalarCost; 3388 } 3389 case Instruction::FNeg: 3390 case Instruction::Add: 3391 case Instruction::FAdd: 3392 case Instruction::Sub: 3393 case Instruction::FSub: 3394 case Instruction::Mul: 3395 case Instruction::FMul: 3396 case Instruction::UDiv: 3397 case Instruction::SDiv: 3398 case Instruction::FDiv: 3399 case Instruction::URem: 3400 case Instruction::SRem: 3401 case Instruction::FRem: 3402 case Instruction::Shl: 3403 case Instruction::LShr: 3404 case Instruction::AShr: 3405 case Instruction::And: 3406 case Instruction::Or: 3407 case Instruction::Xor: { 3408 // Certain instructions can be cheaper to vectorize if they have a 3409 // constant second vector operand. 3410 TargetTransformInfo::OperandValueKind Op1VK = 3411 TargetTransformInfo::OK_AnyValue; 3412 TargetTransformInfo::OperandValueKind Op2VK = 3413 TargetTransformInfo::OK_UniformConstantValue; 3414 TargetTransformInfo::OperandValueProperties Op1VP = 3415 TargetTransformInfo::OP_None; 3416 TargetTransformInfo::OperandValueProperties Op2VP = 3417 TargetTransformInfo::OP_PowerOf2; 3418 3419 // If all operands are exactly the same ConstantInt then set the 3420 // operand kind to OK_UniformConstantValue. 3421 // If instead not all operands are constants, then set the operand kind 3422 // to OK_AnyValue. If all operands are constants but not the same, 3423 // then set the operand kind to OK_NonUniformConstantValue. 3424 ConstantInt *CInt0 = nullptr; 3425 for (unsigned i = 0, e = VL.size(); i < e; ++i) { 3426 const Instruction *I = cast<Instruction>(VL[i]); 3427 unsigned OpIdx = isa<BinaryOperator>(I) ? 1 : 0; 3428 ConstantInt *CInt = dyn_cast<ConstantInt>(I->getOperand(OpIdx)); 3429 if (!CInt) { 3430 Op2VK = TargetTransformInfo::OK_AnyValue; 3431 Op2VP = TargetTransformInfo::OP_None; 3432 break; 3433 } 3434 if (Op2VP == TargetTransformInfo::OP_PowerOf2 && 3435 !CInt->getValue().isPowerOf2()) 3436 Op2VP = TargetTransformInfo::OP_None; 3437 if (i == 0) { 3438 CInt0 = CInt; 3439 continue; 3440 } 3441 if (CInt0 != CInt) 3442 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue; 3443 } 3444 3445 SmallVector<const Value *, 4> Operands(VL0->operand_values()); 3446 int ScalarEltCost = TTI->getArithmeticInstrCost( 3447 E->getOpcode(), ScalarTy, Op1VK, Op2VK, Op1VP, Op2VP, Operands, VL0); 3448 if (NeedToShuffleReuses) { 3449 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3450 } 3451 int ScalarCost = VecTy->getNumElements() * ScalarEltCost; 3452 int VecCost = TTI->getArithmeticInstrCost( 3453 E->getOpcode(), VecTy, Op1VK, Op2VK, Op1VP, Op2VP, Operands, VL0); 3454 return ReuseShuffleCost + VecCost - ScalarCost; 3455 } 3456 case Instruction::GetElementPtr: { 3457 TargetTransformInfo::OperandValueKind Op1VK = 3458 TargetTransformInfo::OK_AnyValue; 3459 TargetTransformInfo::OperandValueKind Op2VK = 3460 TargetTransformInfo::OK_UniformConstantValue; 3461 3462 int ScalarEltCost = 3463 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK); 3464 if (NeedToShuffleReuses) { 3465 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3466 } 3467 int ScalarCost = VecTy->getNumElements() * ScalarEltCost; 3468 int VecCost = 3469 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK); 3470 return ReuseShuffleCost + VecCost - ScalarCost; 3471 } 3472 case Instruction::Load: { 3473 // Cost of wide load - cost of scalar loads. 3474 MaybeAlign alignment(cast<LoadInst>(VL0)->getAlignment()); 3475 int ScalarEltCost = 3476 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0); 3477 if (NeedToShuffleReuses) { 3478 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3479 } 3480 int ScalarLdCost = VecTy->getNumElements() * ScalarEltCost; 3481 int VecLdCost = 3482 TTI->getMemoryOpCost(Instruction::Load, VecTy, alignment, 0, VL0); 3483 if (!E->ReorderIndices.empty()) { 3484 // TODO: Merge this shuffle with the ReuseShuffleCost. 3485 VecLdCost += TTI->getShuffleCost( 3486 TargetTransformInfo::SK_PermuteSingleSrc, VecTy); 3487 } 3488 return ReuseShuffleCost + VecLdCost - ScalarLdCost; 3489 } 3490 case Instruction::Store: { 3491 // We know that we can merge the stores. Calculate the cost. 3492 bool IsReorder = !E->ReorderIndices.empty(); 3493 auto *SI = 3494 cast<StoreInst>(IsReorder ? VL[E->ReorderIndices.front()] : VL0); 3495 MaybeAlign Alignment(SI->getAlignment()); 3496 int ScalarEltCost = 3497 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, Alignment, 0, VL0); 3498 if (NeedToShuffleReuses) 3499 ReuseShuffleCost = -(ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3500 int ScalarStCost = VecTy->getNumElements() * ScalarEltCost; 3501 int VecStCost = TTI->getMemoryOpCost(Instruction::Store, 3502 VecTy, Alignment, 0, VL0); 3503 if (IsReorder) { 3504 // TODO: Merge this shuffle with the ReuseShuffleCost. 3505 VecStCost += TTI->getShuffleCost( 3506 TargetTransformInfo::SK_PermuteSingleSrc, VecTy); 3507 } 3508 return ReuseShuffleCost + VecStCost - ScalarStCost; 3509 } 3510 case Instruction::Call: { 3511 CallInst *CI = cast<CallInst>(VL0); 3512 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 3513 3514 // Calculate the cost of the scalar and vector calls. 3515 SmallVector<Type *, 4> ScalarTys; 3516 for (unsigned op = 0, opc = CI->getNumArgOperands(); op != opc; ++op) 3517 ScalarTys.push_back(CI->getArgOperand(op)->getType()); 3518 3519 FastMathFlags FMF; 3520 if (auto *FPMO = dyn_cast<FPMathOperator>(CI)) 3521 FMF = FPMO->getFastMathFlags(); 3522 3523 int ScalarEltCost = 3524 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF); 3525 if (NeedToShuffleReuses) { 3526 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost; 3527 } 3528 int ScalarCallCost = VecTy->getNumElements() * ScalarEltCost; 3529 3530 SmallVector<Value *, 4> Args(CI->arg_operands()); 3531 int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF, 3532 VecTy->getNumElements()); 3533 3534 LLVM_DEBUG(dbgs() << "SLP: Call cost " << VecCallCost - ScalarCallCost 3535 << " (" << VecCallCost << "-" << ScalarCallCost << ")" 3536 << " for " << *CI << "\n"); 3537 3538 return ReuseShuffleCost + VecCallCost - ScalarCallCost; 3539 } 3540 case Instruction::ShuffleVector: { 3541 assert(E->isAltShuffle() && 3542 ((Instruction::isBinaryOp(E->getOpcode()) && 3543 Instruction::isBinaryOp(E->getAltOpcode())) || 3544 (Instruction::isCast(E->getOpcode()) && 3545 Instruction::isCast(E->getAltOpcode()))) && 3546 "Invalid Shuffle Vector Operand"); 3547 int ScalarCost = 0; 3548 if (NeedToShuffleReuses) { 3549 for (unsigned Idx : E->ReuseShuffleIndices) { 3550 Instruction *I = cast<Instruction>(VL[Idx]); 3551 ReuseShuffleCost -= TTI->getInstructionCost( 3552 I, TargetTransformInfo::TCK_RecipThroughput); 3553 } 3554 for (Value *V : VL) { 3555 Instruction *I = cast<Instruction>(V); 3556 ReuseShuffleCost += TTI->getInstructionCost( 3557 I, TargetTransformInfo::TCK_RecipThroughput); 3558 } 3559 } 3560 for (Value *V : VL) { 3561 Instruction *I = cast<Instruction>(V); 3562 assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode"); 3563 ScalarCost += TTI->getInstructionCost( 3564 I, TargetTransformInfo::TCK_RecipThroughput); 3565 } 3566 // VecCost is equal to sum of the cost of creating 2 vectors 3567 // and the cost of creating shuffle. 3568 int VecCost = 0; 3569 if (Instruction::isBinaryOp(E->getOpcode())) { 3570 VecCost = TTI->getArithmeticInstrCost(E->getOpcode(), VecTy); 3571 VecCost += TTI->getArithmeticInstrCost(E->getAltOpcode(), VecTy); 3572 } else { 3573 Type *Src0SclTy = E->getMainOp()->getOperand(0)->getType(); 3574 Type *Src1SclTy = E->getAltOp()->getOperand(0)->getType(); 3575 VectorType *Src0Ty = VectorType::get(Src0SclTy, VL.size()); 3576 VectorType *Src1Ty = VectorType::get(Src1SclTy, VL.size()); 3577 VecCost = TTI->getCastInstrCost(E->getOpcode(), VecTy, Src0Ty); 3578 VecCost += TTI->getCastInstrCost(E->getAltOpcode(), VecTy, Src1Ty); 3579 } 3580 VecCost += TTI->getShuffleCost(TargetTransformInfo::SK_Select, VecTy, 0); 3581 return ReuseShuffleCost + VecCost - ScalarCost; 3582 } 3583 default: 3584 llvm_unreachable("Unknown instruction"); 3585 } 3586 } 3587 3588 bool BoUpSLP::isFullyVectorizableTinyTree() const { 3589 LLVM_DEBUG(dbgs() << "SLP: Check whether the tree with height " 3590 << VectorizableTree.size() << " is fully vectorizable .\n"); 3591 3592 // We only handle trees of heights 1 and 2. 3593 if (VectorizableTree.size() == 1 && 3594 VectorizableTree[0]->State == TreeEntry::Vectorize) 3595 return true; 3596 3597 if (VectorizableTree.size() != 2) 3598 return false; 3599 3600 // Handle splat and all-constants stores. 3601 if (VectorizableTree[0]->State == TreeEntry::Vectorize && 3602 (allConstant(VectorizableTree[1]->Scalars) || 3603 isSplat(VectorizableTree[1]->Scalars))) 3604 return true; 3605 3606 // Gathering cost would be too much for tiny trees. 3607 if (VectorizableTree[0]->State == TreeEntry::NeedToGather || 3608 VectorizableTree[1]->State == TreeEntry::NeedToGather) 3609 return false; 3610 3611 return true; 3612 } 3613 3614 bool BoUpSLP::isLoadCombineReductionCandidate(unsigned RdxOpcode) const { 3615 if (RdxOpcode != Instruction::Or) 3616 return false; 3617 3618 unsigned NumElts = VectorizableTree[0]->Scalars.size(); 3619 Value *FirstReduced = VectorizableTree[0]->Scalars[0]; 3620 3621 // Look past the reduction to find a source value. Arbitrarily follow the 3622 // path through operand 0 of any 'or'. Also, peek through optional 3623 // shift-left-by-constant. 3624 Value *ZextLoad = FirstReduced; 3625 while (match(ZextLoad, m_Or(m_Value(), m_Value())) || 3626 match(ZextLoad, m_Shl(m_Value(), m_Constant()))) 3627 ZextLoad = cast<BinaryOperator>(ZextLoad)->getOperand(0); 3628 3629 // Check if the input to the reduction is an extended load. 3630 Value *LoadPtr; 3631 if (!match(ZextLoad, m_ZExt(m_Load(m_Value(LoadPtr))))) 3632 return false; 3633 3634 // Require that the total load bit width is a legal integer type. 3635 // For example, <8 x i8> --> i64 is a legal integer on a 64-bit target. 3636 // But <16 x i8> --> i128 is not, so the backend probably can't reduce it. 3637 Type *SrcTy = LoadPtr->getType()->getPointerElementType(); 3638 unsigned LoadBitWidth = SrcTy->getIntegerBitWidth() * NumElts; 3639 LLVMContext &Context = FirstReduced->getContext(); 3640 if (!TTI->isTypeLegal(IntegerType::get(Context, LoadBitWidth))) 3641 return false; 3642 3643 // Everything matched - assume that we can fold the whole sequence using 3644 // load combining. 3645 LLVM_DEBUG(dbgs() << "SLP: Assume load combining for scalar reduction of " 3646 << *(cast<Instruction>(FirstReduced)) << "\n"); 3647 3648 return true; 3649 } 3650 3651 bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() const { 3652 // We can vectorize the tree if its size is greater than or equal to the 3653 // minimum size specified by the MinTreeSize command line option. 3654 if (VectorizableTree.size() >= MinTreeSize) 3655 return false; 3656 3657 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we 3658 // can vectorize it if we can prove it fully vectorizable. 3659 if (isFullyVectorizableTinyTree()) 3660 return false; 3661 3662 assert(VectorizableTree.empty() 3663 ? ExternalUses.empty() 3664 : true && "We shouldn't have any external users"); 3665 3666 // Otherwise, we can't vectorize the tree. It is both tiny and not fully 3667 // vectorizable. 3668 return true; 3669 } 3670 3671 int BoUpSLP::getSpillCost() const { 3672 // Walk from the bottom of the tree to the top, tracking which values are 3673 // live. When we see a call instruction that is not part of our tree, 3674 // query TTI to see if there is a cost to keeping values live over it 3675 // (for example, if spills and fills are required). 3676 unsigned BundleWidth = VectorizableTree.front()->Scalars.size(); 3677 int Cost = 0; 3678 3679 SmallPtrSet<Instruction*, 4> LiveValues; 3680 Instruction *PrevInst = nullptr; 3681 3682 for (const auto &TEPtr : VectorizableTree) { 3683 Instruction *Inst = dyn_cast<Instruction>(TEPtr->Scalars[0]); 3684 if (!Inst) 3685 continue; 3686 3687 if (!PrevInst) { 3688 PrevInst = Inst; 3689 continue; 3690 } 3691 3692 // Update LiveValues. 3693 LiveValues.erase(PrevInst); 3694 for (auto &J : PrevInst->operands()) { 3695 if (isa<Instruction>(&*J) && getTreeEntry(&*J)) 3696 LiveValues.insert(cast<Instruction>(&*J)); 3697 } 3698 3699 LLVM_DEBUG({ 3700 dbgs() << "SLP: #LV: " << LiveValues.size(); 3701 for (auto *X : LiveValues) 3702 dbgs() << " " << X->getName(); 3703 dbgs() << ", Looking at "; 3704 Inst->dump(); 3705 }); 3706 3707 // Now find the sequence of instructions between PrevInst and Inst. 3708 unsigned NumCalls = 0; 3709 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(), 3710 PrevInstIt = 3711 PrevInst->getIterator().getReverse(); 3712 while (InstIt != PrevInstIt) { 3713 if (PrevInstIt == PrevInst->getParent()->rend()) { 3714 PrevInstIt = Inst->getParent()->rbegin(); 3715 continue; 3716 } 3717 3718 // Debug information does not impact spill cost. 3719 if ((isa<CallInst>(&*PrevInstIt) && 3720 !isa<DbgInfoIntrinsic>(&*PrevInstIt)) && 3721 &*PrevInstIt != PrevInst) 3722 NumCalls++; 3723 3724 ++PrevInstIt; 3725 } 3726 3727 if (NumCalls) { 3728 SmallVector<Type*, 4> V; 3729 for (auto *II : LiveValues) 3730 V.push_back(VectorType::get(II->getType(), BundleWidth)); 3731 Cost += NumCalls * TTI->getCostOfKeepingLiveOverCall(V); 3732 } 3733 3734 PrevInst = Inst; 3735 } 3736 3737 return Cost; 3738 } 3739 3740 int BoUpSLP::getTreeCost() { 3741 int Cost = 0; 3742 LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size " 3743 << VectorizableTree.size() << ".\n"); 3744 3745 unsigned BundleWidth = VectorizableTree[0]->Scalars.size(); 3746 3747 for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) { 3748 TreeEntry &TE = *VectorizableTree[I].get(); 3749 3750 // We create duplicate tree entries for gather sequences that have multiple 3751 // uses. However, we should not compute the cost of duplicate sequences. 3752 // For example, if we have a build vector (i.e., insertelement sequence) 3753 // that is used by more than one vector instruction, we only need to 3754 // compute the cost of the insertelement instructions once. The redundant 3755 // instructions will be eliminated by CSE. 3756 // 3757 // We should consider not creating duplicate tree entries for gather 3758 // sequences, and instead add additional edges to the tree representing 3759 // their uses. Since such an approach results in fewer total entries, 3760 // existing heuristics based on tree size may yield different results. 3761 // 3762 if (TE.State == TreeEntry::NeedToGather && 3763 std::any_of(std::next(VectorizableTree.begin(), I + 1), 3764 VectorizableTree.end(), 3765 [TE](const std::unique_ptr<TreeEntry> &EntryPtr) { 3766 return EntryPtr->State == TreeEntry::NeedToGather && 3767 EntryPtr->isSame(TE.Scalars); 3768 })) 3769 continue; 3770 3771 int C = getEntryCost(&TE); 3772 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C 3773 << " for bundle that starts with " << *TE.Scalars[0] 3774 << ".\n"); 3775 Cost += C; 3776 } 3777 3778 SmallPtrSet<Value *, 16> ExtractCostCalculated; 3779 int ExtractCost = 0; 3780 for (ExternalUser &EU : ExternalUses) { 3781 // We only add extract cost once for the same scalar. 3782 if (!ExtractCostCalculated.insert(EU.Scalar).second) 3783 continue; 3784 3785 // Uses by ephemeral values are free (because the ephemeral value will be 3786 // removed prior to code generation, and so the extraction will be 3787 // removed as well). 3788 if (EphValues.count(EU.User)) 3789 continue; 3790 3791 // If we plan to rewrite the tree in a smaller type, we will need to sign 3792 // extend the extracted value back to the original type. Here, we account 3793 // for the extract and the added cost of the sign extend if needed. 3794 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth); 3795 auto *ScalarRoot = VectorizableTree[0]->Scalars[0]; 3796 if (MinBWs.count(ScalarRoot)) { 3797 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first); 3798 auto Extend = 3799 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt; 3800 VecTy = VectorType::get(MinTy, BundleWidth); 3801 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(), 3802 VecTy, EU.Lane); 3803 } else { 3804 ExtractCost += 3805 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane); 3806 } 3807 } 3808 3809 int SpillCost = getSpillCost(); 3810 Cost += SpillCost + ExtractCost; 3811 3812 std::string Str; 3813 { 3814 raw_string_ostream OS(Str); 3815 OS << "SLP: Spill Cost = " << SpillCost << ".\n" 3816 << "SLP: Extract Cost = " << ExtractCost << ".\n" 3817 << "SLP: Total Cost = " << Cost << ".\n"; 3818 } 3819 LLVM_DEBUG(dbgs() << Str); 3820 3821 if (ViewSLPTree) 3822 ViewGraph(this, "SLP" + F->getName(), false, Str); 3823 3824 return Cost; 3825 } 3826 3827 int BoUpSLP::getGatherCost(Type *Ty, 3828 const DenseSet<unsigned> &ShuffledIndices) const { 3829 int Cost = 0; 3830 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i) 3831 if (!ShuffledIndices.count(i)) 3832 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i); 3833 if (!ShuffledIndices.empty()) 3834 Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, Ty); 3835 return Cost; 3836 } 3837 3838 int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) const { 3839 // Find the type of the operands in VL. 3840 Type *ScalarTy = VL[0]->getType(); 3841 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0])) 3842 ScalarTy = SI->getValueOperand()->getType(); 3843 VectorType *VecTy = VectorType::get(ScalarTy, VL.size()); 3844 // Find the cost of inserting/extracting values from the vector. 3845 // Check if the same elements are inserted several times and count them as 3846 // shuffle candidates. 3847 DenseSet<unsigned> ShuffledElements; 3848 DenseSet<Value *> UniqueElements; 3849 // Iterate in reverse order to consider insert elements with the high cost. 3850 for (unsigned I = VL.size(); I > 0; --I) { 3851 unsigned Idx = I - 1; 3852 if (!UniqueElements.insert(VL[Idx]).second) 3853 ShuffledElements.insert(Idx); 3854 } 3855 return getGatherCost(VecTy, ShuffledElements); 3856 } 3857 3858 // Perform operand reordering on the instructions in VL and return the reordered 3859 // operands in Left and Right. 3860 void BoUpSLP::reorderInputsAccordingToOpcode(ArrayRef<Value *> VL, 3861 SmallVectorImpl<Value *> &Left, 3862 SmallVectorImpl<Value *> &Right, 3863 const DataLayout &DL, 3864 ScalarEvolution &SE, 3865 const BoUpSLP &R) { 3866 if (VL.empty()) 3867 return; 3868 VLOperands Ops(VL, DL, SE, R); 3869 // Reorder the operands in place. 3870 Ops.reorder(); 3871 Left = Ops.getVL(0); 3872 Right = Ops.getVL(1); 3873 } 3874 3875 void BoUpSLP::setInsertPointAfterBundle(TreeEntry *E) { 3876 // Get the basic block this bundle is in. All instructions in the bundle 3877 // should be in this block. 3878 auto *Front = E->getMainOp(); 3879 auto *BB = Front->getParent(); 3880 assert(llvm::all_of(make_range(E->Scalars.begin(), E->Scalars.end()), 3881 [=](Value *V) -> bool { 3882 auto *I = cast<Instruction>(V); 3883 return !E->isOpcodeOrAlt(I) || I->getParent() == BB; 3884 })); 3885 3886 // The last instruction in the bundle in program order. 3887 Instruction *LastInst = nullptr; 3888 3889 // Find the last instruction. The common case should be that BB has been 3890 // scheduled, and the last instruction is VL.back(). So we start with 3891 // VL.back() and iterate over schedule data until we reach the end of the 3892 // bundle. The end of the bundle is marked by null ScheduleData. 3893 if (BlocksSchedules.count(BB)) { 3894 auto *Bundle = 3895 BlocksSchedules[BB]->getScheduleData(E->isOneOf(E->Scalars.back())); 3896 if (Bundle && Bundle->isPartOfBundle()) 3897 for (; Bundle; Bundle = Bundle->NextInBundle) 3898 if (Bundle->OpValue == Bundle->Inst) 3899 LastInst = Bundle->Inst; 3900 } 3901 3902 // LastInst can still be null at this point if there's either not an entry 3903 // for BB in BlocksSchedules or there's no ScheduleData available for 3904 // VL.back(). This can be the case if buildTree_rec aborts for various 3905 // reasons (e.g., the maximum recursion depth is reached, the maximum region 3906 // size is reached, etc.). ScheduleData is initialized in the scheduling 3907 // "dry-run". 3908 // 3909 // If this happens, we can still find the last instruction by brute force. We 3910 // iterate forwards from Front (inclusive) until we either see all 3911 // instructions in the bundle or reach the end of the block. If Front is the 3912 // last instruction in program order, LastInst will be set to Front, and we 3913 // will visit all the remaining instructions in the block. 3914 // 3915 // One of the reasons we exit early from buildTree_rec is to place an upper 3916 // bound on compile-time. Thus, taking an additional compile-time hit here is 3917 // not ideal. However, this should be exceedingly rare since it requires that 3918 // we both exit early from buildTree_rec and that the bundle be out-of-order 3919 // (causing us to iterate all the way to the end of the block). 3920 if (!LastInst) { 3921 SmallPtrSet<Value *, 16> Bundle(E->Scalars.begin(), E->Scalars.end()); 3922 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) { 3923 if (Bundle.erase(&I) && E->isOpcodeOrAlt(&I)) 3924 LastInst = &I; 3925 if (Bundle.empty()) 3926 break; 3927 } 3928 } 3929 assert(LastInst && "Failed to find last instruction in bundle"); 3930 3931 // Set the insertion point after the last instruction in the bundle. Set the 3932 // debug location to Front. 3933 Builder.SetInsertPoint(BB, ++LastInst->getIterator()); 3934 Builder.SetCurrentDebugLocation(Front->getDebugLoc()); 3935 } 3936 3937 Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) { 3938 Value *Vec = UndefValue::get(Ty); 3939 // Generate the 'InsertElement' instruction. 3940 for (unsigned i = 0; i < Ty->getNumElements(); ++i) { 3941 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i)); 3942 if (auto *Insrt = dyn_cast<InsertElementInst>(Vec)) { 3943 GatherSeq.insert(Insrt); 3944 CSEBlocks.insert(Insrt->getParent()); 3945 3946 // Add to our 'need-to-extract' list. 3947 if (TreeEntry *E = getTreeEntry(VL[i])) { 3948 // Find which lane we need to extract. 3949 int FoundLane = -1; 3950 for (unsigned Lane = 0, LE = E->Scalars.size(); Lane != LE; ++Lane) { 3951 // Is this the lane of the scalar that we are looking for ? 3952 if (E->Scalars[Lane] == VL[i]) { 3953 FoundLane = Lane; 3954 break; 3955 } 3956 } 3957 assert(FoundLane >= 0 && "Could not find the correct lane"); 3958 if (!E->ReuseShuffleIndices.empty()) { 3959 FoundLane = 3960 std::distance(E->ReuseShuffleIndices.begin(), 3961 llvm::find(E->ReuseShuffleIndices, FoundLane)); 3962 } 3963 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane)); 3964 } 3965 } 3966 } 3967 3968 return Vec; 3969 } 3970 3971 Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) { 3972 InstructionsState S = getSameOpcode(VL); 3973 if (S.getOpcode()) { 3974 if (TreeEntry *E = getTreeEntry(S.OpValue)) { 3975 if (E->isSame(VL)) { 3976 Value *V = vectorizeTree(E); 3977 if (VL.size() == E->Scalars.size() && !E->ReuseShuffleIndices.empty()) { 3978 // We need to get the vectorized value but without shuffle. 3979 if (auto *SV = dyn_cast<ShuffleVectorInst>(V)) { 3980 V = SV->getOperand(0); 3981 } else { 3982 // Reshuffle to get only unique values. 3983 SmallVector<unsigned, 4> UniqueIdxs; 3984 SmallSet<unsigned, 4> UsedIdxs; 3985 for(unsigned Idx : E->ReuseShuffleIndices) 3986 if (UsedIdxs.insert(Idx).second) 3987 UniqueIdxs.emplace_back(Idx); 3988 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()), 3989 UniqueIdxs); 3990 } 3991 } 3992 return V; 3993 } 3994 } 3995 } 3996 3997 Type *ScalarTy = S.OpValue->getType(); 3998 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue)) 3999 ScalarTy = SI->getValueOperand()->getType(); 4000 4001 // Check that every instruction appears once in this bundle. 4002 SmallVector<unsigned, 4> ReuseShuffleIndicies; 4003 SmallVector<Value *, 4> UniqueValues; 4004 if (VL.size() > 2) { 4005 DenseMap<Value *, unsigned> UniquePositions; 4006 for (Value *V : VL) { 4007 auto Res = UniquePositions.try_emplace(V, UniqueValues.size()); 4008 ReuseShuffleIndicies.emplace_back(Res.first->second); 4009 if (Res.second || isa<Constant>(V)) 4010 UniqueValues.emplace_back(V); 4011 } 4012 // Do not shuffle single element or if number of unique values is not power 4013 // of 2. 4014 if (UniqueValues.size() == VL.size() || UniqueValues.size() <= 1 || 4015 !llvm::isPowerOf2_32(UniqueValues.size())) 4016 ReuseShuffleIndicies.clear(); 4017 else 4018 VL = UniqueValues; 4019 } 4020 VectorType *VecTy = VectorType::get(ScalarTy, VL.size()); 4021 4022 Value *V = Gather(VL, VecTy); 4023 if (!ReuseShuffleIndicies.empty()) { 4024 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4025 ReuseShuffleIndicies, "shuffle"); 4026 if (auto *I = dyn_cast<Instruction>(V)) { 4027 GatherSeq.insert(I); 4028 CSEBlocks.insert(I->getParent()); 4029 } 4030 } 4031 return V; 4032 } 4033 4034 static void inversePermutation(ArrayRef<unsigned> Indices, 4035 SmallVectorImpl<unsigned> &Mask) { 4036 Mask.clear(); 4037 const unsigned E = Indices.size(); 4038 Mask.resize(E); 4039 for (unsigned I = 0; I < E; ++I) 4040 Mask[Indices[I]] = I; 4041 } 4042 4043 Value *BoUpSLP::vectorizeTree(TreeEntry *E) { 4044 IRBuilder<>::InsertPointGuard Guard(Builder); 4045 4046 if (E->VectorizedValue) { 4047 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n"); 4048 return E->VectorizedValue; 4049 } 4050 4051 Instruction *VL0 = E->getMainOp(); 4052 Type *ScalarTy = VL0->getType(); 4053 if (StoreInst *SI = dyn_cast<StoreInst>(VL0)) 4054 ScalarTy = SI->getValueOperand()->getType(); 4055 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size()); 4056 4057 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty(); 4058 4059 if (E->State == TreeEntry::NeedToGather) { 4060 setInsertPointAfterBundle(E); 4061 auto *V = Gather(E->Scalars, VecTy); 4062 if (NeedToShuffleReuses) { 4063 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4064 E->ReuseShuffleIndices, "shuffle"); 4065 if (auto *I = dyn_cast<Instruction>(V)) { 4066 GatherSeq.insert(I); 4067 CSEBlocks.insert(I->getParent()); 4068 } 4069 } 4070 E->VectorizedValue = V; 4071 return V; 4072 } 4073 4074 unsigned ShuffleOrOp = 4075 E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode(); 4076 switch (ShuffleOrOp) { 4077 case Instruction::PHI: { 4078 auto *PH = cast<PHINode>(VL0); 4079 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI()); 4080 Builder.SetCurrentDebugLocation(PH->getDebugLoc()); 4081 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues()); 4082 Value *V = NewPhi; 4083 if (NeedToShuffleReuses) { 4084 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4085 E->ReuseShuffleIndices, "shuffle"); 4086 } 4087 E->VectorizedValue = V; 4088 4089 // PHINodes may have multiple entries from the same block. We want to 4090 // visit every block once. 4091 SmallPtrSet<BasicBlock*, 4> VisitedBBs; 4092 4093 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) { 4094 ValueList Operands; 4095 BasicBlock *IBB = PH->getIncomingBlock(i); 4096 4097 if (!VisitedBBs.insert(IBB).second) { 4098 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB); 4099 continue; 4100 } 4101 4102 Builder.SetInsertPoint(IBB->getTerminator()); 4103 Builder.SetCurrentDebugLocation(PH->getDebugLoc()); 4104 Value *Vec = vectorizeTree(E->getOperand(i)); 4105 NewPhi->addIncoming(Vec, IBB); 4106 } 4107 4108 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && 4109 "Invalid number of incoming values"); 4110 return V; 4111 } 4112 4113 case Instruction::ExtractElement: { 4114 if (E->State == TreeEntry::Vectorize) { 4115 Value *V = E->getSingleOperand(0); 4116 if (!E->ReorderIndices.empty()) { 4117 OrdersType Mask; 4118 inversePermutation(E->ReorderIndices, Mask); 4119 Builder.SetInsertPoint(VL0); 4120 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), Mask, 4121 "reorder_shuffle"); 4122 } 4123 if (NeedToShuffleReuses) { 4124 // TODO: Merge this shuffle with the ReorderShuffleMask. 4125 if (E->ReorderIndices.empty()) 4126 Builder.SetInsertPoint(VL0); 4127 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4128 E->ReuseShuffleIndices, "shuffle"); 4129 } 4130 E->VectorizedValue = V; 4131 return V; 4132 } 4133 setInsertPointAfterBundle(E); 4134 auto *V = Gather(E->Scalars, VecTy); 4135 if (NeedToShuffleReuses) { 4136 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4137 E->ReuseShuffleIndices, "shuffle"); 4138 if (auto *I = dyn_cast<Instruction>(V)) { 4139 GatherSeq.insert(I); 4140 CSEBlocks.insert(I->getParent()); 4141 } 4142 } 4143 E->VectorizedValue = V; 4144 return V; 4145 } 4146 case Instruction::ExtractValue: { 4147 if (E->State == TreeEntry::Vectorize) { 4148 LoadInst *LI = cast<LoadInst>(E->getSingleOperand(0)); 4149 Builder.SetInsertPoint(LI); 4150 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace()); 4151 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy); 4152 LoadInst *V = Builder.CreateAlignedLoad(VecTy, Ptr, LI->getAlignment()); 4153 Value *NewV = propagateMetadata(V, E->Scalars); 4154 if (!E->ReorderIndices.empty()) { 4155 OrdersType Mask; 4156 inversePermutation(E->ReorderIndices, Mask); 4157 NewV = Builder.CreateShuffleVector(NewV, UndefValue::get(VecTy), Mask, 4158 "reorder_shuffle"); 4159 } 4160 if (NeedToShuffleReuses) { 4161 // TODO: Merge this shuffle with the ReorderShuffleMask. 4162 NewV = Builder.CreateShuffleVector( 4163 NewV, UndefValue::get(VecTy), E->ReuseShuffleIndices, "shuffle"); 4164 } 4165 E->VectorizedValue = NewV; 4166 return NewV; 4167 } 4168 setInsertPointAfterBundle(E); 4169 auto *V = Gather(E->Scalars, VecTy); 4170 if (NeedToShuffleReuses) { 4171 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4172 E->ReuseShuffleIndices, "shuffle"); 4173 if (auto *I = dyn_cast<Instruction>(V)) { 4174 GatherSeq.insert(I); 4175 CSEBlocks.insert(I->getParent()); 4176 } 4177 } 4178 E->VectorizedValue = V; 4179 return V; 4180 } 4181 case Instruction::ZExt: 4182 case Instruction::SExt: 4183 case Instruction::FPToUI: 4184 case Instruction::FPToSI: 4185 case Instruction::FPExt: 4186 case Instruction::PtrToInt: 4187 case Instruction::IntToPtr: 4188 case Instruction::SIToFP: 4189 case Instruction::UIToFP: 4190 case Instruction::Trunc: 4191 case Instruction::FPTrunc: 4192 case Instruction::BitCast: { 4193 setInsertPointAfterBundle(E); 4194 4195 Value *InVec = vectorizeTree(E->getOperand(0)); 4196 4197 if (E->VectorizedValue) { 4198 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4199 return E->VectorizedValue; 4200 } 4201 4202 auto *CI = cast<CastInst>(VL0); 4203 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy); 4204 if (NeedToShuffleReuses) { 4205 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4206 E->ReuseShuffleIndices, "shuffle"); 4207 } 4208 E->VectorizedValue = V; 4209 ++NumVectorInstructions; 4210 return V; 4211 } 4212 case Instruction::FCmp: 4213 case Instruction::ICmp: { 4214 setInsertPointAfterBundle(E); 4215 4216 Value *L = vectorizeTree(E->getOperand(0)); 4217 Value *R = vectorizeTree(E->getOperand(1)); 4218 4219 if (E->VectorizedValue) { 4220 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4221 return E->VectorizedValue; 4222 } 4223 4224 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate(); 4225 Value *V; 4226 if (E->getOpcode() == Instruction::FCmp) 4227 V = Builder.CreateFCmp(P0, L, R); 4228 else 4229 V = Builder.CreateICmp(P0, L, R); 4230 4231 propagateIRFlags(V, E->Scalars, VL0); 4232 if (NeedToShuffleReuses) { 4233 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4234 E->ReuseShuffleIndices, "shuffle"); 4235 } 4236 E->VectorizedValue = V; 4237 ++NumVectorInstructions; 4238 return V; 4239 } 4240 case Instruction::Select: { 4241 setInsertPointAfterBundle(E); 4242 4243 Value *Cond = vectorizeTree(E->getOperand(0)); 4244 Value *True = vectorizeTree(E->getOperand(1)); 4245 Value *False = vectorizeTree(E->getOperand(2)); 4246 4247 if (E->VectorizedValue) { 4248 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4249 return E->VectorizedValue; 4250 } 4251 4252 Value *V = Builder.CreateSelect(Cond, True, False); 4253 if (NeedToShuffleReuses) { 4254 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4255 E->ReuseShuffleIndices, "shuffle"); 4256 } 4257 E->VectorizedValue = V; 4258 ++NumVectorInstructions; 4259 return V; 4260 } 4261 case Instruction::FNeg: { 4262 setInsertPointAfterBundle(E); 4263 4264 Value *Op = vectorizeTree(E->getOperand(0)); 4265 4266 if (E->VectorizedValue) { 4267 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4268 return E->VectorizedValue; 4269 } 4270 4271 Value *V = Builder.CreateUnOp( 4272 static_cast<Instruction::UnaryOps>(E->getOpcode()), Op); 4273 propagateIRFlags(V, E->Scalars, VL0); 4274 if (auto *I = dyn_cast<Instruction>(V)) 4275 V = propagateMetadata(I, E->Scalars); 4276 4277 if (NeedToShuffleReuses) { 4278 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4279 E->ReuseShuffleIndices, "shuffle"); 4280 } 4281 E->VectorizedValue = V; 4282 ++NumVectorInstructions; 4283 4284 return V; 4285 } 4286 case Instruction::Add: 4287 case Instruction::FAdd: 4288 case Instruction::Sub: 4289 case Instruction::FSub: 4290 case Instruction::Mul: 4291 case Instruction::FMul: 4292 case Instruction::UDiv: 4293 case Instruction::SDiv: 4294 case Instruction::FDiv: 4295 case Instruction::URem: 4296 case Instruction::SRem: 4297 case Instruction::FRem: 4298 case Instruction::Shl: 4299 case Instruction::LShr: 4300 case Instruction::AShr: 4301 case Instruction::And: 4302 case Instruction::Or: 4303 case Instruction::Xor: { 4304 setInsertPointAfterBundle(E); 4305 4306 Value *LHS = vectorizeTree(E->getOperand(0)); 4307 Value *RHS = vectorizeTree(E->getOperand(1)); 4308 4309 if (E->VectorizedValue) { 4310 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4311 return E->VectorizedValue; 4312 } 4313 4314 Value *V = Builder.CreateBinOp( 4315 static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS, 4316 RHS); 4317 propagateIRFlags(V, E->Scalars, VL0); 4318 if (auto *I = dyn_cast<Instruction>(V)) 4319 V = propagateMetadata(I, E->Scalars); 4320 4321 if (NeedToShuffleReuses) { 4322 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4323 E->ReuseShuffleIndices, "shuffle"); 4324 } 4325 E->VectorizedValue = V; 4326 ++NumVectorInstructions; 4327 4328 return V; 4329 } 4330 case Instruction::Load: { 4331 // Loads are inserted at the head of the tree because we don't want to 4332 // sink them all the way down past store instructions. 4333 bool IsReorder = E->updateStateIfReorder(); 4334 if (IsReorder) 4335 VL0 = E->getMainOp(); 4336 setInsertPointAfterBundle(E); 4337 4338 LoadInst *LI = cast<LoadInst>(VL0); 4339 Type *ScalarLoadTy = LI->getType(); 4340 unsigned AS = LI->getPointerAddressSpace(); 4341 4342 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(), 4343 VecTy->getPointerTo(AS)); 4344 4345 // The pointer operand uses an in-tree scalar so we add the new BitCast to 4346 // ExternalUses list to make sure that an extract will be generated in the 4347 // future. 4348 Value *PO = LI->getPointerOperand(); 4349 if (getTreeEntry(PO)) 4350 ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0)); 4351 4352 MaybeAlign Alignment = MaybeAlign(LI->getAlignment()); 4353 LI = Builder.CreateLoad(VecTy, VecPtr); 4354 if (!Alignment) 4355 Alignment = MaybeAlign(DL->getABITypeAlignment(ScalarLoadTy)); 4356 LI->setAlignment(Alignment); 4357 Value *V = propagateMetadata(LI, E->Scalars); 4358 if (IsReorder) { 4359 OrdersType Mask; 4360 inversePermutation(E->ReorderIndices, Mask); 4361 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()), 4362 Mask, "reorder_shuffle"); 4363 } 4364 if (NeedToShuffleReuses) { 4365 // TODO: Merge this shuffle with the ReorderShuffleMask. 4366 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4367 E->ReuseShuffleIndices, "shuffle"); 4368 } 4369 E->VectorizedValue = V; 4370 ++NumVectorInstructions; 4371 return V; 4372 } 4373 case Instruction::Store: { 4374 bool IsReorder = !E->ReorderIndices.empty(); 4375 auto *SI = cast<StoreInst>( 4376 IsReorder ? E->Scalars[E->ReorderIndices.front()] : VL0); 4377 unsigned Alignment = SI->getAlignment(); 4378 unsigned AS = SI->getPointerAddressSpace(); 4379 4380 setInsertPointAfterBundle(E); 4381 4382 Value *VecValue = vectorizeTree(E->getOperand(0)); 4383 if (IsReorder) { 4384 OrdersType Mask; 4385 inversePermutation(E->ReorderIndices, Mask); 4386 VecValue = Builder.CreateShuffleVector( 4387 VecValue, UndefValue::get(VecValue->getType()), E->ReorderIndices, 4388 "reorder_shuffle"); 4389 } 4390 Value *ScalarPtr = SI->getPointerOperand(); 4391 Value *VecPtr = Builder.CreateBitCast( 4392 ScalarPtr, VecValue->getType()->getPointerTo(AS)); 4393 StoreInst *ST = Builder.CreateStore(VecValue, VecPtr); 4394 4395 // The pointer operand uses an in-tree scalar, so add the new BitCast to 4396 // ExternalUses to make sure that an extract will be generated in the 4397 // future. 4398 if (getTreeEntry(ScalarPtr)) 4399 ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0)); 4400 4401 if (!Alignment) 4402 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType()); 4403 4404 ST->setAlignment(Align(Alignment)); 4405 Value *V = propagateMetadata(ST, E->Scalars); 4406 if (NeedToShuffleReuses) { 4407 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4408 E->ReuseShuffleIndices, "shuffle"); 4409 } 4410 E->VectorizedValue = V; 4411 ++NumVectorInstructions; 4412 return V; 4413 } 4414 case Instruction::GetElementPtr: { 4415 setInsertPointAfterBundle(E); 4416 4417 Value *Op0 = vectorizeTree(E->getOperand(0)); 4418 4419 std::vector<Value *> OpVecs; 4420 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e; 4421 ++j) { 4422 ValueList &VL = E->getOperand(j); 4423 // Need to cast all elements to the same type before vectorization to 4424 // avoid crash. 4425 Type *VL0Ty = VL0->getOperand(j)->getType(); 4426 Type *Ty = llvm::all_of( 4427 VL, [VL0Ty](Value *V) { return VL0Ty == V->getType(); }) 4428 ? VL0Ty 4429 : DL->getIndexType(cast<GetElementPtrInst>(VL0) 4430 ->getPointerOperandType() 4431 ->getScalarType()); 4432 for (Value *&V : VL) { 4433 auto *CI = cast<ConstantInt>(V); 4434 V = ConstantExpr::getIntegerCast(CI, Ty, 4435 CI->getValue().isSignBitSet()); 4436 } 4437 Value *OpVec = vectorizeTree(VL); 4438 OpVecs.push_back(OpVec); 4439 } 4440 4441 Value *V = Builder.CreateGEP( 4442 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs); 4443 if (Instruction *I = dyn_cast<Instruction>(V)) 4444 V = propagateMetadata(I, E->Scalars); 4445 4446 if (NeedToShuffleReuses) { 4447 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4448 E->ReuseShuffleIndices, "shuffle"); 4449 } 4450 E->VectorizedValue = V; 4451 ++NumVectorInstructions; 4452 4453 return V; 4454 } 4455 case Instruction::Call: { 4456 CallInst *CI = cast<CallInst>(VL0); 4457 setInsertPointAfterBundle(E); 4458 4459 Intrinsic::ID IID = Intrinsic::not_intrinsic; 4460 if (Function *FI = CI->getCalledFunction()) 4461 IID = FI->getIntrinsicID(); 4462 4463 Value *ScalarArg = nullptr; 4464 std::vector<Value *> OpVecs; 4465 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) { 4466 ValueList OpVL; 4467 // Some intrinsics have scalar arguments. This argument should not be 4468 // vectorized. 4469 if (hasVectorInstrinsicScalarOpd(IID, j)) { 4470 CallInst *CEI = cast<CallInst>(VL0); 4471 ScalarArg = CEI->getArgOperand(j); 4472 OpVecs.push_back(CEI->getArgOperand(j)); 4473 continue; 4474 } 4475 4476 Value *OpVec = vectorizeTree(E->getOperand(j)); 4477 LLVM_DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n"); 4478 OpVecs.push_back(OpVec); 4479 } 4480 4481 Module *M = F->getParent(); 4482 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI); 4483 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) }; 4484 Function *CF = Intrinsic::getDeclaration(M, ID, Tys); 4485 SmallVector<OperandBundleDef, 1> OpBundles; 4486 CI->getOperandBundlesAsDefs(OpBundles); 4487 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles); 4488 4489 // The scalar argument uses an in-tree scalar so we add the new vectorized 4490 // call to ExternalUses list to make sure that an extract will be 4491 // generated in the future. 4492 if (ScalarArg && getTreeEntry(ScalarArg)) 4493 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0)); 4494 4495 propagateIRFlags(V, E->Scalars, VL0); 4496 if (NeedToShuffleReuses) { 4497 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4498 E->ReuseShuffleIndices, "shuffle"); 4499 } 4500 E->VectorizedValue = V; 4501 ++NumVectorInstructions; 4502 return V; 4503 } 4504 case Instruction::ShuffleVector: { 4505 assert(E->isAltShuffle() && 4506 ((Instruction::isBinaryOp(E->getOpcode()) && 4507 Instruction::isBinaryOp(E->getAltOpcode())) || 4508 (Instruction::isCast(E->getOpcode()) && 4509 Instruction::isCast(E->getAltOpcode()))) && 4510 "Invalid Shuffle Vector Operand"); 4511 4512 Value *LHS = nullptr, *RHS = nullptr; 4513 if (Instruction::isBinaryOp(E->getOpcode())) { 4514 setInsertPointAfterBundle(E); 4515 LHS = vectorizeTree(E->getOperand(0)); 4516 RHS = vectorizeTree(E->getOperand(1)); 4517 } else { 4518 setInsertPointAfterBundle(E); 4519 LHS = vectorizeTree(E->getOperand(0)); 4520 } 4521 4522 if (E->VectorizedValue) { 4523 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); 4524 return E->VectorizedValue; 4525 } 4526 4527 Value *V0, *V1; 4528 if (Instruction::isBinaryOp(E->getOpcode())) { 4529 V0 = Builder.CreateBinOp( 4530 static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS, RHS); 4531 V1 = Builder.CreateBinOp( 4532 static_cast<Instruction::BinaryOps>(E->getAltOpcode()), LHS, RHS); 4533 } else { 4534 V0 = Builder.CreateCast( 4535 static_cast<Instruction::CastOps>(E->getOpcode()), LHS, VecTy); 4536 V1 = Builder.CreateCast( 4537 static_cast<Instruction::CastOps>(E->getAltOpcode()), LHS, VecTy); 4538 } 4539 4540 // Create shuffle to take alternate operations from the vector. 4541 // Also, gather up main and alt scalar ops to propagate IR flags to 4542 // each vector operation. 4543 ValueList OpScalars, AltScalars; 4544 unsigned e = E->Scalars.size(); 4545 SmallVector<Constant *, 8> Mask(e); 4546 for (unsigned i = 0; i < e; ++i) { 4547 auto *OpInst = cast<Instruction>(E->Scalars[i]); 4548 assert(E->isOpcodeOrAlt(OpInst) && "Unexpected main/alternate opcode"); 4549 if (OpInst->getOpcode() == E->getAltOpcode()) { 4550 Mask[i] = Builder.getInt32(e + i); 4551 AltScalars.push_back(E->Scalars[i]); 4552 } else { 4553 Mask[i] = Builder.getInt32(i); 4554 OpScalars.push_back(E->Scalars[i]); 4555 } 4556 } 4557 4558 Value *ShuffleMask = ConstantVector::get(Mask); 4559 propagateIRFlags(V0, OpScalars); 4560 propagateIRFlags(V1, AltScalars); 4561 4562 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask); 4563 if (Instruction *I = dyn_cast<Instruction>(V)) 4564 V = propagateMetadata(I, E->Scalars); 4565 if (NeedToShuffleReuses) { 4566 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), 4567 E->ReuseShuffleIndices, "shuffle"); 4568 } 4569 E->VectorizedValue = V; 4570 ++NumVectorInstructions; 4571 4572 return V; 4573 } 4574 default: 4575 llvm_unreachable("unknown inst"); 4576 } 4577 return nullptr; 4578 } 4579 4580 Value *BoUpSLP::vectorizeTree() { 4581 ExtraValueToDebugLocsMap ExternallyUsedValues; 4582 return vectorizeTree(ExternallyUsedValues); 4583 } 4584 4585 Value * 4586 BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) { 4587 // All blocks must be scheduled before any instructions are inserted. 4588 for (auto &BSIter : BlocksSchedules) { 4589 scheduleBlock(BSIter.second.get()); 4590 } 4591 4592 Builder.SetInsertPoint(&F->getEntryBlock().front()); 4593 auto *VectorRoot = vectorizeTree(VectorizableTree[0].get()); 4594 4595 // If the vectorized tree can be rewritten in a smaller type, we truncate the 4596 // vectorized root. InstCombine will then rewrite the entire expression. We 4597 // sign extend the extracted values below. 4598 auto *ScalarRoot = VectorizableTree[0]->Scalars[0]; 4599 if (MinBWs.count(ScalarRoot)) { 4600 if (auto *I = dyn_cast<Instruction>(VectorRoot)) 4601 Builder.SetInsertPoint(&*++BasicBlock::iterator(I)); 4602 auto BundleWidth = VectorizableTree[0]->Scalars.size(); 4603 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first); 4604 auto *VecTy = VectorType::get(MinTy, BundleWidth); 4605 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy); 4606 VectorizableTree[0]->VectorizedValue = Trunc; 4607 } 4608 4609 LLVM_DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() 4610 << " values .\n"); 4611 4612 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type 4613 // specified by ScalarType. 4614 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) { 4615 if (!MinBWs.count(ScalarRoot)) 4616 return Ex; 4617 if (MinBWs[ScalarRoot].second) 4618 return Builder.CreateSExt(Ex, ScalarType); 4619 return Builder.CreateZExt(Ex, ScalarType); 4620 }; 4621 4622 // Extract all of the elements with the external uses. 4623 for (const auto &ExternalUse : ExternalUses) { 4624 Value *Scalar = ExternalUse.Scalar; 4625 llvm::User *User = ExternalUse.User; 4626 4627 // Skip users that we already RAUW. This happens when one instruction 4628 // has multiple uses of the same value. 4629 if (User && !is_contained(Scalar->users(), User)) 4630 continue; 4631 TreeEntry *E = getTreeEntry(Scalar); 4632 assert(E && "Invalid scalar"); 4633 assert(E->State == TreeEntry::Vectorize && "Extracting from a gather list"); 4634 4635 Value *Vec = E->VectorizedValue; 4636 assert(Vec && "Can't find vectorizable value"); 4637 4638 Value *Lane = Builder.getInt32(ExternalUse.Lane); 4639 // If User == nullptr, the Scalar is used as extra arg. Generate 4640 // ExtractElement instruction and update the record for this scalar in 4641 // ExternallyUsedValues. 4642 if (!User) { 4643 assert(ExternallyUsedValues.count(Scalar) && 4644 "Scalar with nullptr as an external user must be registered in " 4645 "ExternallyUsedValues map"); 4646 if (auto *VecI = dyn_cast<Instruction>(Vec)) { 4647 Builder.SetInsertPoint(VecI->getParent(), 4648 std::next(VecI->getIterator())); 4649 } else { 4650 Builder.SetInsertPoint(&F->getEntryBlock().front()); 4651 } 4652 Value *Ex = Builder.CreateExtractElement(Vec, Lane); 4653 Ex = extend(ScalarRoot, Ex, Scalar->getType()); 4654 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent()); 4655 auto &Locs = ExternallyUsedValues[Scalar]; 4656 ExternallyUsedValues.insert({Ex, Locs}); 4657 ExternallyUsedValues.erase(Scalar); 4658 // Required to update internally referenced instructions. 4659 Scalar->replaceAllUsesWith(Ex); 4660 continue; 4661 } 4662 4663 // Generate extracts for out-of-tree users. 4664 // Find the insertion point for the extractelement lane. 4665 if (auto *VecI = dyn_cast<Instruction>(Vec)) { 4666 if (PHINode *PH = dyn_cast<PHINode>(User)) { 4667 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) { 4668 if (PH->getIncomingValue(i) == Scalar) { 4669 Instruction *IncomingTerminator = 4670 PH->getIncomingBlock(i)->getTerminator(); 4671 if (isa<CatchSwitchInst>(IncomingTerminator)) { 4672 Builder.SetInsertPoint(VecI->getParent(), 4673 std::next(VecI->getIterator())); 4674 } else { 4675 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator()); 4676 } 4677 Value *Ex = Builder.CreateExtractElement(Vec, Lane); 4678 Ex = extend(ScalarRoot, Ex, Scalar->getType()); 4679 CSEBlocks.insert(PH->getIncomingBlock(i)); 4680 PH->setOperand(i, Ex); 4681 } 4682 } 4683 } else { 4684 Builder.SetInsertPoint(cast<Instruction>(User)); 4685 Value *Ex = Builder.CreateExtractElement(Vec, Lane); 4686 Ex = extend(ScalarRoot, Ex, Scalar->getType()); 4687 CSEBlocks.insert(cast<Instruction>(User)->getParent()); 4688 User->replaceUsesOfWith(Scalar, Ex); 4689 } 4690 } else { 4691 Builder.SetInsertPoint(&F->getEntryBlock().front()); 4692 Value *Ex = Builder.CreateExtractElement(Vec, Lane); 4693 Ex = extend(ScalarRoot, Ex, Scalar->getType()); 4694 CSEBlocks.insert(&F->getEntryBlock()); 4695 User->replaceUsesOfWith(Scalar, Ex); 4696 } 4697 4698 LLVM_DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n"); 4699 } 4700 4701 // For each vectorized value: 4702 for (auto &TEPtr : VectorizableTree) { 4703 TreeEntry *Entry = TEPtr.get(); 4704 4705 // No need to handle users of gathered values. 4706 if (Entry->State == TreeEntry::NeedToGather) 4707 continue; 4708 4709 assert(Entry->VectorizedValue && "Can't find vectorizable value"); 4710 4711 // For each lane: 4712 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) { 4713 Value *Scalar = Entry->Scalars[Lane]; 4714 4715 #ifndef NDEBUG 4716 Type *Ty = Scalar->getType(); 4717 if (!Ty->isVoidTy()) { 4718 for (User *U : Scalar->users()) { 4719 LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n"); 4720 4721 // It is legal to delete users in the ignorelist. 4722 assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) && 4723 "Deleting out-of-tree value"); 4724 } 4725 } 4726 #endif 4727 LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n"); 4728 eraseInstruction(cast<Instruction>(Scalar)); 4729 } 4730 } 4731 4732 Builder.ClearInsertionPoint(); 4733 4734 return VectorizableTree[0]->VectorizedValue; 4735 } 4736 4737 void BoUpSLP::optimizeGatherSequence() { 4738 LLVM_DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size() 4739 << " gather sequences instructions.\n"); 4740 // LICM InsertElementInst sequences. 4741 for (Instruction *I : GatherSeq) { 4742 if (isDeleted(I)) 4743 continue; 4744 4745 // Check if this block is inside a loop. 4746 Loop *L = LI->getLoopFor(I->getParent()); 4747 if (!L) 4748 continue; 4749 4750 // Check if it has a preheader. 4751 BasicBlock *PreHeader = L->getLoopPreheader(); 4752 if (!PreHeader) 4753 continue; 4754 4755 // If the vector or the element that we insert into it are 4756 // instructions that are defined in this basic block then we can't 4757 // hoist this instruction. 4758 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0)); 4759 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1)); 4760 if (Op0 && L->contains(Op0)) 4761 continue; 4762 if (Op1 && L->contains(Op1)) 4763 continue; 4764 4765 // We can hoist this instruction. Move it to the pre-header. 4766 I->moveBefore(PreHeader->getTerminator()); 4767 } 4768 4769 // Make a list of all reachable blocks in our CSE queue. 4770 SmallVector<const DomTreeNode *, 8> CSEWorkList; 4771 CSEWorkList.reserve(CSEBlocks.size()); 4772 for (BasicBlock *BB : CSEBlocks) 4773 if (DomTreeNode *N = DT->getNode(BB)) { 4774 assert(DT->isReachableFromEntry(N)); 4775 CSEWorkList.push_back(N); 4776 } 4777 4778 // Sort blocks by domination. This ensures we visit a block after all blocks 4779 // dominating it are visited. 4780 llvm::stable_sort(CSEWorkList, 4781 [this](const DomTreeNode *A, const DomTreeNode *B) { 4782 return DT->properlyDominates(A, B); 4783 }); 4784 4785 // Perform O(N^2) search over the gather sequences and merge identical 4786 // instructions. TODO: We can further optimize this scan if we split the 4787 // instructions into different buckets based on the insert lane. 4788 SmallVector<Instruction *, 16> Visited; 4789 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) { 4790 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && 4791 "Worklist not sorted properly!"); 4792 BasicBlock *BB = (*I)->getBlock(); 4793 // For all instructions in blocks containing gather sequences: 4794 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) { 4795 Instruction *In = &*it++; 4796 if (isDeleted(In)) 4797 continue; 4798 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In)) 4799 continue; 4800 4801 // Check if we can replace this instruction with any of the 4802 // visited instructions. 4803 for (Instruction *v : Visited) { 4804 if (In->isIdenticalTo(v) && 4805 DT->dominates(v->getParent(), In->getParent())) { 4806 In->replaceAllUsesWith(v); 4807 eraseInstruction(In); 4808 In = nullptr; 4809 break; 4810 } 4811 } 4812 if (In) { 4813 assert(!is_contained(Visited, In)); 4814 Visited.push_back(In); 4815 } 4816 } 4817 } 4818 CSEBlocks.clear(); 4819 GatherSeq.clear(); 4820 } 4821 4822 // Groups the instructions to a bundle (which is then a single scheduling entity) 4823 // and schedules instructions until the bundle gets ready. 4824 Optional<BoUpSLP::ScheduleData *> 4825 BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, 4826 const InstructionsState &S) { 4827 if (isa<PHINode>(S.OpValue)) 4828 return nullptr; 4829 4830 // Initialize the instruction bundle. 4831 Instruction *OldScheduleEnd = ScheduleEnd; 4832 ScheduleData *PrevInBundle = nullptr; 4833 ScheduleData *Bundle = nullptr; 4834 bool ReSchedule = false; 4835 LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.OpValue << "\n"); 4836 4837 // Make sure that the scheduling region contains all 4838 // instructions of the bundle. 4839 for (Value *V : VL) { 4840 if (!extendSchedulingRegion(V, S)) 4841 return None; 4842 } 4843 4844 for (Value *V : VL) { 4845 ScheduleData *BundleMember = getScheduleData(V); 4846 assert(BundleMember && 4847 "no ScheduleData for bundle member (maybe not in same basic block)"); 4848 if (BundleMember->IsScheduled) { 4849 // A bundle member was scheduled as single instruction before and now 4850 // needs to be scheduled as part of the bundle. We just get rid of the 4851 // existing schedule. 4852 LLVM_DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember 4853 << " was already scheduled\n"); 4854 ReSchedule = true; 4855 } 4856 assert(BundleMember->isSchedulingEntity() && 4857 "bundle member already part of other bundle"); 4858 if (PrevInBundle) { 4859 PrevInBundle->NextInBundle = BundleMember; 4860 } else { 4861 Bundle = BundleMember; 4862 } 4863 BundleMember->UnscheduledDepsInBundle = 0; 4864 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps; 4865 4866 // Group the instructions to a bundle. 4867 BundleMember->FirstInBundle = Bundle; 4868 PrevInBundle = BundleMember; 4869 } 4870 if (ScheduleEnd != OldScheduleEnd) { 4871 // The scheduling region got new instructions at the lower end (or it is a 4872 // new region for the first bundle). This makes it necessary to 4873 // recalculate all dependencies. 4874 // It is seldom that this needs to be done a second time after adding the 4875 // initial bundle to the region. 4876 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) { 4877 doForAllOpcodes(I, [](ScheduleData *SD) { 4878 SD->clearDependencies(); 4879 }); 4880 } 4881 ReSchedule = true; 4882 } 4883 if (ReSchedule) { 4884 resetSchedule(); 4885 initialFillReadyList(ReadyInsts); 4886 } 4887 assert(Bundle && "Failed to find schedule bundle"); 4888 4889 LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block " 4890 << BB->getName() << "\n"); 4891 4892 calculateDependencies(Bundle, true, SLP); 4893 4894 // Now try to schedule the new bundle. As soon as the bundle is "ready" it 4895 // means that there are no cyclic dependencies and we can schedule it. 4896 // Note that's important that we don't "schedule" the bundle yet (see 4897 // cancelScheduling). 4898 while (!Bundle->isReady() && !ReadyInsts.empty()) { 4899 4900 ScheduleData *pickedSD = ReadyInsts.back(); 4901 ReadyInsts.pop_back(); 4902 4903 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) { 4904 schedule(pickedSD, ReadyInsts); 4905 } 4906 } 4907 if (!Bundle->isReady()) { 4908 cancelScheduling(VL, S.OpValue); 4909 return None; 4910 } 4911 return Bundle; 4912 } 4913 4914 void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL, 4915 Value *OpValue) { 4916 if (isa<PHINode>(OpValue)) 4917 return; 4918 4919 ScheduleData *Bundle = getScheduleData(OpValue); 4920 LLVM_DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n"); 4921 assert(!Bundle->IsScheduled && 4922 "Can't cancel bundle which is already scheduled"); 4923 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && 4924 "tried to unbundle something which is not a bundle"); 4925 4926 // Un-bundle: make single instructions out of the bundle. 4927 ScheduleData *BundleMember = Bundle; 4928 while (BundleMember) { 4929 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links"); 4930 BundleMember->FirstInBundle = BundleMember; 4931 ScheduleData *Next = BundleMember->NextInBundle; 4932 BundleMember->NextInBundle = nullptr; 4933 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps; 4934 if (BundleMember->UnscheduledDepsInBundle == 0) { 4935 ReadyInsts.insert(BundleMember); 4936 } 4937 BundleMember = Next; 4938 } 4939 } 4940 4941 BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() { 4942 // Allocate a new ScheduleData for the instruction. 4943 if (ChunkPos >= ChunkSize) { 4944 ScheduleDataChunks.push_back(std::make_unique<ScheduleData[]>(ChunkSize)); 4945 ChunkPos = 0; 4946 } 4947 return &(ScheduleDataChunks.back()[ChunkPos++]); 4948 } 4949 4950 bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V, 4951 const InstructionsState &S) { 4952 if (getScheduleData(V, isOneOf(S, V))) 4953 return true; 4954 Instruction *I = dyn_cast<Instruction>(V); 4955 assert(I && "bundle member must be an instruction"); 4956 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled"); 4957 auto &&CheckSheduleForI = [this, &S](Instruction *I) -> bool { 4958 ScheduleData *ISD = getScheduleData(I); 4959 if (!ISD) 4960 return false; 4961 assert(isInSchedulingRegion(ISD) && 4962 "ScheduleData not in scheduling region"); 4963 ScheduleData *SD = allocateScheduleDataChunks(); 4964 SD->Inst = I; 4965 SD->init(SchedulingRegionID, S.OpValue); 4966 ExtraScheduleDataMap[I][S.OpValue] = SD; 4967 return true; 4968 }; 4969 if (CheckSheduleForI(I)) 4970 return true; 4971 if (!ScheduleStart) { 4972 // It's the first instruction in the new region. 4973 initScheduleData(I, I->getNextNode(), nullptr, nullptr); 4974 ScheduleStart = I; 4975 ScheduleEnd = I->getNextNode(); 4976 if (isOneOf(S, I) != I) 4977 CheckSheduleForI(I); 4978 assert(ScheduleEnd && "tried to vectorize a terminator?"); 4979 LLVM_DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n"); 4980 return true; 4981 } 4982 // Search up and down at the same time, because we don't know if the new 4983 // instruction is above or below the existing scheduling region. 4984 BasicBlock::reverse_iterator UpIter = 4985 ++ScheduleStart->getIterator().getReverse(); 4986 BasicBlock::reverse_iterator UpperEnd = BB->rend(); 4987 BasicBlock::iterator DownIter = ScheduleEnd->getIterator(); 4988 BasicBlock::iterator LowerEnd = BB->end(); 4989 while (true) { 4990 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) { 4991 LLVM_DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n"); 4992 return false; 4993 } 4994 4995 if (UpIter != UpperEnd) { 4996 if (&*UpIter == I) { 4997 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion); 4998 ScheduleStart = I; 4999 if (isOneOf(S, I) != I) 5000 CheckSheduleForI(I); 5001 LLVM_DEBUG(dbgs() << "SLP: extend schedule region start to " << *I 5002 << "\n"); 5003 return true; 5004 } 5005 ++UpIter; 5006 } 5007 if (DownIter != LowerEnd) { 5008 if (&*DownIter == I) { 5009 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion, 5010 nullptr); 5011 ScheduleEnd = I->getNextNode(); 5012 if (isOneOf(S, I) != I) 5013 CheckSheduleForI(I); 5014 assert(ScheduleEnd && "tried to vectorize a terminator?"); 5015 LLVM_DEBUG(dbgs() << "SLP: extend schedule region end to " << *I 5016 << "\n"); 5017 return true; 5018 } 5019 ++DownIter; 5020 } 5021 assert((UpIter != UpperEnd || DownIter != LowerEnd) && 5022 "instruction not found in block"); 5023 } 5024 return true; 5025 } 5026 5027 void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI, 5028 Instruction *ToI, 5029 ScheduleData *PrevLoadStore, 5030 ScheduleData *NextLoadStore) { 5031 ScheduleData *CurrentLoadStore = PrevLoadStore; 5032 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) { 5033 ScheduleData *SD = ScheduleDataMap[I]; 5034 if (!SD) { 5035 SD = allocateScheduleDataChunks(); 5036 ScheduleDataMap[I] = SD; 5037 SD->Inst = I; 5038 } 5039 assert(!isInSchedulingRegion(SD) && 5040 "new ScheduleData already in scheduling region"); 5041 SD->init(SchedulingRegionID, I); 5042 5043 if (I->mayReadOrWriteMemory() && 5044 (!isa<IntrinsicInst>(I) || 5045 cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) { 5046 // Update the linked list of memory accessing instructions. 5047 if (CurrentLoadStore) { 5048 CurrentLoadStore->NextLoadStore = SD; 5049 } else { 5050 FirstLoadStoreInRegion = SD; 5051 } 5052 CurrentLoadStore = SD; 5053 } 5054 } 5055 if (NextLoadStore) { 5056 if (CurrentLoadStore) 5057 CurrentLoadStore->NextLoadStore = NextLoadStore; 5058 } else { 5059 LastLoadStoreInRegion = CurrentLoadStore; 5060 } 5061 } 5062 5063 void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD, 5064 bool InsertInReadyList, 5065 BoUpSLP *SLP) { 5066 assert(SD->isSchedulingEntity()); 5067 5068 SmallVector<ScheduleData *, 10> WorkList; 5069 WorkList.push_back(SD); 5070 5071 while (!WorkList.empty()) { 5072 ScheduleData *SD = WorkList.back(); 5073 WorkList.pop_back(); 5074 5075 ScheduleData *BundleMember = SD; 5076 while (BundleMember) { 5077 assert(isInSchedulingRegion(BundleMember)); 5078 if (!BundleMember->hasValidDependencies()) { 5079 5080 LLVM_DEBUG(dbgs() << "SLP: update deps of " << *BundleMember 5081 << "\n"); 5082 BundleMember->Dependencies = 0; 5083 BundleMember->resetUnscheduledDeps(); 5084 5085 // Handle def-use chain dependencies. 5086 if (BundleMember->OpValue != BundleMember->Inst) { 5087 ScheduleData *UseSD = getScheduleData(BundleMember->Inst); 5088 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) { 5089 BundleMember->Dependencies++; 5090 ScheduleData *DestBundle = UseSD->FirstInBundle; 5091 if (!DestBundle->IsScheduled) 5092 BundleMember->incrementUnscheduledDeps(1); 5093 if (!DestBundle->hasValidDependencies()) 5094 WorkList.push_back(DestBundle); 5095 } 5096 } else { 5097 for (User *U : BundleMember->Inst->users()) { 5098 if (isa<Instruction>(U)) { 5099 ScheduleData *UseSD = getScheduleData(U); 5100 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) { 5101 BundleMember->Dependencies++; 5102 ScheduleData *DestBundle = UseSD->FirstInBundle; 5103 if (!DestBundle->IsScheduled) 5104 BundleMember->incrementUnscheduledDeps(1); 5105 if (!DestBundle->hasValidDependencies()) 5106 WorkList.push_back(DestBundle); 5107 } 5108 } else { 5109 // I'm not sure if this can ever happen. But we need to be safe. 5110 // This lets the instruction/bundle never be scheduled and 5111 // eventually disable vectorization. 5112 BundleMember->Dependencies++; 5113 BundleMember->incrementUnscheduledDeps(1); 5114 } 5115 } 5116 } 5117 5118 // Handle the memory dependencies. 5119 ScheduleData *DepDest = BundleMember->NextLoadStore; 5120 if (DepDest) { 5121 Instruction *SrcInst = BundleMember->Inst; 5122 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA); 5123 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory(); 5124 unsigned numAliased = 0; 5125 unsigned DistToSrc = 1; 5126 5127 while (DepDest) { 5128 assert(isInSchedulingRegion(DepDest)); 5129 5130 // We have two limits to reduce the complexity: 5131 // 1) AliasedCheckLimit: It's a small limit to reduce calls to 5132 // SLP->isAliased (which is the expensive part in this loop). 5133 // 2) MaxMemDepDistance: It's for very large blocks and it aborts 5134 // the whole loop (even if the loop is fast, it's quadratic). 5135 // It's important for the loop break condition (see below) to 5136 // check this limit even between two read-only instructions. 5137 if (DistToSrc >= MaxMemDepDistance || 5138 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) && 5139 (numAliased >= AliasedCheckLimit || 5140 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) { 5141 5142 // We increment the counter only if the locations are aliased 5143 // (instead of counting all alias checks). This gives a better 5144 // balance between reduced runtime and accurate dependencies. 5145 numAliased++; 5146 5147 DepDest->MemoryDependencies.push_back(BundleMember); 5148 BundleMember->Dependencies++; 5149 ScheduleData *DestBundle = DepDest->FirstInBundle; 5150 if (!DestBundle->IsScheduled) { 5151 BundleMember->incrementUnscheduledDeps(1); 5152 } 5153 if (!DestBundle->hasValidDependencies()) { 5154 WorkList.push_back(DestBundle); 5155 } 5156 } 5157 DepDest = DepDest->NextLoadStore; 5158 5159 // Example, explaining the loop break condition: Let's assume our 5160 // starting instruction is i0 and MaxMemDepDistance = 3. 5161 // 5162 // +--------v--v--v 5163 // i0,i1,i2,i3,i4,i5,i6,i7,i8 5164 // +--------^--^--^ 5165 // 5166 // MaxMemDepDistance let us stop alias-checking at i3 and we add 5167 // dependencies from i0 to i3,i4,.. (even if they are not aliased). 5168 // Previously we already added dependencies from i3 to i6,i7,i8 5169 // (because of MaxMemDepDistance). As we added a dependency from 5170 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8 5171 // and we can abort this loop at i6. 5172 if (DistToSrc >= 2 * MaxMemDepDistance) 5173 break; 5174 DistToSrc++; 5175 } 5176 } 5177 } 5178 BundleMember = BundleMember->NextInBundle; 5179 } 5180 if (InsertInReadyList && SD->isReady()) { 5181 ReadyInsts.push_back(SD); 5182 LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst 5183 << "\n"); 5184 } 5185 } 5186 } 5187 5188 void BoUpSLP::BlockScheduling::resetSchedule() { 5189 assert(ScheduleStart && 5190 "tried to reset schedule on block which has not been scheduled"); 5191 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) { 5192 doForAllOpcodes(I, [&](ScheduleData *SD) { 5193 assert(isInSchedulingRegion(SD) && 5194 "ScheduleData not in scheduling region"); 5195 SD->IsScheduled = false; 5196 SD->resetUnscheduledDeps(); 5197 }); 5198 } 5199 ReadyInsts.clear(); 5200 } 5201 5202 void BoUpSLP::scheduleBlock(BlockScheduling *BS) { 5203 if (!BS->ScheduleStart) 5204 return; 5205 5206 LLVM_DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n"); 5207 5208 BS->resetSchedule(); 5209 5210 // For the real scheduling we use a more sophisticated ready-list: it is 5211 // sorted by the original instruction location. This lets the final schedule 5212 // be as close as possible to the original instruction order. 5213 struct ScheduleDataCompare { 5214 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const { 5215 return SD2->SchedulingPriority < SD1->SchedulingPriority; 5216 } 5217 }; 5218 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts; 5219 5220 // Ensure that all dependency data is updated and fill the ready-list with 5221 // initial instructions. 5222 int Idx = 0; 5223 int NumToSchedule = 0; 5224 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd; 5225 I = I->getNextNode()) { 5226 BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) { 5227 assert(SD->isPartOfBundle() == 5228 (getTreeEntry(SD->Inst) != nullptr) && 5229 "scheduler and vectorizer bundle mismatch"); 5230 SD->FirstInBundle->SchedulingPriority = Idx++; 5231 if (SD->isSchedulingEntity()) { 5232 BS->calculateDependencies(SD, false, this); 5233 NumToSchedule++; 5234 } 5235 }); 5236 } 5237 BS->initialFillReadyList(ReadyInsts); 5238 5239 Instruction *LastScheduledInst = BS->ScheduleEnd; 5240 5241 // Do the "real" scheduling. 5242 while (!ReadyInsts.empty()) { 5243 ScheduleData *picked = *ReadyInsts.begin(); 5244 ReadyInsts.erase(ReadyInsts.begin()); 5245 5246 // Move the scheduled instruction(s) to their dedicated places, if not 5247 // there yet. 5248 ScheduleData *BundleMember = picked; 5249 while (BundleMember) { 5250 Instruction *pickedInst = BundleMember->Inst; 5251 if (LastScheduledInst->getNextNode() != pickedInst) { 5252 BS->BB->getInstList().remove(pickedInst); 5253 BS->BB->getInstList().insert(LastScheduledInst->getIterator(), 5254 pickedInst); 5255 } 5256 LastScheduledInst = pickedInst; 5257 BundleMember = BundleMember->NextInBundle; 5258 } 5259 5260 BS->schedule(picked, ReadyInsts); 5261 NumToSchedule--; 5262 } 5263 assert(NumToSchedule == 0 && "could not schedule all instructions"); 5264 5265 // Avoid duplicate scheduling of the block. 5266 BS->ScheduleStart = nullptr; 5267 } 5268 5269 unsigned BoUpSLP::getVectorElementSize(Value *V) const { 5270 // If V is a store, just return the width of the stored value without 5271 // traversing the expression tree. This is the common case. 5272 if (auto *Store = dyn_cast<StoreInst>(V)) 5273 return DL->getTypeSizeInBits(Store->getValueOperand()->getType()); 5274 5275 // If V is not a store, we can traverse the expression tree to find loads 5276 // that feed it. The type of the loaded value may indicate a more suitable 5277 // width than V's type. We want to base the vector element size on the width 5278 // of memory operations where possible. 5279 SmallVector<Instruction *, 16> Worklist; 5280 SmallPtrSet<Instruction *, 16> Visited; 5281 if (auto *I = dyn_cast<Instruction>(V)) 5282 Worklist.push_back(I); 5283 5284 // Traverse the expression tree in bottom-up order looking for loads. If we 5285 // encounter an instruction we don't yet handle, we give up. 5286 auto MaxWidth = 0u; 5287 auto FoundUnknownInst = false; 5288 while (!Worklist.empty() && !FoundUnknownInst) { 5289 auto *I = Worklist.pop_back_val(); 5290 Visited.insert(I); 5291 5292 // We should only be looking at scalar instructions here. If the current 5293 // instruction has a vector type, give up. 5294 auto *Ty = I->getType(); 5295 if (isa<VectorType>(Ty)) 5296 FoundUnknownInst = true; 5297 5298 // If the current instruction is a load, update MaxWidth to reflect the 5299 // width of the loaded value. 5300 else if (isa<LoadInst>(I)) 5301 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty)); 5302 5303 // Otherwise, we need to visit the operands of the instruction. We only 5304 // handle the interesting cases from buildTree here. If an operand is an 5305 // instruction we haven't yet visited, we add it to the worklist. 5306 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) || 5307 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) { 5308 for (Use &U : I->operands()) 5309 if (auto *J = dyn_cast<Instruction>(U.get())) 5310 if (!Visited.count(J)) 5311 Worklist.push_back(J); 5312 } 5313 5314 // If we don't yet handle the instruction, give up. 5315 else 5316 FoundUnknownInst = true; 5317 } 5318 5319 // If we didn't encounter a memory access in the expression tree, or if we 5320 // gave up for some reason, just return the width of V. 5321 if (!MaxWidth || FoundUnknownInst) 5322 return DL->getTypeSizeInBits(V->getType()); 5323 5324 // Otherwise, return the maximum width we found. 5325 return MaxWidth; 5326 } 5327 5328 // Determine if a value V in a vectorizable expression Expr can be demoted to a 5329 // smaller type with a truncation. We collect the values that will be demoted 5330 // in ToDemote and additional roots that require investigating in Roots. 5331 static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr, 5332 SmallVectorImpl<Value *> &ToDemote, 5333 SmallVectorImpl<Value *> &Roots) { 5334 // We can always demote constants. 5335 if (isa<Constant>(V)) { 5336 ToDemote.push_back(V); 5337 return true; 5338 } 5339 5340 // If the value is not an instruction in the expression with only one use, it 5341 // cannot be demoted. 5342 auto *I = dyn_cast<Instruction>(V); 5343 if (!I || !I->hasOneUse() || !Expr.count(I)) 5344 return false; 5345 5346 switch (I->getOpcode()) { 5347 5348 // We can always demote truncations and extensions. Since truncations can 5349 // seed additional demotion, we save the truncated value. 5350 case Instruction::Trunc: 5351 Roots.push_back(I->getOperand(0)); 5352 break; 5353 case Instruction::ZExt: 5354 case Instruction::SExt: 5355 break; 5356 5357 // We can demote certain binary operations if we can demote both of their 5358 // operands. 5359 case Instruction::Add: 5360 case Instruction::Sub: 5361 case Instruction::Mul: 5362 case Instruction::And: 5363 case Instruction::Or: 5364 case Instruction::Xor: 5365 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) || 5366 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots)) 5367 return false; 5368 break; 5369 5370 // We can demote selects if we can demote their true and false values. 5371 case Instruction::Select: { 5372 SelectInst *SI = cast<SelectInst>(I); 5373 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) || 5374 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots)) 5375 return false; 5376 break; 5377 } 5378 5379 // We can demote phis if we can demote all their incoming operands. Note that 5380 // we don't need to worry about cycles since we ensure single use above. 5381 case Instruction::PHI: { 5382 PHINode *PN = cast<PHINode>(I); 5383 for (Value *IncValue : PN->incoming_values()) 5384 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots)) 5385 return false; 5386 break; 5387 } 5388 5389 // Otherwise, conservatively give up. 5390 default: 5391 return false; 5392 } 5393 5394 // Record the value that we can demote. 5395 ToDemote.push_back(V); 5396 return true; 5397 } 5398 5399 void BoUpSLP::computeMinimumValueSizes() { 5400 // If there are no external uses, the expression tree must be rooted by a 5401 // store. We can't demote in-memory values, so there is nothing to do here. 5402 if (ExternalUses.empty()) 5403 return; 5404 5405 // We only attempt to truncate integer expressions. 5406 auto &TreeRoot = VectorizableTree[0]->Scalars; 5407 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType()); 5408 if (!TreeRootIT) 5409 return; 5410 5411 // If the expression is not rooted by a store, these roots should have 5412 // external uses. We will rely on InstCombine to rewrite the expression in 5413 // the narrower type. However, InstCombine only rewrites single-use values. 5414 // This means that if a tree entry other than a root is used externally, it 5415 // must have multiple uses and InstCombine will not rewrite it. The code 5416 // below ensures that only the roots are used externally. 5417 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end()); 5418 for (auto &EU : ExternalUses) 5419 if (!Expr.erase(EU.Scalar)) 5420 return; 5421 if (!Expr.empty()) 5422 return; 5423 5424 // Collect the scalar values of the vectorizable expression. We will use this 5425 // context to determine which values can be demoted. If we see a truncation, 5426 // we mark it as seeding another demotion. 5427 for (auto &EntryPtr : VectorizableTree) 5428 Expr.insert(EntryPtr->Scalars.begin(), EntryPtr->Scalars.end()); 5429 5430 // Ensure the roots of the vectorizable tree don't form a cycle. They must 5431 // have a single external user that is not in the vectorizable tree. 5432 for (auto *Root : TreeRoot) 5433 if (!Root->hasOneUse() || Expr.count(*Root->user_begin())) 5434 return; 5435 5436 // Conservatively determine if we can actually truncate the roots of the 5437 // expression. Collect the values that can be demoted in ToDemote and 5438 // additional roots that require investigating in Roots. 5439 SmallVector<Value *, 32> ToDemote; 5440 SmallVector<Value *, 4> Roots; 5441 for (auto *Root : TreeRoot) 5442 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots)) 5443 return; 5444 5445 // The maximum bit width required to represent all the values that can be 5446 // demoted without loss of precision. It would be safe to truncate the roots 5447 // of the expression to this width. 5448 auto MaxBitWidth = 8u; 5449 5450 // We first check if all the bits of the roots are demanded. If they're not, 5451 // we can truncate the roots to this narrower type. 5452 for (auto *Root : TreeRoot) { 5453 auto Mask = DB->getDemandedBits(cast<Instruction>(Root)); 5454 MaxBitWidth = std::max<unsigned>( 5455 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth); 5456 } 5457 5458 // True if the roots can be zero-extended back to their original type, rather 5459 // than sign-extended. We know that if the leading bits are not demanded, we 5460 // can safely zero-extend. So we initialize IsKnownPositive to True. 5461 bool IsKnownPositive = true; 5462 5463 // If all the bits of the roots are demanded, we can try a little harder to 5464 // compute a narrower type. This can happen, for example, if the roots are 5465 // getelementptr indices. InstCombine promotes these indices to the pointer 5466 // width. Thus, all their bits are technically demanded even though the 5467 // address computation might be vectorized in a smaller type. 5468 // 5469 // We start by looking at each entry that can be demoted. We compute the 5470 // maximum bit width required to store the scalar by using ValueTracking to 5471 // compute the number of high-order bits we can truncate. 5472 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) && 5473 llvm::all_of(TreeRoot, [](Value *R) { 5474 assert(R->hasOneUse() && "Root should have only one use!"); 5475 return isa<GetElementPtrInst>(R->user_back()); 5476 })) { 5477 MaxBitWidth = 8u; 5478 5479 // Determine if the sign bit of all the roots is known to be zero. If not, 5480 // IsKnownPositive is set to False. 5481 IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) { 5482 KnownBits Known = computeKnownBits(R, *DL); 5483 return Known.isNonNegative(); 5484 }); 5485 5486 // Determine the maximum number of bits required to store the scalar 5487 // values. 5488 for (auto *Scalar : ToDemote) { 5489 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT); 5490 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType()); 5491 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth); 5492 } 5493 5494 // If we can't prove that the sign bit is zero, we must add one to the 5495 // maximum bit width to account for the unknown sign bit. This preserves 5496 // the existing sign bit so we can safely sign-extend the root back to the 5497 // original type. Otherwise, if we know the sign bit is zero, we will 5498 // zero-extend the root instead. 5499 // 5500 // FIXME: This is somewhat suboptimal, as there will be cases where adding 5501 // one to the maximum bit width will yield a larger-than-necessary 5502 // type. In general, we need to add an extra bit only if we can't 5503 // prove that the upper bit of the original type is equal to the 5504 // upper bit of the proposed smaller type. If these two bits are the 5505 // same (either zero or one) we know that sign-extending from the 5506 // smaller type will result in the same value. Here, since we can't 5507 // yet prove this, we are just making the proposed smaller type 5508 // larger to ensure correctness. 5509 if (!IsKnownPositive) 5510 ++MaxBitWidth; 5511 } 5512 5513 // Round MaxBitWidth up to the next power-of-two. 5514 if (!isPowerOf2_64(MaxBitWidth)) 5515 MaxBitWidth = NextPowerOf2(MaxBitWidth); 5516 5517 // If the maximum bit width we compute is less than the with of the roots' 5518 // type, we can proceed with the narrowing. Otherwise, do nothing. 5519 if (MaxBitWidth >= TreeRootIT->getBitWidth()) 5520 return; 5521 5522 // If we can truncate the root, we must collect additional values that might 5523 // be demoted as a result. That is, those seeded by truncations we will 5524 // modify. 5525 while (!Roots.empty()) 5526 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots); 5527 5528 // Finally, map the values we can demote to the maximum bit with we computed. 5529 for (auto *Scalar : ToDemote) 5530 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive); 5531 } 5532 5533 namespace { 5534 5535 /// The SLPVectorizer Pass. 5536 struct SLPVectorizer : public FunctionPass { 5537 SLPVectorizerPass Impl; 5538 5539 /// Pass identification, replacement for typeid 5540 static char ID; 5541 5542 explicit SLPVectorizer() : FunctionPass(ID) { 5543 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry()); 5544 } 5545 5546 bool doInitialization(Module &M) override { 5547 return false; 5548 } 5549 5550 bool runOnFunction(Function &F) override { 5551 if (skipFunction(F)) 5552 return false; 5553 5554 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE(); 5555 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 5556 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>(); 5557 auto *TLI = TLIP ? &TLIP->getTLI(F) : nullptr; 5558 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 5559 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); 5560 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); 5561 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); 5562 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits(); 5563 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE(); 5564 5565 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE); 5566 } 5567 5568 void getAnalysisUsage(AnalysisUsage &AU) const override { 5569 FunctionPass::getAnalysisUsage(AU); 5570 AU.addRequired<AssumptionCacheTracker>(); 5571 AU.addRequired<ScalarEvolutionWrapperPass>(); 5572 AU.addRequired<AAResultsWrapperPass>(); 5573 AU.addRequired<TargetTransformInfoWrapperPass>(); 5574 AU.addRequired<LoopInfoWrapperPass>(); 5575 AU.addRequired<DominatorTreeWrapperPass>(); 5576 AU.addRequired<DemandedBitsWrapperPass>(); 5577 AU.addRequired<OptimizationRemarkEmitterWrapperPass>(); 5578 AU.addPreserved<LoopInfoWrapperPass>(); 5579 AU.addPreserved<DominatorTreeWrapperPass>(); 5580 AU.addPreserved<AAResultsWrapperPass>(); 5581 AU.addPreserved<GlobalsAAWrapperPass>(); 5582 AU.setPreservesCFG(); 5583 } 5584 }; 5585 5586 } // end anonymous namespace 5587 5588 PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) { 5589 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F); 5590 auto *TTI = &AM.getResult<TargetIRAnalysis>(F); 5591 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F); 5592 auto *AA = &AM.getResult<AAManager>(F); 5593 auto *LI = &AM.getResult<LoopAnalysis>(F); 5594 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F); 5595 auto *AC = &AM.getResult<AssumptionAnalysis>(F); 5596 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F); 5597 auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F); 5598 5599 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE); 5600 if (!Changed) 5601 return PreservedAnalyses::all(); 5602 5603 PreservedAnalyses PA; 5604 PA.preserveSet<CFGAnalyses>(); 5605 PA.preserve<AAManager>(); 5606 PA.preserve<GlobalsAA>(); 5607 return PA; 5608 } 5609 5610 bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_, 5611 TargetTransformInfo *TTI_, 5612 TargetLibraryInfo *TLI_, AliasAnalysis *AA_, 5613 LoopInfo *LI_, DominatorTree *DT_, 5614 AssumptionCache *AC_, DemandedBits *DB_, 5615 OptimizationRemarkEmitter *ORE_) { 5616 SE = SE_; 5617 TTI = TTI_; 5618 TLI = TLI_; 5619 AA = AA_; 5620 LI = LI_; 5621 DT = DT_; 5622 AC = AC_; 5623 DB = DB_; 5624 DL = &F.getParent()->getDataLayout(); 5625 5626 Stores.clear(); 5627 GEPs.clear(); 5628 bool Changed = false; 5629 5630 // If the target claims to have no vector registers don't attempt 5631 // vectorization. 5632 if (!TTI->getNumberOfRegisters(TTI->getRegisterClassForType(true))) 5633 return false; 5634 5635 // Don't vectorize when the attribute NoImplicitFloat is used. 5636 if (F.hasFnAttribute(Attribute::NoImplicitFloat)) 5637 return false; 5638 5639 LLVM_DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n"); 5640 5641 // Use the bottom up slp vectorizer to construct chains that start with 5642 // store instructions. 5643 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_); 5644 5645 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to 5646 // delete instructions. 5647 5648 // Scan the blocks in the function in post order. 5649 for (auto BB : post_order(&F.getEntryBlock())) { 5650 collectSeedInstructions(BB); 5651 5652 // Vectorize trees that end at stores. 5653 if (!Stores.empty()) { 5654 LLVM_DEBUG(dbgs() << "SLP: Found stores for " << Stores.size() 5655 << " underlying objects.\n"); 5656 Changed |= vectorizeStoreChains(R); 5657 } 5658 5659 // Vectorize trees that end at reductions. 5660 Changed |= vectorizeChainsInBlock(BB, R); 5661 5662 // Vectorize the index computations of getelementptr instructions. This 5663 // is primarily intended to catch gather-like idioms ending at 5664 // non-consecutive loads. 5665 if (!GEPs.empty()) { 5666 LLVM_DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size() 5667 << " underlying objects.\n"); 5668 Changed |= vectorizeGEPIndices(BB, R); 5669 } 5670 } 5671 5672 if (Changed) { 5673 R.optimizeGatherSequence(); 5674 LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n"); 5675 LLVM_DEBUG(verifyFunction(F)); 5676 } 5677 return Changed; 5678 } 5679 5680 bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R, 5681 unsigned Idx) { 5682 LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << Chain.size() 5683 << "\n"); 5684 const unsigned Sz = R.getVectorElementSize(Chain[0]); 5685 const unsigned MinVF = R.getMinVecRegSize() / Sz; 5686 unsigned VF = Chain.size(); 5687 5688 if (!isPowerOf2_32(Sz) || !isPowerOf2_32(VF) || VF < 2 || VF < MinVF) 5689 return false; 5690 5691 LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << Idx 5692 << "\n"); 5693 5694 R.buildTree(Chain); 5695 Optional<ArrayRef<unsigned>> Order = R.bestOrder(); 5696 // TODO: Handle orders of size less than number of elements in the vector. 5697 if (Order && Order->size() == Chain.size()) { 5698 // TODO: reorder tree nodes without tree rebuilding. 5699 SmallVector<Value *, 4> ReorderedOps(Chain.rbegin(), Chain.rend()); 5700 llvm::transform(*Order, ReorderedOps.begin(), 5701 [Chain](const unsigned Idx) { return Chain[Idx]; }); 5702 R.buildTree(ReorderedOps); 5703 } 5704 if (R.isTreeTinyAndNotFullyVectorizable()) 5705 return false; 5706 5707 R.computeMinimumValueSizes(); 5708 5709 int Cost = R.getTreeCost(); 5710 5711 LLVM_DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n"); 5712 if (Cost < -SLPCostThreshold) { 5713 LLVM_DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n"); 5714 5715 using namespace ore; 5716 5717 R.getORE()->emit(OptimizationRemark(SV_NAME, "StoresVectorized", 5718 cast<StoreInst>(Chain[0])) 5719 << "Stores SLP vectorized with cost " << NV("Cost", Cost) 5720 << " and with tree size " 5721 << NV("TreeSize", R.getTreeSize())); 5722 5723 R.vectorizeTree(); 5724 return true; 5725 } 5726 5727 return false; 5728 } 5729 5730 bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores, 5731 BoUpSLP &R) { 5732 // We may run into multiple chains that merge into a single chain. We mark the 5733 // stores that we vectorized so that we don't visit the same store twice. 5734 BoUpSLP::ValueSet VectorizedStores; 5735 bool Changed = false; 5736 5737 int E = Stores.size(); 5738 SmallBitVector Tails(E, false); 5739 SmallVector<int, 16> ConsecutiveChain(E, E + 1); 5740 int MaxIter = MaxStoreLookup.getValue(); 5741 int IterCnt; 5742 auto &&FindConsecutiveAccess = [this, &Stores, &Tails, &IterCnt, MaxIter, 5743 &ConsecutiveChain](int K, int Idx) { 5744 if (IterCnt >= MaxIter) 5745 return true; 5746 ++IterCnt; 5747 if (!isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) 5748 return false; 5749 5750 Tails.set(Idx); 5751 ConsecutiveChain[K] = Idx; 5752 return true; 5753 }; 5754 // Do a quadratic search on all of the given stores in reverse order and find 5755 // all of the pairs of stores that follow each other. 5756 for (int Idx = E - 1; Idx >= 0; --Idx) { 5757 // If a store has multiple consecutive store candidates, search according 5758 // to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ... 5759 // This is because usually pairing with immediate succeeding or preceding 5760 // candidate create the best chance to find slp vectorization opportunity. 5761 const int MaxLookDepth = std::max(E - Idx, Idx + 1); 5762 IterCnt = 0; 5763 for (int Offset = 1, F = MaxLookDepth; Offset < F; ++Offset) 5764 if ((Idx >= Offset && FindConsecutiveAccess(Idx - Offset, Idx)) || 5765 (Idx + Offset < E && FindConsecutiveAccess(Idx + Offset, Idx))) 5766 break; 5767 } 5768 5769 // For stores that start but don't end a link in the chain: 5770 for (int Cnt = E; Cnt > 0; --Cnt) { 5771 int I = Cnt - 1; 5772 if (ConsecutiveChain[I] == E + 1 || Tails.test(I)) 5773 continue; 5774 // We found a store instr that starts a chain. Now follow the chain and try 5775 // to vectorize it. 5776 BoUpSLP::ValueList Operands; 5777 // Collect the chain into a list. 5778 while (I != E + 1 && !VectorizedStores.count(Stores[I])) { 5779 Operands.push_back(Stores[I]); 5780 // Move to the next value in the chain. 5781 I = ConsecutiveChain[I]; 5782 } 5783 5784 // If a vector register can't hold 1 element, we are done. 5785 unsigned MaxVecRegSize = R.getMaxVecRegSize(); 5786 unsigned EltSize = R.getVectorElementSize(Stores[0]); 5787 if (MaxVecRegSize % EltSize != 0) 5788 continue; 5789 5790 unsigned MaxElts = MaxVecRegSize / EltSize; 5791 // FIXME: Is division-by-2 the correct step? Should we assert that the 5792 // register size is a power-of-2? 5793 unsigned StartIdx = 0; 5794 for (unsigned Size = llvm::PowerOf2Ceil(MaxElts); Size >= 2; Size /= 2) { 5795 for (unsigned Cnt = StartIdx, E = Operands.size(); Cnt + Size <= E;) { 5796 ArrayRef<Value *> Slice = makeArrayRef(Operands).slice(Cnt, Size); 5797 if (!VectorizedStores.count(Slice.front()) && 5798 !VectorizedStores.count(Slice.back()) && 5799 vectorizeStoreChain(Slice, R, Cnt)) { 5800 // Mark the vectorized stores so that we don't vectorize them again. 5801 VectorizedStores.insert(Slice.begin(), Slice.end()); 5802 Changed = true; 5803 // If we vectorized initial block, no need to try to vectorize it 5804 // again. 5805 if (Cnt == StartIdx) 5806 StartIdx += Size; 5807 Cnt += Size; 5808 continue; 5809 } 5810 ++Cnt; 5811 } 5812 // Check if the whole array was vectorized already - exit. 5813 if (StartIdx >= Operands.size()) 5814 break; 5815 } 5816 } 5817 5818 return Changed; 5819 } 5820 5821 void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) { 5822 // Initialize the collections. We will make a single pass over the block. 5823 Stores.clear(); 5824 GEPs.clear(); 5825 5826 // Visit the store and getelementptr instructions in BB and organize them in 5827 // Stores and GEPs according to the underlying objects of their pointer 5828 // operands. 5829 for (Instruction &I : *BB) { 5830 // Ignore store instructions that are volatile or have a pointer operand 5831 // that doesn't point to a scalar type. 5832 if (auto *SI = dyn_cast<StoreInst>(&I)) { 5833 if (!SI->isSimple()) 5834 continue; 5835 if (!isValidElementType(SI->getValueOperand()->getType())) 5836 continue; 5837 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI); 5838 } 5839 5840 // Ignore getelementptr instructions that have more than one index, a 5841 // constant index, or a pointer operand that doesn't point to a scalar 5842 // type. 5843 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) { 5844 auto Idx = GEP->idx_begin()->get(); 5845 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx)) 5846 continue; 5847 if (!isValidElementType(Idx->getType())) 5848 continue; 5849 if (GEP->getType()->isVectorTy()) 5850 continue; 5851 GEPs[GEP->getPointerOperand()].push_back(GEP); 5852 } 5853 } 5854 } 5855 5856 bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) { 5857 if (!A || !B) 5858 return false; 5859 Value *VL[] = { A, B }; 5860 return tryToVectorizeList(VL, R, /*UserCost=*/0, true); 5861 } 5862 5863 bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R, 5864 int UserCost, bool AllowReorder) { 5865 if (VL.size() < 2) 5866 return false; 5867 5868 LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " 5869 << VL.size() << ".\n"); 5870 5871 // Check that all of the parts are scalar instructions of the same type, 5872 // we permit an alternate opcode via InstructionsState. 5873 InstructionsState S = getSameOpcode(VL); 5874 if (!S.getOpcode()) 5875 return false; 5876 5877 Instruction *I0 = cast<Instruction>(S.OpValue); 5878 unsigned Sz = R.getVectorElementSize(I0); 5879 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz); 5880 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF); 5881 if (MaxVF < 2) { 5882 R.getORE()->emit([&]() { 5883 return OptimizationRemarkMissed(SV_NAME, "SmallVF", I0) 5884 << "Cannot SLP vectorize list: vectorization factor " 5885 << "less than 2 is not supported"; 5886 }); 5887 return false; 5888 } 5889 5890 for (Value *V : VL) { 5891 Type *Ty = V->getType(); 5892 if (!isValidElementType(Ty)) { 5893 // NOTE: the following will give user internal llvm type name, which may 5894 // not be useful. 5895 R.getORE()->emit([&]() { 5896 std::string type_str; 5897 llvm::raw_string_ostream rso(type_str); 5898 Ty->print(rso); 5899 return OptimizationRemarkMissed(SV_NAME, "UnsupportedType", I0) 5900 << "Cannot SLP vectorize list: type " 5901 << rso.str() + " is unsupported by vectorizer"; 5902 }); 5903 return false; 5904 } 5905 } 5906 5907 bool Changed = false; 5908 bool CandidateFound = false; 5909 int MinCost = SLPCostThreshold; 5910 5911 unsigned NextInst = 0, MaxInst = VL.size(); 5912 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF; VF /= 2) { 5913 // No actual vectorization should happen, if number of parts is the same as 5914 // provided vectorization factor (i.e. the scalar type is used for vector 5915 // code during codegen). 5916 auto *VecTy = VectorType::get(VL[0]->getType(), VF); 5917 if (TTI->getNumberOfParts(VecTy) == VF) 5918 continue; 5919 for (unsigned I = NextInst; I < MaxInst; ++I) { 5920 unsigned OpsWidth = 0; 5921 5922 if (I + VF > MaxInst) 5923 OpsWidth = MaxInst - I; 5924 else 5925 OpsWidth = VF; 5926 5927 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2) 5928 break; 5929 5930 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth); 5931 // Check that a previous iteration of this loop did not delete the Value. 5932 if (llvm::any_of(Ops, [&R](Value *V) { 5933 auto *I = dyn_cast<Instruction>(V); 5934 return I && R.isDeleted(I); 5935 })) 5936 continue; 5937 5938 LLVM_DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations " 5939 << "\n"); 5940 5941 R.buildTree(Ops); 5942 Optional<ArrayRef<unsigned>> Order = R.bestOrder(); 5943 // TODO: check if we can allow reordering for more cases. 5944 if (AllowReorder && Order) { 5945 // TODO: reorder tree nodes without tree rebuilding. 5946 // Conceptually, there is nothing actually preventing us from trying to 5947 // reorder a larger list. In fact, we do exactly this when vectorizing 5948 // reductions. However, at this point, we only expect to get here when 5949 // there are exactly two operations. 5950 assert(Ops.size() == 2); 5951 Value *ReorderedOps[] = {Ops[1], Ops[0]}; 5952 R.buildTree(ReorderedOps, None); 5953 } 5954 if (R.isTreeTinyAndNotFullyVectorizable()) 5955 continue; 5956 5957 R.computeMinimumValueSizes(); 5958 int Cost = R.getTreeCost() - UserCost; 5959 CandidateFound = true; 5960 MinCost = std::min(MinCost, Cost); 5961 5962 if (Cost < -SLPCostThreshold) { 5963 LLVM_DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n"); 5964 R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList", 5965 cast<Instruction>(Ops[0])) 5966 << "SLP vectorized with cost " << ore::NV("Cost", Cost) 5967 << " and with tree size " 5968 << ore::NV("TreeSize", R.getTreeSize())); 5969 5970 R.vectorizeTree(); 5971 // Move to the next bundle. 5972 I += VF - 1; 5973 NextInst = I + 1; 5974 Changed = true; 5975 } 5976 } 5977 } 5978 5979 if (!Changed && CandidateFound) { 5980 R.getORE()->emit([&]() { 5981 return OptimizationRemarkMissed(SV_NAME, "NotBeneficial", I0) 5982 << "List vectorization was possible but not beneficial with cost " 5983 << ore::NV("Cost", MinCost) << " >= " 5984 << ore::NV("Treshold", -SLPCostThreshold); 5985 }); 5986 } else if (!Changed) { 5987 R.getORE()->emit([&]() { 5988 return OptimizationRemarkMissed(SV_NAME, "NotPossible", I0) 5989 << "Cannot SLP vectorize list: vectorization was impossible" 5990 << " with available vectorization factors"; 5991 }); 5992 } 5993 return Changed; 5994 } 5995 5996 bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) { 5997 if (!I) 5998 return false; 5999 6000 if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I)) 6001 return false; 6002 6003 Value *P = I->getParent(); 6004 6005 // Vectorize in current basic block only. 6006 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0)); 6007 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1)); 6008 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P) 6009 return false; 6010 6011 // Try to vectorize V. 6012 if (tryToVectorizePair(Op0, Op1, R)) 6013 return true; 6014 6015 auto *A = dyn_cast<BinaryOperator>(Op0); 6016 auto *B = dyn_cast<BinaryOperator>(Op1); 6017 // Try to skip B. 6018 if (B && B->hasOneUse()) { 6019 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0)); 6020 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1)); 6021 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R)) 6022 return true; 6023 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R)) 6024 return true; 6025 } 6026 6027 // Try to skip A. 6028 if (A && A->hasOneUse()) { 6029 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0)); 6030 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1)); 6031 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R)) 6032 return true; 6033 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R)) 6034 return true; 6035 } 6036 return false; 6037 } 6038 6039 /// Generate a shuffle mask to be used in a reduction tree. 6040 /// 6041 /// \param VecLen The length of the vector to be reduced. 6042 /// \param NumEltsToRdx The number of elements that should be reduced in the 6043 /// vector. 6044 /// \param IsPairwise Whether the reduction is a pairwise or splitting 6045 /// reduction. A pairwise reduction will generate a mask of 6046 /// <0,2,...> or <1,3,..> while a splitting reduction will generate 6047 /// <2,3, undef,undef> for a vector of 4 and NumElts = 2. 6048 /// \param IsLeft True will generate a mask of even elements, odd otherwise. 6049 static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx, 6050 bool IsPairwise, bool IsLeft, 6051 IRBuilder<> &Builder) { 6052 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask"); 6053 6054 SmallVector<Constant *, 32> ShuffleMask( 6055 VecLen, UndefValue::get(Builder.getInt32Ty())); 6056 6057 if (IsPairwise) 6058 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right). 6059 for (unsigned i = 0; i != NumEltsToRdx; ++i) 6060 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft); 6061 else 6062 // Move the upper half of the vector to the lower half. 6063 for (unsigned i = 0; i != NumEltsToRdx; ++i) 6064 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i); 6065 6066 return ConstantVector::get(ShuffleMask); 6067 } 6068 6069 namespace { 6070 6071 /// Model horizontal reductions. 6072 /// 6073 /// A horizontal reduction is a tree of reduction operations (currently add and 6074 /// fadd) that has operations that can be put into a vector as its leaf. 6075 /// For example, this tree: 6076 /// 6077 /// mul mul mul mul 6078 /// \ / \ / 6079 /// + + 6080 /// \ / 6081 /// + 6082 /// This tree has "mul" as its reduced values and "+" as its reduction 6083 /// operations. A reduction might be feeding into a store or a binary operation 6084 /// feeding a phi. 6085 /// ... 6086 /// \ / 6087 /// + 6088 /// | 6089 /// phi += 6090 /// 6091 /// Or: 6092 /// ... 6093 /// \ / 6094 /// + 6095 /// | 6096 /// *p = 6097 /// 6098 class HorizontalReduction { 6099 using ReductionOpsType = SmallVector<Value *, 16>; 6100 using ReductionOpsListType = SmallVector<ReductionOpsType, 2>; 6101 ReductionOpsListType ReductionOps; 6102 SmallVector<Value *, 32> ReducedVals; 6103 // Use map vector to make stable output. 6104 MapVector<Instruction *, Value *> ExtraArgs; 6105 6106 /// Kind of the reduction data. 6107 enum ReductionKind { 6108 RK_None, /// Not a reduction. 6109 RK_Arithmetic, /// Binary reduction data. 6110 RK_Min, /// Minimum reduction data. 6111 RK_UMin, /// Unsigned minimum reduction data. 6112 RK_Max, /// Maximum reduction data. 6113 RK_UMax, /// Unsigned maximum reduction data. 6114 }; 6115 6116 /// Contains info about operation, like its opcode, left and right operands. 6117 class OperationData { 6118 /// Opcode of the instruction. 6119 unsigned Opcode = 0; 6120 6121 /// Left operand of the reduction operation. 6122 Value *LHS = nullptr; 6123 6124 /// Right operand of the reduction operation. 6125 Value *RHS = nullptr; 6126 6127 /// Kind of the reduction operation. 6128 ReductionKind Kind = RK_None; 6129 6130 /// True if float point min/max reduction has no NaNs. 6131 bool NoNaN = false; 6132 6133 /// Checks if the reduction operation can be vectorized. 6134 bool isVectorizable() const { 6135 return LHS && RHS && 6136 // We currently only support add/mul/logical && min/max reductions. 6137 ((Kind == RK_Arithmetic && 6138 (Opcode == Instruction::Add || Opcode == Instruction::FAdd || 6139 Opcode == Instruction::Mul || Opcode == Instruction::FMul || 6140 Opcode == Instruction::And || Opcode == Instruction::Or || 6141 Opcode == Instruction::Xor)) || 6142 ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) && 6143 (Kind == RK_Min || Kind == RK_Max)) || 6144 (Opcode == Instruction::ICmp && 6145 (Kind == RK_UMin || Kind == RK_UMax))); 6146 } 6147 6148 /// Creates reduction operation with the current opcode. 6149 Value *createOp(IRBuilder<> &Builder, const Twine &Name) const { 6150 assert(isVectorizable() && 6151 "Expected add|fadd or min/max reduction operation."); 6152 Value *Cmp = nullptr; 6153 switch (Kind) { 6154 case RK_Arithmetic: 6155 return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS, 6156 Name); 6157 case RK_Min: 6158 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS) 6159 : Builder.CreateFCmpOLT(LHS, RHS); 6160 return Builder.CreateSelect(Cmp, LHS, RHS, Name); 6161 case RK_Max: 6162 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS) 6163 : Builder.CreateFCmpOGT(LHS, RHS); 6164 return Builder.CreateSelect(Cmp, LHS, RHS, Name); 6165 case RK_UMin: 6166 assert(Opcode == Instruction::ICmp && "Expected integer types."); 6167 Cmp = Builder.CreateICmpULT(LHS, RHS); 6168 return Builder.CreateSelect(Cmp, LHS, RHS, Name); 6169 case RK_UMax: 6170 assert(Opcode == Instruction::ICmp && "Expected integer types."); 6171 Cmp = Builder.CreateICmpUGT(LHS, RHS); 6172 return Builder.CreateSelect(Cmp, LHS, RHS, Name); 6173 case RK_None: 6174 break; 6175 } 6176 llvm_unreachable("Unknown reduction operation."); 6177 } 6178 6179 public: 6180 explicit OperationData() = default; 6181 6182 /// Construction for reduced values. They are identified by opcode only and 6183 /// don't have associated LHS/RHS values. 6184 explicit OperationData(Value *V) { 6185 if (auto *I = dyn_cast<Instruction>(V)) 6186 Opcode = I->getOpcode(); 6187 } 6188 6189 /// Constructor for reduction operations with opcode and its left and 6190 /// right operands. 6191 OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind, 6192 bool NoNaN = false) 6193 : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) { 6194 assert(Kind != RK_None && "One of the reduction operations is expected."); 6195 } 6196 6197 explicit operator bool() const { return Opcode; } 6198 6199 /// Return true if this operation is any kind of minimum or maximum. 6200 bool isMinMax() const { 6201 switch (Kind) { 6202 case RK_Arithmetic: 6203 return false; 6204 case RK_Min: 6205 case RK_Max: 6206 case RK_UMin: 6207 case RK_UMax: 6208 return true; 6209 case RK_None: 6210 break; 6211 } 6212 llvm_unreachable("Reduction kind is not set"); 6213 } 6214 6215 /// Get the index of the first operand. 6216 unsigned getFirstOperandIndex() const { 6217 assert(!!*this && "The opcode is not set."); 6218 // We allow calling this before 'Kind' is set, so handle that specially. 6219 if (Kind == RK_None) 6220 return 0; 6221 return isMinMax() ? 1 : 0; 6222 } 6223 6224 /// Total number of operands in the reduction operation. 6225 unsigned getNumberOfOperands() const { 6226 assert(Kind != RK_None && !!*this && LHS && RHS && 6227 "Expected reduction operation."); 6228 return isMinMax() ? 3 : 2; 6229 } 6230 6231 /// Checks if the operation has the same parent as \p P. 6232 bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const { 6233 assert(Kind != RK_None && !!*this && LHS && RHS && 6234 "Expected reduction operation."); 6235 if (!IsRedOp) 6236 return I->getParent() == P; 6237 if (isMinMax()) { 6238 // SelectInst must be used twice while the condition op must have single 6239 // use only. 6240 auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition()); 6241 return I->getParent() == P && Cmp && Cmp->getParent() == P; 6242 } 6243 // Arithmetic reduction operation must be used once only. 6244 return I->getParent() == P; 6245 } 6246 6247 /// Expected number of uses for reduction operations/reduced values. 6248 bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const { 6249 assert(Kind != RK_None && !!*this && LHS && RHS && 6250 "Expected reduction operation."); 6251 if (isMinMax()) 6252 return I->hasNUses(2) && 6253 (!IsReductionOp || 6254 cast<SelectInst>(I)->getCondition()->hasOneUse()); 6255 return I->hasOneUse(); 6256 } 6257 6258 /// Initializes the list of reduction operations. 6259 void initReductionOps(ReductionOpsListType &ReductionOps) { 6260 assert(Kind != RK_None && !!*this && LHS && RHS && 6261 "Expected reduction operation."); 6262 if (isMinMax()) 6263 ReductionOps.assign(2, ReductionOpsType()); 6264 else 6265 ReductionOps.assign(1, ReductionOpsType()); 6266 } 6267 6268 /// Add all reduction operations for the reduction instruction \p I. 6269 void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) { 6270 assert(Kind != RK_None && !!*this && LHS && RHS && 6271 "Expected reduction operation."); 6272 if (isMinMax()) { 6273 ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition()); 6274 ReductionOps[1].emplace_back(I); 6275 } else { 6276 ReductionOps[0].emplace_back(I); 6277 } 6278 } 6279 6280 /// Checks if instruction is associative and can be vectorized. 6281 bool isAssociative(Instruction *I) const { 6282 assert(Kind != RK_None && *this && LHS && RHS && 6283 "Expected reduction operation."); 6284 switch (Kind) { 6285 case RK_Arithmetic: 6286 return I->isAssociative(); 6287 case RK_Min: 6288 case RK_Max: 6289 return Opcode == Instruction::ICmp || 6290 cast<Instruction>(I->getOperand(0))->isFast(); 6291 case RK_UMin: 6292 case RK_UMax: 6293 assert(Opcode == Instruction::ICmp && 6294 "Only integer compare operation is expected."); 6295 return true; 6296 case RK_None: 6297 break; 6298 } 6299 llvm_unreachable("Reduction kind is not set"); 6300 } 6301 6302 /// Checks if the reduction operation can be vectorized. 6303 bool isVectorizable(Instruction *I) const { 6304 return isVectorizable() && isAssociative(I); 6305 } 6306 6307 /// Checks if two operation data are both a reduction op or both a reduced 6308 /// value. 6309 bool operator==(const OperationData &OD) const { 6310 assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && 6311 "One of the comparing operations is incorrect."); 6312 return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode); 6313 } 6314 bool operator!=(const OperationData &OD) const { return !(*this == OD); } 6315 void clear() { 6316 Opcode = 0; 6317 LHS = nullptr; 6318 RHS = nullptr; 6319 Kind = RK_None; 6320 NoNaN = false; 6321 } 6322 6323 /// Get the opcode of the reduction operation. 6324 unsigned getOpcode() const { 6325 assert(isVectorizable() && "Expected vectorizable operation."); 6326 return Opcode; 6327 } 6328 6329 /// Get kind of reduction data. 6330 ReductionKind getKind() const { return Kind; } 6331 Value *getLHS() const { return LHS; } 6332 Value *getRHS() const { return RHS; } 6333 Type *getConditionType() const { 6334 return isMinMax() ? CmpInst::makeCmpResultType(LHS->getType()) : nullptr; 6335 } 6336 6337 /// Creates reduction operation with the current opcode with the IR flags 6338 /// from \p ReductionOps. 6339 Value *createOp(IRBuilder<> &Builder, const Twine &Name, 6340 const ReductionOpsListType &ReductionOps) const { 6341 assert(isVectorizable() && 6342 "Expected add|fadd or min/max reduction operation."); 6343 auto *Op = createOp(Builder, Name); 6344 switch (Kind) { 6345 case RK_Arithmetic: 6346 propagateIRFlags(Op, ReductionOps[0]); 6347 return Op; 6348 case RK_Min: 6349 case RK_Max: 6350 case RK_UMin: 6351 case RK_UMax: 6352 if (auto *SI = dyn_cast<SelectInst>(Op)) 6353 propagateIRFlags(SI->getCondition(), ReductionOps[0]); 6354 propagateIRFlags(Op, ReductionOps[1]); 6355 return Op; 6356 case RK_None: 6357 break; 6358 } 6359 llvm_unreachable("Unknown reduction operation."); 6360 } 6361 /// Creates reduction operation with the current opcode with the IR flags 6362 /// from \p I. 6363 Value *createOp(IRBuilder<> &Builder, const Twine &Name, 6364 Instruction *I) const { 6365 assert(isVectorizable() && 6366 "Expected add|fadd or min/max reduction operation."); 6367 auto *Op = createOp(Builder, Name); 6368 switch (Kind) { 6369 case RK_Arithmetic: 6370 propagateIRFlags(Op, I); 6371 return Op; 6372 case RK_Min: 6373 case RK_Max: 6374 case RK_UMin: 6375 case RK_UMax: 6376 if (auto *SI = dyn_cast<SelectInst>(Op)) { 6377 propagateIRFlags(SI->getCondition(), 6378 cast<SelectInst>(I)->getCondition()); 6379 } 6380 propagateIRFlags(Op, I); 6381 return Op; 6382 case RK_None: 6383 break; 6384 } 6385 llvm_unreachable("Unknown reduction operation."); 6386 } 6387 6388 TargetTransformInfo::ReductionFlags getFlags() const { 6389 TargetTransformInfo::ReductionFlags Flags; 6390 Flags.NoNaN = NoNaN; 6391 switch (Kind) { 6392 case RK_Arithmetic: 6393 break; 6394 case RK_Min: 6395 Flags.IsSigned = Opcode == Instruction::ICmp; 6396 Flags.IsMaxOp = false; 6397 break; 6398 case RK_Max: 6399 Flags.IsSigned = Opcode == Instruction::ICmp; 6400 Flags.IsMaxOp = true; 6401 break; 6402 case RK_UMin: 6403 Flags.IsSigned = false; 6404 Flags.IsMaxOp = false; 6405 break; 6406 case RK_UMax: 6407 Flags.IsSigned = false; 6408 Flags.IsMaxOp = true; 6409 break; 6410 case RK_None: 6411 llvm_unreachable("Reduction kind is not set"); 6412 } 6413 return Flags; 6414 } 6415 }; 6416 6417 WeakTrackingVH ReductionRoot; 6418 6419 /// The operation data of the reduction operation. 6420 OperationData ReductionData; 6421 6422 /// The operation data of the values we perform a reduction on. 6423 OperationData ReducedValueData; 6424 6425 /// Should we model this reduction as a pairwise reduction tree or a tree that 6426 /// splits the vector in halves and adds those halves. 6427 bool IsPairwiseReduction = false; 6428 6429 /// Checks if the ParentStackElem.first should be marked as a reduction 6430 /// operation with an extra argument or as extra argument itself. 6431 void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem, 6432 Value *ExtraArg) { 6433 if (ExtraArgs.count(ParentStackElem.first)) { 6434 ExtraArgs[ParentStackElem.first] = nullptr; 6435 // We ran into something like: 6436 // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg. 6437 // The whole ParentStackElem.first should be considered as an extra value 6438 // in this case. 6439 // Do not perform analysis of remaining operands of ParentStackElem.first 6440 // instruction, this whole instruction is an extra argument. 6441 ParentStackElem.second = ParentStackElem.first->getNumOperands(); 6442 } else { 6443 // We ran into something like: 6444 // ParentStackElem.first += ... + ExtraArg + ... 6445 ExtraArgs[ParentStackElem.first] = ExtraArg; 6446 } 6447 } 6448 6449 static OperationData getOperationData(Value *V) { 6450 if (!V) 6451 return OperationData(); 6452 6453 Value *LHS; 6454 Value *RHS; 6455 if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) { 6456 return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS, 6457 RK_Arithmetic); 6458 } 6459 if (auto *Select = dyn_cast<SelectInst>(V)) { 6460 // Look for a min/max pattern. 6461 if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) { 6462 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin); 6463 } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) { 6464 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min); 6465 } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) || 6466 m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) { 6467 return OperationData( 6468 Instruction::FCmp, LHS, RHS, RK_Min, 6469 cast<Instruction>(Select->getCondition())->hasNoNaNs()); 6470 } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) { 6471 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax); 6472 } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) { 6473 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max); 6474 } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) || 6475 m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) { 6476 return OperationData( 6477 Instruction::FCmp, LHS, RHS, RK_Max, 6478 cast<Instruction>(Select->getCondition())->hasNoNaNs()); 6479 } else { 6480 // Try harder: look for min/max pattern based on instructions producing 6481 // same values such as: select ((cmp Inst1, Inst2), Inst1, Inst2). 6482 // During the intermediate stages of SLP, it's very common to have 6483 // pattern like this (since optimizeGatherSequence is run only once 6484 // at the end): 6485 // %1 = extractelement <2 x i32> %a, i32 0 6486 // %2 = extractelement <2 x i32> %a, i32 1 6487 // %cond = icmp sgt i32 %1, %2 6488 // %3 = extractelement <2 x i32> %a, i32 0 6489 // %4 = extractelement <2 x i32> %a, i32 1 6490 // %select = select i1 %cond, i32 %3, i32 %4 6491 CmpInst::Predicate Pred; 6492 Instruction *L1; 6493 Instruction *L2; 6494 6495 LHS = Select->getTrueValue(); 6496 RHS = Select->getFalseValue(); 6497 Value *Cond = Select->getCondition(); 6498 6499 // TODO: Support inverse predicates. 6500 if (match(Cond, m_Cmp(Pred, m_Specific(LHS), m_Instruction(L2)))) { 6501 if (!isa<ExtractElementInst>(RHS) || 6502 !L2->isIdenticalTo(cast<Instruction>(RHS))) 6503 return OperationData(V); 6504 } else if (match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Specific(RHS)))) { 6505 if (!isa<ExtractElementInst>(LHS) || 6506 !L1->isIdenticalTo(cast<Instruction>(LHS))) 6507 return OperationData(V); 6508 } else { 6509 if (!isa<ExtractElementInst>(LHS) || !isa<ExtractElementInst>(RHS)) 6510 return OperationData(V); 6511 if (!match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Instruction(L2))) || 6512 !L1->isIdenticalTo(cast<Instruction>(LHS)) || 6513 !L2->isIdenticalTo(cast<Instruction>(RHS))) 6514 return OperationData(V); 6515 } 6516 switch (Pred) { 6517 default: 6518 return OperationData(V); 6519 6520 case CmpInst::ICMP_ULT: 6521 case CmpInst::ICMP_ULE: 6522 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin); 6523 6524 case CmpInst::ICMP_SLT: 6525 case CmpInst::ICMP_SLE: 6526 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min); 6527 6528 case CmpInst::FCMP_OLT: 6529 case CmpInst::FCMP_OLE: 6530 case CmpInst::FCMP_ULT: 6531 case CmpInst::FCMP_ULE: 6532 return OperationData(Instruction::FCmp, LHS, RHS, RK_Min, 6533 cast<Instruction>(Cond)->hasNoNaNs()); 6534 6535 case CmpInst::ICMP_UGT: 6536 case CmpInst::ICMP_UGE: 6537 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax); 6538 6539 case CmpInst::ICMP_SGT: 6540 case CmpInst::ICMP_SGE: 6541 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max); 6542 6543 case CmpInst::FCMP_OGT: 6544 case CmpInst::FCMP_OGE: 6545 case CmpInst::FCMP_UGT: 6546 case CmpInst::FCMP_UGE: 6547 return OperationData(Instruction::FCmp, LHS, RHS, RK_Max, 6548 cast<Instruction>(Cond)->hasNoNaNs()); 6549 } 6550 } 6551 } 6552 return OperationData(V); 6553 } 6554 6555 public: 6556 HorizontalReduction() = default; 6557 6558 /// Try to find a reduction tree. 6559 bool matchAssociativeReduction(PHINode *Phi, Instruction *B) { 6560 assert((!Phi || is_contained(Phi->operands(), B)) && 6561 "Thi phi needs to use the binary operator"); 6562 6563 ReductionData = getOperationData(B); 6564 6565 // We could have a initial reductions that is not an add. 6566 // r *= v1 + v2 + v3 + v4 6567 // In such a case start looking for a tree rooted in the first '+'. 6568 if (Phi) { 6569 if (ReductionData.getLHS() == Phi) { 6570 Phi = nullptr; 6571 B = dyn_cast<Instruction>(ReductionData.getRHS()); 6572 ReductionData = getOperationData(B); 6573 } else if (ReductionData.getRHS() == Phi) { 6574 Phi = nullptr; 6575 B = dyn_cast<Instruction>(ReductionData.getLHS()); 6576 ReductionData = getOperationData(B); 6577 } 6578 } 6579 6580 if (!ReductionData.isVectorizable(B)) 6581 return false; 6582 6583 Type *Ty = B->getType(); 6584 if (!isValidElementType(Ty)) 6585 return false; 6586 if (!Ty->isIntOrIntVectorTy() && !Ty->isFPOrFPVectorTy()) 6587 return false; 6588 6589 ReducedValueData.clear(); 6590 ReductionRoot = B; 6591 6592 // Post order traverse the reduction tree starting at B. We only handle true 6593 // trees containing only binary operators. 6594 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack; 6595 Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex())); 6596 ReductionData.initReductionOps(ReductionOps); 6597 while (!Stack.empty()) { 6598 Instruction *TreeN = Stack.back().first; 6599 unsigned EdgeToVist = Stack.back().second++; 6600 OperationData OpData = getOperationData(TreeN); 6601 bool IsReducedValue = OpData != ReductionData; 6602 6603 // Postorder vist. 6604 if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) { 6605 if (IsReducedValue) 6606 ReducedVals.push_back(TreeN); 6607 else { 6608 auto I = ExtraArgs.find(TreeN); 6609 if (I != ExtraArgs.end() && !I->second) { 6610 // Check if TreeN is an extra argument of its parent operation. 6611 if (Stack.size() <= 1) { 6612 // TreeN can't be an extra argument as it is a root reduction 6613 // operation. 6614 return false; 6615 } 6616 // Yes, TreeN is an extra argument, do not add it to a list of 6617 // reduction operations. 6618 // Stack[Stack.size() - 2] always points to the parent operation. 6619 markExtraArg(Stack[Stack.size() - 2], TreeN); 6620 ExtraArgs.erase(TreeN); 6621 } else 6622 ReductionData.addReductionOps(TreeN, ReductionOps); 6623 } 6624 // Retract. 6625 Stack.pop_back(); 6626 continue; 6627 } 6628 6629 // Visit left or right. 6630 Value *NextV = TreeN->getOperand(EdgeToVist); 6631 if (NextV != Phi) { 6632 auto *I = dyn_cast<Instruction>(NextV); 6633 OpData = getOperationData(I); 6634 // Continue analysis if the next operand is a reduction operation or 6635 // (possibly) a reduced value. If the reduced value opcode is not set, 6636 // the first met operation != reduction operation is considered as the 6637 // reduced value class. 6638 if (I && (!ReducedValueData || OpData == ReducedValueData || 6639 OpData == ReductionData)) { 6640 const bool IsReductionOperation = OpData == ReductionData; 6641 // Only handle trees in the current basic block. 6642 if (!ReductionData.hasSameParent(I, B->getParent(), 6643 IsReductionOperation)) { 6644 // I is an extra argument for TreeN (its parent operation). 6645 markExtraArg(Stack.back(), I); 6646 continue; 6647 } 6648 6649 // Each tree node needs to have minimal number of users except for the 6650 // ultimate reduction. 6651 if (!ReductionData.hasRequiredNumberOfUses(I, 6652 OpData == ReductionData) && 6653 I != B) { 6654 // I is an extra argument for TreeN (its parent operation). 6655 markExtraArg(Stack.back(), I); 6656 continue; 6657 } 6658 6659 if (IsReductionOperation) { 6660 // We need to be able to reassociate the reduction operations. 6661 if (!OpData.isAssociative(I)) { 6662 // I is an extra argument for TreeN (its parent operation). 6663 markExtraArg(Stack.back(), I); 6664 continue; 6665 } 6666 } else if (ReducedValueData && 6667 ReducedValueData != OpData) { 6668 // Make sure that the opcodes of the operations that we are going to 6669 // reduce match. 6670 // I is an extra argument for TreeN (its parent operation). 6671 markExtraArg(Stack.back(), I); 6672 continue; 6673 } else if (!ReducedValueData) 6674 ReducedValueData = OpData; 6675 6676 Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex())); 6677 continue; 6678 } 6679 } 6680 // NextV is an extra argument for TreeN (its parent operation). 6681 markExtraArg(Stack.back(), NextV); 6682 } 6683 return true; 6684 } 6685 6686 /// Attempt to vectorize the tree found by 6687 /// matchAssociativeReduction. 6688 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) { 6689 if (ReducedVals.empty()) 6690 return false; 6691 6692 // If there is a sufficient number of reduction values, reduce 6693 // to a nearby power-of-2. Can safely generate oversized 6694 // vectors and rely on the backend to split them to legal sizes. 6695 unsigned NumReducedVals = ReducedVals.size(); 6696 if (NumReducedVals < 4) 6697 return false; 6698 6699 unsigned ReduxWidth = PowerOf2Floor(NumReducedVals); 6700 6701 Value *VectorizedTree = nullptr; 6702 6703 // FIXME: Fast-math-flags should be set based on the instructions in the 6704 // reduction (not all of 'fast' are required). 6705 IRBuilder<> Builder(cast<Instruction>(ReductionRoot)); 6706 FastMathFlags Unsafe; 6707 Unsafe.setFast(); 6708 Builder.setFastMathFlags(Unsafe); 6709 unsigned i = 0; 6710 6711 BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues; 6712 // The same extra argument may be used several time, so log each attempt 6713 // to use it. 6714 for (auto &Pair : ExtraArgs) { 6715 assert(Pair.first && "DebugLoc must be set."); 6716 ExternallyUsedValues[Pair.second].push_back(Pair.first); 6717 } 6718 6719 // The compare instruction of a min/max is the insertion point for new 6720 // instructions and may be replaced with a new compare instruction. 6721 auto getCmpForMinMaxReduction = [](Instruction *RdxRootInst) { 6722 assert(isa<SelectInst>(RdxRootInst) && 6723 "Expected min/max reduction to have select root instruction"); 6724 Value *ScalarCond = cast<SelectInst>(RdxRootInst)->getCondition(); 6725 assert(isa<Instruction>(ScalarCond) && 6726 "Expected min/max reduction to have compare condition"); 6727 return cast<Instruction>(ScalarCond); 6728 }; 6729 6730 // The reduction root is used as the insertion point for new instructions, 6731 // so set it as externally used to prevent it from being deleted. 6732 ExternallyUsedValues[ReductionRoot]; 6733 SmallVector<Value *, 16> IgnoreList; 6734 for (auto &V : ReductionOps) 6735 IgnoreList.append(V.begin(), V.end()); 6736 while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) { 6737 auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth); 6738 V.buildTree(VL, ExternallyUsedValues, IgnoreList); 6739 Optional<ArrayRef<unsigned>> Order = V.bestOrder(); 6740 // TODO: Handle orders of size less than number of elements in the vector. 6741 if (Order && Order->size() == VL.size()) { 6742 // TODO: reorder tree nodes without tree rebuilding. 6743 SmallVector<Value *, 4> ReorderedOps(VL.size()); 6744 llvm::transform(*Order, ReorderedOps.begin(), 6745 [VL](const unsigned Idx) { return VL[Idx]; }); 6746 V.buildTree(ReorderedOps, ExternallyUsedValues, IgnoreList); 6747 } 6748 if (V.isTreeTinyAndNotFullyVectorizable()) 6749 break; 6750 if (V.isLoadCombineReductionCandidate(ReductionData.getOpcode())) 6751 break; 6752 6753 V.computeMinimumValueSizes(); 6754 6755 // Estimate cost. 6756 int TreeCost = V.getTreeCost(); 6757 int ReductionCost = getReductionCost(TTI, ReducedVals[i], ReduxWidth); 6758 int Cost = TreeCost + ReductionCost; 6759 if (Cost >= -SLPCostThreshold) { 6760 V.getORE()->emit([&]() { 6761 return OptimizationRemarkMissed( 6762 SV_NAME, "HorSLPNotBeneficial", cast<Instruction>(VL[0])) 6763 << "Vectorizing horizontal reduction is possible" 6764 << "but not beneficial with cost " 6765 << ore::NV("Cost", Cost) << " and threshold " 6766 << ore::NV("Threshold", -SLPCostThreshold); 6767 }); 6768 break; 6769 } 6770 6771 LLVM_DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" 6772 << Cost << ". (HorRdx)\n"); 6773 V.getORE()->emit([&]() { 6774 return OptimizationRemark( 6775 SV_NAME, "VectorizedHorizontalReduction", cast<Instruction>(VL[0])) 6776 << "Vectorized horizontal reduction with cost " 6777 << ore::NV("Cost", Cost) << " and with tree size " 6778 << ore::NV("TreeSize", V.getTreeSize()); 6779 }); 6780 6781 // Vectorize a tree. 6782 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc(); 6783 Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues); 6784 6785 // Emit a reduction. For min/max, the root is a select, but the insertion 6786 // point is the compare condition of that select. 6787 Instruction *RdxRootInst = cast<Instruction>(ReductionRoot); 6788 if (ReductionData.isMinMax()) 6789 Builder.SetInsertPoint(getCmpForMinMaxReduction(RdxRootInst)); 6790 else 6791 Builder.SetInsertPoint(RdxRootInst); 6792 6793 Value *ReducedSubTree = 6794 emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI); 6795 if (VectorizedTree) { 6796 Builder.SetCurrentDebugLocation(Loc); 6797 OperationData VectReductionData(ReductionData.getOpcode(), 6798 VectorizedTree, ReducedSubTree, 6799 ReductionData.getKind()); 6800 VectorizedTree = 6801 VectReductionData.createOp(Builder, "op.rdx", ReductionOps); 6802 } else 6803 VectorizedTree = ReducedSubTree; 6804 i += ReduxWidth; 6805 ReduxWidth = PowerOf2Floor(NumReducedVals - i); 6806 } 6807 6808 if (VectorizedTree) { 6809 // Finish the reduction. 6810 for (; i < NumReducedVals; ++i) { 6811 auto *I = cast<Instruction>(ReducedVals[i]); 6812 Builder.SetCurrentDebugLocation(I->getDebugLoc()); 6813 OperationData VectReductionData(ReductionData.getOpcode(), 6814 VectorizedTree, I, 6815 ReductionData.getKind()); 6816 VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps); 6817 } 6818 for (auto &Pair : ExternallyUsedValues) { 6819 // Add each externally used value to the final reduction. 6820 for (auto *I : Pair.second) { 6821 Builder.SetCurrentDebugLocation(I->getDebugLoc()); 6822 OperationData VectReductionData(ReductionData.getOpcode(), 6823 VectorizedTree, Pair.first, 6824 ReductionData.getKind()); 6825 VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I); 6826 } 6827 } 6828 6829 // Update users. For a min/max reduction that ends with a compare and 6830 // select, we also have to RAUW for the compare instruction feeding the 6831 // reduction root. That's because the original compare may have extra uses 6832 // besides the final select of the reduction. 6833 if (ReductionData.isMinMax()) { 6834 if (auto *VecSelect = dyn_cast<SelectInst>(VectorizedTree)) { 6835 Instruction *ScalarCmp = 6836 getCmpForMinMaxReduction(cast<Instruction>(ReductionRoot)); 6837 ScalarCmp->replaceAllUsesWith(VecSelect->getCondition()); 6838 } 6839 } 6840 ReductionRoot->replaceAllUsesWith(VectorizedTree); 6841 6842 // Mark all scalar reduction ops for deletion, they are replaced by the 6843 // vector reductions. 6844 V.eraseInstructions(IgnoreList); 6845 } 6846 return VectorizedTree != nullptr; 6847 } 6848 6849 unsigned numReductionValues() const { 6850 return ReducedVals.size(); 6851 } 6852 6853 private: 6854 /// Calculate the cost of a reduction. 6855 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal, 6856 unsigned ReduxWidth) { 6857 Type *ScalarTy = FirstReducedVal->getType(); 6858 Type *VecTy = VectorType::get(ScalarTy, ReduxWidth); 6859 6860 int PairwiseRdxCost; 6861 int SplittingRdxCost; 6862 switch (ReductionData.getKind()) { 6863 case RK_Arithmetic: 6864 PairwiseRdxCost = 6865 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy, 6866 /*IsPairwiseForm=*/true); 6867 SplittingRdxCost = 6868 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy, 6869 /*IsPairwiseForm=*/false); 6870 break; 6871 case RK_Min: 6872 case RK_Max: 6873 case RK_UMin: 6874 case RK_UMax: { 6875 Type *VecCondTy = CmpInst::makeCmpResultType(VecTy); 6876 bool IsUnsigned = ReductionData.getKind() == RK_UMin || 6877 ReductionData.getKind() == RK_UMax; 6878 PairwiseRdxCost = 6879 TTI->getMinMaxReductionCost(VecTy, VecCondTy, 6880 /*IsPairwiseForm=*/true, IsUnsigned); 6881 SplittingRdxCost = 6882 TTI->getMinMaxReductionCost(VecTy, VecCondTy, 6883 /*IsPairwiseForm=*/false, IsUnsigned); 6884 break; 6885 } 6886 case RK_None: 6887 llvm_unreachable("Expected arithmetic or min/max reduction operation"); 6888 } 6889 6890 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost; 6891 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost; 6892 6893 int ScalarReduxCost = 0; 6894 switch (ReductionData.getKind()) { 6895 case RK_Arithmetic: 6896 ScalarReduxCost = 6897 TTI->getArithmeticInstrCost(ReductionData.getOpcode(), ScalarTy); 6898 break; 6899 case RK_Min: 6900 case RK_Max: 6901 case RK_UMin: 6902 case RK_UMax: 6903 ScalarReduxCost = 6904 TTI->getCmpSelInstrCost(ReductionData.getOpcode(), ScalarTy) + 6905 TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy, 6906 CmpInst::makeCmpResultType(ScalarTy)); 6907 break; 6908 case RK_None: 6909 llvm_unreachable("Expected arithmetic or min/max reduction operation"); 6910 } 6911 ScalarReduxCost *= (ReduxWidth - 1); 6912 6913 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCost 6914 << " for reduction that starts with " << *FirstReducedVal 6915 << " (It is a " 6916 << (IsPairwiseReduction ? "pairwise" : "splitting") 6917 << " reduction)\n"); 6918 6919 return VecReduxCost - ScalarReduxCost; 6920 } 6921 6922 /// Emit a horizontal reduction of the vectorized value. 6923 Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder, 6924 unsigned ReduxWidth, const TargetTransformInfo *TTI) { 6925 assert(VectorizedValue && "Need to have a vectorized tree node"); 6926 assert(isPowerOf2_32(ReduxWidth) && 6927 "We only handle power-of-two reductions for now"); 6928 6929 if (!IsPairwiseReduction) { 6930 // FIXME: The builder should use an FMF guard. It should not be hard-coded 6931 // to 'fast'. 6932 assert(Builder.getFastMathFlags().isFast() && "Expected 'fast' FMF"); 6933 return createSimpleTargetReduction( 6934 Builder, TTI, ReductionData.getOpcode(), VectorizedValue, 6935 ReductionData.getFlags(), ReductionOps.back()); 6936 } 6937 6938 Value *TmpVec = VectorizedValue; 6939 for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) { 6940 Value *LeftMask = 6941 createRdxShuffleMask(ReduxWidth, i, true, true, Builder); 6942 Value *RightMask = 6943 createRdxShuffleMask(ReduxWidth, i, true, false, Builder); 6944 6945 Value *LeftShuf = Builder.CreateShuffleVector( 6946 TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l"); 6947 Value *RightShuf = Builder.CreateShuffleVector( 6948 TmpVec, UndefValue::get(TmpVec->getType()), (RightMask), 6949 "rdx.shuf.r"); 6950 OperationData VectReductionData(ReductionData.getOpcode(), LeftShuf, 6951 RightShuf, ReductionData.getKind()); 6952 TmpVec = VectReductionData.createOp(Builder, "op.rdx", ReductionOps); 6953 } 6954 6955 // The result is in the first element of the vector. 6956 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0)); 6957 } 6958 }; 6959 6960 } // end anonymous namespace 6961 6962 /// Recognize construction of vectors like 6963 /// %ra = insertelement <4 x float> undef, float %s0, i32 0 6964 /// %rb = insertelement <4 x float> %ra, float %s1, i32 1 6965 /// %rc = insertelement <4 x float> %rb, float %s2, i32 2 6966 /// %rd = insertelement <4 x float> %rc, float %s3, i32 3 6967 /// starting from the last insertelement or insertvalue instruction. 6968 /// 6969 /// Also recognize aggregates like {<2 x float>, <2 x float>}, 6970 /// {{float, float}, {float, float}}, [2 x {float, float}] and so on. 6971 /// See llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll for examples. 6972 /// 6973 /// Assume LastInsertInst is of InsertElementInst or InsertValueInst type. 6974 /// 6975 /// \return true if it matches. 6976 static bool findBuildAggregate(Value *LastInsertInst, TargetTransformInfo *TTI, 6977 SmallVectorImpl<Value *> &BuildVectorOpds, 6978 int &UserCost) { 6979 assert((isa<InsertElementInst>(LastInsertInst) || 6980 isa<InsertValueInst>(LastInsertInst)) && 6981 "Expected insertelement or insertvalue instruction!"); 6982 UserCost = 0; 6983 do { 6984 Value *InsertedOperand; 6985 if (auto *IE = dyn_cast<InsertElementInst>(LastInsertInst)) { 6986 InsertedOperand = IE->getOperand(1); 6987 LastInsertInst = IE->getOperand(0); 6988 if (auto *CI = dyn_cast<ConstantInt>(IE->getOperand(2))) { 6989 UserCost += TTI->getVectorInstrCost(Instruction::InsertElement, 6990 IE->getType(), CI->getZExtValue()); 6991 } 6992 } else { 6993 auto *IV = cast<InsertValueInst>(LastInsertInst); 6994 InsertedOperand = IV->getInsertedValueOperand(); 6995 LastInsertInst = IV->getAggregateOperand(); 6996 } 6997 if (isa<InsertElementInst>(InsertedOperand) || 6998 isa<InsertValueInst>(InsertedOperand)) { 6999 int TmpUserCost; 7000 SmallVector<Value *, 8> TmpBuildVectorOpds; 7001 if (!findBuildAggregate(InsertedOperand, TTI, TmpBuildVectorOpds, 7002 TmpUserCost)) 7003 return false; 7004 BuildVectorOpds.append(TmpBuildVectorOpds.rbegin(), 7005 TmpBuildVectorOpds.rend()); 7006 UserCost += TmpUserCost; 7007 } else { 7008 BuildVectorOpds.push_back(InsertedOperand); 7009 } 7010 if (isa<UndefValue>(LastInsertInst)) 7011 break; 7012 if ((!isa<InsertValueInst>(LastInsertInst) && 7013 !isa<InsertElementInst>(LastInsertInst)) || 7014 !LastInsertInst->hasOneUse()) 7015 return false; 7016 } while (true); 7017 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end()); 7018 return true; 7019 } 7020 7021 static bool PhiTypeSorterFunc(Value *V, Value *V2) { 7022 return V->getType() < V2->getType(); 7023 } 7024 7025 /// Try and get a reduction value from a phi node. 7026 /// 7027 /// Given a phi node \p P in a block \p ParentBB, consider possible reductions 7028 /// if they come from either \p ParentBB or a containing loop latch. 7029 /// 7030 /// \returns A candidate reduction value if possible, or \code nullptr \endcode 7031 /// if not possible. 7032 static Value *getReductionValue(const DominatorTree *DT, PHINode *P, 7033 BasicBlock *ParentBB, LoopInfo *LI) { 7034 // There are situations where the reduction value is not dominated by the 7035 // reduction phi. Vectorizing such cases has been reported to cause 7036 // miscompiles. See PR25787. 7037 auto DominatedReduxValue = [&](Value *R) { 7038 return isa<Instruction>(R) && 7039 DT->dominates(P->getParent(), cast<Instruction>(R)->getParent()); 7040 }; 7041 7042 Value *Rdx = nullptr; 7043 7044 // Return the incoming value if it comes from the same BB as the phi node. 7045 if (P->getIncomingBlock(0) == ParentBB) { 7046 Rdx = P->getIncomingValue(0); 7047 } else if (P->getIncomingBlock(1) == ParentBB) { 7048 Rdx = P->getIncomingValue(1); 7049 } 7050 7051 if (Rdx && DominatedReduxValue(Rdx)) 7052 return Rdx; 7053 7054 // Otherwise, check whether we have a loop latch to look at. 7055 Loop *BBL = LI->getLoopFor(ParentBB); 7056 if (!BBL) 7057 return nullptr; 7058 BasicBlock *BBLatch = BBL->getLoopLatch(); 7059 if (!BBLatch) 7060 return nullptr; 7061 7062 // There is a loop latch, return the incoming value if it comes from 7063 // that. This reduction pattern occasionally turns up. 7064 if (P->getIncomingBlock(0) == BBLatch) { 7065 Rdx = P->getIncomingValue(0); 7066 } else if (P->getIncomingBlock(1) == BBLatch) { 7067 Rdx = P->getIncomingValue(1); 7068 } 7069 7070 if (Rdx && DominatedReduxValue(Rdx)) 7071 return Rdx; 7072 7073 return nullptr; 7074 } 7075 7076 /// Attempt to reduce a horizontal reduction. 7077 /// If it is legal to match a horizontal reduction feeding the phi node \a P 7078 /// with reduction operators \a Root (or one of its operands) in a basic block 7079 /// \a BB, then check if it can be done. If horizontal reduction is not found 7080 /// and root instruction is a binary operation, vectorization of the operands is 7081 /// attempted. 7082 /// \returns true if a horizontal reduction was matched and reduced or operands 7083 /// of one of the binary instruction were vectorized. 7084 /// \returns false if a horizontal reduction was not matched (or not possible) 7085 /// or no vectorization of any binary operation feeding \a Root instruction was 7086 /// performed. 7087 static bool tryToVectorizeHorReductionOrInstOperands( 7088 PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R, 7089 TargetTransformInfo *TTI, 7090 const function_ref<bool(Instruction *, BoUpSLP &)> Vectorize) { 7091 if (!ShouldVectorizeHor) 7092 return false; 7093 7094 if (!Root) 7095 return false; 7096 7097 if (Root->getParent() != BB || isa<PHINode>(Root)) 7098 return false; 7099 // Start analysis starting from Root instruction. If horizontal reduction is 7100 // found, try to vectorize it. If it is not a horizontal reduction or 7101 // vectorization is not possible or not effective, and currently analyzed 7102 // instruction is a binary operation, try to vectorize the operands, using 7103 // pre-order DFS traversal order. If the operands were not vectorized, repeat 7104 // the same procedure considering each operand as a possible root of the 7105 // horizontal reduction. 7106 // Interrupt the process if the Root instruction itself was vectorized or all 7107 // sub-trees not higher that RecursionMaxDepth were analyzed/vectorized. 7108 SmallVector<std::pair<Instruction *, unsigned>, 8> Stack(1, {Root, 0}); 7109 SmallPtrSet<Value *, 8> VisitedInstrs; 7110 bool Res = false; 7111 while (!Stack.empty()) { 7112 Instruction *Inst; 7113 unsigned Level; 7114 std::tie(Inst, Level) = Stack.pop_back_val(); 7115 auto *BI = dyn_cast<BinaryOperator>(Inst); 7116 auto *SI = dyn_cast<SelectInst>(Inst); 7117 if (BI || SI) { 7118 HorizontalReduction HorRdx; 7119 if (HorRdx.matchAssociativeReduction(P, Inst)) { 7120 if (HorRdx.tryToReduce(R, TTI)) { 7121 Res = true; 7122 // Set P to nullptr to avoid re-analysis of phi node in 7123 // matchAssociativeReduction function unless this is the root node. 7124 P = nullptr; 7125 continue; 7126 } 7127 } 7128 if (P && BI) { 7129 Inst = dyn_cast<Instruction>(BI->getOperand(0)); 7130 if (Inst == P) 7131 Inst = dyn_cast<Instruction>(BI->getOperand(1)); 7132 if (!Inst) { 7133 // Set P to nullptr to avoid re-analysis of phi node in 7134 // matchAssociativeReduction function unless this is the root node. 7135 P = nullptr; 7136 continue; 7137 } 7138 } 7139 } 7140 // Set P to nullptr to avoid re-analysis of phi node in 7141 // matchAssociativeReduction function unless this is the root node. 7142 P = nullptr; 7143 if (Vectorize(Inst, R)) { 7144 Res = true; 7145 continue; 7146 } 7147 7148 // Try to vectorize operands. 7149 // Continue analysis for the instruction from the same basic block only to 7150 // save compile time. 7151 if (++Level < RecursionMaxDepth) 7152 for (auto *Op : Inst->operand_values()) 7153 if (VisitedInstrs.insert(Op).second) 7154 if (auto *I = dyn_cast<Instruction>(Op)) 7155 if (!isa<PHINode>(I) && !R.isDeleted(I) && I->getParent() == BB) 7156 Stack.emplace_back(I, Level); 7157 } 7158 return Res; 7159 } 7160 7161 bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V, 7162 BasicBlock *BB, BoUpSLP &R, 7163 TargetTransformInfo *TTI) { 7164 if (!V) 7165 return false; 7166 auto *I = dyn_cast<Instruction>(V); 7167 if (!I) 7168 return false; 7169 7170 if (!isa<BinaryOperator>(I)) 7171 P = nullptr; 7172 // Try to match and vectorize a horizontal reduction. 7173 auto &&ExtraVectorization = [this](Instruction *I, BoUpSLP &R) -> bool { 7174 return tryToVectorize(I, R); 7175 }; 7176 return tryToVectorizeHorReductionOrInstOperands(P, I, BB, R, TTI, 7177 ExtraVectorization); 7178 } 7179 7180 bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI, 7181 BasicBlock *BB, BoUpSLP &R) { 7182 int UserCost = 0; 7183 const DataLayout &DL = BB->getModule()->getDataLayout(); 7184 if (!R.canMapToVector(IVI->getType(), DL)) 7185 return false; 7186 7187 SmallVector<Value *, 16> BuildVectorOpds; 7188 if (!findBuildAggregate(IVI, TTI, BuildVectorOpds, UserCost)) 7189 return false; 7190 7191 LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n"); 7192 // Aggregate value is unlikely to be processed in vector register, we need to 7193 // extract scalars into scalar registers, so NeedExtraction is set true. 7194 return tryToVectorizeList(BuildVectorOpds, R, UserCost); 7195 } 7196 7197 bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI, 7198 BasicBlock *BB, BoUpSLP &R) { 7199 int UserCost; 7200 SmallVector<Value *, 16> BuildVectorOpds; 7201 if (!findBuildAggregate(IEI, TTI, BuildVectorOpds, UserCost) || 7202 (llvm::all_of(BuildVectorOpds, 7203 [](Value *V) { return isa<ExtractElementInst>(V); }) && 7204 isShuffle(BuildVectorOpds))) 7205 return false; 7206 7207 // Vectorize starting with the build vector operands ignoring the BuildVector 7208 // instructions for the purpose of scheduling and user extraction. 7209 return tryToVectorizeList(BuildVectorOpds, R, UserCost); 7210 } 7211 7212 bool SLPVectorizerPass::vectorizeCmpInst(CmpInst *CI, BasicBlock *BB, 7213 BoUpSLP &R) { 7214 if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R)) 7215 return true; 7216 7217 bool OpsChanged = false; 7218 for (int Idx = 0; Idx < 2; ++Idx) { 7219 OpsChanged |= 7220 vectorizeRootInstruction(nullptr, CI->getOperand(Idx), BB, R, TTI); 7221 } 7222 return OpsChanged; 7223 } 7224 7225 bool SLPVectorizerPass::vectorizeSimpleInstructions( 7226 SmallVectorImpl<Instruction *> &Instructions, BasicBlock *BB, BoUpSLP &R) { 7227 bool OpsChanged = false; 7228 for (auto *I : reverse(Instructions)) { 7229 if (R.isDeleted(I)) 7230 continue; 7231 if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I)) 7232 OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R); 7233 else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I)) 7234 OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R); 7235 else if (auto *CI = dyn_cast<CmpInst>(I)) 7236 OpsChanged |= vectorizeCmpInst(CI, BB, R); 7237 } 7238 Instructions.clear(); 7239 return OpsChanged; 7240 } 7241 7242 bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) { 7243 bool Changed = false; 7244 SmallVector<Value *, 4> Incoming; 7245 SmallPtrSet<Value *, 16> VisitedInstrs; 7246 7247 bool HaveVectorizedPhiNodes = true; 7248 while (HaveVectorizedPhiNodes) { 7249 HaveVectorizedPhiNodes = false; 7250 7251 // Collect the incoming values from the PHIs. 7252 Incoming.clear(); 7253 for (Instruction &I : *BB) { 7254 PHINode *P = dyn_cast<PHINode>(&I); 7255 if (!P) 7256 break; 7257 7258 if (!VisitedInstrs.count(P) && !R.isDeleted(P)) 7259 Incoming.push_back(P); 7260 } 7261 7262 // Sort by type. 7263 llvm::stable_sort(Incoming, PhiTypeSorterFunc); 7264 7265 // Try to vectorize elements base on their type. 7266 for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(), 7267 E = Incoming.end(); 7268 IncIt != E;) { 7269 7270 // Look for the next elements with the same type. 7271 SmallVector<Value *, 4>::iterator SameTypeIt = IncIt; 7272 while (SameTypeIt != E && 7273 (*SameTypeIt)->getType() == (*IncIt)->getType()) { 7274 VisitedInstrs.insert(*SameTypeIt); 7275 ++SameTypeIt; 7276 } 7277 7278 // Try to vectorize them. 7279 unsigned NumElts = (SameTypeIt - IncIt); 7280 LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize starting at PHIs (" 7281 << NumElts << ")\n"); 7282 // The order in which the phi nodes appear in the program does not matter. 7283 // So allow tryToVectorizeList to reorder them if it is beneficial. This 7284 // is done when there are exactly two elements since tryToVectorizeList 7285 // asserts that there are only two values when AllowReorder is true. 7286 bool AllowReorder = NumElts == 2; 7287 if (NumElts > 1 && tryToVectorizeList(makeArrayRef(IncIt, NumElts), R, 7288 /*UserCost=*/0, AllowReorder)) { 7289 // Success start over because instructions might have been changed. 7290 HaveVectorizedPhiNodes = true; 7291 Changed = true; 7292 break; 7293 } 7294 7295 // Start over at the next instruction of a different type (or the end). 7296 IncIt = SameTypeIt; 7297 } 7298 } 7299 7300 VisitedInstrs.clear(); 7301 7302 SmallVector<Instruction *, 8> PostProcessInstructions; 7303 SmallDenseSet<Instruction *, 4> KeyNodes; 7304 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) { 7305 // Skip instructions marked for the deletion. 7306 if (R.isDeleted(&*it)) 7307 continue; 7308 // We may go through BB multiple times so skip the one we have checked. 7309 if (!VisitedInstrs.insert(&*it).second) { 7310 if (it->use_empty() && KeyNodes.count(&*it) > 0 && 7311 vectorizeSimpleInstructions(PostProcessInstructions, BB, R)) { 7312 // We would like to start over since some instructions are deleted 7313 // and the iterator may become invalid value. 7314 Changed = true; 7315 it = BB->begin(); 7316 e = BB->end(); 7317 } 7318 continue; 7319 } 7320 7321 if (isa<DbgInfoIntrinsic>(it)) 7322 continue; 7323 7324 // Try to vectorize reductions that use PHINodes. 7325 if (PHINode *P = dyn_cast<PHINode>(it)) { 7326 // Check that the PHI is a reduction PHI. 7327 if (P->getNumIncomingValues() != 2) 7328 return Changed; 7329 7330 // Try to match and vectorize a horizontal reduction. 7331 if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R, 7332 TTI)) { 7333 Changed = true; 7334 it = BB->begin(); 7335 e = BB->end(); 7336 continue; 7337 } 7338 continue; 7339 } 7340 7341 // Ran into an instruction without users, like terminator, or function call 7342 // with ignored return value, store. Ignore unused instructions (basing on 7343 // instruction type, except for CallInst and InvokeInst). 7344 if (it->use_empty() && (it->getType()->isVoidTy() || isa<CallInst>(it) || 7345 isa<InvokeInst>(it))) { 7346 KeyNodes.insert(&*it); 7347 bool OpsChanged = false; 7348 if (ShouldStartVectorizeHorAtStore || !isa<StoreInst>(it)) { 7349 for (auto *V : it->operand_values()) { 7350 // Try to match and vectorize a horizontal reduction. 7351 OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI); 7352 } 7353 } 7354 // Start vectorization of post-process list of instructions from the 7355 // top-tree instructions to try to vectorize as many instructions as 7356 // possible. 7357 OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R); 7358 if (OpsChanged) { 7359 // We would like to start over since some instructions are deleted 7360 // and the iterator may become invalid value. 7361 Changed = true; 7362 it = BB->begin(); 7363 e = BB->end(); 7364 continue; 7365 } 7366 } 7367 7368 if (isa<InsertElementInst>(it) || isa<CmpInst>(it) || 7369 isa<InsertValueInst>(it)) 7370 PostProcessInstructions.push_back(&*it); 7371 } 7372 7373 return Changed; 7374 } 7375 7376 bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) { 7377 auto Changed = false; 7378 for (auto &Entry : GEPs) { 7379 // If the getelementptr list has fewer than two elements, there's nothing 7380 // to do. 7381 if (Entry.second.size() < 2) 7382 continue; 7383 7384 LLVM_DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length " 7385 << Entry.second.size() << ".\n"); 7386 7387 // Process the GEP list in chunks suitable for the target's supported 7388 // vector size. If a vector register can't hold 1 element, we are done. 7389 unsigned MaxVecRegSize = R.getMaxVecRegSize(); 7390 unsigned EltSize = R.getVectorElementSize(Entry.second[0]); 7391 if (MaxVecRegSize < EltSize) 7392 continue; 7393 7394 unsigned MaxElts = MaxVecRegSize / EltSize; 7395 for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += MaxElts) { 7396 auto Len = std::min<unsigned>(BE - BI, MaxElts); 7397 auto GEPList = makeArrayRef(&Entry.second[BI], Len); 7398 7399 // Initialize a set a candidate getelementptrs. Note that we use a 7400 // SetVector here to preserve program order. If the index computations 7401 // are vectorizable and begin with loads, we want to minimize the chance 7402 // of having to reorder them later. 7403 SetVector<Value *> Candidates(GEPList.begin(), GEPList.end()); 7404 7405 // Some of the candidates may have already been vectorized after we 7406 // initially collected them. If so, they are marked as deleted, so remove 7407 // them from the set of candidates. 7408 Candidates.remove_if( 7409 [&R](Value *I) { return R.isDeleted(cast<Instruction>(I)); }); 7410 7411 // Remove from the set of candidates all pairs of getelementptrs with 7412 // constant differences. Such getelementptrs are likely not good 7413 // candidates for vectorization in a bottom-up phase since one can be 7414 // computed from the other. We also ensure all candidate getelementptr 7415 // indices are unique. 7416 for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) { 7417 auto *GEPI = GEPList[I]; 7418 if (!Candidates.count(GEPI)) 7419 continue; 7420 auto *SCEVI = SE->getSCEV(GEPList[I]); 7421 for (int J = I + 1; J < E && Candidates.size() > 1; ++J) { 7422 auto *GEPJ = GEPList[J]; 7423 auto *SCEVJ = SE->getSCEV(GEPList[J]); 7424 if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) { 7425 Candidates.remove(GEPI); 7426 Candidates.remove(GEPJ); 7427 } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) { 7428 Candidates.remove(GEPJ); 7429 } 7430 } 7431 } 7432 7433 // We break out of the above computation as soon as we know there are 7434 // fewer than two candidates remaining. 7435 if (Candidates.size() < 2) 7436 continue; 7437 7438 // Add the single, non-constant index of each candidate to the bundle. We 7439 // ensured the indices met these constraints when we originally collected 7440 // the getelementptrs. 7441 SmallVector<Value *, 16> Bundle(Candidates.size()); 7442 auto BundleIndex = 0u; 7443 for (auto *V : Candidates) { 7444 auto *GEP = cast<GetElementPtrInst>(V); 7445 auto *GEPIdx = GEP->idx_begin()->get(); 7446 assert(GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx)); 7447 Bundle[BundleIndex++] = GEPIdx; 7448 } 7449 7450 // Try and vectorize the indices. We are currently only interested in 7451 // gather-like cases of the form: 7452 // 7453 // ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ... 7454 // 7455 // where the loads of "a", the loads of "b", and the subtractions can be 7456 // performed in parallel. It's likely that detecting this pattern in a 7457 // bottom-up phase will be simpler and less costly than building a 7458 // full-blown top-down phase beginning at the consecutive loads. 7459 Changed |= tryToVectorizeList(Bundle, R); 7460 } 7461 } 7462 return Changed; 7463 } 7464 7465 bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) { 7466 bool Changed = false; 7467 // Attempt to sort and vectorize each of the store-groups. 7468 for (StoreListMap::iterator it = Stores.begin(), e = Stores.end(); it != e; 7469 ++it) { 7470 if (it->second.size() < 2) 7471 continue; 7472 7473 LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " 7474 << it->second.size() << ".\n"); 7475 7476 Changed |= vectorizeStores(it->second, R); 7477 } 7478 return Changed; 7479 } 7480 7481 char SLPVectorizer::ID = 0; 7482 7483 static const char lv_name[] = "SLP Vectorizer"; 7484 7485 INITIALIZE_PASS_BEGIN(SLPVectorizer, SV_NAME, lv_name, false, false) 7486 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 7487 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) 7488 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 7489 INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass) 7490 INITIALIZE_PASS_DEPENDENCY(LoopSimplify) 7491 INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass) 7492 INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass) 7493 INITIALIZE_PASS_END(SLPVectorizer, SV_NAME, lv_name, false, false) 7494 7495 Pass *llvm::createSLPVectorizerPass() { return new SLPVectorizer(); } 7496