1 //===- llvm/Analysis/LoopAccessAnalysis.h -----------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines the interface for the loop memory dependence framework that 10 // was originally developed for the Loop Vectorizer. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_ANALYSIS_LOOPACCESSANALYSIS_H 15 #define LLVM_ANALYSIS_LOOPACCESSANALYSIS_H 16 17 #include "llvm/ADT/EquivalenceClasses.h" 18 #include "llvm/Analysis/LoopAnalysisManager.h" 19 #include "llvm/Analysis/ScalarEvolutionExpressions.h" 20 #include "llvm/IR/DiagnosticInfo.h" 21 #include "llvm/Pass.h" 22 23 namespace llvm { 24 25 class AAResults; 26 class DataLayout; 27 class Loop; 28 class LoopAccessInfo; 29 class raw_ostream; 30 class SCEV; 31 class SCEVUnionPredicate; 32 class Value; 33 34 /// Collection of parameters shared beetween the Loop Vectorizer and the 35 /// Loop Access Analysis. 36 struct VectorizerParams { 37 /// Maximum SIMD width. 38 static const unsigned MaxVectorWidth; 39 40 /// VF as overridden by the user. 41 static unsigned VectorizationFactor; 42 /// Interleave factor as overridden by the user. 43 static unsigned VectorizationInterleave; 44 /// True if force-vector-interleave was specified by the user. 45 static bool isInterleaveForced(); 46 47 /// \When performing memory disambiguation checks at runtime do not 48 /// make more than this number of comparisons. 49 static unsigned RuntimeMemoryCheckThreshold; 50 }; 51 52 /// Checks memory dependences among accesses to the same underlying 53 /// object to determine whether there vectorization is legal or not (and at 54 /// which vectorization factor). 55 /// 56 /// Note: This class will compute a conservative dependence for access to 57 /// different underlying pointers. Clients, such as the loop vectorizer, will 58 /// sometimes deal these potential dependencies by emitting runtime checks. 59 /// 60 /// We use the ScalarEvolution framework to symbolically evalutate access 61 /// functions pairs. Since we currently don't restructure the loop we can rely 62 /// on the program order of memory accesses to determine their safety. 63 /// At the moment we will only deem accesses as safe for: 64 /// * A negative constant distance assuming program order. 65 /// 66 /// Safe: tmp = a[i + 1]; OR a[i + 1] = x; 67 /// a[i] = tmp; y = a[i]; 68 /// 69 /// The latter case is safe because later checks guarantuee that there can't 70 /// be a cycle through a phi node (that is, we check that "x" and "y" is not 71 /// the same variable: a header phi can only be an induction or a reduction, a 72 /// reduction can't have a memory sink, an induction can't have a memory 73 /// source). This is important and must not be violated (or we have to 74 /// resort to checking for cycles through memory). 75 /// 76 /// * A positive constant distance assuming program order that is bigger 77 /// than the biggest memory access. 78 /// 79 /// tmp = a[i] OR b[i] = x 80 /// a[i+2] = tmp y = b[i+2]; 81 /// 82 /// Safe distance: 2 x sizeof(a[0]), and 2 x sizeof(b[0]), respectively. 83 /// 84 /// * Zero distances and all accesses have the same size. 85 /// 86 class MemoryDepChecker { 87 public: 88 typedef PointerIntPair<Value *, 1, bool> MemAccessInfo; 89 typedef SmallVector<MemAccessInfo, 8> MemAccessInfoList; 90 /// Set of potential dependent memory accesses. 91 typedef EquivalenceClasses<MemAccessInfo> DepCandidates; 92 93 /// Type to keep track of the status of the dependence check. The order of 94 /// the elements is important and has to be from most permissive to least 95 /// permissive. 96 enum class VectorizationSafetyStatus { 97 // Can vectorize safely without RT checks. All dependences are known to be 98 // safe. 99 Safe, 100 // Can possibly vectorize with RT checks to overcome unknown dependencies. 101 PossiblySafeWithRtChecks, 102 // Cannot vectorize due to known unsafe dependencies. 103 Unsafe, 104 }; 105 106 /// Dependece between memory access instructions. 107 struct Dependence { 108 /// The type of the dependence. 109 enum DepType { 110 // No dependence. 111 NoDep, 112 // We couldn't determine the direction or the distance. 113 Unknown, 114 // Lexically forward. 115 // 116 // FIXME: If we only have loop-independent forward dependences (e.g. a 117 // read and write of A[i]), LAA will locally deem the dependence "safe" 118 // without querying the MemoryDepChecker. Therefore we can miss 119 // enumerating loop-independent forward dependences in 120 // getDependences. Note that as soon as there are different 121 // indices used to access the same array, the MemoryDepChecker *is* 122 // queried and the dependence list is complete. 123 Forward, 124 // Forward, but if vectorized, is likely to prevent store-to-load 125 // forwarding. 126 ForwardButPreventsForwarding, 127 // Lexically backward. 128 Backward, 129 // Backward, but the distance allows a vectorization factor of 130 // MaxSafeDepDistBytes. 131 BackwardVectorizable, 132 // Same, but may prevent store-to-load forwarding. 133 BackwardVectorizableButPreventsForwarding 134 }; 135 136 /// String version of the types. 137 static const char *DepName[]; 138 139 /// Index of the source of the dependence in the InstMap vector. 140 unsigned Source; 141 /// Index of the destination of the dependence in the InstMap vector. 142 unsigned Destination; 143 /// The type of the dependence. 144 DepType Type; 145 146 Dependence(unsigned Source, unsigned Destination, DepType Type) 147 : Source(Source), Destination(Destination), Type(Type) {} 148 149 /// Return the source instruction of the dependence. 150 Instruction *getSource(const LoopAccessInfo &LAI) const; 151 /// Return the destination instruction of the dependence. 152 Instruction *getDestination(const LoopAccessInfo &LAI) const; 153 154 /// Dependence types that don't prevent vectorization. 155 static VectorizationSafetyStatus isSafeForVectorization(DepType Type); 156 157 /// Lexically forward dependence. 158 bool isForward() const; 159 /// Lexically backward dependence. 160 bool isBackward() const; 161 162 /// May be a lexically backward dependence type (includes Unknown). 163 bool isPossiblyBackward() const; 164 165 /// Print the dependence. \p Instr is used to map the instruction 166 /// indices to instructions. 167 void print(raw_ostream &OS, unsigned Depth, 168 const SmallVectorImpl<Instruction *> &Instrs) const; 169 }; 170 171 MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L) 172 : PSE(PSE), InnermostLoop(L) {} 173 174 /// Register the location (instructions are given increasing numbers) 175 /// of a write access. 176 void addAccess(StoreInst *SI); 177 178 /// Register the location (instructions are given increasing numbers) 179 /// of a write access. 180 void addAccess(LoadInst *LI); 181 182 /// Check whether the dependencies between the accesses are safe. 183 /// 184 /// Only checks sets with elements in \p CheckDeps. 185 bool areDepsSafe(DepCandidates &AccessSets, MemAccessInfoList &CheckDeps, 186 const ValueToValueMap &Strides); 187 188 /// No memory dependence was encountered that would inhibit 189 /// vectorization. 190 bool isSafeForVectorization() const { 191 return Status == VectorizationSafetyStatus::Safe; 192 } 193 194 /// Return true if the number of elements that are safe to operate on 195 /// simultaneously is not bounded. 196 bool isSafeForAnyVectorWidth() const { 197 return MaxSafeVectorWidthInBits == UINT_MAX; 198 } 199 200 /// The maximum number of bytes of a vector register we can vectorize 201 /// the accesses safely with. 202 uint64_t getMaxSafeDepDistBytes() { return MaxSafeDepDistBytes; } 203 204 /// Return the number of elements that are safe to operate on 205 /// simultaneously, multiplied by the size of the element in bits. 206 uint64_t getMaxSafeVectorWidthInBits() const { 207 return MaxSafeVectorWidthInBits; 208 } 209 210 /// In same cases when the dependency check fails we can still 211 /// vectorize the loop with a dynamic array access check. 212 bool shouldRetryWithRuntimeCheck() const { 213 return FoundNonConstantDistanceDependence && 214 Status == VectorizationSafetyStatus::PossiblySafeWithRtChecks; 215 } 216 217 /// Returns the memory dependences. If null is returned we exceeded 218 /// the MaxDependences threshold and this information is not 219 /// available. 220 const SmallVectorImpl<Dependence> *getDependences() const { 221 return RecordDependences ? &Dependences : nullptr; 222 } 223 224 void clearDependences() { Dependences.clear(); } 225 226 /// The vector of memory access instructions. The indices are used as 227 /// instruction identifiers in the Dependence class. 228 const SmallVectorImpl<Instruction *> &getMemoryInstructions() const { 229 return InstMap; 230 } 231 232 /// Generate a mapping between the memory instructions and their 233 /// indices according to program order. 234 DenseMap<Instruction *, unsigned> generateInstructionOrderMap() const { 235 DenseMap<Instruction *, unsigned> OrderMap; 236 237 for (unsigned I = 0; I < InstMap.size(); ++I) 238 OrderMap[InstMap[I]] = I; 239 240 return OrderMap; 241 } 242 243 /// Find the set of instructions that read or write via \p Ptr. 244 SmallVector<Instruction *, 4> getInstructionsForAccess(Value *Ptr, 245 bool isWrite) const; 246 247 /// Return the program order indices for the access location (Ptr, IsWrite). 248 /// Returns an empty ArrayRef if there are no accesses for the location. 249 ArrayRef<unsigned> getOrderForAccess(Value *Ptr, bool IsWrite) const { 250 auto I = Accesses.find({Ptr, IsWrite}); 251 if (I != Accesses.end()) 252 return I->second; 253 return {}; 254 } 255 256 const Loop *getInnermostLoop() const { return InnermostLoop; } 257 258 private: 259 /// A wrapper around ScalarEvolution, used to add runtime SCEV checks, and 260 /// applies dynamic knowledge to simplify SCEV expressions and convert them 261 /// to a more usable form. We need this in case assumptions about SCEV 262 /// expressions need to be made in order to avoid unknown dependences. For 263 /// example we might assume a unit stride for a pointer in order to prove 264 /// that a memory access is strided and doesn't wrap. 265 PredicatedScalarEvolution &PSE; 266 const Loop *InnermostLoop; 267 268 /// Maps access locations (ptr, read/write) to program order. 269 DenseMap<MemAccessInfo, std::vector<unsigned> > Accesses; 270 271 /// Memory access instructions in program order. 272 SmallVector<Instruction *, 16> InstMap; 273 274 /// The program order index to be used for the next instruction. 275 unsigned AccessIdx = 0; 276 277 // We can access this many bytes in parallel safely. 278 uint64_t MaxSafeDepDistBytes = 0; 279 280 /// Number of elements (from consecutive iterations) that are safe to 281 /// operate on simultaneously, multiplied by the size of the element in bits. 282 /// The size of the element is taken from the memory access that is most 283 /// restrictive. 284 uint64_t MaxSafeVectorWidthInBits = -1U; 285 286 /// If we see a non-constant dependence distance we can still try to 287 /// vectorize this loop with runtime checks. 288 bool FoundNonConstantDistanceDependence = false; 289 290 /// Result of the dependence checks, indicating whether the checked 291 /// dependences are safe for vectorization, require RT checks or are known to 292 /// be unsafe. 293 VectorizationSafetyStatus Status = VectorizationSafetyStatus::Safe; 294 295 //// True if Dependences reflects the dependences in the 296 //// loop. If false we exceeded MaxDependences and 297 //// Dependences is invalid. 298 bool RecordDependences = true; 299 300 /// Memory dependences collected during the analysis. Only valid if 301 /// RecordDependences is true. 302 SmallVector<Dependence, 8> Dependences; 303 304 /// Check whether there is a plausible dependence between the two 305 /// accesses. 306 /// 307 /// Access \p A must happen before \p B in program order. The two indices 308 /// identify the index into the program order map. 309 /// 310 /// This function checks whether there is a plausible dependence (or the 311 /// absence of such can't be proved) between the two accesses. If there is a 312 /// plausible dependence but the dependence distance is bigger than one 313 /// element access it records this distance in \p MaxSafeDepDistBytes (if this 314 /// distance is smaller than any other distance encountered so far). 315 /// Otherwise, this function returns true signaling a possible dependence. 316 Dependence::DepType isDependent(const MemAccessInfo &A, unsigned AIdx, 317 const MemAccessInfo &B, unsigned BIdx, 318 const ValueToValueMap &Strides); 319 320 /// Check whether the data dependence could prevent store-load 321 /// forwarding. 322 /// 323 /// \return false if we shouldn't vectorize at all or avoid larger 324 /// vectorization factors by limiting MaxSafeDepDistBytes. 325 bool couldPreventStoreLoadForward(uint64_t Distance, uint64_t TypeByteSize); 326 327 /// Updates the current safety status with \p S. We can go from Safe to 328 /// either PossiblySafeWithRtChecks or Unsafe and from 329 /// PossiblySafeWithRtChecks to Unsafe. 330 void mergeInStatus(VectorizationSafetyStatus S); 331 }; 332 333 class RuntimePointerChecking; 334 /// A grouping of pointers. A single memcheck is required between 335 /// two groups. 336 struct RuntimeCheckingPtrGroup { 337 /// Create a new pointer checking group containing a single 338 /// pointer, with index \p Index in RtCheck. 339 RuntimeCheckingPtrGroup(unsigned Index, RuntimePointerChecking &RtCheck); 340 341 /// Tries to add the pointer recorded in RtCheck at index 342 /// \p Index to this pointer checking group. We can only add a pointer 343 /// to a checking group if we will still be able to get 344 /// the upper and lower bounds of the check. Returns true in case 345 /// of success, false otherwise. 346 bool addPointer(unsigned Index, RuntimePointerChecking &RtCheck); 347 bool addPointer(unsigned Index, const SCEV *Start, const SCEV *End, 348 unsigned AS, bool NeedsFreeze, ScalarEvolution &SE); 349 350 /// The SCEV expression which represents the upper bound of all the 351 /// pointers in this group. 352 const SCEV *High; 353 /// The SCEV expression which represents the lower bound of all the 354 /// pointers in this group. 355 const SCEV *Low; 356 /// Indices of all the pointers that constitute this grouping. 357 SmallVector<unsigned, 2> Members; 358 /// Address space of the involved pointers. 359 unsigned AddressSpace; 360 /// Whether the pointer needs to be frozen after expansion, e.g. because it 361 /// may be poison outside the loop. 362 bool NeedsFreeze = false; 363 }; 364 365 /// A memcheck which made up of a pair of grouped pointers. 366 typedef std::pair<const RuntimeCheckingPtrGroup *, 367 const RuntimeCheckingPtrGroup *> 368 RuntimePointerCheck; 369 370 struct PointerDiffInfo { 371 const SCEV *SrcStart; 372 const SCEV *SinkStart; 373 unsigned AccessSize; 374 bool NeedsFreeze; 375 376 PointerDiffInfo(const SCEV *SrcStart, const SCEV *SinkStart, 377 unsigned AccessSize, bool NeedsFreeze) 378 : SrcStart(SrcStart), SinkStart(SinkStart), AccessSize(AccessSize), 379 NeedsFreeze(NeedsFreeze) {} 380 }; 381 382 /// Holds information about the memory runtime legality checks to verify 383 /// that a group of pointers do not overlap. 384 class RuntimePointerChecking { 385 friend struct RuntimeCheckingPtrGroup; 386 387 public: 388 struct PointerInfo { 389 /// Holds the pointer value that we need to check. 390 TrackingVH<Value> PointerValue; 391 /// Holds the smallest byte address accessed by the pointer throughout all 392 /// iterations of the loop. 393 const SCEV *Start; 394 /// Holds the largest byte address accessed by the pointer throughout all 395 /// iterations of the loop, plus 1. 396 const SCEV *End; 397 /// Holds the information if this pointer is used for writing to memory. 398 bool IsWritePtr; 399 /// Holds the id of the set of pointers that could be dependent because of a 400 /// shared underlying object. 401 unsigned DependencySetId; 402 /// Holds the id of the disjoint alias set to which this pointer belongs. 403 unsigned AliasSetId; 404 /// SCEV for the access. 405 const SCEV *Expr; 406 /// True if the pointer expressions needs to be frozen after expansion. 407 bool NeedsFreeze; 408 409 PointerInfo(Value *PointerValue, const SCEV *Start, const SCEV *End, 410 bool IsWritePtr, unsigned DependencySetId, unsigned AliasSetId, 411 const SCEV *Expr, bool NeedsFreeze) 412 : PointerValue(PointerValue), Start(Start), End(End), 413 IsWritePtr(IsWritePtr), DependencySetId(DependencySetId), 414 AliasSetId(AliasSetId), Expr(Expr), NeedsFreeze(NeedsFreeze) {} 415 }; 416 417 RuntimePointerChecking(MemoryDepChecker &DC, ScalarEvolution *SE) 418 : DC(DC), SE(SE) {} 419 420 /// Reset the state of the pointer runtime information. 421 void reset() { 422 Need = false; 423 Pointers.clear(); 424 Checks.clear(); 425 } 426 427 /// Insert a pointer and calculate the start and end SCEVs. 428 /// We need \p PSE in order to compute the SCEV expression of the pointer 429 /// according to the assumptions that we've made during the analysis. 430 /// The method might also version the pointer stride according to \p Strides, 431 /// and add new predicates to \p PSE. 432 void insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr, Type *AccessTy, 433 bool WritePtr, unsigned DepSetId, unsigned ASId, 434 PredicatedScalarEvolution &PSE, bool NeedsFreeze); 435 436 /// No run-time memory checking is necessary. 437 bool empty() const { return Pointers.empty(); } 438 439 /// Generate the checks and store it. This also performs the grouping 440 /// of pointers to reduce the number of memchecks necessary. 441 void generateChecks(MemoryDepChecker::DepCandidates &DepCands, 442 bool UseDependencies); 443 444 /// Returns the checks that generateChecks created. They can be used to ensure 445 /// no read/write accesses overlap across all loop iterations. 446 const SmallVectorImpl<RuntimePointerCheck> &getChecks() const { 447 return Checks; 448 } 449 450 // Returns an optional list of (pointer-difference expressions, access size) 451 // pairs that can be used to prove that there are no vectorization-preventing 452 // dependencies at runtime. There are is a vectorization-preventing dependency 453 // if any pointer-difference is <u VF * InterleaveCount * access size. Returns 454 // None if pointer-difference checks cannot be used. 455 Optional<ArrayRef<PointerDiffInfo>> getDiffChecks() const { 456 if (!CanUseDiffCheck) 457 return None; 458 return {DiffChecks}; 459 } 460 461 /// Decide if we need to add a check between two groups of pointers, 462 /// according to needsChecking. 463 bool needsChecking(const RuntimeCheckingPtrGroup &M, 464 const RuntimeCheckingPtrGroup &N) const; 465 466 /// Returns the number of run-time checks required according to 467 /// needsChecking. 468 unsigned getNumberOfChecks() const { return Checks.size(); } 469 470 /// Print the list run-time memory checks necessary. 471 void print(raw_ostream &OS, unsigned Depth = 0) const; 472 473 /// Print \p Checks. 474 void printChecks(raw_ostream &OS, 475 const SmallVectorImpl<RuntimePointerCheck> &Checks, 476 unsigned Depth = 0) const; 477 478 /// This flag indicates if we need to add the runtime check. 479 bool Need = false; 480 481 /// Information about the pointers that may require checking. 482 SmallVector<PointerInfo, 2> Pointers; 483 484 /// Holds a partitioning of pointers into "check groups". 485 SmallVector<RuntimeCheckingPtrGroup, 2> CheckingGroups; 486 487 /// Check if pointers are in the same partition 488 /// 489 /// \p PtrToPartition contains the partition number for pointers (-1 if the 490 /// pointer belongs to multiple partitions). 491 static bool 492 arePointersInSamePartition(const SmallVectorImpl<int> &PtrToPartition, 493 unsigned PtrIdx1, unsigned PtrIdx2); 494 495 /// Decide whether we need to issue a run-time check for pointer at 496 /// index \p I and \p J to prove their independence. 497 bool needsChecking(unsigned I, unsigned J) const; 498 499 /// Return PointerInfo for pointer at index \p PtrIdx. 500 const PointerInfo &getPointerInfo(unsigned PtrIdx) const { 501 return Pointers[PtrIdx]; 502 } 503 504 ScalarEvolution *getSE() const { return SE; } 505 506 private: 507 /// Groups pointers such that a single memcheck is required 508 /// between two different groups. This will clear the CheckingGroups vector 509 /// and re-compute it. We will only group dependecies if \p UseDependencies 510 /// is true, otherwise we will create a separate group for each pointer. 511 void groupChecks(MemoryDepChecker::DepCandidates &DepCands, 512 bool UseDependencies); 513 514 /// Generate the checks and return them. 515 SmallVector<RuntimePointerCheck, 4> generateChecks(); 516 517 /// Try to create add a new (pointer-difference, access size) pair to 518 /// DiffCheck for checking groups \p CGI and \p CGJ. If pointer-difference 519 /// checks cannot be used for the groups, set CanUseDiffCheck to false. 520 void tryToCreateDiffCheck(const RuntimeCheckingPtrGroup &CGI, 521 const RuntimeCheckingPtrGroup &CGJ); 522 523 MemoryDepChecker &DC; 524 525 /// Holds a pointer to the ScalarEvolution analysis. 526 ScalarEvolution *SE; 527 528 /// Set of run-time checks required to establish independence of 529 /// otherwise may-aliasing pointers in the loop. 530 SmallVector<RuntimePointerCheck, 4> Checks; 531 532 /// Flag indicating if pointer-difference checks can be used 533 bool CanUseDiffCheck = true; 534 535 /// A list of (pointer-difference, access size) pairs that can be used to 536 /// prove that there are no vectorization-preventing dependencies. 537 SmallVector<PointerDiffInfo> DiffChecks; 538 }; 539 540 /// Drive the analysis of memory accesses in the loop 541 /// 542 /// This class is responsible for analyzing the memory accesses of a loop. It 543 /// collects the accesses and then its main helper the AccessAnalysis class 544 /// finds and categorizes the dependences in buildDependenceSets. 545 /// 546 /// For memory dependences that can be analyzed at compile time, it determines 547 /// whether the dependence is part of cycle inhibiting vectorization. This work 548 /// is delegated to the MemoryDepChecker class. 549 /// 550 /// For memory dependences that cannot be determined at compile time, it 551 /// generates run-time checks to prove independence. This is done by 552 /// AccessAnalysis::canCheckPtrAtRT and the checks are maintained by the 553 /// RuntimePointerCheck class. 554 /// 555 /// If pointers can wrap or can't be expressed as affine AddRec expressions by 556 /// ScalarEvolution, we will generate run-time checks by emitting a 557 /// SCEVUnionPredicate. 558 /// 559 /// Checks for both memory dependences and the SCEV predicates contained in the 560 /// PSE must be emitted in order for the results of this analysis to be valid. 561 class LoopAccessInfo { 562 public: 563 LoopAccessInfo(Loop *L, ScalarEvolution *SE, const TargetLibraryInfo *TLI, 564 AAResults *AA, DominatorTree *DT, LoopInfo *LI); 565 566 /// Return true we can analyze the memory accesses in the loop and there are 567 /// no memory dependence cycles. 568 bool canVectorizeMemory() const { return CanVecMem; } 569 570 /// Return true if there is a convergent operation in the loop. There may 571 /// still be reported runtime pointer checks that would be required, but it is 572 /// not legal to insert them. 573 bool hasConvergentOp() const { return HasConvergentOp; } 574 575 const RuntimePointerChecking *getRuntimePointerChecking() const { 576 return PtrRtChecking.get(); 577 } 578 579 /// Number of memchecks required to prove independence of otherwise 580 /// may-alias pointers. 581 unsigned getNumRuntimePointerChecks() const { 582 return PtrRtChecking->getNumberOfChecks(); 583 } 584 585 /// Return true if the block BB needs to be predicated in order for the loop 586 /// to be vectorized. 587 static bool blockNeedsPredication(BasicBlock *BB, Loop *TheLoop, 588 DominatorTree *DT); 589 590 /// Returns true if the value V is uniform within the loop. 591 bool isUniform(Value *V) const; 592 593 uint64_t getMaxSafeDepDistBytes() const { return MaxSafeDepDistBytes; } 594 unsigned getNumStores() const { return NumStores; } 595 unsigned getNumLoads() const { return NumLoads;} 596 597 /// The diagnostics report generated for the analysis. E.g. why we 598 /// couldn't analyze the loop. 599 const OptimizationRemarkAnalysis *getReport() const { return Report.get(); } 600 601 /// the Memory Dependence Checker which can determine the 602 /// loop-independent and loop-carried dependences between memory accesses. 603 const MemoryDepChecker &getDepChecker() const { return *DepChecker; } 604 605 /// Return the list of instructions that use \p Ptr to read or write 606 /// memory. 607 SmallVector<Instruction *, 4> getInstructionsForAccess(Value *Ptr, 608 bool isWrite) const { 609 return DepChecker->getInstructionsForAccess(Ptr, isWrite); 610 } 611 612 /// If an access has a symbolic strides, this maps the pointer value to 613 /// the stride symbol. 614 const ValueToValueMap &getSymbolicStrides() const { return SymbolicStrides; } 615 616 /// Pointer has a symbolic stride. 617 bool hasStride(Value *V) const { return StrideSet.count(V); } 618 619 /// Print the information about the memory accesses in the loop. 620 void print(raw_ostream &OS, unsigned Depth = 0) const; 621 622 /// If the loop has memory dependence involving an invariant address, i.e. two 623 /// stores or a store and a load, then return true, else return false. 624 bool hasDependenceInvolvingLoopInvariantAddress() const { 625 return HasDependenceInvolvingLoopInvariantAddress; 626 } 627 628 /// Return the list of stores to invariant addresses. 629 const ArrayRef<StoreInst *> getStoresToInvariantAddresses() const { 630 return StoresToInvariantAddresses; 631 } 632 633 /// Used to add runtime SCEV checks. Simplifies SCEV expressions and converts 634 /// them to a more usable form. All SCEV expressions during the analysis 635 /// should be re-written (and therefore simplified) according to PSE. 636 /// A user of LoopAccessAnalysis will need to emit the runtime checks 637 /// associated with this predicate. 638 const PredicatedScalarEvolution &getPSE() const { return *PSE; } 639 640 private: 641 /// Analyze the loop. 642 void analyzeLoop(AAResults *AA, LoopInfo *LI, 643 const TargetLibraryInfo *TLI, DominatorTree *DT); 644 645 /// Check if the structure of the loop allows it to be analyzed by this 646 /// pass. 647 bool canAnalyzeLoop(); 648 649 /// Save the analysis remark. 650 /// 651 /// LAA does not directly emits the remarks. Instead it stores it which the 652 /// client can retrieve and presents as its own analysis 653 /// (e.g. -Rpass-analysis=loop-vectorize). 654 OptimizationRemarkAnalysis &recordAnalysis(StringRef RemarkName, 655 Instruction *Instr = nullptr); 656 657 /// Collect memory access with loop invariant strides. 658 /// 659 /// Looks for accesses like "a[i * StrideA]" where "StrideA" is loop 660 /// invariant. 661 void collectStridedAccess(Value *LoadOrStoreInst); 662 663 // Emits the first unsafe memory dependence in a loop. 664 // Emits nothing if there are no unsafe dependences 665 // or if the dependences were not recorded. 666 void emitUnsafeDependenceRemark(); 667 668 std::unique_ptr<PredicatedScalarEvolution> PSE; 669 670 /// We need to check that all of the pointers in this list are disjoint 671 /// at runtime. Using std::unique_ptr to make using move ctor simpler. 672 std::unique_ptr<RuntimePointerChecking> PtrRtChecking; 673 674 /// the Memory Dependence Checker which can determine the 675 /// loop-independent and loop-carried dependences between memory accesses. 676 std::unique_ptr<MemoryDepChecker> DepChecker; 677 678 Loop *TheLoop; 679 680 unsigned NumLoads = 0; 681 unsigned NumStores = 0; 682 683 uint64_t MaxSafeDepDistBytes = -1; 684 685 /// Cache the result of analyzeLoop. 686 bool CanVecMem = false; 687 bool HasConvergentOp = false; 688 689 /// Indicator that there are non vectorizable stores to a uniform address. 690 bool HasDependenceInvolvingLoopInvariantAddress = false; 691 692 /// List of stores to invariant addresses. 693 SmallVector<StoreInst *> StoresToInvariantAddresses; 694 695 /// The diagnostics report generated for the analysis. E.g. why we 696 /// couldn't analyze the loop. 697 std::unique_ptr<OptimizationRemarkAnalysis> Report; 698 699 /// If an access has a symbolic strides, this maps the pointer value to 700 /// the stride symbol. 701 ValueToValueMap SymbolicStrides; 702 703 /// Set of symbolic strides values. 704 SmallPtrSet<Value *, 8> StrideSet; 705 }; 706 707 Value *stripIntegerCast(Value *V); 708 709 /// Return the SCEV corresponding to a pointer with the symbolic stride 710 /// replaced with constant one, assuming the SCEV predicate associated with 711 /// \p PSE is true. 712 /// 713 /// If necessary this method will version the stride of the pointer according 714 /// to \p PtrToStride and therefore add further predicates to \p PSE. 715 /// 716 /// \p PtrToStride provides the mapping between the pointer value and its 717 /// stride as collected by LoopVectorizationLegality::collectStridedAccess. 718 const SCEV *replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE, 719 const ValueToValueMap &PtrToStride, 720 Value *Ptr); 721 722 /// If the pointer has a constant stride return it in units of the access type 723 /// size. Otherwise return zero. 724 /// 725 /// Ensure that it does not wrap in the address space, assuming the predicate 726 /// associated with \p PSE is true. 727 /// 728 /// If necessary this method will version the stride of the pointer according 729 /// to \p PtrToStride and therefore add further predicates to \p PSE. 730 /// The \p Assume parameter indicates if we are allowed to make additional 731 /// run-time assumptions. 732 int64_t getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, 733 const Loop *Lp, 734 const ValueToValueMap &StridesMap = ValueToValueMap(), 735 bool Assume = false, bool ShouldCheckWrap = true); 736 737 /// Returns the distance between the pointers \p PtrA and \p PtrB iff they are 738 /// compatible and it is possible to calculate the distance between them. This 739 /// is a simple API that does not depend on the analysis pass. 740 /// \param StrictCheck Ensure that the calculated distance matches the 741 /// type-based one after all the bitcasts removal in the provided pointers. 742 Optional<int> getPointersDiff(Type *ElemTyA, Value *PtrA, Type *ElemTyB, 743 Value *PtrB, const DataLayout &DL, 744 ScalarEvolution &SE, bool StrictCheck = false, 745 bool CheckType = true); 746 747 /// Attempt to sort the pointers in \p VL and return the sorted indices 748 /// in \p SortedIndices, if reordering is required. 749 /// 750 /// Returns 'true' if sorting is legal, otherwise returns 'false'. 751 /// 752 /// For example, for a given \p VL of memory accesses in program order, a[i+4], 753 /// a[i+0], a[i+1] and a[i+7], this function will sort the \p VL and save the 754 /// sorted indices in \p SortedIndices as a[i+0], a[i+1], a[i+4], a[i+7] and 755 /// saves the mask for actual memory accesses in program order in 756 /// \p SortedIndices as <1,2,0,3> 757 bool sortPtrAccesses(ArrayRef<Value *> VL, Type *ElemTy, const DataLayout &DL, 758 ScalarEvolution &SE, 759 SmallVectorImpl<unsigned> &SortedIndices); 760 761 /// Returns true if the memory operations \p A and \p B are consecutive. 762 /// This is a simple API that does not depend on the analysis pass. 763 bool isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL, 764 ScalarEvolution &SE, bool CheckType = true); 765 766 /// This analysis provides dependence information for the memory accesses 767 /// of a loop. 768 /// 769 /// It runs the analysis for a loop on demand. This can be initiated by 770 /// querying the loop access info via LAA::getInfo. getInfo return a 771 /// LoopAccessInfo object. See this class for the specifics of what information 772 /// is provided. 773 class LoopAccessLegacyAnalysis : public FunctionPass { 774 public: 775 static char ID; 776 777 LoopAccessLegacyAnalysis(); 778 779 bool runOnFunction(Function &F) override; 780 781 void getAnalysisUsage(AnalysisUsage &AU) const override; 782 783 /// Query the result of the loop access information for the loop \p L. 784 /// 785 /// If there is no cached result available run the analysis. 786 const LoopAccessInfo &getInfo(Loop *L); 787 788 void releaseMemory() override { 789 // Invalidate the cache when the pass is freed. 790 LoopAccessInfoMap.clear(); 791 } 792 793 /// Print the result of the analysis when invoked with -analyze. 794 void print(raw_ostream &OS, const Module *M = nullptr) const override; 795 796 private: 797 /// The cache. 798 DenseMap<Loop *, std::unique_ptr<LoopAccessInfo>> LoopAccessInfoMap; 799 800 // The used analysis passes. 801 ScalarEvolution *SE = nullptr; 802 const TargetLibraryInfo *TLI = nullptr; 803 AAResults *AA = nullptr; 804 DominatorTree *DT = nullptr; 805 LoopInfo *LI = nullptr; 806 }; 807 808 /// This analysis provides dependence information for the memory 809 /// accesses of a loop. 810 /// 811 /// It runs the analysis for a loop on demand. This can be initiated by 812 /// querying the loop access info via AM.getResult<LoopAccessAnalysis>. 813 /// getResult return a LoopAccessInfo object. See this class for the 814 /// specifics of what information is provided. 815 class LoopAccessAnalysis 816 : public AnalysisInfoMixin<LoopAccessAnalysis> { 817 friend AnalysisInfoMixin<LoopAccessAnalysis>; 818 static AnalysisKey Key; 819 820 public: 821 typedef LoopAccessInfo Result; 822 823 Result run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR); 824 }; 825 826 inline Instruction *MemoryDepChecker::Dependence::getSource( 827 const LoopAccessInfo &LAI) const { 828 return LAI.getDepChecker().getMemoryInstructions()[Source]; 829 } 830 831 inline Instruction *MemoryDepChecker::Dependence::getDestination( 832 const LoopAccessInfo &LAI) const { 833 return LAI.getDepChecker().getMemoryInstructions()[Destination]; 834 } 835 836 } // End llvm namespace 837 838 #endif 839