1 //===- llvm/Analysis/LoopAccessAnalysis.h -----------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interface for the loop memory dependence framework that
10 // was originally developed for the Loop Vectorizer.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #ifndef LLVM_ANALYSIS_LOOPACCESSANALYSIS_H
15 #define LLVM_ANALYSIS_LOOPACCESSANALYSIS_H
16
17 #include "llvm/ADT/EquivalenceClasses.h"
18 #include "llvm/Analysis/LoopAnalysisManager.h"
19 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
20 #include "llvm/IR/DiagnosticInfo.h"
21 #include "llvm/Pass.h"
22
23 namespace llvm {
24
25 class AAResults;
26 class DataLayout;
27 class Loop;
28 class LoopAccessInfo;
29 class OptimizationRemarkEmitter;
30 class raw_ostream;
31 class SCEV;
32 class SCEVUnionPredicate;
33 class Value;
34
35 /// Collection of parameters shared beetween the Loop Vectorizer and the
36 /// Loop Access Analysis.
37 struct VectorizerParams {
38 /// Maximum SIMD width.
39 static const unsigned MaxVectorWidth;
40
41 /// VF as overridden by the user.
42 static unsigned VectorizationFactor;
43 /// Interleave factor as overridden by the user.
44 static unsigned VectorizationInterleave;
45 /// True if force-vector-interleave was specified by the user.
46 static bool isInterleaveForced();
47
48 /// \When performing memory disambiguation checks at runtime do not
49 /// make more than this number of comparisons.
50 static unsigned RuntimeMemoryCheckThreshold;
51 };
52
53 /// Checks memory dependences among accesses to the same underlying
54 /// object to determine whether there vectorization is legal or not (and at
55 /// which vectorization factor).
56 ///
57 /// Note: This class will compute a conservative dependence for access to
58 /// different underlying pointers. Clients, such as the loop vectorizer, will
59 /// sometimes deal these potential dependencies by emitting runtime checks.
60 ///
61 /// We use the ScalarEvolution framework to symbolically evalutate access
62 /// functions pairs. Since we currently don't restructure the loop we can rely
63 /// on the program order of memory accesses to determine their safety.
64 /// At the moment we will only deem accesses as safe for:
65 /// * A negative constant distance assuming program order.
66 ///
67 /// Safe: tmp = a[i + 1]; OR a[i + 1] = x;
68 /// a[i] = tmp; y = a[i];
69 ///
70 /// The latter case is safe because later checks guarantuee that there can't
71 /// be a cycle through a phi node (that is, we check that "x" and "y" is not
72 /// the same variable: a header phi can only be an induction or a reduction, a
73 /// reduction can't have a memory sink, an induction can't have a memory
74 /// source). This is important and must not be violated (or we have to
75 /// resort to checking for cycles through memory).
76 ///
77 /// * A positive constant distance assuming program order that is bigger
78 /// than the biggest memory access.
79 ///
80 /// tmp = a[i] OR b[i] = x
81 /// a[i+2] = tmp y = b[i+2];
82 ///
83 /// Safe distance: 2 x sizeof(a[0]), and 2 x sizeof(b[0]), respectively.
84 ///
85 /// * Zero distances and all accesses have the same size.
86 ///
87 class MemoryDepChecker {
88 public:
89 typedef PointerIntPair<Value *, 1, bool> MemAccessInfo;
90 typedef SmallVector<MemAccessInfo, 8> MemAccessInfoList;
91 /// Set of potential dependent memory accesses.
92 typedef EquivalenceClasses<MemAccessInfo> DepCandidates;
93
94 /// Type to keep track of the status of the dependence check. The order of
95 /// the elements is important and has to be from most permissive to least
96 /// permissive.
97 enum class VectorizationSafetyStatus {
98 // Can vectorize safely without RT checks. All dependences are known to be
99 // safe.
100 Safe,
101 // Can possibly vectorize with RT checks to overcome unknown dependencies.
102 PossiblySafeWithRtChecks,
103 // Cannot vectorize due to known unsafe dependencies.
104 Unsafe,
105 };
106
107 /// Dependece between memory access instructions.
108 struct Dependence {
109 /// The type of the dependence.
110 enum DepType {
111 // No dependence.
112 NoDep,
113 // We couldn't determine the direction or the distance.
114 Unknown,
115 // Lexically forward.
116 //
117 // FIXME: If we only have loop-independent forward dependences (e.g. a
118 // read and write of A[i]), LAA will locally deem the dependence "safe"
119 // without querying the MemoryDepChecker. Therefore we can miss
120 // enumerating loop-independent forward dependences in
121 // getDependences. Note that as soon as there are different
122 // indices used to access the same array, the MemoryDepChecker *is*
123 // queried and the dependence list is complete.
124 Forward,
125 // Forward, but if vectorized, is likely to prevent store-to-load
126 // forwarding.
127 ForwardButPreventsForwarding,
128 // Lexically backward.
129 Backward,
130 // Backward, but the distance allows a vectorization factor of
131 // MaxSafeDepDistBytes.
132 BackwardVectorizable,
133 // Same, but may prevent store-to-load forwarding.
134 BackwardVectorizableButPreventsForwarding
135 };
136
137 /// String version of the types.
138 static const char *DepName[];
139
140 /// Index of the source of the dependence in the InstMap vector.
141 unsigned Source;
142 /// Index of the destination of the dependence in the InstMap vector.
143 unsigned Destination;
144 /// The type of the dependence.
145 DepType Type;
146
DependenceDependence147 Dependence(unsigned Source, unsigned Destination, DepType Type)
148 : Source(Source), Destination(Destination), Type(Type) {}
149
150 /// Return the source instruction of the dependence.
151 Instruction *getSource(const LoopAccessInfo &LAI) const;
152 /// Return the destination instruction of the dependence.
153 Instruction *getDestination(const LoopAccessInfo &LAI) const;
154
155 /// Dependence types that don't prevent vectorization.
156 static VectorizationSafetyStatus isSafeForVectorization(DepType Type);
157
158 /// Lexically forward dependence.
159 bool isForward() const;
160 /// Lexically backward dependence.
161 bool isBackward() const;
162
163 /// May be a lexically backward dependence type (includes Unknown).
164 bool isPossiblyBackward() const;
165
166 /// Print the dependence. \p Instr is used to map the instruction
167 /// indices to instructions.
168 void print(raw_ostream &OS, unsigned Depth,
169 const SmallVectorImpl<Instruction *> &Instrs) const;
170 };
171
MemoryDepChecker(PredicatedScalarEvolution & PSE,const Loop * L)172 MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L)
173 : PSE(PSE), InnermostLoop(L), AccessIdx(0), MaxSafeDepDistBytes(0),
174 MaxSafeVectorWidthInBits(-1U),
175 FoundNonConstantDistanceDependence(false),
176 Status(VectorizationSafetyStatus::Safe), RecordDependences(true) {}
177
178 /// Register the location (instructions are given increasing numbers)
179 /// of a write access.
addAccess(StoreInst * SI)180 void addAccess(StoreInst *SI) {
181 Value *Ptr = SI->getPointerOperand();
182 Accesses[MemAccessInfo(Ptr, true)].push_back(AccessIdx);
183 InstMap.push_back(SI);
184 ++AccessIdx;
185 }
186
187 /// Register the location (instructions are given increasing numbers)
188 /// of a write access.
addAccess(LoadInst * LI)189 void addAccess(LoadInst *LI) {
190 Value *Ptr = LI->getPointerOperand();
191 Accesses[MemAccessInfo(Ptr, false)].push_back(AccessIdx);
192 InstMap.push_back(LI);
193 ++AccessIdx;
194 }
195
196 /// Check whether the dependencies between the accesses are safe.
197 ///
198 /// Only checks sets with elements in \p CheckDeps.
199 bool areDepsSafe(DepCandidates &AccessSets, MemAccessInfoList &CheckDeps,
200 const ValueToValueMap &Strides);
201
202 /// No memory dependence was encountered that would inhibit
203 /// vectorization.
isSafeForVectorization()204 bool isSafeForVectorization() const {
205 return Status == VectorizationSafetyStatus::Safe;
206 }
207
208 /// Return true if the number of elements that are safe to operate on
209 /// simultaneously is not bounded.
isSafeForAnyVectorWidth()210 bool isSafeForAnyVectorWidth() const {
211 return MaxSafeVectorWidthInBits == UINT_MAX;
212 }
213
214 /// The maximum number of bytes of a vector register we can vectorize
215 /// the accesses safely with.
getMaxSafeDepDistBytes()216 uint64_t getMaxSafeDepDistBytes() { return MaxSafeDepDistBytes; }
217
218 /// Return the number of elements that are safe to operate on
219 /// simultaneously, multiplied by the size of the element in bits.
getMaxSafeVectorWidthInBits()220 uint64_t getMaxSafeVectorWidthInBits() const {
221 return MaxSafeVectorWidthInBits;
222 }
223
224 /// In same cases when the dependency check fails we can still
225 /// vectorize the loop with a dynamic array access check.
shouldRetryWithRuntimeCheck()226 bool shouldRetryWithRuntimeCheck() const {
227 return FoundNonConstantDistanceDependence &&
228 Status == VectorizationSafetyStatus::PossiblySafeWithRtChecks;
229 }
230
231 /// Returns the memory dependences. If null is returned we exceeded
232 /// the MaxDependences threshold and this information is not
233 /// available.
getDependences()234 const SmallVectorImpl<Dependence> *getDependences() const {
235 return RecordDependences ? &Dependences : nullptr;
236 }
237
clearDependences()238 void clearDependences() { Dependences.clear(); }
239
240 /// The vector of memory access instructions. The indices are used as
241 /// instruction identifiers in the Dependence class.
getMemoryInstructions()242 const SmallVectorImpl<Instruction *> &getMemoryInstructions() const {
243 return InstMap;
244 }
245
246 /// Generate a mapping between the memory instructions and their
247 /// indices according to program order.
generateInstructionOrderMap()248 DenseMap<Instruction *, unsigned> generateInstructionOrderMap() const {
249 DenseMap<Instruction *, unsigned> OrderMap;
250
251 for (unsigned I = 0; I < InstMap.size(); ++I)
252 OrderMap[InstMap[I]] = I;
253
254 return OrderMap;
255 }
256
257 /// Find the set of instructions that read or write via \p Ptr.
258 SmallVector<Instruction *, 4> getInstructionsForAccess(Value *Ptr,
259 bool isWrite) const;
260
261 private:
262 /// A wrapper around ScalarEvolution, used to add runtime SCEV checks, and
263 /// applies dynamic knowledge to simplify SCEV expressions and convert them
264 /// to a more usable form. We need this in case assumptions about SCEV
265 /// expressions need to be made in order to avoid unknown dependences. For
266 /// example we might assume a unit stride for a pointer in order to prove
267 /// that a memory access is strided and doesn't wrap.
268 PredicatedScalarEvolution &PSE;
269 const Loop *InnermostLoop;
270
271 /// Maps access locations (ptr, read/write) to program order.
272 DenseMap<MemAccessInfo, std::vector<unsigned> > Accesses;
273
274 /// Memory access instructions in program order.
275 SmallVector<Instruction *, 16> InstMap;
276
277 /// The program order index to be used for the next instruction.
278 unsigned AccessIdx;
279
280 // We can access this many bytes in parallel safely.
281 uint64_t MaxSafeDepDistBytes;
282
283 /// Number of elements (from consecutive iterations) that are safe to
284 /// operate on simultaneously, multiplied by the size of the element in bits.
285 /// The size of the element is taken from the memory access that is most
286 /// restrictive.
287 uint64_t MaxSafeVectorWidthInBits;
288
289 /// If we see a non-constant dependence distance we can still try to
290 /// vectorize this loop with runtime checks.
291 bool FoundNonConstantDistanceDependence;
292
293 /// Result of the dependence checks, indicating whether the checked
294 /// dependences are safe for vectorization, require RT checks or are known to
295 /// be unsafe.
296 VectorizationSafetyStatus Status;
297
298 //// True if Dependences reflects the dependences in the
299 //// loop. If false we exceeded MaxDependences and
300 //// Dependences is invalid.
301 bool RecordDependences;
302
303 /// Memory dependences collected during the analysis. Only valid if
304 /// RecordDependences is true.
305 SmallVector<Dependence, 8> Dependences;
306
307 /// Check whether there is a plausible dependence between the two
308 /// accesses.
309 ///
310 /// Access \p A must happen before \p B in program order. The two indices
311 /// identify the index into the program order map.
312 ///
313 /// This function checks whether there is a plausible dependence (or the
314 /// absence of such can't be proved) between the two accesses. If there is a
315 /// plausible dependence but the dependence distance is bigger than one
316 /// element access it records this distance in \p MaxSafeDepDistBytes (if this
317 /// distance is smaller than any other distance encountered so far).
318 /// Otherwise, this function returns true signaling a possible dependence.
319 Dependence::DepType isDependent(const MemAccessInfo &A, unsigned AIdx,
320 const MemAccessInfo &B, unsigned BIdx,
321 const ValueToValueMap &Strides);
322
323 /// Check whether the data dependence could prevent store-load
324 /// forwarding.
325 ///
326 /// \return false if we shouldn't vectorize at all or avoid larger
327 /// vectorization factors by limiting MaxSafeDepDistBytes.
328 bool couldPreventStoreLoadForward(uint64_t Distance, uint64_t TypeByteSize);
329
330 /// Updates the current safety status with \p S. We can go from Safe to
331 /// either PossiblySafeWithRtChecks or Unsafe and from
332 /// PossiblySafeWithRtChecks to Unsafe.
333 void mergeInStatus(VectorizationSafetyStatus S);
334 };
335
336 class RuntimePointerChecking;
337 /// A grouping of pointers. A single memcheck is required between
338 /// two groups.
339 struct RuntimeCheckingPtrGroup {
340 /// Create a new pointer checking group containing a single
341 /// pointer, with index \p Index in RtCheck.
342 RuntimeCheckingPtrGroup(unsigned Index, RuntimePointerChecking &RtCheck);
343
RuntimeCheckingPtrGroupRuntimeCheckingPtrGroup344 RuntimeCheckingPtrGroup(unsigned Index, const SCEV *Start, const SCEV *End,
345 unsigned AS)
346 : High(End), Low(Start), AddressSpace(AS) {
347 Members.push_back(Index);
348 }
349
350 /// Tries to add the pointer recorded in RtCheck at index
351 /// \p Index to this pointer checking group. We can only add a pointer
352 /// to a checking group if we will still be able to get
353 /// the upper and lower bounds of the check. Returns true in case
354 /// of success, false otherwise.
355 bool addPointer(unsigned Index, RuntimePointerChecking &RtCheck);
356 bool addPointer(unsigned Index, const SCEV *Start, const SCEV *End,
357 unsigned AS, ScalarEvolution &SE);
358
359 /// The SCEV expression which represents the upper bound of all the
360 /// pointers in this group.
361 const SCEV *High;
362 /// The SCEV expression which represents the lower bound of all the
363 /// pointers in this group.
364 const SCEV *Low;
365 /// Indices of all the pointers that constitute this grouping.
366 SmallVector<unsigned, 2> Members;
367 /// Address space of the involved pointers.
368 unsigned AddressSpace;
369 };
370
371 /// A memcheck which made up of a pair of grouped pointers.
372 typedef std::pair<const RuntimeCheckingPtrGroup *,
373 const RuntimeCheckingPtrGroup *>
374 RuntimePointerCheck;
375
376 /// Holds information about the memory runtime legality checks to verify
377 /// that a group of pointers do not overlap.
378 class RuntimePointerChecking {
379 friend struct RuntimeCheckingPtrGroup;
380
381 public:
382 struct PointerInfo {
383 /// Holds the pointer value that we need to check.
384 TrackingVH<Value> PointerValue;
385 /// Holds the smallest byte address accessed by the pointer throughout all
386 /// iterations of the loop.
387 const SCEV *Start;
388 /// Holds the largest byte address accessed by the pointer throughout all
389 /// iterations of the loop, plus 1.
390 const SCEV *End;
391 /// Holds the information if this pointer is used for writing to memory.
392 bool IsWritePtr;
393 /// Holds the id of the set of pointers that could be dependent because of a
394 /// shared underlying object.
395 unsigned DependencySetId;
396 /// Holds the id of the disjoint alias set to which this pointer belongs.
397 unsigned AliasSetId;
398 /// SCEV for the access.
399 const SCEV *Expr;
400
PointerInfoPointerInfo401 PointerInfo(Value *PointerValue, const SCEV *Start, const SCEV *End,
402 bool IsWritePtr, unsigned DependencySetId, unsigned AliasSetId,
403 const SCEV *Expr)
404 : PointerValue(PointerValue), Start(Start), End(End),
405 IsWritePtr(IsWritePtr), DependencySetId(DependencySetId),
406 AliasSetId(AliasSetId), Expr(Expr) {}
407 };
408
RuntimePointerChecking(ScalarEvolution * SE)409 RuntimePointerChecking(ScalarEvolution *SE) : Need(false), SE(SE) {}
410
411 /// Reset the state of the pointer runtime information.
reset()412 void reset() {
413 Need = false;
414 Pointers.clear();
415 Checks.clear();
416 }
417
418 /// Insert a pointer and calculate the start and end SCEVs.
419 /// We need \p PSE in order to compute the SCEV expression of the pointer
420 /// according to the assumptions that we've made during the analysis.
421 /// The method might also version the pointer stride according to \p Strides,
422 /// and add new predicates to \p PSE.
423 void insert(Loop *Lp, Value *Ptr, bool WritePtr, unsigned DepSetId,
424 unsigned ASId, const ValueToValueMap &Strides,
425 PredicatedScalarEvolution &PSE);
426
427 /// No run-time memory checking is necessary.
empty()428 bool empty() const { return Pointers.empty(); }
429
430 /// Generate the checks and store it. This also performs the grouping
431 /// of pointers to reduce the number of memchecks necessary.
432 void generateChecks(MemoryDepChecker::DepCandidates &DepCands,
433 bool UseDependencies);
434
435 /// Returns the checks that generateChecks created.
getChecks()436 const SmallVectorImpl<RuntimePointerCheck> &getChecks() const {
437 return Checks;
438 }
439
440 /// Decide if we need to add a check between two groups of pointers,
441 /// according to needsChecking.
442 bool needsChecking(const RuntimeCheckingPtrGroup &M,
443 const RuntimeCheckingPtrGroup &N) const;
444
445 /// Returns the number of run-time checks required according to
446 /// needsChecking.
getNumberOfChecks()447 unsigned getNumberOfChecks() const { return Checks.size(); }
448
449 /// Print the list run-time memory checks necessary.
450 void print(raw_ostream &OS, unsigned Depth = 0) const;
451
452 /// Print \p Checks.
453 void printChecks(raw_ostream &OS,
454 const SmallVectorImpl<RuntimePointerCheck> &Checks,
455 unsigned Depth = 0) const;
456
457 /// This flag indicates if we need to add the runtime check.
458 bool Need;
459
460 /// Information about the pointers that may require checking.
461 SmallVector<PointerInfo, 2> Pointers;
462
463 /// Holds a partitioning of pointers into "check groups".
464 SmallVector<RuntimeCheckingPtrGroup, 2> CheckingGroups;
465
466 /// Check if pointers are in the same partition
467 ///
468 /// \p PtrToPartition contains the partition number for pointers (-1 if the
469 /// pointer belongs to multiple partitions).
470 static bool
471 arePointersInSamePartition(const SmallVectorImpl<int> &PtrToPartition,
472 unsigned PtrIdx1, unsigned PtrIdx2);
473
474 /// Decide whether we need to issue a run-time check for pointer at
475 /// index \p I and \p J to prove their independence.
476 bool needsChecking(unsigned I, unsigned J) const;
477
478 /// Return PointerInfo for pointer at index \p PtrIdx.
getPointerInfo(unsigned PtrIdx)479 const PointerInfo &getPointerInfo(unsigned PtrIdx) const {
480 return Pointers[PtrIdx];
481 }
482
getSE()483 ScalarEvolution *getSE() const { return SE; }
484
485 private:
486 /// Groups pointers such that a single memcheck is required
487 /// between two different groups. This will clear the CheckingGroups vector
488 /// and re-compute it. We will only group dependecies if \p UseDependencies
489 /// is true, otherwise we will create a separate group for each pointer.
490 void groupChecks(MemoryDepChecker::DepCandidates &DepCands,
491 bool UseDependencies);
492
493 /// Generate the checks and return them.
494 SmallVector<RuntimePointerCheck, 4> generateChecks() const;
495
496 /// Holds a pointer to the ScalarEvolution analysis.
497 ScalarEvolution *SE;
498
499 /// Set of run-time checks required to establish independence of
500 /// otherwise may-aliasing pointers in the loop.
501 SmallVector<RuntimePointerCheck, 4> Checks;
502 };
503
504 /// Drive the analysis of memory accesses in the loop
505 ///
506 /// This class is responsible for analyzing the memory accesses of a loop. It
507 /// collects the accesses and then its main helper the AccessAnalysis class
508 /// finds and categorizes the dependences in buildDependenceSets.
509 ///
510 /// For memory dependences that can be analyzed at compile time, it determines
511 /// whether the dependence is part of cycle inhibiting vectorization. This work
512 /// is delegated to the MemoryDepChecker class.
513 ///
514 /// For memory dependences that cannot be determined at compile time, it
515 /// generates run-time checks to prove independence. This is done by
516 /// AccessAnalysis::canCheckPtrAtRT and the checks are maintained by the
517 /// RuntimePointerCheck class.
518 ///
519 /// If pointers can wrap or can't be expressed as affine AddRec expressions by
520 /// ScalarEvolution, we will generate run-time checks by emitting a
521 /// SCEVUnionPredicate.
522 ///
523 /// Checks for both memory dependences and the SCEV predicates contained in the
524 /// PSE must be emitted in order for the results of this analysis to be valid.
525 class LoopAccessInfo {
526 public:
527 LoopAccessInfo(Loop *L, ScalarEvolution *SE, const TargetLibraryInfo *TLI,
528 AAResults *AA, DominatorTree *DT, LoopInfo *LI);
529
530 /// Return true we can analyze the memory accesses in the loop and there are
531 /// no memory dependence cycles.
canVectorizeMemory()532 bool canVectorizeMemory() const { return CanVecMem; }
533
534 /// Return true if there is a convergent operation in the loop. There may
535 /// still be reported runtime pointer checks that would be required, but it is
536 /// not legal to insert them.
hasConvergentOp()537 bool hasConvergentOp() const { return HasConvergentOp; }
538
getRuntimePointerChecking()539 const RuntimePointerChecking *getRuntimePointerChecking() const {
540 return PtrRtChecking.get();
541 }
542
543 /// Number of memchecks required to prove independence of otherwise
544 /// may-alias pointers.
getNumRuntimePointerChecks()545 unsigned getNumRuntimePointerChecks() const {
546 return PtrRtChecking->getNumberOfChecks();
547 }
548
549 /// Return true if the block BB needs to be predicated in order for the loop
550 /// to be vectorized.
551 static bool blockNeedsPredication(BasicBlock *BB, Loop *TheLoop,
552 DominatorTree *DT);
553
554 /// Returns true if the value V is uniform within the loop.
555 bool isUniform(Value *V) const;
556
getMaxSafeDepDistBytes()557 uint64_t getMaxSafeDepDistBytes() const { return MaxSafeDepDistBytes; }
getNumStores()558 unsigned getNumStores() const { return NumStores; }
getNumLoads()559 unsigned getNumLoads() const { return NumLoads;}
560
561 /// The diagnostics report generated for the analysis. E.g. why we
562 /// couldn't analyze the loop.
getReport()563 const OptimizationRemarkAnalysis *getReport() const { return Report.get(); }
564
565 /// the Memory Dependence Checker which can determine the
566 /// loop-independent and loop-carried dependences between memory accesses.
getDepChecker()567 const MemoryDepChecker &getDepChecker() const { return *DepChecker; }
568
569 /// Return the list of instructions that use \p Ptr to read or write
570 /// memory.
getInstructionsForAccess(Value * Ptr,bool isWrite)571 SmallVector<Instruction *, 4> getInstructionsForAccess(Value *Ptr,
572 bool isWrite) const {
573 return DepChecker->getInstructionsForAccess(Ptr, isWrite);
574 }
575
576 /// If an access has a symbolic strides, this maps the pointer value to
577 /// the stride symbol.
getSymbolicStrides()578 const ValueToValueMap &getSymbolicStrides() const { return SymbolicStrides; }
579
580 /// Pointer has a symbolic stride.
hasStride(Value * V)581 bool hasStride(Value *V) const { return StrideSet.count(V); }
582
583 /// Print the information about the memory accesses in the loop.
584 void print(raw_ostream &OS, unsigned Depth = 0) const;
585
586 /// If the loop has memory dependence involving an invariant address, i.e. two
587 /// stores or a store and a load, then return true, else return false.
hasDependenceInvolvingLoopInvariantAddress()588 bool hasDependenceInvolvingLoopInvariantAddress() const {
589 return HasDependenceInvolvingLoopInvariantAddress;
590 }
591
592 /// Used to add runtime SCEV checks. Simplifies SCEV expressions and converts
593 /// them to a more usable form. All SCEV expressions during the analysis
594 /// should be re-written (and therefore simplified) according to PSE.
595 /// A user of LoopAccessAnalysis will need to emit the runtime checks
596 /// associated with this predicate.
getPSE()597 const PredicatedScalarEvolution &getPSE() const { return *PSE; }
598
599 private:
600 /// Analyze the loop.
601 void analyzeLoop(AAResults *AA, LoopInfo *LI,
602 const TargetLibraryInfo *TLI, DominatorTree *DT);
603
604 /// Check if the structure of the loop allows it to be analyzed by this
605 /// pass.
606 bool canAnalyzeLoop();
607
608 /// Save the analysis remark.
609 ///
610 /// LAA does not directly emits the remarks. Instead it stores it which the
611 /// client can retrieve and presents as its own analysis
612 /// (e.g. -Rpass-analysis=loop-vectorize).
613 OptimizationRemarkAnalysis &recordAnalysis(StringRef RemarkName,
614 Instruction *Instr = nullptr);
615
616 /// Collect memory access with loop invariant strides.
617 ///
618 /// Looks for accesses like "a[i * StrideA]" where "StrideA" is loop
619 /// invariant.
620 void collectStridedAccess(Value *LoadOrStoreInst);
621
622 std::unique_ptr<PredicatedScalarEvolution> PSE;
623
624 /// We need to check that all of the pointers in this list are disjoint
625 /// at runtime. Using std::unique_ptr to make using move ctor simpler.
626 std::unique_ptr<RuntimePointerChecking> PtrRtChecking;
627
628 /// the Memory Dependence Checker which can determine the
629 /// loop-independent and loop-carried dependences between memory accesses.
630 std::unique_ptr<MemoryDepChecker> DepChecker;
631
632 Loop *TheLoop;
633
634 unsigned NumLoads;
635 unsigned NumStores;
636
637 uint64_t MaxSafeDepDistBytes;
638
639 /// Cache the result of analyzeLoop.
640 bool CanVecMem;
641 bool HasConvergentOp;
642
643 /// Indicator that there are non vectorizable stores to a uniform address.
644 bool HasDependenceInvolvingLoopInvariantAddress;
645
646 /// The diagnostics report generated for the analysis. E.g. why we
647 /// couldn't analyze the loop.
648 std::unique_ptr<OptimizationRemarkAnalysis> Report;
649
650 /// If an access has a symbolic strides, this maps the pointer value to
651 /// the stride symbol.
652 ValueToValueMap SymbolicStrides;
653
654 /// Set of symbolic strides values.
655 SmallPtrSet<Value *, 8> StrideSet;
656 };
657
658 Value *stripIntegerCast(Value *V);
659
660 /// Return the SCEV corresponding to a pointer with the symbolic stride
661 /// replaced with constant one, assuming the SCEV predicate associated with
662 /// \p PSE is true.
663 ///
664 /// If necessary this method will version the stride of the pointer according
665 /// to \p PtrToStride and therefore add further predicates to \p PSE.
666 ///
667 /// If \p OrigPtr is not null, use it to look up the stride value instead of \p
668 /// Ptr. \p PtrToStride provides the mapping between the pointer value and its
669 /// stride as collected by LoopVectorizationLegality::collectStridedAccess.
670 const SCEV *replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
671 const ValueToValueMap &PtrToStride,
672 Value *Ptr, Value *OrigPtr = nullptr);
673
674 /// If the pointer has a constant stride return it in units of its
675 /// element size. Otherwise return zero.
676 ///
677 /// Ensure that it does not wrap in the address space, assuming the predicate
678 /// associated with \p PSE is true.
679 ///
680 /// If necessary this method will version the stride of the pointer according
681 /// to \p PtrToStride and therefore add further predicates to \p PSE.
682 /// The \p Assume parameter indicates if we are allowed to make additional
683 /// run-time assumptions.
684 int64_t getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, const Loop *Lp,
685 const ValueToValueMap &StridesMap = ValueToValueMap(),
686 bool Assume = false, bool ShouldCheckWrap = true);
687
688 /// Returns the distance between the pointers \p PtrA and \p PtrB iff they are
689 /// compatible and it is possible to calculate the distance between them. This
690 /// is a simple API that does not depend on the analysis pass.
691 /// \param StrictCheck Ensure that the calculated distance matches the
692 /// type-based one after all the bitcasts removal in the provided pointers.
693 Optional<int> getPointersDiff(Type *ElemTyA, Value *PtrA, Type *ElemTyB,
694 Value *PtrB, const DataLayout &DL,
695 ScalarEvolution &SE, bool StrictCheck = false,
696 bool CheckType = true);
697
698 /// Attempt to sort the pointers in \p VL and return the sorted indices
699 /// in \p SortedIndices, if reordering is required.
700 ///
701 /// Returns 'true' if sorting is legal, otherwise returns 'false'.
702 ///
703 /// For example, for a given \p VL of memory accesses in program order, a[i+4],
704 /// a[i+0], a[i+1] and a[i+7], this function will sort the \p VL and save the
705 /// sorted indices in \p SortedIndices as a[i+0], a[i+1], a[i+4], a[i+7] and
706 /// saves the mask for actual memory accesses in program order in
707 /// \p SortedIndices as <1,2,0,3>
708 bool sortPtrAccesses(ArrayRef<Value *> VL, Type *ElemTy, const DataLayout &DL,
709 ScalarEvolution &SE,
710 SmallVectorImpl<unsigned> &SortedIndices);
711
712 /// Returns true if the memory operations \p A and \p B are consecutive.
713 /// This is a simple API that does not depend on the analysis pass.
714 bool isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL,
715 ScalarEvolution &SE, bool CheckType = true);
716
717 /// This analysis provides dependence information for the memory accesses
718 /// of a loop.
719 ///
720 /// It runs the analysis for a loop on demand. This can be initiated by
721 /// querying the loop access info via LAA::getInfo. getInfo return a
722 /// LoopAccessInfo object. See this class for the specifics of what information
723 /// is provided.
724 class LoopAccessLegacyAnalysis : public FunctionPass {
725 public:
726 static char ID;
727
728 LoopAccessLegacyAnalysis();
729
730 bool runOnFunction(Function &F) override;
731
732 void getAnalysisUsage(AnalysisUsage &AU) const override;
733
734 /// Query the result of the loop access information for the loop \p L.
735 ///
736 /// If there is no cached result available run the analysis.
737 const LoopAccessInfo &getInfo(Loop *L);
738
releaseMemory()739 void releaseMemory() override {
740 // Invalidate the cache when the pass is freed.
741 LoopAccessInfoMap.clear();
742 }
743
744 /// Print the result of the analysis when invoked with -analyze.
745 void print(raw_ostream &OS, const Module *M = nullptr) const override;
746
747 private:
748 /// The cache.
749 DenseMap<Loop *, std::unique_ptr<LoopAccessInfo>> LoopAccessInfoMap;
750
751 // The used analysis passes.
752 ScalarEvolution *SE = nullptr;
753 const TargetLibraryInfo *TLI = nullptr;
754 AAResults *AA = nullptr;
755 DominatorTree *DT = nullptr;
756 LoopInfo *LI = nullptr;
757 };
758
759 /// This analysis provides dependence information for the memory
760 /// accesses of a loop.
761 ///
762 /// It runs the analysis for a loop on demand. This can be initiated by
763 /// querying the loop access info via AM.getResult<LoopAccessAnalysis>.
764 /// getResult return a LoopAccessInfo object. See this class for the
765 /// specifics of what information is provided.
766 class LoopAccessAnalysis
767 : public AnalysisInfoMixin<LoopAccessAnalysis> {
768 friend AnalysisInfoMixin<LoopAccessAnalysis>;
769 static AnalysisKey Key;
770
771 public:
772 typedef LoopAccessInfo Result;
773
774 Result run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR);
775 };
776
getSource(const LoopAccessInfo & LAI)777 inline Instruction *MemoryDepChecker::Dependence::getSource(
778 const LoopAccessInfo &LAI) const {
779 return LAI.getDepChecker().getMemoryInstructions()[Source];
780 }
781
getDestination(const LoopAccessInfo & LAI)782 inline Instruction *MemoryDepChecker::Dependence::getDestination(
783 const LoopAccessInfo &LAI) const {
784 return LAI.getDepChecker().getMemoryInstructions()[Destination];
785 }
786
787 } // End llvm namespace
788
789 #endif
790