1 //===- MemorySSA.h - Build Memory SSA ---------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file exposes an interface to building/using memory SSA to 11 /// walk memory instructions using a use/def graph. 12 /// 13 /// Memory SSA class builds an SSA form that links together memory access 14 /// instructions such as loads, stores, atomics, and calls. Additionally, it 15 /// does a trivial form of "heap versioning" Every time the memory state changes 16 /// in the program, we generate a new heap version. It generates 17 /// MemoryDef/Uses/Phis that are overlayed on top of the existing instructions. 18 /// 19 /// As a trivial example, 20 /// define i32 @main() #0 { 21 /// entry: 22 /// %call = call noalias i8* @_Znwm(i64 4) #2 23 /// %0 = bitcast i8* %call to i32* 24 /// %call1 = call noalias i8* @_Znwm(i64 4) #2 25 /// %1 = bitcast i8* %call1 to i32* 26 /// store i32 5, i32* %0, align 4 27 /// store i32 7, i32* %1, align 4 28 /// %2 = load i32* %0, align 4 29 /// %3 = load i32* %1, align 4 30 /// %add = add nsw i32 %2, %3 31 /// ret i32 %add 32 /// } 33 /// 34 /// Will become 35 /// define i32 @main() #0 { 36 /// entry: 37 /// ; 1 = MemoryDef(0) 38 /// %call = call noalias i8* @_Znwm(i64 4) #3 39 /// %2 = bitcast i8* %call to i32* 40 /// ; 2 = MemoryDef(1) 41 /// %call1 = call noalias i8* @_Znwm(i64 4) #3 42 /// %4 = bitcast i8* %call1 to i32* 43 /// ; 3 = MemoryDef(2) 44 /// store i32 5, i32* %2, align 4 45 /// ; 4 = MemoryDef(3) 46 /// store i32 7, i32* %4, align 4 47 /// ; MemoryUse(3) 48 /// %7 = load i32* %2, align 4 49 /// ; MemoryUse(4) 50 /// %8 = load i32* %4, align 4 51 /// %add = add nsw i32 %7, %8 52 /// ret i32 %add 53 /// } 54 /// 55 /// Given this form, all the stores that could ever effect the load at %8 can be 56 /// gotten by using the MemoryUse associated with it, and walking from use to 57 /// def until you hit the top of the function. 58 /// 59 /// Each def also has a list of users associated with it, so you can walk from 60 /// both def to users, and users to defs. Note that we disambiguate MemoryUses, 61 /// but not the RHS of MemoryDefs. You can see this above at %7, which would 62 /// otherwise be a MemoryUse(4). Being disambiguated means that for a given 63 /// store, all the MemoryUses on its use lists are may-aliases of that store 64 /// (but the MemoryDefs on its use list may not be). 65 /// 66 /// MemoryDefs are not disambiguated because it would require multiple reaching 67 /// definitions, which would require multiple phis, and multiple memoryaccesses 68 /// per instruction. 69 // 70 //===----------------------------------------------------------------------===// 71 72 #ifndef LLVM_ANALYSIS_MEMORYSSA_H 73 #define LLVM_ANALYSIS_MEMORYSSA_H 74 75 #include "llvm/ADT/DenseMap.h" 76 #include "llvm/ADT/GraphTraits.h" 77 #include "llvm/ADT/SmallPtrSet.h" 78 #include "llvm/ADT/SmallVector.h" 79 #include "llvm/ADT/ilist.h" 80 #include "llvm/ADT/ilist_node.h" 81 #include "llvm/ADT/iterator.h" 82 #include "llvm/ADT/iterator_range.h" 83 #include "llvm/ADT/simple_ilist.h" 84 #include "llvm/Analysis/AliasAnalysis.h" 85 #include "llvm/Analysis/MemoryLocation.h" 86 #include "llvm/Analysis/PHITransAddr.h" 87 #include "llvm/IR/BasicBlock.h" 88 #include "llvm/IR/DerivedUser.h" 89 #include "llvm/IR/Dominators.h" 90 #include "llvm/IR/Module.h" 91 #include "llvm/IR/Operator.h" 92 #include "llvm/IR/Type.h" 93 #include "llvm/IR/Use.h" 94 #include "llvm/IR/User.h" 95 #include "llvm/IR/Value.h" 96 #include "llvm/IR/ValueHandle.h" 97 #include "llvm/Pass.h" 98 #include "llvm/Support/Casting.h" 99 #include "llvm/Support/CommandLine.h" 100 #include <algorithm> 101 #include <cassert> 102 #include <cstddef> 103 #include <iterator> 104 #include <memory> 105 #include <utility> 106 107 namespace llvm { 108 109 /// Enables memory ssa as a dependency for loop passes. 110 extern cl::opt<bool> EnableMSSALoopDependency; 111 112 class AllocaInst; 113 class Function; 114 class Instruction; 115 class MemoryAccess; 116 class MemorySSAWalker; 117 class LLVMContext; 118 class raw_ostream; 119 120 namespace MSSAHelpers { 121 122 struct AllAccessTag {}; 123 struct DefsOnlyTag {}; 124 125 } // end namespace MSSAHelpers 126 127 enum : unsigned { 128 // Used to signify what the default invalid ID is for MemoryAccess's 129 // getID() 130 INVALID_MEMORYACCESS_ID = -1U 131 }; 132 133 template <class T> class memoryaccess_def_iterator_base; 134 using memoryaccess_def_iterator = memoryaccess_def_iterator_base<MemoryAccess>; 135 using const_memoryaccess_def_iterator = 136 memoryaccess_def_iterator_base<const MemoryAccess>; 137 138 // The base for all memory accesses. All memory accesses in a block are 139 // linked together using an intrusive list. 140 class MemoryAccess 141 : public DerivedUser, 142 public ilist_node<MemoryAccess, ilist_tag<MSSAHelpers::AllAccessTag>>, 143 public ilist_node<MemoryAccess, ilist_tag<MSSAHelpers::DefsOnlyTag>> { 144 public: 145 using AllAccessType = 146 ilist_node<MemoryAccess, ilist_tag<MSSAHelpers::AllAccessTag>>; 147 using DefsOnlyType = 148 ilist_node<MemoryAccess, ilist_tag<MSSAHelpers::DefsOnlyTag>>; 149 150 MemoryAccess(const MemoryAccess &) = delete; 151 MemoryAccess &operator=(const MemoryAccess &) = delete; 152 153 void *operator new(size_t) = delete; 154 155 // Methods for support type inquiry through isa, cast, and 156 // dyn_cast 157 static bool classof(const Value *V) { 158 unsigned ID = V->getValueID(); 159 return ID == MemoryUseVal || ID == MemoryPhiVal || ID == MemoryDefVal; 160 } 161 162 BasicBlock *getBlock() const { return Block; } 163 164 void print(raw_ostream &OS) const; 165 void dump() const; 166 167 /// The user iterators for a memory access 168 using iterator = user_iterator; 169 using const_iterator = const_user_iterator; 170 171 /// This iterator walks over all of the defs in a given 172 /// MemoryAccess. For MemoryPhi nodes, this walks arguments. For 173 /// MemoryUse/MemoryDef, this walks the defining access. 174 memoryaccess_def_iterator defs_begin(); 175 const_memoryaccess_def_iterator defs_begin() const; 176 memoryaccess_def_iterator defs_end(); 177 const_memoryaccess_def_iterator defs_end() const; 178 179 /// Get the iterators for the all access list and the defs only list 180 /// We default to the all access list. 181 AllAccessType::self_iterator getIterator() { 182 return this->AllAccessType::getIterator(); 183 } 184 AllAccessType::const_self_iterator getIterator() const { 185 return this->AllAccessType::getIterator(); 186 } 187 AllAccessType::reverse_self_iterator getReverseIterator() { 188 return this->AllAccessType::getReverseIterator(); 189 } 190 AllAccessType::const_reverse_self_iterator getReverseIterator() const { 191 return this->AllAccessType::getReverseIterator(); 192 } 193 DefsOnlyType::self_iterator getDefsIterator() { 194 return this->DefsOnlyType::getIterator(); 195 } 196 DefsOnlyType::const_self_iterator getDefsIterator() const { 197 return this->DefsOnlyType::getIterator(); 198 } 199 DefsOnlyType::reverse_self_iterator getReverseDefsIterator() { 200 return this->DefsOnlyType::getReverseIterator(); 201 } 202 DefsOnlyType::const_reverse_self_iterator getReverseDefsIterator() const { 203 return this->DefsOnlyType::getReverseIterator(); 204 } 205 206 protected: 207 friend class MemoryDef; 208 friend class MemoryPhi; 209 friend class MemorySSA; 210 friend class MemoryUse; 211 friend class MemoryUseOrDef; 212 213 /// Used by MemorySSA to change the block of a MemoryAccess when it is 214 /// moved. 215 void setBlock(BasicBlock *BB) { Block = BB; } 216 217 /// Used for debugging and tracking things about MemoryAccesses. 218 /// Guaranteed unique among MemoryAccesses, no guarantees otherwise. 219 inline unsigned getID() const; 220 221 MemoryAccess(LLVMContext &C, unsigned Vty, DeleteValueTy DeleteValue, 222 BasicBlock *BB, unsigned NumOperands) 223 : DerivedUser(Type::getVoidTy(C), Vty, nullptr, NumOperands, DeleteValue), 224 Block(BB) {} 225 226 // Use deleteValue() to delete a generic MemoryAccess. 227 ~MemoryAccess() = default; 228 229 private: 230 BasicBlock *Block; 231 }; 232 233 template <> 234 struct ilist_alloc_traits<MemoryAccess> { 235 static void deleteNode(MemoryAccess *MA) { MA->deleteValue(); } 236 }; 237 238 inline raw_ostream &operator<<(raw_ostream &OS, const MemoryAccess &MA) { 239 MA.print(OS); 240 return OS; 241 } 242 243 /// Class that has the common methods + fields of memory uses/defs. It's 244 /// a little awkward to have, but there are many cases where we want either a 245 /// use or def, and there are many cases where uses are needed (defs aren't 246 /// acceptable), and vice-versa. 247 /// 248 /// This class should never be instantiated directly; make a MemoryUse or 249 /// MemoryDef instead. 250 class MemoryUseOrDef : public MemoryAccess { 251 public: 252 void *operator new(size_t) = delete; 253 254 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(MemoryAccess); 255 256 /// Get the instruction that this MemoryUse represents. 257 Instruction *getMemoryInst() const { return MemoryInstruction; } 258 259 /// Get the access that produces the memory state used by this Use. 260 MemoryAccess *getDefiningAccess() const { return getOperand(0); } 261 262 static bool classof(const Value *MA) { 263 return MA->getValueID() == MemoryUseVal || MA->getValueID() == MemoryDefVal; 264 } 265 266 // Sadly, these have to be public because they are needed in some of the 267 // iterators. 268 inline bool isOptimized() const; 269 inline MemoryAccess *getOptimized() const; 270 inline void setOptimized(MemoryAccess *); 271 272 // Retrieve AliasResult type of the optimized access. Ideally this would be 273 // returned by the caching walker and may go away in the future. 274 Optional<AliasResult> getOptimizedAccessType() const { 275 return isOptimized() ? OptimizedAccessAlias : None; 276 } 277 278 /// Reset the ID of what this MemoryUse was optimized to, causing it to 279 /// be rewalked by the walker if necessary. 280 /// This really should only be called by tests. 281 inline void resetOptimized(); 282 283 protected: 284 friend class MemorySSA; 285 friend class MemorySSAUpdater; 286 287 MemoryUseOrDef(LLVMContext &C, MemoryAccess *DMA, unsigned Vty, 288 DeleteValueTy DeleteValue, Instruction *MI, BasicBlock *BB, 289 unsigned NumOperands) 290 : MemoryAccess(C, Vty, DeleteValue, BB, NumOperands), 291 MemoryInstruction(MI), OptimizedAccessAlias(MayAlias) { 292 setDefiningAccess(DMA); 293 } 294 295 // Use deleteValue() to delete a generic MemoryUseOrDef. 296 ~MemoryUseOrDef() = default; 297 298 void setOptimizedAccessType(Optional<AliasResult> AR) { 299 OptimizedAccessAlias = AR; 300 } 301 302 void setDefiningAccess(MemoryAccess *DMA, bool Optimized = false, 303 Optional<AliasResult> AR = MayAlias) { 304 if (!Optimized) { 305 setOperand(0, DMA); 306 return; 307 } 308 setOptimized(DMA); 309 setOptimizedAccessType(AR); 310 } 311 312 private: 313 Instruction *MemoryInstruction; 314 Optional<AliasResult> OptimizedAccessAlias; 315 }; 316 317 /// Represents read-only accesses to memory 318 /// 319 /// In particular, the set of Instructions that will be represented by 320 /// MemoryUse's is exactly the set of Instructions for which 321 /// AliasAnalysis::getModRefInfo returns "Ref". 322 class MemoryUse final : public MemoryUseOrDef { 323 public: 324 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(MemoryAccess); 325 326 MemoryUse(LLVMContext &C, MemoryAccess *DMA, Instruction *MI, BasicBlock *BB) 327 : MemoryUseOrDef(C, DMA, MemoryUseVal, deleteMe, MI, BB, 328 /*NumOperands=*/1) {} 329 330 // allocate space for exactly one operand 331 void *operator new(size_t s) { return User::operator new(s, 1); } 332 333 static bool classof(const Value *MA) { 334 return MA->getValueID() == MemoryUseVal; 335 } 336 337 void print(raw_ostream &OS) const; 338 339 void setOptimized(MemoryAccess *DMA) { 340 OptimizedID = DMA->getID(); 341 setOperand(0, DMA); 342 } 343 344 bool isOptimized() const { 345 return getDefiningAccess() && OptimizedID == getDefiningAccess()->getID(); 346 } 347 348 MemoryAccess *getOptimized() const { 349 return getDefiningAccess(); 350 } 351 352 void resetOptimized() { 353 OptimizedID = INVALID_MEMORYACCESS_ID; 354 } 355 356 protected: 357 friend class MemorySSA; 358 359 private: 360 static void deleteMe(DerivedUser *Self); 361 362 unsigned OptimizedID = INVALID_MEMORYACCESS_ID; 363 }; 364 365 template <> 366 struct OperandTraits<MemoryUse> : public FixedNumOperandTraits<MemoryUse, 1> {}; 367 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(MemoryUse, MemoryAccess) 368 369 /// Represents a read-write access to memory, whether it is a must-alias, 370 /// or a may-alias. 371 /// 372 /// In particular, the set of Instructions that will be represented by 373 /// MemoryDef's is exactly the set of Instructions for which 374 /// AliasAnalysis::getModRefInfo returns "Mod" or "ModRef". 375 /// Note that, in order to provide def-def chains, all defs also have a use 376 /// associated with them. This use points to the nearest reaching 377 /// MemoryDef/MemoryPhi. 378 class MemoryDef final : public MemoryUseOrDef { 379 public: 380 friend class MemorySSA; 381 382 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(MemoryAccess); 383 384 MemoryDef(LLVMContext &C, MemoryAccess *DMA, Instruction *MI, BasicBlock *BB, 385 unsigned Ver) 386 : MemoryUseOrDef(C, DMA, MemoryDefVal, deleteMe, MI, BB, 387 /*NumOperands=*/2), 388 ID(Ver) {} 389 390 // allocate space for exactly two operands 391 void *operator new(size_t s) { return User::operator new(s, 2); } 392 393 static bool classof(const Value *MA) { 394 return MA->getValueID() == MemoryDefVal; 395 } 396 397 void setOptimized(MemoryAccess *MA) { 398 setOperand(1, MA); 399 OptimizedID = MA->getID(); 400 } 401 402 MemoryAccess *getOptimized() const { 403 return cast_or_null<MemoryAccess>(getOperand(1)); 404 } 405 406 bool isOptimized() const { 407 return getOptimized() && OptimizedID == getOptimized()->getID(); 408 } 409 410 void resetOptimized() { 411 OptimizedID = INVALID_MEMORYACCESS_ID; 412 setOperand(1, nullptr); 413 } 414 415 void print(raw_ostream &OS) const; 416 417 unsigned getID() const { return ID; } 418 419 private: 420 static void deleteMe(DerivedUser *Self); 421 422 const unsigned ID; 423 unsigned OptimizedID = INVALID_MEMORYACCESS_ID; 424 }; 425 426 template <> 427 struct OperandTraits<MemoryDef> : public FixedNumOperandTraits<MemoryDef, 2> {}; 428 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(MemoryDef, MemoryAccess) 429 430 template <> 431 struct OperandTraits<MemoryUseOrDef> { 432 static Use *op_begin(MemoryUseOrDef *MUD) { 433 if (auto *MU = dyn_cast<MemoryUse>(MUD)) 434 return OperandTraits<MemoryUse>::op_begin(MU); 435 return OperandTraits<MemoryDef>::op_begin(cast<MemoryDef>(MUD)); 436 } 437 438 static Use *op_end(MemoryUseOrDef *MUD) { 439 if (auto *MU = dyn_cast<MemoryUse>(MUD)) 440 return OperandTraits<MemoryUse>::op_end(MU); 441 return OperandTraits<MemoryDef>::op_end(cast<MemoryDef>(MUD)); 442 } 443 444 static unsigned operands(const MemoryUseOrDef *MUD) { 445 if (const auto *MU = dyn_cast<MemoryUse>(MUD)) 446 return OperandTraits<MemoryUse>::operands(MU); 447 return OperandTraits<MemoryDef>::operands(cast<MemoryDef>(MUD)); 448 } 449 }; 450 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(MemoryUseOrDef, MemoryAccess) 451 452 /// Represents phi nodes for memory accesses. 453 /// 454 /// These have the same semantic as regular phi nodes, with the exception that 455 /// only one phi will ever exist in a given basic block. 456 /// Guaranteeing one phi per block means guaranteeing there is only ever one 457 /// valid reaching MemoryDef/MemoryPHI along each path to the phi node. 458 /// This is ensured by not allowing disambiguation of the RHS of a MemoryDef or 459 /// a MemoryPhi's operands. 460 /// That is, given 461 /// if (a) { 462 /// store %a 463 /// store %b 464 /// } 465 /// it *must* be transformed into 466 /// if (a) { 467 /// 1 = MemoryDef(liveOnEntry) 468 /// store %a 469 /// 2 = MemoryDef(1) 470 /// store %b 471 /// } 472 /// and *not* 473 /// if (a) { 474 /// 1 = MemoryDef(liveOnEntry) 475 /// store %a 476 /// 2 = MemoryDef(liveOnEntry) 477 /// store %b 478 /// } 479 /// even if the two stores do not conflict. Otherwise, both 1 and 2 reach the 480 /// end of the branch, and if there are not two phi nodes, one will be 481 /// disconnected completely from the SSA graph below that point. 482 /// Because MemoryUse's do not generate new definitions, they do not have this 483 /// issue. 484 class MemoryPhi final : public MemoryAccess { 485 // allocate space for exactly zero operands 486 void *operator new(size_t s) { return User::operator new(s); } 487 488 public: 489 /// Provide fast operand accessors 490 DECLARE_TRANSPARENT_OPERAND_ACCESSORS(MemoryAccess); 491 492 MemoryPhi(LLVMContext &C, BasicBlock *BB, unsigned Ver, unsigned NumPreds = 0) 493 : MemoryAccess(C, MemoryPhiVal, deleteMe, BB, 0), ID(Ver), 494 ReservedSpace(NumPreds) { 495 allocHungoffUses(ReservedSpace); 496 } 497 498 // Block iterator interface. This provides access to the list of incoming 499 // basic blocks, which parallels the list of incoming values. 500 using block_iterator = BasicBlock **; 501 using const_block_iterator = BasicBlock *const *; 502 503 block_iterator block_begin() { 504 return reinterpret_cast<block_iterator>(op_begin() + ReservedSpace); 505 } 506 507 const_block_iterator block_begin() const { 508 return reinterpret_cast<const_block_iterator>(op_begin() + ReservedSpace); 509 } 510 511 block_iterator block_end() { return block_begin() + getNumOperands(); } 512 513 const_block_iterator block_end() const { 514 return block_begin() + getNumOperands(); 515 } 516 517 iterator_range<block_iterator> blocks() { 518 return make_range(block_begin(), block_end()); 519 } 520 521 iterator_range<const_block_iterator> blocks() const { 522 return make_range(block_begin(), block_end()); 523 } 524 525 op_range incoming_values() { return operands(); } 526 527 const_op_range incoming_values() const { return operands(); } 528 529 /// Return the number of incoming edges 530 unsigned getNumIncomingValues() const { return getNumOperands(); } 531 532 /// Return incoming value number x 533 MemoryAccess *getIncomingValue(unsigned I) const { return getOperand(I); } 534 void setIncomingValue(unsigned I, MemoryAccess *V) { 535 assert(V && "PHI node got a null value!"); 536 setOperand(I, V); 537 } 538 539 static unsigned getOperandNumForIncomingValue(unsigned I) { return I; } 540 static unsigned getIncomingValueNumForOperand(unsigned I) { return I; } 541 542 /// Return incoming basic block number @p i. 543 BasicBlock *getIncomingBlock(unsigned I) const { return block_begin()[I]; } 544 545 /// Return incoming basic block corresponding 546 /// to an operand of the PHI. 547 BasicBlock *getIncomingBlock(const Use &U) const { 548 assert(this == U.getUser() && "Iterator doesn't point to PHI's Uses?"); 549 return getIncomingBlock(unsigned(&U - op_begin())); 550 } 551 552 /// Return incoming basic block corresponding 553 /// to value use iterator. 554 BasicBlock *getIncomingBlock(MemoryAccess::const_user_iterator I) const { 555 return getIncomingBlock(I.getUse()); 556 } 557 558 void setIncomingBlock(unsigned I, BasicBlock *BB) { 559 assert(BB && "PHI node got a null basic block!"); 560 block_begin()[I] = BB; 561 } 562 563 /// Add an incoming value to the end of the PHI list 564 void addIncoming(MemoryAccess *V, BasicBlock *BB) { 565 if (getNumOperands() == ReservedSpace) 566 growOperands(); // Get more space! 567 // Initialize some new operands. 568 setNumHungOffUseOperands(getNumOperands() + 1); 569 setIncomingValue(getNumOperands() - 1, V); 570 setIncomingBlock(getNumOperands() - 1, BB); 571 } 572 573 /// Return the first index of the specified basic 574 /// block in the value list for this PHI. Returns -1 if no instance. 575 int getBasicBlockIndex(const BasicBlock *BB) const { 576 for (unsigned I = 0, E = getNumOperands(); I != E; ++I) 577 if (block_begin()[I] == BB) 578 return I; 579 return -1; 580 } 581 582 MemoryAccess *getIncomingValueForBlock(const BasicBlock *BB) const { 583 int Idx = getBasicBlockIndex(BB); 584 assert(Idx >= 0 && "Invalid basic block argument!"); 585 return getIncomingValue(Idx); 586 } 587 588 // After deleting incoming position I, the order of incoming may be changed. 589 void unorderedDeleteIncoming(unsigned I) { 590 unsigned E = getNumOperands(); 591 assert(I < E && "Cannot remove out of bounds Phi entry."); 592 // MemoryPhi must have at least two incoming values, otherwise the MemoryPhi 593 // itself should be deleted. 594 assert(E >= 2 && "Cannot only remove incoming values in MemoryPhis with " 595 "at least 2 values."); 596 setIncomingValue(I, getIncomingValue(E - 1)); 597 setIncomingBlock(I, block_begin()[E - 1]); 598 setOperand(E - 1, nullptr); 599 block_begin()[E - 1] = nullptr; 600 setNumHungOffUseOperands(getNumOperands() - 1); 601 } 602 603 // After deleting entries that satisfy Pred, remaining entries may have 604 // changed order. 605 template <typename Fn> void unorderedDeleteIncomingIf(Fn &&Pred) { 606 for (unsigned I = 0, E = getNumOperands(); I != E; ++I) 607 if (Pred(getIncomingValue(I), getIncomingBlock(I))) { 608 unorderedDeleteIncoming(I); 609 E = getNumOperands(); 610 --I; 611 } 612 assert(getNumOperands() >= 1 && 613 "Cannot remove all incoming blocks in a MemoryPhi."); 614 } 615 616 // After deleting incoming block BB, the incoming blocks order may be changed. 617 void unorderedDeleteIncomingBlock(const BasicBlock *BB) { 618 unorderedDeleteIncomingIf( 619 [&](const MemoryAccess *, const BasicBlock *B) { return BB == B; }); 620 } 621 622 // After deleting incoming memory access MA, the incoming accesses order may 623 // be changed. 624 void unorderedDeleteIncomingValue(const MemoryAccess *MA) { 625 unorderedDeleteIncomingIf( 626 [&](const MemoryAccess *M, const BasicBlock *) { return MA == M; }); 627 } 628 629 static bool classof(const Value *V) { 630 return V->getValueID() == MemoryPhiVal; 631 } 632 633 void print(raw_ostream &OS) const; 634 635 unsigned getID() const { return ID; } 636 637 protected: 638 friend class MemorySSA; 639 640 /// this is more complicated than the generic 641 /// User::allocHungoffUses, because we have to allocate Uses for the incoming 642 /// values and pointers to the incoming blocks, all in one allocation. 643 void allocHungoffUses(unsigned N) { 644 User::allocHungoffUses(N, /* IsPhi */ true); 645 } 646 647 private: 648 // For debugging only 649 const unsigned ID; 650 unsigned ReservedSpace; 651 652 /// This grows the operand list in response to a push_back style of 653 /// operation. This grows the number of ops by 1.5 times. 654 void growOperands() { 655 unsigned E = getNumOperands(); 656 // 2 op PHI nodes are VERY common, so reserve at least enough for that. 657 ReservedSpace = std::max(E + E / 2, 2u); 658 growHungoffUses(ReservedSpace, /* IsPhi */ true); 659 } 660 661 static void deleteMe(DerivedUser *Self); 662 }; 663 664 inline unsigned MemoryAccess::getID() const { 665 assert((isa<MemoryDef>(this) || isa<MemoryPhi>(this)) && 666 "only memory defs and phis have ids"); 667 if (const auto *MD = dyn_cast<MemoryDef>(this)) 668 return MD->getID(); 669 return cast<MemoryPhi>(this)->getID(); 670 } 671 672 inline bool MemoryUseOrDef::isOptimized() const { 673 if (const auto *MD = dyn_cast<MemoryDef>(this)) 674 return MD->isOptimized(); 675 return cast<MemoryUse>(this)->isOptimized(); 676 } 677 678 inline MemoryAccess *MemoryUseOrDef::getOptimized() const { 679 if (const auto *MD = dyn_cast<MemoryDef>(this)) 680 return MD->getOptimized(); 681 return cast<MemoryUse>(this)->getOptimized(); 682 } 683 684 inline void MemoryUseOrDef::setOptimized(MemoryAccess *MA) { 685 if (auto *MD = dyn_cast<MemoryDef>(this)) 686 MD->setOptimized(MA); 687 else 688 cast<MemoryUse>(this)->setOptimized(MA); 689 } 690 691 inline void MemoryUseOrDef::resetOptimized() { 692 if (auto *MD = dyn_cast<MemoryDef>(this)) 693 MD->resetOptimized(); 694 else 695 cast<MemoryUse>(this)->resetOptimized(); 696 } 697 698 template <> struct OperandTraits<MemoryPhi> : public HungoffOperandTraits<2> {}; 699 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(MemoryPhi, MemoryAccess) 700 701 /// Encapsulates MemorySSA, including all data associated with memory 702 /// accesses. 703 class MemorySSA { 704 public: 705 MemorySSA(Function &, AliasAnalysis *, DominatorTree *); 706 707 // MemorySSA must remain where it's constructed; Walkers it creates store 708 // pointers to it. 709 MemorySSA(MemorySSA &&) = delete; 710 711 ~MemorySSA(); 712 713 MemorySSAWalker *getWalker(); 714 MemorySSAWalker *getSkipSelfWalker(); 715 716 /// Given a memory Mod/Ref'ing instruction, get the MemorySSA 717 /// access associated with it. If passed a basic block gets the memory phi 718 /// node that exists for that block, if there is one. Otherwise, this will get 719 /// a MemoryUseOrDef. 720 MemoryUseOrDef *getMemoryAccess(const Instruction *I) const { 721 return cast_or_null<MemoryUseOrDef>(ValueToMemoryAccess.lookup(I)); 722 } 723 724 MemoryPhi *getMemoryAccess(const BasicBlock *BB) const { 725 return cast_or_null<MemoryPhi>(ValueToMemoryAccess.lookup(cast<Value>(BB))); 726 } 727 728 DominatorTree &getDomTree() const { return *DT; } 729 730 void dump() const; 731 void print(raw_ostream &) const; 732 733 /// Return true if \p MA represents the live on entry value 734 /// 735 /// Loads and stores from pointer arguments and other global values may be 736 /// defined by memory operations that do not occur in the current function, so 737 /// they may be live on entry to the function. MemorySSA represents such 738 /// memory state by the live on entry definition, which is guaranteed to occur 739 /// before any other memory access in the function. 740 inline bool isLiveOnEntryDef(const MemoryAccess *MA) const { 741 return MA == LiveOnEntryDef.get(); 742 } 743 744 inline MemoryAccess *getLiveOnEntryDef() const { 745 return LiveOnEntryDef.get(); 746 } 747 748 // Sadly, iplists, by default, owns and deletes pointers added to the 749 // list. It's not currently possible to have two iplists for the same type, 750 // where one owns the pointers, and one does not. This is because the traits 751 // are per-type, not per-tag. If this ever changes, we should make the 752 // DefList an iplist. 753 using AccessList = iplist<MemoryAccess, ilist_tag<MSSAHelpers::AllAccessTag>>; 754 using DefsList = 755 simple_ilist<MemoryAccess, ilist_tag<MSSAHelpers::DefsOnlyTag>>; 756 757 /// Return the list of MemoryAccess's for a given basic block. 758 /// 759 /// This list is not modifiable by the user. 760 const AccessList *getBlockAccesses(const BasicBlock *BB) const { 761 return getWritableBlockAccesses(BB); 762 } 763 764 /// Return the list of MemoryDef's and MemoryPhi's for a given basic 765 /// block. 766 /// 767 /// This list is not modifiable by the user. 768 const DefsList *getBlockDefs(const BasicBlock *BB) const { 769 return getWritableBlockDefs(BB); 770 } 771 772 /// Given two memory accesses in the same basic block, determine 773 /// whether MemoryAccess \p A dominates MemoryAccess \p B. 774 bool locallyDominates(const MemoryAccess *A, const MemoryAccess *B) const; 775 776 /// Given two memory accesses in potentially different blocks, 777 /// determine whether MemoryAccess \p A dominates MemoryAccess \p B. 778 bool dominates(const MemoryAccess *A, const MemoryAccess *B) const; 779 780 /// Given a MemoryAccess and a Use, determine whether MemoryAccess \p A 781 /// dominates Use \p B. 782 bool dominates(const MemoryAccess *A, const Use &B) const; 783 784 /// Verify that MemorySSA is self consistent (IE definitions dominate 785 /// all uses, uses appear in the right places). This is used by unit tests. 786 void verifyMemorySSA() const; 787 788 /// Used in various insertion functions to specify whether we are talking 789 /// about the beginning or end of a block. 790 enum InsertionPlace { Beginning, End, BeforeTerminator }; 791 792 protected: 793 // Used by Memory SSA annotater, dumpers, and wrapper pass 794 friend class MemorySSAAnnotatedWriter; 795 friend class MemorySSAPrinterLegacyPass; 796 friend class MemorySSAUpdater; 797 798 void verifyOrderingDominationAndDefUses(Function &F) const; 799 void verifyDominationNumbers(const Function &F) const; 800 void verifyPrevDefInPhis(Function &F) const; 801 802 // This is used by the use optimizer and updater. 803 AccessList *getWritableBlockAccesses(const BasicBlock *BB) const { 804 auto It = PerBlockAccesses.find(BB); 805 return It == PerBlockAccesses.end() ? nullptr : It->second.get(); 806 } 807 808 // This is used by the use optimizer and updater. 809 DefsList *getWritableBlockDefs(const BasicBlock *BB) const { 810 auto It = PerBlockDefs.find(BB); 811 return It == PerBlockDefs.end() ? nullptr : It->second.get(); 812 } 813 814 // These is used by the updater to perform various internal MemorySSA 815 // machinsations. They do not always leave the IR in a correct state, and 816 // relies on the updater to fixup what it breaks, so it is not public. 817 818 void moveTo(MemoryUseOrDef *What, BasicBlock *BB, AccessList::iterator Where); 819 void moveTo(MemoryAccess *What, BasicBlock *BB, InsertionPlace Point); 820 821 // Rename the dominator tree branch rooted at BB. 822 void renamePass(BasicBlock *BB, MemoryAccess *IncomingVal, 823 SmallPtrSetImpl<BasicBlock *> &Visited) { 824 renamePass(DT->getNode(BB), IncomingVal, Visited, true, true); 825 } 826 827 void removeFromLookups(MemoryAccess *); 828 void removeFromLists(MemoryAccess *, bool ShouldDelete = true); 829 void insertIntoListsForBlock(MemoryAccess *, const BasicBlock *, 830 InsertionPlace); 831 void insertIntoListsBefore(MemoryAccess *, const BasicBlock *, 832 AccessList::iterator); 833 MemoryUseOrDef *createDefinedAccess(Instruction *, MemoryAccess *, 834 const MemoryUseOrDef *Template = nullptr, 835 bool CreationMustSucceed = true); 836 837 private: 838 template <class AliasAnalysisType> class ClobberWalkerBase; 839 template <class AliasAnalysisType> class CachingWalker; 840 template <class AliasAnalysisType> class SkipSelfWalker; 841 class OptimizeUses; 842 843 CachingWalker<AliasAnalysis> *getWalkerImpl(); 844 void buildMemorySSA(BatchAAResults &BAA); 845 846 void prepareForMoveTo(MemoryAccess *, BasicBlock *); 847 void verifyUseInDefs(MemoryAccess *, MemoryAccess *) const; 848 849 using AccessMap = DenseMap<const BasicBlock *, std::unique_ptr<AccessList>>; 850 using DefsMap = DenseMap<const BasicBlock *, std::unique_ptr<DefsList>>; 851 852 void markUnreachableAsLiveOnEntry(BasicBlock *BB); 853 MemoryPhi *createMemoryPhi(BasicBlock *BB); 854 template <typename AliasAnalysisType> 855 MemoryUseOrDef *createNewAccess(Instruction *, AliasAnalysisType *, 856 const MemoryUseOrDef *Template = nullptr); 857 void placePHINodes(const SmallPtrSetImpl<BasicBlock *> &); 858 MemoryAccess *renameBlock(BasicBlock *, MemoryAccess *, bool); 859 void renameSuccessorPhis(BasicBlock *, MemoryAccess *, bool); 860 void renamePass(DomTreeNode *, MemoryAccess *IncomingVal, 861 SmallPtrSetImpl<BasicBlock *> &Visited, 862 bool SkipVisited = false, bool RenameAllUses = false); 863 AccessList *getOrCreateAccessList(const BasicBlock *); 864 DefsList *getOrCreateDefsList(const BasicBlock *); 865 void renumberBlock(const BasicBlock *) const; 866 AliasAnalysis *AA; 867 DominatorTree *DT; 868 Function &F; 869 870 // Memory SSA mappings 871 DenseMap<const Value *, MemoryAccess *> ValueToMemoryAccess; 872 873 // These two mappings contain the main block to access/def mappings for 874 // MemorySSA. The list contained in PerBlockAccesses really owns all the 875 // MemoryAccesses. 876 // Both maps maintain the invariant that if a block is found in them, the 877 // corresponding list is not empty, and if a block is not found in them, the 878 // corresponding list is empty. 879 AccessMap PerBlockAccesses; 880 DefsMap PerBlockDefs; 881 std::unique_ptr<MemoryAccess, ValueDeleter> LiveOnEntryDef; 882 883 // Domination mappings 884 // Note that the numbering is local to a block, even though the map is 885 // global. 886 mutable SmallPtrSet<const BasicBlock *, 16> BlockNumberingValid; 887 mutable DenseMap<const MemoryAccess *, unsigned long> BlockNumbering; 888 889 // Memory SSA building info 890 std::unique_ptr<ClobberWalkerBase<AliasAnalysis>> WalkerBase; 891 std::unique_ptr<CachingWalker<AliasAnalysis>> Walker; 892 std::unique_ptr<SkipSelfWalker<AliasAnalysis>> SkipWalker; 893 unsigned NextID; 894 }; 895 896 // Internal MemorySSA utils, for use by MemorySSA classes and walkers 897 class MemorySSAUtil { 898 protected: 899 friend class GVNHoist; 900 friend class MemorySSAWalker; 901 902 // This function should not be used by new passes. 903 static bool defClobbersUseOrDef(MemoryDef *MD, const MemoryUseOrDef *MU, 904 AliasAnalysis &AA); 905 }; 906 907 // This pass does eager building and then printing of MemorySSA. It is used by 908 // the tests to be able to build, dump, and verify Memory SSA. 909 class MemorySSAPrinterLegacyPass : public FunctionPass { 910 public: 911 MemorySSAPrinterLegacyPass(); 912 913 bool runOnFunction(Function &) override; 914 void getAnalysisUsage(AnalysisUsage &AU) const override; 915 916 static char ID; 917 }; 918 919 /// An analysis that produces \c MemorySSA for a function. 920 /// 921 class MemorySSAAnalysis : public AnalysisInfoMixin<MemorySSAAnalysis> { 922 friend AnalysisInfoMixin<MemorySSAAnalysis>; 923 924 static AnalysisKey Key; 925 926 public: 927 // Wrap MemorySSA result to ensure address stability of internal MemorySSA 928 // pointers after construction. Use a wrapper class instead of plain 929 // unique_ptr<MemorySSA> to avoid build breakage on MSVC. 930 struct Result { 931 Result(std::unique_ptr<MemorySSA> &&MSSA) : MSSA(std::move(MSSA)) {} 932 933 MemorySSA &getMSSA() { return *MSSA.get(); } 934 935 std::unique_ptr<MemorySSA> MSSA; 936 937 bool invalidate(Function &F, const PreservedAnalyses &PA, 938 FunctionAnalysisManager::Invalidator &Inv); 939 }; 940 941 Result run(Function &F, FunctionAnalysisManager &AM); 942 }; 943 944 /// Printer pass for \c MemorySSA. 945 class MemorySSAPrinterPass : public PassInfoMixin<MemorySSAPrinterPass> { 946 raw_ostream &OS; 947 948 public: 949 explicit MemorySSAPrinterPass(raw_ostream &OS) : OS(OS) {} 950 951 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); 952 }; 953 954 /// Verifier pass for \c MemorySSA. 955 struct MemorySSAVerifierPass : PassInfoMixin<MemorySSAVerifierPass> { 956 PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); 957 }; 958 959 /// Legacy analysis pass which computes \c MemorySSA. 960 class MemorySSAWrapperPass : public FunctionPass { 961 public: 962 MemorySSAWrapperPass(); 963 964 static char ID; 965 966 bool runOnFunction(Function &) override; 967 void releaseMemory() override; 968 MemorySSA &getMSSA() { return *MSSA; } 969 const MemorySSA &getMSSA() const { return *MSSA; } 970 971 void getAnalysisUsage(AnalysisUsage &AU) const override; 972 973 void verifyAnalysis() const override; 974 void print(raw_ostream &OS, const Module *M = nullptr) const override; 975 976 private: 977 std::unique_ptr<MemorySSA> MSSA; 978 }; 979 980 /// This is the generic walker interface for walkers of MemorySSA. 981 /// Walkers are used to be able to further disambiguate the def-use chains 982 /// MemorySSA gives you, or otherwise produce better info than MemorySSA gives 983 /// you. 984 /// In particular, while the def-use chains provide basic information, and are 985 /// guaranteed to give, for example, the nearest may-aliasing MemoryDef for a 986 /// MemoryUse as AliasAnalysis considers it, a user mant want better or other 987 /// information. In particular, they may want to use SCEV info to further 988 /// disambiguate memory accesses, or they may want the nearest dominating 989 /// may-aliasing MemoryDef for a call or a store. This API enables a 990 /// standardized interface to getting and using that info. 991 class MemorySSAWalker { 992 public: 993 MemorySSAWalker(MemorySSA *); 994 virtual ~MemorySSAWalker() = default; 995 996 using MemoryAccessSet = SmallVector<MemoryAccess *, 8>; 997 998 /// Given a memory Mod/Ref/ModRef'ing instruction, calling this 999 /// will give you the nearest dominating MemoryAccess that Mod's the location 1000 /// the instruction accesses (by skipping any def which AA can prove does not 1001 /// alias the location(s) accessed by the instruction given). 1002 /// 1003 /// Note that this will return a single access, and it must dominate the 1004 /// Instruction, so if an operand of a MemoryPhi node Mod's the instruction, 1005 /// this will return the MemoryPhi, not the operand. This means that 1006 /// given: 1007 /// if (a) { 1008 /// 1 = MemoryDef(liveOnEntry) 1009 /// store %a 1010 /// } else { 1011 /// 2 = MemoryDef(liveOnEntry) 1012 /// store %b 1013 /// } 1014 /// 3 = MemoryPhi(2, 1) 1015 /// MemoryUse(3) 1016 /// load %a 1017 /// 1018 /// calling this API on load(%a) will return the MemoryPhi, not the MemoryDef 1019 /// in the if (a) branch. 1020 MemoryAccess *getClobberingMemoryAccess(const Instruction *I) { 1021 MemoryAccess *MA = MSSA->getMemoryAccess(I); 1022 assert(MA && "Handed an instruction that MemorySSA doesn't recognize?"); 1023 return getClobberingMemoryAccess(MA); 1024 } 1025 1026 /// Does the same thing as getClobberingMemoryAccess(const Instruction *I), 1027 /// but takes a MemoryAccess instead of an Instruction. 1028 virtual MemoryAccess *getClobberingMemoryAccess(MemoryAccess *) = 0; 1029 1030 /// Given a potentially clobbering memory access and a new location, 1031 /// calling this will give you the nearest dominating clobbering MemoryAccess 1032 /// (by skipping non-aliasing def links). 1033 /// 1034 /// This version of the function is mainly used to disambiguate phi translated 1035 /// pointers, where the value of a pointer may have changed from the initial 1036 /// memory access. Note that this expects to be handed either a MemoryUse, 1037 /// or an already potentially clobbering access. Unlike the above API, if 1038 /// given a MemoryDef that clobbers the pointer as the starting access, it 1039 /// will return that MemoryDef, whereas the above would return the clobber 1040 /// starting from the use side of the memory def. 1041 virtual MemoryAccess *getClobberingMemoryAccess(MemoryAccess *, 1042 const MemoryLocation &) = 0; 1043 1044 /// Given a memory access, invalidate anything this walker knows about 1045 /// that access. 1046 /// This API is used by walkers that store information to perform basic cache 1047 /// invalidation. This will be called by MemorySSA at appropriate times for 1048 /// the walker it uses or returns. 1049 virtual void invalidateInfo(MemoryAccess *) {} 1050 1051 protected: 1052 friend class MemorySSA; // For updating MSSA pointer in MemorySSA move 1053 // constructor. 1054 MemorySSA *MSSA; 1055 }; 1056 1057 /// A MemorySSAWalker that does no alias queries, or anything else. It 1058 /// simply returns the links as they were constructed by the builder. 1059 class DoNothingMemorySSAWalker final : public MemorySSAWalker { 1060 public: 1061 // Keep the overrides below from hiding the Instruction overload of 1062 // getClobberingMemoryAccess. 1063 using MemorySSAWalker::getClobberingMemoryAccess; 1064 1065 MemoryAccess *getClobberingMemoryAccess(MemoryAccess *) override; 1066 MemoryAccess *getClobberingMemoryAccess(MemoryAccess *, 1067 const MemoryLocation &) override; 1068 }; 1069 1070 using MemoryAccessPair = std::pair<MemoryAccess *, MemoryLocation>; 1071 using ConstMemoryAccessPair = std::pair<const MemoryAccess *, MemoryLocation>; 1072 1073 /// Iterator base class used to implement const and non-const iterators 1074 /// over the defining accesses of a MemoryAccess. 1075 template <class T> 1076 class memoryaccess_def_iterator_base 1077 : public iterator_facade_base<memoryaccess_def_iterator_base<T>, 1078 std::forward_iterator_tag, T, ptrdiff_t, T *, 1079 T *> { 1080 using BaseT = typename memoryaccess_def_iterator_base::iterator_facade_base; 1081 1082 public: 1083 memoryaccess_def_iterator_base(T *Start) : Access(Start) {} 1084 memoryaccess_def_iterator_base() = default; 1085 1086 bool operator==(const memoryaccess_def_iterator_base &Other) const { 1087 return Access == Other.Access && (!Access || ArgNo == Other.ArgNo); 1088 } 1089 1090 // This is a bit ugly, but for MemoryPHI's, unlike PHINodes, you can't get the 1091 // block from the operand in constant time (In a PHINode, the uselist has 1092 // both, so it's just subtraction). We provide it as part of the 1093 // iterator to avoid callers having to linear walk to get the block. 1094 // If the operation becomes constant time on MemoryPHI's, this bit of 1095 // abstraction breaking should be removed. 1096 BasicBlock *getPhiArgBlock() const { 1097 MemoryPhi *MP = dyn_cast<MemoryPhi>(Access); 1098 assert(MP && "Tried to get phi arg block when not iterating over a PHI"); 1099 return MP->getIncomingBlock(ArgNo); 1100 } 1101 1102 typename BaseT::iterator::pointer operator*() const { 1103 assert(Access && "Tried to access past the end of our iterator"); 1104 // Go to the first argument for phis, and the defining access for everything 1105 // else. 1106 if (const MemoryPhi *MP = dyn_cast<MemoryPhi>(Access)) 1107 return MP->getIncomingValue(ArgNo); 1108 return cast<MemoryUseOrDef>(Access)->getDefiningAccess(); 1109 } 1110 1111 using BaseT::operator++; 1112 memoryaccess_def_iterator_base &operator++() { 1113 assert(Access && "Hit end of iterator"); 1114 if (const MemoryPhi *MP = dyn_cast<MemoryPhi>(Access)) { 1115 if (++ArgNo >= MP->getNumIncomingValues()) { 1116 ArgNo = 0; 1117 Access = nullptr; 1118 } 1119 } else { 1120 Access = nullptr; 1121 } 1122 return *this; 1123 } 1124 1125 private: 1126 T *Access = nullptr; 1127 unsigned ArgNo = 0; 1128 }; 1129 1130 inline memoryaccess_def_iterator MemoryAccess::defs_begin() { 1131 return memoryaccess_def_iterator(this); 1132 } 1133 1134 inline const_memoryaccess_def_iterator MemoryAccess::defs_begin() const { 1135 return const_memoryaccess_def_iterator(this); 1136 } 1137 1138 inline memoryaccess_def_iterator MemoryAccess::defs_end() { 1139 return memoryaccess_def_iterator(); 1140 } 1141 1142 inline const_memoryaccess_def_iterator MemoryAccess::defs_end() const { 1143 return const_memoryaccess_def_iterator(); 1144 } 1145 1146 /// GraphTraits for a MemoryAccess, which walks defs in the normal case, 1147 /// and uses in the inverse case. 1148 template <> struct GraphTraits<MemoryAccess *> { 1149 using NodeRef = MemoryAccess *; 1150 using ChildIteratorType = memoryaccess_def_iterator; 1151 1152 static NodeRef getEntryNode(NodeRef N) { return N; } 1153 static ChildIteratorType child_begin(NodeRef N) { return N->defs_begin(); } 1154 static ChildIteratorType child_end(NodeRef N) { return N->defs_end(); } 1155 }; 1156 1157 template <> struct GraphTraits<Inverse<MemoryAccess *>> { 1158 using NodeRef = MemoryAccess *; 1159 using ChildIteratorType = MemoryAccess::iterator; 1160 1161 static NodeRef getEntryNode(NodeRef N) { return N; } 1162 static ChildIteratorType child_begin(NodeRef N) { return N->user_begin(); } 1163 static ChildIteratorType child_end(NodeRef N) { return N->user_end(); } 1164 }; 1165 1166 /// Provide an iterator that walks defs, giving both the memory access, 1167 /// and the current pointer location, updating the pointer location as it 1168 /// changes due to phi node translation. 1169 /// 1170 /// This iterator, while somewhat specialized, is what most clients actually 1171 /// want when walking upwards through MemorySSA def chains. It takes a pair of 1172 /// <MemoryAccess,MemoryLocation>, and walks defs, properly translating the 1173 /// memory location through phi nodes for the user. 1174 class upward_defs_iterator 1175 : public iterator_facade_base<upward_defs_iterator, 1176 std::forward_iterator_tag, 1177 const MemoryAccessPair> { 1178 using BaseT = upward_defs_iterator::iterator_facade_base; 1179 1180 public: 1181 upward_defs_iterator(const MemoryAccessPair &Info, DominatorTree *DT, 1182 bool *PerformedPhiTranslation = nullptr) 1183 : DefIterator(Info.first), Location(Info.second), 1184 OriginalAccess(Info.first), DT(DT), 1185 PerformedPhiTranslation(PerformedPhiTranslation) { 1186 CurrentPair.first = nullptr; 1187 1188 WalkingPhi = Info.first && isa<MemoryPhi>(Info.first); 1189 fillInCurrentPair(); 1190 } 1191 1192 upward_defs_iterator() { CurrentPair.first = nullptr; } 1193 1194 bool operator==(const upward_defs_iterator &Other) const { 1195 return DefIterator == Other.DefIterator; 1196 } 1197 1198 BaseT::iterator::reference operator*() const { 1199 assert(DefIterator != OriginalAccess->defs_end() && 1200 "Tried to access past the end of our iterator"); 1201 return CurrentPair; 1202 } 1203 1204 using BaseT::operator++; 1205 upward_defs_iterator &operator++() { 1206 assert(DefIterator != OriginalAccess->defs_end() && 1207 "Tried to access past the end of the iterator"); 1208 ++DefIterator; 1209 if (DefIterator != OriginalAccess->defs_end()) 1210 fillInCurrentPair(); 1211 return *this; 1212 } 1213 1214 BasicBlock *getPhiArgBlock() const { return DefIterator.getPhiArgBlock(); } 1215 1216 private: 1217 /// Returns true if \p Ptr is guaranteed to be loop invariant for any possible 1218 /// loop. In particular, this guarantees that it only references a single 1219 /// MemoryLocation during execution of the containing function. 1220 bool IsGuaranteedLoopInvariant(Value *Ptr) const; 1221 1222 void fillInCurrentPair() { 1223 CurrentPair.first = *DefIterator; 1224 CurrentPair.second = Location; 1225 if (WalkingPhi && Location.Ptr) { 1226 // Mark size as unknown, if the location is not guaranteed to be 1227 // loop-invariant for any possible loop in the function. Setting the size 1228 // to unknown guarantees that any memory accesses that access locations 1229 // after the pointer are considered as clobbers, which is important to 1230 // catch loop carried dependences. 1231 if (Location.Ptr && 1232 !IsGuaranteedLoopInvariant(const_cast<Value *>(Location.Ptr))) 1233 CurrentPair.second = 1234 Location.getWithNewSize(LocationSize::beforeOrAfterPointer()); 1235 PHITransAddr Translator( 1236 const_cast<Value *>(Location.Ptr), 1237 OriginalAccess->getBlock()->getModule()->getDataLayout(), nullptr); 1238 1239 if (!Translator.PHITranslateValue(OriginalAccess->getBlock(), 1240 DefIterator.getPhiArgBlock(), DT, 1241 true)) { 1242 Value *TransAddr = Translator.getAddr(); 1243 if (TransAddr != Location.Ptr) { 1244 CurrentPair.second = CurrentPair.second.getWithNewPtr(TransAddr); 1245 1246 if (TransAddr && 1247 !IsGuaranteedLoopInvariant(const_cast<Value *>(TransAddr))) 1248 CurrentPair.second = CurrentPair.second.getWithNewSize( 1249 LocationSize::beforeOrAfterPointer()); 1250 1251 if (PerformedPhiTranslation) 1252 *PerformedPhiTranslation = true; 1253 } 1254 } 1255 } 1256 } 1257 1258 MemoryAccessPair CurrentPair; 1259 memoryaccess_def_iterator DefIterator; 1260 MemoryLocation Location; 1261 MemoryAccess *OriginalAccess = nullptr; 1262 DominatorTree *DT = nullptr; 1263 bool WalkingPhi = false; 1264 bool *PerformedPhiTranslation = nullptr; 1265 }; 1266 1267 inline upward_defs_iterator 1268 upward_defs_begin(const MemoryAccessPair &Pair, DominatorTree &DT, 1269 bool *PerformedPhiTranslation = nullptr) { 1270 return upward_defs_iterator(Pair, &DT, PerformedPhiTranslation); 1271 } 1272 1273 inline upward_defs_iterator upward_defs_end() { return upward_defs_iterator(); } 1274 1275 inline iterator_range<upward_defs_iterator> 1276 upward_defs(const MemoryAccessPair &Pair, DominatorTree &DT) { 1277 return make_range(upward_defs_begin(Pair, DT), upward_defs_end()); 1278 } 1279 1280 /// Walks the defining accesses of MemoryDefs. Stops after we hit something that 1281 /// has no defining use (e.g. a MemoryPhi or liveOnEntry). Note that, when 1282 /// comparing against a null def_chain_iterator, this will compare equal only 1283 /// after walking said Phi/liveOnEntry. 1284 /// 1285 /// The UseOptimizedChain flag specifies whether to walk the clobbering 1286 /// access chain, or all the accesses. 1287 /// 1288 /// Normally, MemoryDef are all just def/use linked together, so a def_chain on 1289 /// a MemoryDef will walk all MemoryDefs above it in the program until it hits 1290 /// a phi node. The optimized chain walks the clobbering access of a store. 1291 /// So if you are just trying to find, given a store, what the next 1292 /// thing that would clobber the same memory is, you want the optimized chain. 1293 template <class T, bool UseOptimizedChain = false> 1294 struct def_chain_iterator 1295 : public iterator_facade_base<def_chain_iterator<T, UseOptimizedChain>, 1296 std::forward_iterator_tag, MemoryAccess *> { 1297 def_chain_iterator() : MA(nullptr) {} 1298 def_chain_iterator(T MA) : MA(MA) {} 1299 1300 T operator*() const { return MA; } 1301 1302 def_chain_iterator &operator++() { 1303 // N.B. liveOnEntry has a null defining access. 1304 if (auto *MUD = dyn_cast<MemoryUseOrDef>(MA)) { 1305 if (UseOptimizedChain && MUD->isOptimized()) 1306 MA = MUD->getOptimized(); 1307 else 1308 MA = MUD->getDefiningAccess(); 1309 } else { 1310 MA = nullptr; 1311 } 1312 1313 return *this; 1314 } 1315 1316 bool operator==(const def_chain_iterator &O) const { return MA == O.MA; } 1317 1318 private: 1319 T MA; 1320 }; 1321 1322 template <class T> 1323 inline iterator_range<def_chain_iterator<T>> 1324 def_chain(T MA, MemoryAccess *UpTo = nullptr) { 1325 #ifdef EXPENSIVE_CHECKS 1326 assert((!UpTo || find(def_chain(MA), UpTo) != def_chain_iterator<T>()) && 1327 "UpTo isn't in the def chain!"); 1328 #endif 1329 return make_range(def_chain_iterator<T>(MA), def_chain_iterator<T>(UpTo)); 1330 } 1331 1332 template <class T> 1333 inline iterator_range<def_chain_iterator<T, true>> optimized_def_chain(T MA) { 1334 return make_range(def_chain_iterator<T, true>(MA), 1335 def_chain_iterator<T, true>(nullptr)); 1336 } 1337 1338 } // end namespace llvm 1339 1340 #endif // LLVM_ANALYSIS_MEMORYSSA_H 1341