1 //===- Metadata.cpp - Implement Metadata classes --------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the Metadata classes. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "LLVMContextImpl.h" 14 #include "MetadataImpl.h" 15 #include "SymbolTableListTraitsImpl.h" 16 #include "llvm/ADT/APFloat.h" 17 #include "llvm/ADT/APInt.h" 18 #include "llvm/ADT/ArrayRef.h" 19 #include "llvm/ADT/DenseSet.h" 20 #include "llvm/ADT/None.h" 21 #include "llvm/ADT/STLExtras.h" 22 #include "llvm/ADT/SetVector.h" 23 #include "llvm/ADT/SmallPtrSet.h" 24 #include "llvm/ADT/SmallSet.h" 25 #include "llvm/ADT/SmallVector.h" 26 #include "llvm/ADT/StringMap.h" 27 #include "llvm/ADT/StringRef.h" 28 #include "llvm/ADT/Twine.h" 29 #include "llvm/IR/Argument.h" 30 #include "llvm/IR/BasicBlock.h" 31 #include "llvm/IR/Constant.h" 32 #include "llvm/IR/ConstantRange.h" 33 #include "llvm/IR/Constants.h" 34 #include "llvm/IR/DebugInfoMetadata.h" 35 #include "llvm/IR/DebugLoc.h" 36 #include "llvm/IR/Function.h" 37 #include "llvm/IR/GlobalObject.h" 38 #include "llvm/IR/GlobalVariable.h" 39 #include "llvm/IR/Instruction.h" 40 #include "llvm/IR/LLVMContext.h" 41 #include "llvm/IR/Metadata.h" 42 #include "llvm/IR/Module.h" 43 #include "llvm/IR/TrackingMDRef.h" 44 #include "llvm/IR/Type.h" 45 #include "llvm/IR/Value.h" 46 #include "llvm/IR/ValueHandle.h" 47 #include "llvm/Support/Casting.h" 48 #include "llvm/Support/ErrorHandling.h" 49 #include "llvm/Support/MathExtras.h" 50 #include <algorithm> 51 #include <cassert> 52 #include <cstddef> 53 #include <cstdint> 54 #include <iterator> 55 #include <tuple> 56 #include <type_traits> 57 #include <utility> 58 #include <vector> 59 60 using namespace llvm; 61 62 MetadataAsValue::MetadataAsValue(Type *Ty, Metadata *MD) 63 : Value(Ty, MetadataAsValueVal), MD(MD) { 64 track(); 65 } 66 67 MetadataAsValue::~MetadataAsValue() { 68 getType()->getContext().pImpl->MetadataAsValues.erase(MD); 69 untrack(); 70 } 71 72 /// Canonicalize metadata arguments to intrinsics. 73 /// 74 /// To support bitcode upgrades (and assembly semantic sugar) for \a 75 /// MetadataAsValue, we need to canonicalize certain metadata. 76 /// 77 /// - nullptr is replaced by an empty MDNode. 78 /// - An MDNode with a single null operand is replaced by an empty MDNode. 79 /// - An MDNode whose only operand is a \a ConstantAsMetadata gets skipped. 80 /// 81 /// This maintains readability of bitcode from when metadata was a type of 82 /// value, and these bridges were unnecessary. 83 static Metadata *canonicalizeMetadataForValue(LLVMContext &Context, 84 Metadata *MD) { 85 if (!MD) 86 // !{} 87 return MDNode::get(Context, None); 88 89 // Return early if this isn't a single-operand MDNode. 90 auto *N = dyn_cast<MDNode>(MD); 91 if (!N || N->getNumOperands() != 1) 92 return MD; 93 94 if (!N->getOperand(0)) 95 // !{} 96 return MDNode::get(Context, None); 97 98 if (auto *C = dyn_cast<ConstantAsMetadata>(N->getOperand(0))) 99 // Look through the MDNode. 100 return C; 101 102 return MD; 103 } 104 105 MetadataAsValue *MetadataAsValue::get(LLVMContext &Context, Metadata *MD) { 106 MD = canonicalizeMetadataForValue(Context, MD); 107 auto *&Entry = Context.pImpl->MetadataAsValues[MD]; 108 if (!Entry) 109 Entry = new MetadataAsValue(Type::getMetadataTy(Context), MD); 110 return Entry; 111 } 112 113 MetadataAsValue *MetadataAsValue::getIfExists(LLVMContext &Context, 114 Metadata *MD) { 115 MD = canonicalizeMetadataForValue(Context, MD); 116 auto &Store = Context.pImpl->MetadataAsValues; 117 return Store.lookup(MD); 118 } 119 120 void MetadataAsValue::handleChangedMetadata(Metadata *MD) { 121 LLVMContext &Context = getContext(); 122 MD = canonicalizeMetadataForValue(Context, MD); 123 auto &Store = Context.pImpl->MetadataAsValues; 124 125 // Stop tracking the old metadata. 126 Store.erase(this->MD); 127 untrack(); 128 this->MD = nullptr; 129 130 // Start tracking MD, or RAUW if necessary. 131 auto *&Entry = Store[MD]; 132 if (Entry) { 133 replaceAllUsesWith(Entry); 134 delete this; 135 return; 136 } 137 138 this->MD = MD; 139 track(); 140 Entry = this; 141 } 142 143 void MetadataAsValue::track() { 144 if (MD) 145 MetadataTracking::track(&MD, *MD, *this); 146 } 147 148 void MetadataAsValue::untrack() { 149 if (MD) 150 MetadataTracking::untrack(MD); 151 } 152 153 bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) { 154 assert(Ref && "Expected live reference"); 155 assert((Owner || *static_cast<Metadata **>(Ref) == &MD) && 156 "Reference without owner must be direct"); 157 if (auto *R = ReplaceableMetadataImpl::getOrCreate(MD)) { 158 R->addRef(Ref, Owner); 159 return true; 160 } 161 if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD)) { 162 assert(!PH->Use && "Placeholders can only be used once"); 163 assert(!Owner && "Unexpected callback to owner"); 164 PH->Use = static_cast<Metadata **>(Ref); 165 return true; 166 } 167 return false; 168 } 169 170 void MetadataTracking::untrack(void *Ref, Metadata &MD) { 171 assert(Ref && "Expected live reference"); 172 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) 173 R->dropRef(Ref); 174 else if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD)) 175 PH->Use = nullptr; 176 } 177 178 bool MetadataTracking::retrack(void *Ref, Metadata &MD, void *New) { 179 assert(Ref && "Expected live reference"); 180 assert(New && "Expected live reference"); 181 assert(Ref != New && "Expected change"); 182 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) { 183 R->moveRef(Ref, New, MD); 184 return true; 185 } 186 assert(!isa<DistinctMDOperandPlaceholder>(MD) && 187 "Unexpected move of an MDOperand"); 188 assert(!isReplaceable(MD) && 189 "Expected un-replaceable metadata, since we didn't move a reference"); 190 return false; 191 } 192 193 bool MetadataTracking::isReplaceable(const Metadata &MD) { 194 return ReplaceableMetadataImpl::isReplaceable(MD); 195 } 196 197 void ReplaceableMetadataImpl::addRef(void *Ref, OwnerTy Owner) { 198 bool WasInserted = 199 UseMap.insert(std::make_pair(Ref, std::make_pair(Owner, NextIndex))) 200 .second; 201 (void)WasInserted; 202 assert(WasInserted && "Expected to add a reference"); 203 204 ++NextIndex; 205 assert(NextIndex != 0 && "Unexpected overflow"); 206 } 207 208 void ReplaceableMetadataImpl::dropRef(void *Ref) { 209 bool WasErased = UseMap.erase(Ref); 210 (void)WasErased; 211 assert(WasErased && "Expected to drop a reference"); 212 } 213 214 void ReplaceableMetadataImpl::moveRef(void *Ref, void *New, 215 const Metadata &MD) { 216 auto I = UseMap.find(Ref); 217 assert(I != UseMap.end() && "Expected to move a reference"); 218 auto OwnerAndIndex = I->second; 219 UseMap.erase(I); 220 bool WasInserted = UseMap.insert(std::make_pair(New, OwnerAndIndex)).second; 221 (void)WasInserted; 222 assert(WasInserted && "Expected to add a reference"); 223 224 // Check that the references are direct if there's no owner. 225 (void)MD; 226 assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) && 227 "Reference without owner must be direct"); 228 assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) && 229 "Reference without owner must be direct"); 230 } 231 232 void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) { 233 if (UseMap.empty()) 234 return; 235 236 // Copy out uses since UseMap will get touched below. 237 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>; 238 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end()); 239 llvm::sort(Uses, [](const UseTy &L, const UseTy &R) { 240 return L.second.second < R.second.second; 241 }); 242 for (const auto &Pair : Uses) { 243 // Check that this Ref hasn't disappeared after RAUW (when updating a 244 // previous Ref). 245 if (!UseMap.count(Pair.first)) 246 continue; 247 248 OwnerTy Owner = Pair.second.first; 249 if (!Owner) { 250 // Update unowned tracking references directly. 251 Metadata *&Ref = *static_cast<Metadata **>(Pair.first); 252 Ref = MD; 253 if (MD) 254 MetadataTracking::track(Ref); 255 UseMap.erase(Pair.first); 256 continue; 257 } 258 259 // Check for MetadataAsValue. 260 if (Owner.is<MetadataAsValue *>()) { 261 Owner.get<MetadataAsValue *>()->handleChangedMetadata(MD); 262 continue; 263 } 264 265 // There's a Metadata owner -- dispatch. 266 Metadata *OwnerMD = Owner.get<Metadata *>(); 267 switch (OwnerMD->getMetadataID()) { 268 #define HANDLE_METADATA_LEAF(CLASS) \ 269 case Metadata::CLASS##Kind: \ 270 cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \ 271 continue; 272 #include "llvm/IR/Metadata.def" 273 default: 274 llvm_unreachable("Invalid metadata subclass"); 275 } 276 } 277 assert(UseMap.empty() && "Expected all uses to be replaced"); 278 } 279 280 void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) { 281 if (UseMap.empty()) 282 return; 283 284 if (!ResolveUsers) { 285 UseMap.clear(); 286 return; 287 } 288 289 // Copy out uses since UseMap could get touched below. 290 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>; 291 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end()); 292 llvm::sort(Uses, [](const UseTy &L, const UseTy &R) { 293 return L.second.second < R.second.second; 294 }); 295 UseMap.clear(); 296 for (const auto &Pair : Uses) { 297 auto Owner = Pair.second.first; 298 if (!Owner) 299 continue; 300 if (Owner.is<MetadataAsValue *>()) 301 continue; 302 303 // Resolve MDNodes that point at this. 304 auto *OwnerMD = dyn_cast<MDNode>(Owner.get<Metadata *>()); 305 if (!OwnerMD) 306 continue; 307 if (OwnerMD->isResolved()) 308 continue; 309 OwnerMD->decrementUnresolvedOperandCount(); 310 } 311 } 312 313 ReplaceableMetadataImpl *ReplaceableMetadataImpl::getOrCreate(Metadata &MD) { 314 if (auto *N = dyn_cast<MDNode>(&MD)) 315 return N->isResolved() ? nullptr : N->Context.getOrCreateReplaceableUses(); 316 return dyn_cast<ValueAsMetadata>(&MD); 317 } 318 319 ReplaceableMetadataImpl *ReplaceableMetadataImpl::getIfExists(Metadata &MD) { 320 if (auto *N = dyn_cast<MDNode>(&MD)) 321 return N->isResolved() ? nullptr : N->Context.getReplaceableUses(); 322 return dyn_cast<ValueAsMetadata>(&MD); 323 } 324 325 bool ReplaceableMetadataImpl::isReplaceable(const Metadata &MD) { 326 if (auto *N = dyn_cast<MDNode>(&MD)) 327 return !N->isResolved(); 328 return dyn_cast<ValueAsMetadata>(&MD); 329 } 330 331 static DISubprogram *getLocalFunctionMetadata(Value *V) { 332 assert(V && "Expected value"); 333 if (auto *A = dyn_cast<Argument>(V)) { 334 if (auto *Fn = A->getParent()) 335 return Fn->getSubprogram(); 336 return nullptr; 337 } 338 339 if (BasicBlock *BB = cast<Instruction>(V)->getParent()) { 340 if (auto *Fn = BB->getParent()) 341 return Fn->getSubprogram(); 342 return nullptr; 343 } 344 345 return nullptr; 346 } 347 348 ValueAsMetadata *ValueAsMetadata::get(Value *V) { 349 assert(V && "Unexpected null Value"); 350 351 auto &Context = V->getContext(); 352 auto *&Entry = Context.pImpl->ValuesAsMetadata[V]; 353 if (!Entry) { 354 assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) && 355 "Expected constant or function-local value"); 356 assert(!V->IsUsedByMD && "Expected this to be the only metadata use"); 357 V->IsUsedByMD = true; 358 if (auto *C = dyn_cast<Constant>(V)) 359 Entry = new ConstantAsMetadata(C); 360 else 361 Entry = new LocalAsMetadata(V); 362 } 363 364 return Entry; 365 } 366 367 ValueAsMetadata *ValueAsMetadata::getIfExists(Value *V) { 368 assert(V && "Unexpected null Value"); 369 return V->getContext().pImpl->ValuesAsMetadata.lookup(V); 370 } 371 372 void ValueAsMetadata::handleDeletion(Value *V) { 373 assert(V && "Expected valid value"); 374 375 auto &Store = V->getType()->getContext().pImpl->ValuesAsMetadata; 376 auto I = Store.find(V); 377 if (I == Store.end()) 378 return; 379 380 // Remove old entry from the map. 381 ValueAsMetadata *MD = I->second; 382 assert(MD && "Expected valid metadata"); 383 assert(MD->getValue() == V && "Expected valid mapping"); 384 Store.erase(I); 385 386 // Delete the metadata. 387 MD->replaceAllUsesWith(nullptr); 388 delete MD; 389 } 390 391 void ValueAsMetadata::handleRAUW(Value *From, Value *To) { 392 assert(From && "Expected valid value"); 393 assert(To && "Expected valid value"); 394 assert(From != To && "Expected changed value"); 395 assert(From->getType() == To->getType() && "Unexpected type change"); 396 397 LLVMContext &Context = From->getType()->getContext(); 398 auto &Store = Context.pImpl->ValuesAsMetadata; 399 auto I = Store.find(From); 400 if (I == Store.end()) { 401 assert(!From->IsUsedByMD && "Expected From not to be used by metadata"); 402 return; 403 } 404 405 // Remove old entry from the map. 406 assert(From->IsUsedByMD && "Expected From to be used by metadata"); 407 From->IsUsedByMD = false; 408 ValueAsMetadata *MD = I->second; 409 assert(MD && "Expected valid metadata"); 410 assert(MD->getValue() == From && "Expected valid mapping"); 411 Store.erase(I); 412 413 if (isa<LocalAsMetadata>(MD)) { 414 if (auto *C = dyn_cast<Constant>(To)) { 415 // Local became a constant. 416 MD->replaceAllUsesWith(ConstantAsMetadata::get(C)); 417 delete MD; 418 return; 419 } 420 if (getLocalFunctionMetadata(From) && getLocalFunctionMetadata(To) && 421 getLocalFunctionMetadata(From) != getLocalFunctionMetadata(To)) { 422 // DISubprogram changed. 423 MD->replaceAllUsesWith(nullptr); 424 delete MD; 425 return; 426 } 427 } else if (!isa<Constant>(To)) { 428 // Changed to function-local value. 429 MD->replaceAllUsesWith(nullptr); 430 delete MD; 431 return; 432 } 433 434 auto *&Entry = Store[To]; 435 if (Entry) { 436 // The target already exists. 437 MD->replaceAllUsesWith(Entry); 438 delete MD; 439 return; 440 } 441 442 // Update MD in place (and update the map entry). 443 assert(!To->IsUsedByMD && "Expected this to be the only metadata use"); 444 To->IsUsedByMD = true; 445 MD->V = To; 446 Entry = MD; 447 } 448 449 //===----------------------------------------------------------------------===// 450 // MDString implementation. 451 // 452 453 MDString *MDString::get(LLVMContext &Context, StringRef Str) { 454 auto &Store = Context.pImpl->MDStringCache; 455 auto I = Store.try_emplace(Str); 456 auto &MapEntry = I.first->getValue(); 457 if (!I.second) 458 return &MapEntry; 459 MapEntry.Entry = &*I.first; 460 return &MapEntry; 461 } 462 463 StringRef MDString::getString() const { 464 assert(Entry && "Expected to find string map entry"); 465 return Entry->first(); 466 } 467 468 //===----------------------------------------------------------------------===// 469 // MDNode implementation. 470 // 471 472 // Assert that the MDNode types will not be unaligned by the objects 473 // prepended to them. 474 #define HANDLE_MDNODE_LEAF(CLASS) \ 475 static_assert( \ 476 alignof(uint64_t) >= alignof(CLASS), \ 477 "Alignment is insufficient after objects prepended to " #CLASS); 478 #include "llvm/IR/Metadata.def" 479 480 void *MDNode::operator new(size_t Size, unsigned NumOps) { 481 size_t OpSize = NumOps * sizeof(MDOperand); 482 // uint64_t is the most aligned type we need support (ensured by static_assert 483 // above) 484 OpSize = alignTo(OpSize, alignof(uint64_t)); 485 void *Ptr = reinterpret_cast<char *>(::operator new(OpSize + Size)) + OpSize; 486 MDOperand *O = static_cast<MDOperand *>(Ptr); 487 for (MDOperand *E = O - NumOps; O != E; --O) 488 (void)new (O - 1) MDOperand; 489 return Ptr; 490 } 491 492 void MDNode::operator delete(void *Mem) { 493 MDNode *N = static_cast<MDNode *>(Mem); 494 size_t OpSize = N->NumOperands * sizeof(MDOperand); 495 OpSize = alignTo(OpSize, alignof(uint64_t)); 496 497 MDOperand *O = static_cast<MDOperand *>(Mem); 498 for (MDOperand *E = O - N->NumOperands; O != E; --O) 499 (O - 1)->~MDOperand(); 500 ::operator delete(reinterpret_cast<char *>(Mem) - OpSize); 501 } 502 503 MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage, 504 ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2) 505 : Metadata(ID, Storage), NumOperands(Ops1.size() + Ops2.size()), 506 NumUnresolved(0), Context(Context) { 507 unsigned Op = 0; 508 for (Metadata *MD : Ops1) 509 setOperand(Op++, MD); 510 for (Metadata *MD : Ops2) 511 setOperand(Op++, MD); 512 513 if (!isUniqued()) 514 return; 515 516 // Count the unresolved operands. If there are any, RAUW support will be 517 // added lazily on first reference. 518 countUnresolvedOperands(); 519 } 520 521 TempMDNode MDNode::clone() const { 522 switch (getMetadataID()) { 523 default: 524 llvm_unreachable("Invalid MDNode subclass"); 525 #define HANDLE_MDNODE_LEAF(CLASS) \ 526 case CLASS##Kind: \ 527 return cast<CLASS>(this)->cloneImpl(); 528 #include "llvm/IR/Metadata.def" 529 } 530 } 531 532 static bool isOperandUnresolved(Metadata *Op) { 533 if (auto *N = dyn_cast_or_null<MDNode>(Op)) 534 return !N->isResolved(); 535 return false; 536 } 537 538 void MDNode::countUnresolvedOperands() { 539 assert(NumUnresolved == 0 && "Expected unresolved ops to be uncounted"); 540 assert(isUniqued() && "Expected this to be uniqued"); 541 NumUnresolved = count_if(operands(), isOperandUnresolved); 542 } 543 544 void MDNode::makeUniqued() { 545 assert(isTemporary() && "Expected this to be temporary"); 546 assert(!isResolved() && "Expected this to be unresolved"); 547 548 // Enable uniquing callbacks. 549 for (auto &Op : mutable_operands()) 550 Op.reset(Op.get(), this); 551 552 // Make this 'uniqued'. 553 Storage = Uniqued; 554 countUnresolvedOperands(); 555 if (!NumUnresolved) { 556 dropReplaceableUses(); 557 assert(isResolved() && "Expected this to be resolved"); 558 } 559 560 assert(isUniqued() && "Expected this to be uniqued"); 561 } 562 563 void MDNode::makeDistinct() { 564 assert(isTemporary() && "Expected this to be temporary"); 565 assert(!isResolved() && "Expected this to be unresolved"); 566 567 // Drop RAUW support and store as a distinct node. 568 dropReplaceableUses(); 569 storeDistinctInContext(); 570 571 assert(isDistinct() && "Expected this to be distinct"); 572 assert(isResolved() && "Expected this to be resolved"); 573 } 574 575 void MDNode::resolve() { 576 assert(isUniqued() && "Expected this to be uniqued"); 577 assert(!isResolved() && "Expected this to be unresolved"); 578 579 NumUnresolved = 0; 580 dropReplaceableUses(); 581 582 assert(isResolved() && "Expected this to be resolved"); 583 } 584 585 void MDNode::dropReplaceableUses() { 586 assert(!NumUnresolved && "Unexpected unresolved operand"); 587 588 // Drop any RAUW support. 589 if (Context.hasReplaceableUses()) 590 Context.takeReplaceableUses()->resolveAllUses(); 591 } 592 593 void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) { 594 assert(isUniqued() && "Expected this to be uniqued"); 595 assert(NumUnresolved != 0 && "Expected unresolved operands"); 596 597 // Check if an operand was resolved. 598 if (!isOperandUnresolved(Old)) { 599 if (isOperandUnresolved(New)) 600 // An operand was un-resolved! 601 ++NumUnresolved; 602 } else if (!isOperandUnresolved(New)) 603 decrementUnresolvedOperandCount(); 604 } 605 606 void MDNode::decrementUnresolvedOperandCount() { 607 assert(!isResolved() && "Expected this to be unresolved"); 608 if (isTemporary()) 609 return; 610 611 assert(isUniqued() && "Expected this to be uniqued"); 612 if (--NumUnresolved) 613 return; 614 615 // Last unresolved operand has just been resolved. 616 dropReplaceableUses(); 617 assert(isResolved() && "Expected this to become resolved"); 618 } 619 620 void MDNode::resolveCycles() { 621 if (isResolved()) 622 return; 623 624 // Resolve this node immediately. 625 resolve(); 626 627 // Resolve all operands. 628 for (const auto &Op : operands()) { 629 auto *N = dyn_cast_or_null<MDNode>(Op); 630 if (!N) 631 continue; 632 633 assert(!N->isTemporary() && 634 "Expected all forward declarations to be resolved"); 635 if (!N->isResolved()) 636 N->resolveCycles(); 637 } 638 } 639 640 static bool hasSelfReference(MDNode *N) { 641 for (Metadata *MD : N->operands()) 642 if (MD == N) 643 return true; 644 return false; 645 } 646 647 MDNode *MDNode::replaceWithPermanentImpl() { 648 switch (getMetadataID()) { 649 default: 650 // If this type isn't uniquable, replace with a distinct node. 651 return replaceWithDistinctImpl(); 652 653 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 654 case CLASS##Kind: \ 655 break; 656 #include "llvm/IR/Metadata.def" 657 } 658 659 // Even if this type is uniquable, self-references have to be distinct. 660 if (hasSelfReference(this)) 661 return replaceWithDistinctImpl(); 662 return replaceWithUniquedImpl(); 663 } 664 665 MDNode *MDNode::replaceWithUniquedImpl() { 666 // Try to uniquify in place. 667 MDNode *UniquedNode = uniquify(); 668 669 if (UniquedNode == this) { 670 makeUniqued(); 671 return this; 672 } 673 674 // Collision, so RAUW instead. 675 replaceAllUsesWith(UniquedNode); 676 deleteAsSubclass(); 677 return UniquedNode; 678 } 679 680 MDNode *MDNode::replaceWithDistinctImpl() { 681 makeDistinct(); 682 return this; 683 } 684 685 void MDTuple::recalculateHash() { 686 setHash(MDTupleInfo::KeyTy::calculateHash(this)); 687 } 688 689 void MDNode::dropAllReferences() { 690 for (unsigned I = 0, E = NumOperands; I != E; ++I) 691 setOperand(I, nullptr); 692 if (Context.hasReplaceableUses()) { 693 Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false); 694 (void)Context.takeReplaceableUses(); 695 } 696 } 697 698 void MDNode::handleChangedOperand(void *Ref, Metadata *New) { 699 unsigned Op = static_cast<MDOperand *>(Ref) - op_begin(); 700 assert(Op < getNumOperands() && "Expected valid operand"); 701 702 if (!isUniqued()) { 703 // This node is not uniqued. Just set the operand and be done with it. 704 setOperand(Op, New); 705 return; 706 } 707 708 // This node is uniqued. 709 eraseFromStore(); 710 711 Metadata *Old = getOperand(Op); 712 setOperand(Op, New); 713 714 // Drop uniquing for self-reference cycles and deleted constants. 715 if (New == this || (!New && Old && isa<ConstantAsMetadata>(Old))) { 716 if (!isResolved()) 717 resolve(); 718 storeDistinctInContext(); 719 return; 720 } 721 722 // Re-unique the node. 723 auto *Uniqued = uniquify(); 724 if (Uniqued == this) { 725 if (!isResolved()) 726 resolveAfterOperandChange(Old, New); 727 return; 728 } 729 730 // Collision. 731 if (!isResolved()) { 732 // Still unresolved, so RAUW. 733 // 734 // First, clear out all operands to prevent any recursion (similar to 735 // dropAllReferences(), but we still need the use-list). 736 for (unsigned O = 0, E = getNumOperands(); O != E; ++O) 737 setOperand(O, nullptr); 738 if (Context.hasReplaceableUses()) 739 Context.getReplaceableUses()->replaceAllUsesWith(Uniqued); 740 deleteAsSubclass(); 741 return; 742 } 743 744 // Store in non-uniqued form if RAUW isn't possible. 745 storeDistinctInContext(); 746 } 747 748 void MDNode::deleteAsSubclass() { 749 switch (getMetadataID()) { 750 default: 751 llvm_unreachable("Invalid subclass of MDNode"); 752 #define HANDLE_MDNODE_LEAF(CLASS) \ 753 case CLASS##Kind: \ 754 delete cast<CLASS>(this); \ 755 break; 756 #include "llvm/IR/Metadata.def" 757 } 758 } 759 760 template <class T, class InfoT> 761 static T *uniquifyImpl(T *N, DenseSet<T *, InfoT> &Store) { 762 if (T *U = getUniqued(Store, N)) 763 return U; 764 765 Store.insert(N); 766 return N; 767 } 768 769 template <class NodeTy> struct MDNode::HasCachedHash { 770 using Yes = char[1]; 771 using No = char[2]; 772 template <class U, U Val> struct SFINAE {}; 773 774 template <class U> 775 static Yes &check(SFINAE<void (U::*)(unsigned), &U::setHash> *); 776 template <class U> static No &check(...); 777 778 static const bool value = sizeof(check<NodeTy>(nullptr)) == sizeof(Yes); 779 }; 780 781 MDNode *MDNode::uniquify() { 782 assert(!hasSelfReference(this) && "Cannot uniquify a self-referencing node"); 783 784 // Try to insert into uniquing store. 785 switch (getMetadataID()) { 786 default: 787 llvm_unreachable("Invalid or non-uniquable subclass of MDNode"); 788 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 789 case CLASS##Kind: { \ 790 CLASS *SubclassThis = cast<CLASS>(this); \ 791 std::integral_constant<bool, HasCachedHash<CLASS>::value> \ 792 ShouldRecalculateHash; \ 793 dispatchRecalculateHash(SubclassThis, ShouldRecalculateHash); \ 794 return uniquifyImpl(SubclassThis, getContext().pImpl->CLASS##s); \ 795 } 796 #include "llvm/IR/Metadata.def" 797 } 798 } 799 800 void MDNode::eraseFromStore() { 801 switch (getMetadataID()) { 802 default: 803 llvm_unreachable("Invalid or non-uniquable subclass of MDNode"); 804 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \ 805 case CLASS##Kind: \ 806 getContext().pImpl->CLASS##s.erase(cast<CLASS>(this)); \ 807 break; 808 #include "llvm/IR/Metadata.def" 809 } 810 } 811 812 MDTuple *MDTuple::getImpl(LLVMContext &Context, ArrayRef<Metadata *> MDs, 813 StorageType Storage, bool ShouldCreate) { 814 unsigned Hash = 0; 815 if (Storage == Uniqued) { 816 MDTupleInfo::KeyTy Key(MDs); 817 if (auto *N = getUniqued(Context.pImpl->MDTuples, Key)) 818 return N; 819 if (!ShouldCreate) 820 return nullptr; 821 Hash = Key.getHash(); 822 } else { 823 assert(ShouldCreate && "Expected non-uniqued nodes to always be created"); 824 } 825 826 return storeImpl(new (MDs.size()) MDTuple(Context, Storage, Hash, MDs), 827 Storage, Context.pImpl->MDTuples); 828 } 829 830 void MDNode::deleteTemporary(MDNode *N) { 831 assert(N->isTemporary() && "Expected temporary node"); 832 N->replaceAllUsesWith(nullptr); 833 N->deleteAsSubclass(); 834 } 835 836 void MDNode::storeDistinctInContext() { 837 assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses"); 838 assert(!NumUnresolved && "Unexpected unresolved nodes"); 839 Storage = Distinct; 840 assert(isResolved() && "Expected this to be resolved"); 841 842 // Reset the hash. 843 switch (getMetadataID()) { 844 default: 845 llvm_unreachable("Invalid subclass of MDNode"); 846 #define HANDLE_MDNODE_LEAF(CLASS) \ 847 case CLASS##Kind: { \ 848 std::integral_constant<bool, HasCachedHash<CLASS>::value> ShouldResetHash; \ 849 dispatchResetHash(cast<CLASS>(this), ShouldResetHash); \ 850 break; \ 851 } 852 #include "llvm/IR/Metadata.def" 853 } 854 855 getContext().pImpl->DistinctMDNodes.push_back(this); 856 } 857 858 void MDNode::replaceOperandWith(unsigned I, Metadata *New) { 859 if (getOperand(I) == New) 860 return; 861 862 if (!isUniqued()) { 863 setOperand(I, New); 864 return; 865 } 866 867 handleChangedOperand(mutable_begin() + I, New); 868 } 869 870 void MDNode::setOperand(unsigned I, Metadata *New) { 871 assert(I < NumOperands); 872 mutable_begin()[I].reset(New, isUniqued() ? this : nullptr); 873 } 874 875 /// Get a node or a self-reference that looks like it. 876 /// 877 /// Special handling for finding self-references, for use by \a 878 /// MDNode::concatenate() and \a MDNode::intersect() to maintain behaviour from 879 /// when self-referencing nodes were still uniqued. If the first operand has 880 /// the same operands as \c Ops, return the first operand instead. 881 static MDNode *getOrSelfReference(LLVMContext &Context, 882 ArrayRef<Metadata *> Ops) { 883 if (!Ops.empty()) 884 if (MDNode *N = dyn_cast_or_null<MDNode>(Ops[0])) 885 if (N->getNumOperands() == Ops.size() && N == N->getOperand(0)) { 886 for (unsigned I = 1, E = Ops.size(); I != E; ++I) 887 if (Ops[I] != N->getOperand(I)) 888 return MDNode::get(Context, Ops); 889 return N; 890 } 891 892 return MDNode::get(Context, Ops); 893 } 894 895 MDNode *MDNode::concatenate(MDNode *A, MDNode *B) { 896 if (!A) 897 return B; 898 if (!B) 899 return A; 900 901 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end()); 902 MDs.insert(B->op_begin(), B->op_end()); 903 904 // FIXME: This preserves long-standing behaviour, but is it really the right 905 // behaviour? Or was that an unintended side-effect of node uniquing? 906 return getOrSelfReference(A->getContext(), MDs.getArrayRef()); 907 } 908 909 MDNode *MDNode::intersect(MDNode *A, MDNode *B) { 910 if (!A || !B) 911 return nullptr; 912 913 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end()); 914 SmallPtrSet<Metadata *, 4> BSet(B->op_begin(), B->op_end()); 915 MDs.remove_if([&](Metadata *MD) { return !is_contained(BSet, MD); }); 916 917 // FIXME: This preserves long-standing behaviour, but is it really the right 918 // behaviour? Or was that an unintended side-effect of node uniquing? 919 return getOrSelfReference(A->getContext(), MDs.getArrayRef()); 920 } 921 922 MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) { 923 if (!A || !B) 924 return nullptr; 925 926 return concatenate(A, B); 927 } 928 929 MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) { 930 if (!A || !B) 931 return nullptr; 932 933 APFloat AVal = mdconst::extract<ConstantFP>(A->getOperand(0))->getValueAPF(); 934 APFloat BVal = mdconst::extract<ConstantFP>(B->getOperand(0))->getValueAPF(); 935 if (AVal.compare(BVal) == APFloat::cmpLessThan) 936 return A; 937 return B; 938 } 939 940 static bool isContiguous(const ConstantRange &A, const ConstantRange &B) { 941 return A.getUpper() == B.getLower() || A.getLower() == B.getUpper(); 942 } 943 944 static bool canBeMerged(const ConstantRange &A, const ConstantRange &B) { 945 return !A.intersectWith(B).isEmptySet() || isContiguous(A, B); 946 } 947 948 static bool tryMergeRange(SmallVectorImpl<ConstantInt *> &EndPoints, 949 ConstantInt *Low, ConstantInt *High) { 950 ConstantRange NewRange(Low->getValue(), High->getValue()); 951 unsigned Size = EndPoints.size(); 952 APInt LB = EndPoints[Size - 2]->getValue(); 953 APInt LE = EndPoints[Size - 1]->getValue(); 954 ConstantRange LastRange(LB, LE); 955 if (canBeMerged(NewRange, LastRange)) { 956 ConstantRange Union = LastRange.unionWith(NewRange); 957 Type *Ty = High->getType(); 958 EndPoints[Size - 2] = 959 cast<ConstantInt>(ConstantInt::get(Ty, Union.getLower())); 960 EndPoints[Size - 1] = 961 cast<ConstantInt>(ConstantInt::get(Ty, Union.getUpper())); 962 return true; 963 } 964 return false; 965 } 966 967 static void addRange(SmallVectorImpl<ConstantInt *> &EndPoints, 968 ConstantInt *Low, ConstantInt *High) { 969 if (!EndPoints.empty()) 970 if (tryMergeRange(EndPoints, Low, High)) 971 return; 972 973 EndPoints.push_back(Low); 974 EndPoints.push_back(High); 975 } 976 977 MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) { 978 // Given two ranges, we want to compute the union of the ranges. This 979 // is slightly complicated by having to combine the intervals and merge 980 // the ones that overlap. 981 982 if (!A || !B) 983 return nullptr; 984 985 if (A == B) 986 return A; 987 988 // First, walk both lists in order of the lower boundary of each interval. 989 // At each step, try to merge the new interval to the last one we adedd. 990 SmallVector<ConstantInt *, 4> EndPoints; 991 int AI = 0; 992 int BI = 0; 993 int AN = A->getNumOperands() / 2; 994 int BN = B->getNumOperands() / 2; 995 while (AI < AN && BI < BN) { 996 ConstantInt *ALow = mdconst::extract<ConstantInt>(A->getOperand(2 * AI)); 997 ConstantInt *BLow = mdconst::extract<ConstantInt>(B->getOperand(2 * BI)); 998 999 if (ALow->getValue().slt(BLow->getValue())) { 1000 addRange(EndPoints, ALow, 1001 mdconst::extract<ConstantInt>(A->getOperand(2 * AI + 1))); 1002 ++AI; 1003 } else { 1004 addRange(EndPoints, BLow, 1005 mdconst::extract<ConstantInt>(B->getOperand(2 * BI + 1))); 1006 ++BI; 1007 } 1008 } 1009 while (AI < AN) { 1010 addRange(EndPoints, mdconst::extract<ConstantInt>(A->getOperand(2 * AI)), 1011 mdconst::extract<ConstantInt>(A->getOperand(2 * AI + 1))); 1012 ++AI; 1013 } 1014 while (BI < BN) { 1015 addRange(EndPoints, mdconst::extract<ConstantInt>(B->getOperand(2 * BI)), 1016 mdconst::extract<ConstantInt>(B->getOperand(2 * BI + 1))); 1017 ++BI; 1018 } 1019 1020 // If we have more than 2 ranges (4 endpoints) we have to try to merge 1021 // the last and first ones. 1022 unsigned Size = EndPoints.size(); 1023 if (Size > 4) { 1024 ConstantInt *FB = EndPoints[0]; 1025 ConstantInt *FE = EndPoints[1]; 1026 if (tryMergeRange(EndPoints, FB, FE)) { 1027 for (unsigned i = 0; i < Size - 2; ++i) { 1028 EndPoints[i] = EndPoints[i + 2]; 1029 } 1030 EndPoints.resize(Size - 2); 1031 } 1032 } 1033 1034 // If in the end we have a single range, it is possible that it is now the 1035 // full range. Just drop the metadata in that case. 1036 if (EndPoints.size() == 2) { 1037 ConstantRange Range(EndPoints[0]->getValue(), EndPoints[1]->getValue()); 1038 if (Range.isFullSet()) 1039 return nullptr; 1040 } 1041 1042 SmallVector<Metadata *, 4> MDs; 1043 MDs.reserve(EndPoints.size()); 1044 for (auto *I : EndPoints) 1045 MDs.push_back(ConstantAsMetadata::get(I)); 1046 return MDNode::get(A->getContext(), MDs); 1047 } 1048 1049 MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) { 1050 if (!A || !B) 1051 return nullptr; 1052 1053 ConstantInt *AVal = mdconst::extract<ConstantInt>(A->getOperand(0)); 1054 ConstantInt *BVal = mdconst::extract<ConstantInt>(B->getOperand(0)); 1055 if (AVal->getZExtValue() < BVal->getZExtValue()) 1056 return A; 1057 return B; 1058 } 1059 1060 //===----------------------------------------------------------------------===// 1061 // NamedMDNode implementation. 1062 // 1063 1064 static SmallVector<TrackingMDRef, 4> &getNMDOps(void *Operands) { 1065 return *(SmallVector<TrackingMDRef, 4> *)Operands; 1066 } 1067 1068 NamedMDNode::NamedMDNode(const Twine &N) 1069 : Name(N.str()), Operands(new SmallVector<TrackingMDRef, 4>()) {} 1070 1071 NamedMDNode::~NamedMDNode() { 1072 dropAllReferences(); 1073 delete &getNMDOps(Operands); 1074 } 1075 1076 unsigned NamedMDNode::getNumOperands() const { 1077 return (unsigned)getNMDOps(Operands).size(); 1078 } 1079 1080 MDNode *NamedMDNode::getOperand(unsigned i) const { 1081 assert(i < getNumOperands() && "Invalid Operand number!"); 1082 auto *N = getNMDOps(Operands)[i].get(); 1083 return cast_or_null<MDNode>(N); 1084 } 1085 1086 void NamedMDNode::addOperand(MDNode *M) { getNMDOps(Operands).emplace_back(M); } 1087 1088 void NamedMDNode::setOperand(unsigned I, MDNode *New) { 1089 assert(I < getNumOperands() && "Invalid operand number"); 1090 getNMDOps(Operands)[I].reset(New); 1091 } 1092 1093 void NamedMDNode::eraseFromParent() { getParent()->eraseNamedMetadata(this); } 1094 1095 void NamedMDNode::clearOperands() { getNMDOps(Operands).clear(); } 1096 1097 StringRef NamedMDNode::getName() const { return StringRef(Name); } 1098 1099 //===----------------------------------------------------------------------===// 1100 // Instruction Metadata method implementations. 1101 // 1102 void MDAttachmentMap::set(unsigned ID, MDNode &MD) { 1103 for (auto &I : Attachments) 1104 if (I.first == ID) { 1105 I.second.reset(&MD); 1106 return; 1107 } 1108 Attachments.emplace_back(std::piecewise_construct, std::make_tuple(ID), 1109 std::make_tuple(&MD)); 1110 } 1111 1112 bool MDAttachmentMap::erase(unsigned ID) { 1113 if (empty()) 1114 return false; 1115 1116 // Common case is one/last value. 1117 if (Attachments.back().first == ID) { 1118 Attachments.pop_back(); 1119 return true; 1120 } 1121 1122 for (auto I = Attachments.begin(), E = std::prev(Attachments.end()); I != E; 1123 ++I) 1124 if (I->first == ID) { 1125 *I = std::move(Attachments.back()); 1126 Attachments.pop_back(); 1127 return true; 1128 } 1129 1130 return false; 1131 } 1132 1133 MDNode *MDAttachmentMap::lookup(unsigned ID) const { 1134 for (const auto &I : Attachments) 1135 if (I.first == ID) 1136 return I.second; 1137 return nullptr; 1138 } 1139 1140 void MDAttachmentMap::getAll( 1141 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { 1142 Result.append(Attachments.begin(), Attachments.end()); 1143 1144 // Sort the resulting array so it is stable. 1145 if (Result.size() > 1) 1146 array_pod_sort(Result.begin(), Result.end()); 1147 } 1148 1149 void MDGlobalAttachmentMap::insert(unsigned ID, MDNode &MD) { 1150 Attachments.push_back({ID, TrackingMDNodeRef(&MD)}); 1151 } 1152 1153 MDNode *MDGlobalAttachmentMap::lookup(unsigned ID) const { 1154 for (const auto &A : Attachments) 1155 if (A.MDKind == ID) 1156 return A.Node; 1157 return nullptr; 1158 } 1159 1160 void MDGlobalAttachmentMap::get(unsigned ID, 1161 SmallVectorImpl<MDNode *> &Result) const { 1162 for (const auto &A : Attachments) 1163 if (A.MDKind == ID) 1164 Result.push_back(A.Node); 1165 } 1166 1167 bool MDGlobalAttachmentMap::erase(unsigned ID) { 1168 auto I = std::remove_if(Attachments.begin(), Attachments.end(), 1169 [ID](const Attachment &A) { return A.MDKind == ID; }); 1170 bool Changed = I != Attachments.end(); 1171 Attachments.erase(I, Attachments.end()); 1172 return Changed; 1173 } 1174 1175 void MDGlobalAttachmentMap::getAll( 1176 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { 1177 for (const auto &A : Attachments) 1178 Result.emplace_back(A.MDKind, A.Node); 1179 1180 // Sort the resulting array so it is stable with respect to metadata IDs. We 1181 // need to preserve the original insertion order though. 1182 llvm::stable_sort(Result, less_first()); 1183 } 1184 1185 void Instruction::setMetadata(StringRef Kind, MDNode *Node) { 1186 if (!Node && !hasMetadata()) 1187 return; 1188 setMetadata(getContext().getMDKindID(Kind), Node); 1189 } 1190 1191 MDNode *Instruction::getMetadataImpl(StringRef Kind) const { 1192 return getMetadataImpl(getContext().getMDKindID(Kind)); 1193 } 1194 1195 void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) { 1196 if (!hasMetadataHashEntry()) 1197 return; // Nothing to remove! 1198 1199 auto &InstructionMetadata = getContext().pImpl->InstructionMetadata; 1200 1201 SmallSet<unsigned, 4> KnownSet; 1202 KnownSet.insert(KnownIDs.begin(), KnownIDs.end()); 1203 if (KnownSet.empty()) { 1204 // Just drop our entry at the store. 1205 InstructionMetadata.erase(this); 1206 setHasMetadataHashEntry(false); 1207 return; 1208 } 1209 1210 auto &Info = InstructionMetadata[this]; 1211 Info.remove_if([&KnownSet](const std::pair<unsigned, TrackingMDNodeRef> &I) { 1212 return !KnownSet.count(I.first); 1213 }); 1214 1215 if (Info.empty()) { 1216 // Drop our entry at the store. 1217 InstructionMetadata.erase(this); 1218 setHasMetadataHashEntry(false); 1219 } 1220 } 1221 1222 void Instruction::setMetadata(unsigned KindID, MDNode *Node) { 1223 if (!Node && !hasMetadata()) 1224 return; 1225 1226 // Handle 'dbg' as a special case since it is not stored in the hash table. 1227 if (KindID == LLVMContext::MD_dbg) { 1228 DbgLoc = DebugLoc(Node); 1229 return; 1230 } 1231 1232 // Handle the case when we're adding/updating metadata on an instruction. 1233 if (Node) { 1234 auto &Info = getContext().pImpl->InstructionMetadata[this]; 1235 assert(!Info.empty() == hasMetadataHashEntry() && 1236 "HasMetadata bit is wonked"); 1237 if (Info.empty()) 1238 setHasMetadataHashEntry(true); 1239 Info.set(KindID, *Node); 1240 return; 1241 } 1242 1243 // Otherwise, we're removing metadata from an instruction. 1244 assert((hasMetadataHashEntry() == 1245 (getContext().pImpl->InstructionMetadata.count(this) > 0)) && 1246 "HasMetadata bit out of date!"); 1247 if (!hasMetadataHashEntry()) 1248 return; // Nothing to remove! 1249 auto &Info = getContext().pImpl->InstructionMetadata[this]; 1250 1251 // Handle removal of an existing value. 1252 Info.erase(KindID); 1253 1254 if (!Info.empty()) 1255 return; 1256 1257 getContext().pImpl->InstructionMetadata.erase(this); 1258 setHasMetadataHashEntry(false); 1259 } 1260 1261 void Instruction::setAAMetadata(const AAMDNodes &N) { 1262 setMetadata(LLVMContext::MD_tbaa, N.TBAA); 1263 setMetadata(LLVMContext::MD_alias_scope, N.Scope); 1264 setMetadata(LLVMContext::MD_noalias, N.NoAlias); 1265 } 1266 1267 MDNode *Instruction::getMetadataImpl(unsigned KindID) const { 1268 // Handle 'dbg' as a special case since it is not stored in the hash table. 1269 if (KindID == LLVMContext::MD_dbg) 1270 return DbgLoc.getAsMDNode(); 1271 1272 if (!hasMetadataHashEntry()) 1273 return nullptr; 1274 auto &Info = getContext().pImpl->InstructionMetadata[this]; 1275 assert(!Info.empty() && "bit out of sync with hash table"); 1276 1277 return Info.lookup(KindID); 1278 } 1279 1280 void Instruction::getAllMetadataImpl( 1281 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { 1282 Result.clear(); 1283 1284 // Handle 'dbg' as a special case since it is not stored in the hash table. 1285 if (DbgLoc) { 1286 Result.push_back( 1287 std::make_pair((unsigned)LLVMContext::MD_dbg, DbgLoc.getAsMDNode())); 1288 if (!hasMetadataHashEntry()) 1289 return; 1290 } 1291 1292 assert(hasMetadataHashEntry() && 1293 getContext().pImpl->InstructionMetadata.count(this) && 1294 "Shouldn't have called this"); 1295 const auto &Info = getContext().pImpl->InstructionMetadata.find(this)->second; 1296 assert(!Info.empty() && "Shouldn't have called this"); 1297 Info.getAll(Result); 1298 } 1299 1300 void Instruction::getAllMetadataOtherThanDebugLocImpl( 1301 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { 1302 Result.clear(); 1303 assert(hasMetadataHashEntry() && 1304 getContext().pImpl->InstructionMetadata.count(this) && 1305 "Shouldn't have called this"); 1306 const auto &Info = getContext().pImpl->InstructionMetadata.find(this)->second; 1307 assert(!Info.empty() && "Shouldn't have called this"); 1308 Info.getAll(Result); 1309 } 1310 1311 bool Instruction::extractProfMetadata(uint64_t &TrueVal, 1312 uint64_t &FalseVal) const { 1313 assert( 1314 (getOpcode() == Instruction::Br || getOpcode() == Instruction::Select) && 1315 "Looking for branch weights on something besides branch or select"); 1316 1317 auto *ProfileData = getMetadata(LLVMContext::MD_prof); 1318 if (!ProfileData || ProfileData->getNumOperands() != 3) 1319 return false; 1320 1321 auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0)); 1322 if (!ProfDataName || !ProfDataName->getString().equals("branch_weights")) 1323 return false; 1324 1325 auto *CITrue = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(1)); 1326 auto *CIFalse = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2)); 1327 if (!CITrue || !CIFalse) 1328 return false; 1329 1330 TrueVal = CITrue->getValue().getZExtValue(); 1331 FalseVal = CIFalse->getValue().getZExtValue(); 1332 1333 return true; 1334 } 1335 1336 bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) const { 1337 assert((getOpcode() == Instruction::Br || 1338 getOpcode() == Instruction::Select || 1339 getOpcode() == Instruction::Call || 1340 getOpcode() == Instruction::Invoke || 1341 getOpcode() == Instruction::Switch) && 1342 "Looking for branch weights on something besides branch"); 1343 1344 TotalVal = 0; 1345 auto *ProfileData = getMetadata(LLVMContext::MD_prof); 1346 if (!ProfileData) 1347 return false; 1348 1349 auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0)); 1350 if (!ProfDataName) 1351 return false; 1352 1353 if (ProfDataName->getString().equals("branch_weights")) { 1354 TotalVal = 0; 1355 for (unsigned i = 1; i < ProfileData->getNumOperands(); i++) { 1356 auto *V = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(i)); 1357 if (!V) 1358 return false; 1359 TotalVal += V->getValue().getZExtValue(); 1360 } 1361 return true; 1362 } else if (ProfDataName->getString().equals("VP") && 1363 ProfileData->getNumOperands() > 3) { 1364 TotalVal = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2)) 1365 ->getValue() 1366 .getZExtValue(); 1367 return true; 1368 } 1369 return false; 1370 } 1371 1372 void Instruction::clearMetadataHashEntries() { 1373 assert(hasMetadataHashEntry() && "Caller should check"); 1374 getContext().pImpl->InstructionMetadata.erase(this); 1375 setHasMetadataHashEntry(false); 1376 } 1377 1378 void GlobalObject::getMetadata(unsigned KindID, 1379 SmallVectorImpl<MDNode *> &MDs) const { 1380 if (hasMetadata()) 1381 getContext().pImpl->GlobalObjectMetadata[this].get(KindID, MDs); 1382 } 1383 1384 void GlobalObject::getMetadata(StringRef Kind, 1385 SmallVectorImpl<MDNode *> &MDs) const { 1386 if (hasMetadata()) 1387 getMetadata(getContext().getMDKindID(Kind), MDs); 1388 } 1389 1390 void GlobalObject::addMetadata(unsigned KindID, MDNode &MD) { 1391 if (!hasMetadata()) 1392 setHasMetadataHashEntry(true); 1393 1394 getContext().pImpl->GlobalObjectMetadata[this].insert(KindID, MD); 1395 } 1396 1397 void GlobalObject::addMetadata(StringRef Kind, MDNode &MD) { 1398 addMetadata(getContext().getMDKindID(Kind), MD); 1399 } 1400 1401 bool GlobalObject::eraseMetadata(unsigned KindID) { 1402 // Nothing to unset. 1403 if (!hasMetadata()) 1404 return false; 1405 1406 auto &Store = getContext().pImpl->GlobalObjectMetadata[this]; 1407 bool Changed = Store.erase(KindID); 1408 if (Store.empty()) 1409 clearMetadata(); 1410 return Changed; 1411 } 1412 1413 void GlobalObject::getAllMetadata( 1414 SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const { 1415 MDs.clear(); 1416 1417 if (!hasMetadata()) 1418 return; 1419 1420 getContext().pImpl->GlobalObjectMetadata[this].getAll(MDs); 1421 } 1422 1423 void GlobalObject::clearMetadata() { 1424 if (!hasMetadata()) 1425 return; 1426 getContext().pImpl->GlobalObjectMetadata.erase(this); 1427 setHasMetadataHashEntry(false); 1428 } 1429 1430 void GlobalObject::setMetadata(unsigned KindID, MDNode *N) { 1431 eraseMetadata(KindID); 1432 if (N) 1433 addMetadata(KindID, *N); 1434 } 1435 1436 void GlobalObject::setMetadata(StringRef Kind, MDNode *N) { 1437 setMetadata(getContext().getMDKindID(Kind), N); 1438 } 1439 1440 MDNode *GlobalObject::getMetadata(unsigned KindID) const { 1441 if (hasMetadata()) 1442 return getContext().pImpl->GlobalObjectMetadata[this].lookup(KindID); 1443 return nullptr; 1444 } 1445 1446 MDNode *GlobalObject::getMetadata(StringRef Kind) const { 1447 return getMetadata(getContext().getMDKindID(Kind)); 1448 } 1449 1450 void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) { 1451 SmallVector<std::pair<unsigned, MDNode *>, 8> MDs; 1452 Other->getAllMetadata(MDs); 1453 for (auto &MD : MDs) { 1454 // We need to adjust the type metadata offset. 1455 if (Offset != 0 && MD.first == LLVMContext::MD_type) { 1456 auto *OffsetConst = cast<ConstantInt>( 1457 cast<ConstantAsMetadata>(MD.second->getOperand(0))->getValue()); 1458 Metadata *TypeId = MD.second->getOperand(1); 1459 auto *NewOffsetMD = ConstantAsMetadata::get(ConstantInt::get( 1460 OffsetConst->getType(), OffsetConst->getValue() + Offset)); 1461 addMetadata(LLVMContext::MD_type, 1462 *MDNode::get(getContext(), {NewOffsetMD, TypeId})); 1463 continue; 1464 } 1465 // If an offset adjustment was specified we need to modify the DIExpression 1466 // to prepend the adjustment: 1467 // !DIExpression(DW_OP_plus, Offset, [original expr]) 1468 auto *Attachment = MD.second; 1469 if (Offset != 0 && MD.first == LLVMContext::MD_dbg) { 1470 DIGlobalVariable *GV = dyn_cast<DIGlobalVariable>(Attachment); 1471 DIExpression *E = nullptr; 1472 if (!GV) { 1473 auto *GVE = cast<DIGlobalVariableExpression>(Attachment); 1474 GV = GVE->getVariable(); 1475 E = GVE->getExpression(); 1476 } 1477 ArrayRef<uint64_t> OrigElements; 1478 if (E) 1479 OrigElements = E->getElements(); 1480 std::vector<uint64_t> Elements(OrigElements.size() + 2); 1481 Elements[0] = dwarf::DW_OP_plus_uconst; 1482 Elements[1] = Offset; 1483 llvm::copy(OrigElements, Elements.begin() + 2); 1484 E = DIExpression::get(getContext(), Elements); 1485 Attachment = DIGlobalVariableExpression::get(getContext(), GV, E); 1486 } 1487 addMetadata(MD.first, *Attachment); 1488 } 1489 } 1490 1491 void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) { 1492 addMetadata( 1493 LLVMContext::MD_type, 1494 *MDTuple::get(getContext(), 1495 {ConstantAsMetadata::get(ConstantInt::get( 1496 Type::getInt64Ty(getContext()), Offset)), 1497 TypeID})); 1498 } 1499 1500 void Function::setSubprogram(DISubprogram *SP) { 1501 setMetadata(LLVMContext::MD_dbg, SP); 1502 } 1503 1504 DISubprogram *Function::getSubprogram() const { 1505 return cast_or_null<DISubprogram>(getMetadata(LLVMContext::MD_dbg)); 1506 } 1507 1508 bool Function::isDebugInfoForProfiling() const { 1509 if (DISubprogram *SP = getSubprogram()) { 1510 if (DICompileUnit *CU = SP->getUnit()) { 1511 return CU->getDebugInfoForProfiling(); 1512 } 1513 } 1514 return false; 1515 } 1516 1517 void GlobalVariable::addDebugInfo(DIGlobalVariableExpression *GV) { 1518 addMetadata(LLVMContext::MD_dbg, *GV); 1519 } 1520 1521 void GlobalVariable::getDebugInfo( 1522 SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const { 1523 SmallVector<MDNode *, 1> MDs; 1524 getMetadata(LLVMContext::MD_dbg, MDs); 1525 for (MDNode *MD : MDs) 1526 GVs.push_back(cast<DIGlobalVariableExpression>(MD)); 1527 } 1528