1 //===- AArch64StackTagging.cpp - Stack tagging in IR --===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 //===----------------------------------------------------------------------===// 9 10 #include "AArch64.h" 11 #include "AArch64InstrInfo.h" 12 #include "AArch64Subtarget.h" 13 #include "AArch64TargetMachine.h" 14 #include "llvm/ADT/DenseMap.h" 15 #include "llvm/ADT/DepthFirstIterator.h" 16 #include "llvm/ADT/MapVector.h" 17 #include "llvm/ADT/SmallVector.h" 18 #include "llvm/ADT/Statistic.h" 19 #include "llvm/Analysis/AliasAnalysis.h" 20 #include "llvm/Analysis/CFG.h" 21 #include "llvm/Analysis/LoopInfo.h" 22 #include "llvm/Analysis/PostDominators.h" 23 #include "llvm/Analysis/ScalarEvolution.h" 24 #include "llvm/Analysis/ScalarEvolutionExpressions.h" 25 #include "llvm/Analysis/StackSafetyAnalysis.h" 26 #include "llvm/Analysis/ValueTracking.h" 27 #include "llvm/CodeGen/LiveRegUnits.h" 28 #include "llvm/CodeGen/MachineBasicBlock.h" 29 #include "llvm/CodeGen/MachineFunction.h" 30 #include "llvm/CodeGen/MachineFunctionPass.h" 31 #include "llvm/CodeGen/MachineInstr.h" 32 #include "llvm/CodeGen/MachineInstrBuilder.h" 33 #include "llvm/CodeGen/MachineLoopInfo.h" 34 #include "llvm/CodeGen/MachineOperand.h" 35 #include "llvm/CodeGen/MachineRegisterInfo.h" 36 #include "llvm/CodeGen/TargetPassConfig.h" 37 #include "llvm/CodeGen/TargetRegisterInfo.h" 38 #include "llvm/IR/DebugLoc.h" 39 #include "llvm/IR/Dominators.h" 40 #include "llvm/IR/Function.h" 41 #include "llvm/IR/GetElementPtrTypeIterator.h" 42 #include "llvm/IR/IRBuilder.h" 43 #include "llvm/IR/InstIterator.h" 44 #include "llvm/IR/Instruction.h" 45 #include "llvm/IR/Instructions.h" 46 #include "llvm/IR/IntrinsicInst.h" 47 #include "llvm/IR/IntrinsicsAArch64.h" 48 #include "llvm/IR/Metadata.h" 49 #include "llvm/IR/ValueHandle.h" 50 #include "llvm/InitializePasses.h" 51 #include "llvm/Pass.h" 52 #include "llvm/Support/Casting.h" 53 #include "llvm/Support/Debug.h" 54 #include "llvm/Support/raw_ostream.h" 55 #include "llvm/Transforms/Utils/Local.h" 56 #include "llvm/Transforms/Utils/MemoryTaggingSupport.h" 57 #include <cassert> 58 #include <iterator> 59 #include <memory> 60 #include <utility> 61 62 using namespace llvm; 63 64 #define DEBUG_TYPE "aarch64-stack-tagging" 65 66 static cl::opt<bool> ClMergeInit( 67 "stack-tagging-merge-init", cl::Hidden, cl::init(true), 68 cl::desc("merge stack variable initializers with tagging when possible")); 69 70 static cl::opt<bool> 71 ClUseStackSafety("stack-tagging-use-stack-safety", cl::Hidden, 72 cl::init(true), 73 cl::desc("Use Stack Safety analysis results")); 74 75 static cl::opt<unsigned> ClScanLimit("stack-tagging-merge-init-scan-limit", 76 cl::init(40), cl::Hidden); 77 78 static cl::opt<unsigned> 79 ClMergeInitSizeLimit("stack-tagging-merge-init-size-limit", cl::init(272), 80 cl::Hidden); 81 82 static cl::opt<size_t> ClMaxLifetimes( 83 "stack-tagging-max-lifetimes-for-alloca", cl::Hidden, cl::init(3), 84 cl::ReallyHidden, 85 cl::desc("How many lifetime ends to handle for a single alloca."), 86 cl::Optional); 87 88 static const Align kTagGranuleSize = Align(16); 89 90 namespace { 91 92 class InitializerBuilder { 93 uint64_t Size; 94 const DataLayout *DL; 95 Value *BasePtr; 96 Function *SetTagFn; 97 Function *SetTagZeroFn; 98 Function *StgpFn; 99 100 // List of initializers sorted by start offset. 101 struct Range { 102 uint64_t Start, End; 103 Instruction *Inst; 104 }; 105 SmallVector<Range, 4> Ranges; 106 // 8-aligned offset => 8-byte initializer 107 // Missing keys are zero initialized. 108 std::map<uint64_t, Value *> Out; 109 110 public: 111 InitializerBuilder(uint64_t Size, const DataLayout *DL, Value *BasePtr, 112 Function *SetTagFn, Function *SetTagZeroFn, 113 Function *StgpFn) 114 : Size(Size), DL(DL), BasePtr(BasePtr), SetTagFn(SetTagFn), 115 SetTagZeroFn(SetTagZeroFn), StgpFn(StgpFn) {} 116 117 bool addRange(uint64_t Start, uint64_t End, Instruction *Inst) { 118 auto I = 119 llvm::lower_bound(Ranges, Start, [](const Range &LHS, uint64_t RHS) { 120 return LHS.End <= RHS; 121 }); 122 if (I != Ranges.end() && End > I->Start) { 123 // Overlap - bail. 124 return false; 125 } 126 Ranges.insert(I, {Start, End, Inst}); 127 return true; 128 } 129 130 bool addStore(uint64_t Offset, StoreInst *SI, const DataLayout *DL) { 131 int64_t StoreSize = DL->getTypeStoreSize(SI->getOperand(0)->getType()); 132 if (!addRange(Offset, Offset + StoreSize, SI)) 133 return false; 134 IRBuilder<> IRB(SI); 135 applyStore(IRB, Offset, Offset + StoreSize, SI->getOperand(0)); 136 return true; 137 } 138 139 bool addMemSet(uint64_t Offset, MemSetInst *MSI) { 140 uint64_t StoreSize = cast<ConstantInt>(MSI->getLength())->getZExtValue(); 141 if (!addRange(Offset, Offset + StoreSize, MSI)) 142 return false; 143 IRBuilder<> IRB(MSI); 144 applyMemSet(IRB, Offset, Offset + StoreSize, 145 cast<ConstantInt>(MSI->getValue())); 146 return true; 147 } 148 149 void applyMemSet(IRBuilder<> &IRB, int64_t Start, int64_t End, 150 ConstantInt *V) { 151 // Out[] does not distinguish between zero and undef, and we already know 152 // that this memset does not overlap with any other initializer. Nothing to 153 // do for memset(0). 154 if (V->isZero()) 155 return; 156 for (int64_t Offset = Start - Start % 8; Offset < End; Offset += 8) { 157 uint64_t Cst = 0x0101010101010101UL; 158 int LowBits = Offset < Start ? (Start - Offset) * 8 : 0; 159 if (LowBits) 160 Cst = (Cst >> LowBits) << LowBits; 161 int HighBits = End - Offset < 8 ? (8 - (End - Offset)) * 8 : 0; 162 if (HighBits) 163 Cst = (Cst << HighBits) >> HighBits; 164 ConstantInt *C = 165 ConstantInt::get(IRB.getInt64Ty(), Cst * V->getZExtValue()); 166 167 Value *&CurrentV = Out[Offset]; 168 if (!CurrentV) { 169 CurrentV = C; 170 } else { 171 CurrentV = IRB.CreateOr(CurrentV, C); 172 } 173 } 174 } 175 176 // Take a 64-bit slice of the value starting at the given offset (in bytes). 177 // Offset can be negative. Pad with zeroes on both sides when necessary. 178 Value *sliceValue(IRBuilder<> &IRB, Value *V, int64_t Offset) { 179 if (Offset > 0) { 180 V = IRB.CreateLShr(V, Offset * 8); 181 V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty()); 182 } else if (Offset < 0) { 183 V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty()); 184 V = IRB.CreateShl(V, -Offset * 8); 185 } else { 186 V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty()); 187 } 188 return V; 189 } 190 191 void applyStore(IRBuilder<> &IRB, int64_t Start, int64_t End, 192 Value *StoredValue) { 193 StoredValue = flatten(IRB, StoredValue); 194 for (int64_t Offset = Start - Start % 8; Offset < End; Offset += 8) { 195 Value *V = sliceValue(IRB, StoredValue, Offset - Start); 196 Value *&CurrentV = Out[Offset]; 197 if (!CurrentV) { 198 CurrentV = V; 199 } else { 200 CurrentV = IRB.CreateOr(CurrentV, V); 201 } 202 } 203 } 204 205 void generate(IRBuilder<> &IRB) { 206 LLVM_DEBUG(dbgs() << "Combined initializer\n"); 207 // No initializers => the entire allocation is undef. 208 if (Ranges.empty()) { 209 emitUndef(IRB, 0, Size); 210 return; 211 } 212 213 // Look through 8-byte initializer list 16 bytes at a time; 214 // If one of the two 8-byte halfs is non-zero non-undef, emit STGP. 215 // Otherwise, emit zeroes up to next available item. 216 uint64_t LastOffset = 0; 217 for (uint64_t Offset = 0; Offset < Size; Offset += 16) { 218 auto I1 = Out.find(Offset); 219 auto I2 = Out.find(Offset + 8); 220 if (I1 == Out.end() && I2 == Out.end()) 221 continue; 222 223 if (Offset > LastOffset) 224 emitZeroes(IRB, LastOffset, Offset - LastOffset); 225 226 Value *Store1 = I1 == Out.end() ? Constant::getNullValue(IRB.getInt64Ty()) 227 : I1->second; 228 Value *Store2 = I2 == Out.end() ? Constant::getNullValue(IRB.getInt64Ty()) 229 : I2->second; 230 emitPair(IRB, Offset, Store1, Store2); 231 LastOffset = Offset + 16; 232 } 233 234 // memset(0) does not update Out[], therefore the tail can be either undef 235 // or zero. 236 if (LastOffset < Size) 237 emitZeroes(IRB, LastOffset, Size - LastOffset); 238 239 for (const auto &R : Ranges) { 240 R.Inst->eraseFromParent(); 241 } 242 } 243 244 void emitZeroes(IRBuilder<> &IRB, uint64_t Offset, uint64_t Size) { 245 LLVM_DEBUG(dbgs() << " [" << Offset << ", " << Offset + Size 246 << ") zero\n"); 247 Value *Ptr = BasePtr; 248 if (Offset) 249 Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset); 250 IRB.CreateCall(SetTagZeroFn, 251 {Ptr, ConstantInt::get(IRB.getInt64Ty(), Size)}); 252 } 253 254 void emitUndef(IRBuilder<> &IRB, uint64_t Offset, uint64_t Size) { 255 LLVM_DEBUG(dbgs() << " [" << Offset << ", " << Offset + Size 256 << ") undef\n"); 257 Value *Ptr = BasePtr; 258 if (Offset) 259 Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset); 260 IRB.CreateCall(SetTagFn, {Ptr, ConstantInt::get(IRB.getInt64Ty(), Size)}); 261 } 262 263 void emitPair(IRBuilder<> &IRB, uint64_t Offset, Value *A, Value *B) { 264 LLVM_DEBUG(dbgs() << " [" << Offset << ", " << Offset + 16 << "):\n"); 265 LLVM_DEBUG(dbgs() << " " << *A << "\n " << *B << "\n"); 266 Value *Ptr = BasePtr; 267 if (Offset) 268 Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset); 269 IRB.CreateCall(StgpFn, {Ptr, A, B}); 270 } 271 272 Value *flatten(IRBuilder<> &IRB, Value *V) { 273 if (V->getType()->isIntegerTy()) 274 return V; 275 // vector of pointers -> vector of ints 276 if (VectorType *VecTy = dyn_cast<VectorType>(V->getType())) { 277 LLVMContext &Ctx = IRB.getContext(); 278 Type *EltTy = VecTy->getElementType(); 279 if (EltTy->isPointerTy()) { 280 uint32_t EltSize = DL->getTypeSizeInBits(EltTy); 281 auto *NewTy = FixedVectorType::get( 282 IntegerType::get(Ctx, EltSize), 283 cast<FixedVectorType>(VecTy)->getNumElements()); 284 V = IRB.CreatePointerCast(V, NewTy); 285 } 286 } 287 return IRB.CreateBitOrPointerCast( 288 V, IRB.getIntNTy(DL->getTypeStoreSize(V->getType()) * 8)); 289 } 290 }; 291 292 class AArch64StackTagging : public FunctionPass { 293 const bool MergeInit; 294 const bool UseStackSafety; 295 296 public: 297 static char ID; // Pass ID, replacement for typeid 298 299 AArch64StackTagging(bool IsOptNone = false) 300 : FunctionPass(ID), 301 MergeInit(ClMergeInit.getNumOccurrences() ? ClMergeInit : !IsOptNone), 302 UseStackSafety(ClUseStackSafety.getNumOccurrences() ? ClUseStackSafety 303 : !IsOptNone) { 304 initializeAArch64StackTaggingPass(*PassRegistry::getPassRegistry()); 305 } 306 307 void tagAlloca(AllocaInst *AI, Instruction *InsertBefore, Value *Ptr, 308 uint64_t Size); 309 void untagAlloca(AllocaInst *AI, Instruction *InsertBefore, uint64_t Size); 310 311 Instruction *collectInitializers(Instruction *StartInst, Value *StartPtr, 312 uint64_t Size, InitializerBuilder &IB); 313 314 Instruction *insertBaseTaggedPointer( 315 const MapVector<AllocaInst *, memtag::AllocaInfo> &Allocas, 316 const DominatorTree *DT); 317 bool runOnFunction(Function &F) override; 318 319 StringRef getPassName() const override { return "AArch64 Stack Tagging"; } 320 321 private: 322 Function *F = nullptr; 323 Function *SetTagFunc = nullptr; 324 const DataLayout *DL = nullptr; 325 AAResults *AA = nullptr; 326 const StackSafetyGlobalInfo *SSI = nullptr; 327 328 void getAnalysisUsage(AnalysisUsage &AU) const override { 329 AU.setPreservesCFG(); 330 if (UseStackSafety) 331 AU.addRequired<StackSafetyGlobalInfoWrapperPass>(); 332 if (MergeInit) 333 AU.addRequired<AAResultsWrapperPass>(); 334 } 335 }; 336 337 } // end anonymous namespace 338 339 char AArch64StackTagging::ID = 0; 340 341 INITIALIZE_PASS_BEGIN(AArch64StackTagging, DEBUG_TYPE, "AArch64 Stack Tagging", 342 false, false) 343 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 344 INITIALIZE_PASS_DEPENDENCY(StackSafetyGlobalInfoWrapperPass) 345 INITIALIZE_PASS_END(AArch64StackTagging, DEBUG_TYPE, "AArch64 Stack Tagging", 346 false, false) 347 348 FunctionPass *llvm::createAArch64StackTaggingPass(bool IsOptNone) { 349 return new AArch64StackTagging(IsOptNone); 350 } 351 352 Instruction *AArch64StackTagging::collectInitializers(Instruction *StartInst, 353 Value *StartPtr, 354 uint64_t Size, 355 InitializerBuilder &IB) { 356 MemoryLocation AllocaLoc{StartPtr, Size}; 357 Instruction *LastInst = StartInst; 358 BasicBlock::iterator BI(StartInst); 359 360 unsigned Count = 0; 361 for (; Count < ClScanLimit && !BI->isTerminator(); ++BI) { 362 if (!isa<DbgInfoIntrinsic>(*BI)) 363 ++Count; 364 365 if (isNoModRef(AA->getModRefInfo(&*BI, AllocaLoc))) 366 continue; 367 368 if (!isa<StoreInst>(BI) && !isa<MemSetInst>(BI)) { 369 // If the instruction is readnone, ignore it, otherwise bail out. We 370 // don't even allow readonly here because we don't want something like: 371 // A[1] = 2; strlen(A); A[2] = 2; -> memcpy(A, ...); strlen(A). 372 if (BI->mayWriteToMemory() || BI->mayReadFromMemory()) 373 break; 374 continue; 375 } 376 377 if (StoreInst *NextStore = dyn_cast<StoreInst>(BI)) { 378 if (!NextStore->isSimple()) 379 break; 380 381 // Check to see if this store is to a constant offset from the start ptr. 382 std::optional<int64_t> Offset = 383 NextStore->getPointerOperand()->getPointerOffsetFrom(StartPtr, *DL); 384 if (!Offset) 385 break; 386 387 if (!IB.addStore(*Offset, NextStore, DL)) 388 break; 389 LastInst = NextStore; 390 } else { 391 MemSetInst *MSI = cast<MemSetInst>(BI); 392 393 if (MSI->isVolatile() || !isa<ConstantInt>(MSI->getLength())) 394 break; 395 396 if (!isa<ConstantInt>(MSI->getValue())) 397 break; 398 399 // Check to see if this store is to a constant offset from the start ptr. 400 std::optional<int64_t> Offset = 401 MSI->getDest()->getPointerOffsetFrom(StartPtr, *DL); 402 if (!Offset) 403 break; 404 405 if (!IB.addMemSet(*Offset, MSI)) 406 break; 407 LastInst = MSI; 408 } 409 } 410 return LastInst; 411 } 412 413 void AArch64StackTagging::tagAlloca(AllocaInst *AI, Instruction *InsertBefore, 414 Value *Ptr, uint64_t Size) { 415 auto SetTagZeroFunc = 416 Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_settag_zero); 417 auto StgpFunc = 418 Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_stgp); 419 420 InitializerBuilder IB(Size, DL, Ptr, SetTagFunc, SetTagZeroFunc, StgpFunc); 421 bool LittleEndian = 422 Triple(AI->getModule()->getTargetTriple()).isLittleEndian(); 423 // Current implementation of initializer merging assumes little endianness. 424 if (MergeInit && !F->hasOptNone() && LittleEndian && 425 Size < ClMergeInitSizeLimit) { 426 LLVM_DEBUG(dbgs() << "collecting initializers for " << *AI 427 << ", size = " << Size << "\n"); 428 InsertBefore = collectInitializers(InsertBefore, Ptr, Size, IB); 429 } 430 431 IRBuilder<> IRB(InsertBefore); 432 IB.generate(IRB); 433 } 434 435 void AArch64StackTagging::untagAlloca(AllocaInst *AI, Instruction *InsertBefore, 436 uint64_t Size) { 437 IRBuilder<> IRB(InsertBefore); 438 IRB.CreateCall(SetTagFunc, {IRB.CreatePointerCast(AI, IRB.getInt8PtrTy()), 439 ConstantInt::get(IRB.getInt64Ty(), Size)}); 440 } 441 442 Instruction *AArch64StackTagging::insertBaseTaggedPointer( 443 const MapVector<AllocaInst *, memtag::AllocaInfo> &AllocasToInstrument, 444 const DominatorTree *DT) { 445 BasicBlock *PrologueBB = nullptr; 446 // Try sinking IRG as deep as possible to avoid hurting shrink wrap. 447 for (auto &I : AllocasToInstrument) { 448 const memtag::AllocaInfo &Info = I.second; 449 AllocaInst *AI = Info.AI; 450 if (!PrologueBB) { 451 PrologueBB = AI->getParent(); 452 continue; 453 } 454 PrologueBB = DT->findNearestCommonDominator(PrologueBB, AI->getParent()); 455 } 456 assert(PrologueBB); 457 458 IRBuilder<> IRB(&PrologueBB->front()); 459 Function *IRG_SP = 460 Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_irg_sp); 461 Instruction *Base = 462 IRB.CreateCall(IRG_SP, {Constant::getNullValue(IRB.getInt64Ty())}); 463 Base->setName("basetag"); 464 return Base; 465 } 466 467 // FIXME: check for MTE extension 468 bool AArch64StackTagging::runOnFunction(Function &Fn) { 469 if (!Fn.hasFnAttribute(Attribute::SanitizeMemTag)) 470 return false; 471 472 if (UseStackSafety) 473 SSI = &getAnalysis<StackSafetyGlobalInfoWrapperPass>().getResult(); 474 F = &Fn; 475 DL = &Fn.getParent()->getDataLayout(); 476 if (MergeInit) 477 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 478 479 memtag::StackInfoBuilder SIB(SSI); 480 for (Instruction &I : instructions(F)) 481 SIB.visit(I); 482 memtag::StackInfo &SInfo = SIB.get(); 483 484 if (SInfo.AllocasToInstrument.empty()) 485 return false; 486 487 std::unique_ptr<DominatorTree> DeleteDT; 488 DominatorTree *DT = nullptr; 489 if (auto *P = getAnalysisIfAvailable<DominatorTreeWrapperPass>()) 490 DT = &P->getDomTree(); 491 492 if (DT == nullptr) { 493 DeleteDT = std::make_unique<DominatorTree>(*F); 494 DT = DeleteDT.get(); 495 } 496 497 std::unique_ptr<PostDominatorTree> DeletePDT; 498 PostDominatorTree *PDT = nullptr; 499 if (auto *P = getAnalysisIfAvailable<PostDominatorTreeWrapperPass>()) 500 PDT = &P->getPostDomTree(); 501 502 if (PDT == nullptr) { 503 DeletePDT = std::make_unique<PostDominatorTree>(*F); 504 PDT = DeletePDT.get(); 505 } 506 507 std::unique_ptr<LoopInfo> DeleteLI; 508 LoopInfo *LI = nullptr; 509 if (auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>()) { 510 LI = &LIWP->getLoopInfo(); 511 } else { 512 DeleteLI = std::make_unique<LoopInfo>(*DT); 513 LI = DeleteLI.get(); 514 } 515 516 SetTagFunc = 517 Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_settag); 518 519 Instruction *Base = insertBaseTaggedPointer(SInfo.AllocasToInstrument, DT); 520 521 int NextTag = 0; 522 for (auto &I : SInfo.AllocasToInstrument) { 523 memtag::AllocaInfo &Info = I.second; 524 assert(Info.AI && SIB.isInterestingAlloca(*Info.AI)); 525 TrackingVH<Instruction> OldAI = Info.AI; 526 memtag::alignAndPadAlloca(Info, kTagGranuleSize); 527 AllocaInst *AI = Info.AI; 528 int Tag = NextTag; 529 NextTag = (NextTag + 1) % 16; 530 // Replace alloca with tagp(alloca). 531 IRBuilder<> IRB(Info.AI->getNextNode()); 532 Function *TagP = Intrinsic::getDeclaration( 533 F->getParent(), Intrinsic::aarch64_tagp, {Info.AI->getType()}); 534 Instruction *TagPCall = 535 IRB.CreateCall(TagP, {Constant::getNullValue(Info.AI->getType()), Base, 536 ConstantInt::get(IRB.getInt64Ty(), Tag)}); 537 if (Info.AI->hasName()) 538 TagPCall->setName(Info.AI->getName() + ".tag"); 539 Info.AI->replaceAllUsesWith(TagPCall); 540 TagPCall->setOperand(0, Info.AI); 541 542 // Calls to functions that may return twice (e.g. setjmp) confuse the 543 // postdominator analysis, and will leave us to keep memory tagged after 544 // function return. Work around this by always untagging at every return 545 // statement if return_twice functions are called. 546 bool StandardLifetime = 547 SInfo.UnrecognizedLifetimes.empty() && 548 memtag::isStandardLifetime(Info.LifetimeStart, Info.LifetimeEnd, DT, LI, 549 ClMaxLifetimes) && 550 !SInfo.CallsReturnTwice; 551 if (StandardLifetime) { 552 IntrinsicInst *Start = Info.LifetimeStart[0]; 553 uint64_t Size = 554 cast<ConstantInt>(Start->getArgOperand(0))->getZExtValue(); 555 Size = alignTo(Size, kTagGranuleSize); 556 tagAlloca(AI, Start->getNextNode(), Start->getArgOperand(1), Size); 557 558 auto TagEnd = [&](Instruction *Node) { untagAlloca(AI, Node, Size); }; 559 if (!DT || !PDT || 560 !memtag::forAllReachableExits(*DT, *PDT, *LI, Start, Info.LifetimeEnd, 561 SInfo.RetVec, TagEnd)) { 562 for (auto *End : Info.LifetimeEnd) 563 End->eraseFromParent(); 564 } 565 } else { 566 uint64_t Size = *Info.AI->getAllocationSize(*DL); 567 Value *Ptr = IRB.CreatePointerCast(TagPCall, IRB.getInt8PtrTy()); 568 tagAlloca(AI, &*IRB.GetInsertPoint(), Ptr, Size); 569 for (auto *RI : SInfo.RetVec) { 570 untagAlloca(AI, RI, Size); 571 } 572 // We may have inserted tag/untag outside of any lifetime interval. 573 // Remove all lifetime intrinsics for this alloca. 574 for (auto *II : Info.LifetimeStart) 575 II->eraseFromParent(); 576 for (auto *II : Info.LifetimeEnd) 577 II->eraseFromParent(); 578 } 579 580 // Fixup debug intrinsics to point to the new alloca. 581 for (auto *DVI : Info.DbgVariableIntrinsics) 582 DVI->replaceVariableLocationOp(OldAI, Info.AI); 583 } 584 585 // If we have instrumented at least one alloca, all unrecognized lifetime 586 // intrinsics have to go. 587 for (auto *I : SInfo.UnrecognizedLifetimes) 588 I->eraseFromParent(); 589 590 return true; 591 } 592