1 //===- AArch64StackTagging.cpp - Stack tagging in IR --===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //===----------------------------------------------------------------------===//
9 
10 #include "AArch64.h"
11 #include "AArch64InstrInfo.h"
12 #include "AArch64Subtarget.h"
13 #include "AArch64TargetMachine.h"
14 #include "llvm/ADT/DenseMap.h"
15 #include "llvm/ADT/DepthFirstIterator.h"
16 #include "llvm/ADT/MapVector.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/Statistic.h"
19 #include "llvm/Analysis/AliasAnalysis.h"
20 #include "llvm/Analysis/CFG.h"
21 #include "llvm/Analysis/LoopInfo.h"
22 #include "llvm/Analysis/PostDominators.h"
23 #include "llvm/Analysis/ScalarEvolution.h"
24 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
25 #include "llvm/Analysis/StackSafetyAnalysis.h"
26 #include "llvm/Analysis/ValueTracking.h"
27 #include "llvm/CodeGen/LiveRegUnits.h"
28 #include "llvm/CodeGen/MachineBasicBlock.h"
29 #include "llvm/CodeGen/MachineFunction.h"
30 #include "llvm/CodeGen/MachineFunctionPass.h"
31 #include "llvm/CodeGen/MachineInstr.h"
32 #include "llvm/CodeGen/MachineInstrBuilder.h"
33 #include "llvm/CodeGen/MachineLoopInfo.h"
34 #include "llvm/CodeGen/MachineOperand.h"
35 #include "llvm/CodeGen/MachineRegisterInfo.h"
36 #include "llvm/CodeGen/TargetPassConfig.h"
37 #include "llvm/CodeGen/TargetRegisterInfo.h"
38 #include "llvm/IR/DebugLoc.h"
39 #include "llvm/IR/Dominators.h"
40 #include "llvm/IR/Function.h"
41 #include "llvm/IR/GetElementPtrTypeIterator.h"
42 #include "llvm/IR/IRBuilder.h"
43 #include "llvm/IR/InstIterator.h"
44 #include "llvm/IR/Instruction.h"
45 #include "llvm/IR/Instructions.h"
46 #include "llvm/IR/IntrinsicInst.h"
47 #include "llvm/IR/IntrinsicsAArch64.h"
48 #include "llvm/IR/Metadata.h"
49 #include "llvm/IR/ValueHandle.h"
50 #include "llvm/InitializePasses.h"
51 #include "llvm/Pass.h"
52 #include "llvm/Support/Casting.h"
53 #include "llvm/Support/Debug.h"
54 #include "llvm/Support/raw_ostream.h"
55 #include "llvm/Transforms/Utils/Local.h"
56 #include "llvm/Transforms/Utils/MemoryTaggingSupport.h"
57 #include <cassert>
58 #include <iterator>
59 #include <memory>
60 #include <utility>
61 
62 using namespace llvm;
63 
64 #define DEBUG_TYPE "aarch64-stack-tagging"
65 
66 static cl::opt<bool> ClMergeInit(
67     "stack-tagging-merge-init", cl::Hidden, cl::init(true),
68     cl::desc("merge stack variable initializers with tagging when possible"));
69 
70 static cl::opt<bool>
71     ClUseStackSafety("stack-tagging-use-stack-safety", cl::Hidden,
72                      cl::init(true),
73                      cl::desc("Use Stack Safety analysis results"));
74 
75 static cl::opt<unsigned> ClScanLimit("stack-tagging-merge-init-scan-limit",
76                                      cl::init(40), cl::Hidden);
77 
78 static cl::opt<unsigned>
79     ClMergeInitSizeLimit("stack-tagging-merge-init-size-limit", cl::init(272),
80                          cl::Hidden);
81 
82 static cl::opt<size_t> ClMaxLifetimes(
83     "stack-tagging-max-lifetimes-for-alloca", cl::Hidden, cl::init(3),
84     cl::ReallyHidden,
85     cl::desc("How many lifetime ends to handle for a single alloca."),
86     cl::Optional);
87 
88 static const Align kTagGranuleSize = Align(16);
89 
90 namespace {
91 
92 class InitializerBuilder {
93   uint64_t Size;
94   const DataLayout *DL;
95   Value *BasePtr;
96   Function *SetTagFn;
97   Function *SetTagZeroFn;
98   Function *StgpFn;
99 
100   // List of initializers sorted by start offset.
101   struct Range {
102     uint64_t Start, End;
103     Instruction *Inst;
104   };
105   SmallVector<Range, 4> Ranges;
106   // 8-aligned offset => 8-byte initializer
107   // Missing keys are zero initialized.
108   std::map<uint64_t, Value *> Out;
109 
110 public:
111   InitializerBuilder(uint64_t Size, const DataLayout *DL, Value *BasePtr,
112                      Function *SetTagFn, Function *SetTagZeroFn,
113                      Function *StgpFn)
114       : Size(Size), DL(DL), BasePtr(BasePtr), SetTagFn(SetTagFn),
115         SetTagZeroFn(SetTagZeroFn), StgpFn(StgpFn) {}
116 
117   bool addRange(uint64_t Start, uint64_t End, Instruction *Inst) {
118     auto I =
119         llvm::lower_bound(Ranges, Start, [](const Range &LHS, uint64_t RHS) {
120           return LHS.End <= RHS;
121         });
122     if (I != Ranges.end() && End > I->Start) {
123       // Overlap - bail.
124       return false;
125     }
126     Ranges.insert(I, {Start, End, Inst});
127     return true;
128   }
129 
130   bool addStore(uint64_t Offset, StoreInst *SI, const DataLayout *DL) {
131     int64_t StoreSize = DL->getTypeStoreSize(SI->getOperand(0)->getType());
132     if (!addRange(Offset, Offset + StoreSize, SI))
133       return false;
134     IRBuilder<> IRB(SI);
135     applyStore(IRB, Offset, Offset + StoreSize, SI->getOperand(0));
136     return true;
137   }
138 
139   bool addMemSet(uint64_t Offset, MemSetInst *MSI) {
140     uint64_t StoreSize = cast<ConstantInt>(MSI->getLength())->getZExtValue();
141     if (!addRange(Offset, Offset + StoreSize, MSI))
142       return false;
143     IRBuilder<> IRB(MSI);
144     applyMemSet(IRB, Offset, Offset + StoreSize,
145                 cast<ConstantInt>(MSI->getValue()));
146     return true;
147   }
148 
149   void applyMemSet(IRBuilder<> &IRB, int64_t Start, int64_t End,
150                    ConstantInt *V) {
151     // Out[] does not distinguish between zero and undef, and we already know
152     // that this memset does not overlap with any other initializer. Nothing to
153     // do for memset(0).
154     if (V->isZero())
155       return;
156     for (int64_t Offset = Start - Start % 8; Offset < End; Offset += 8) {
157       uint64_t Cst = 0x0101010101010101UL;
158       int LowBits = Offset < Start ? (Start - Offset) * 8 : 0;
159       if (LowBits)
160         Cst = (Cst >> LowBits) << LowBits;
161       int HighBits = End - Offset < 8 ? (8 - (End - Offset)) * 8 : 0;
162       if (HighBits)
163         Cst = (Cst << HighBits) >> HighBits;
164       ConstantInt *C =
165           ConstantInt::get(IRB.getInt64Ty(), Cst * V->getZExtValue());
166 
167       Value *&CurrentV = Out[Offset];
168       if (!CurrentV) {
169         CurrentV = C;
170       } else {
171         CurrentV = IRB.CreateOr(CurrentV, C);
172       }
173     }
174   }
175 
176   // Take a 64-bit slice of the value starting at the given offset (in bytes).
177   // Offset can be negative. Pad with zeroes on both sides when necessary.
178   Value *sliceValue(IRBuilder<> &IRB, Value *V, int64_t Offset) {
179     if (Offset > 0) {
180       V = IRB.CreateLShr(V, Offset * 8);
181       V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty());
182     } else if (Offset < 0) {
183       V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty());
184       V = IRB.CreateShl(V, -Offset * 8);
185     } else {
186       V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty());
187     }
188     return V;
189   }
190 
191   void applyStore(IRBuilder<> &IRB, int64_t Start, int64_t End,
192                   Value *StoredValue) {
193     StoredValue = flatten(IRB, StoredValue);
194     for (int64_t Offset = Start - Start % 8; Offset < End; Offset += 8) {
195       Value *V = sliceValue(IRB, StoredValue, Offset - Start);
196       Value *&CurrentV = Out[Offset];
197       if (!CurrentV) {
198         CurrentV = V;
199       } else {
200         CurrentV = IRB.CreateOr(CurrentV, V);
201       }
202     }
203   }
204 
205   void generate(IRBuilder<> &IRB) {
206     LLVM_DEBUG(dbgs() << "Combined initializer\n");
207     // No initializers => the entire allocation is undef.
208     if (Ranges.empty()) {
209       emitUndef(IRB, 0, Size);
210       return;
211     }
212 
213     // Look through 8-byte initializer list 16 bytes at a time;
214     // If one of the two 8-byte halfs is non-zero non-undef, emit STGP.
215     // Otherwise, emit zeroes up to next available item.
216     uint64_t LastOffset = 0;
217     for (uint64_t Offset = 0; Offset < Size; Offset += 16) {
218       auto I1 = Out.find(Offset);
219       auto I2 = Out.find(Offset + 8);
220       if (I1 == Out.end() && I2 == Out.end())
221         continue;
222 
223       if (Offset > LastOffset)
224         emitZeroes(IRB, LastOffset, Offset - LastOffset);
225 
226       Value *Store1 = I1 == Out.end() ? Constant::getNullValue(IRB.getInt64Ty())
227                                       : I1->second;
228       Value *Store2 = I2 == Out.end() ? Constant::getNullValue(IRB.getInt64Ty())
229                                       : I2->second;
230       emitPair(IRB, Offset, Store1, Store2);
231       LastOffset = Offset + 16;
232     }
233 
234     // memset(0) does not update Out[], therefore the tail can be either undef
235     // or zero.
236     if (LastOffset < Size)
237       emitZeroes(IRB, LastOffset, Size - LastOffset);
238 
239     for (const auto &R : Ranges) {
240       R.Inst->eraseFromParent();
241     }
242   }
243 
244   void emitZeroes(IRBuilder<> &IRB, uint64_t Offset, uint64_t Size) {
245     LLVM_DEBUG(dbgs() << "  [" << Offset << ", " << Offset + Size
246                       << ") zero\n");
247     Value *Ptr = BasePtr;
248     if (Offset)
249       Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset);
250     IRB.CreateCall(SetTagZeroFn,
251                    {Ptr, ConstantInt::get(IRB.getInt64Ty(), Size)});
252   }
253 
254   void emitUndef(IRBuilder<> &IRB, uint64_t Offset, uint64_t Size) {
255     LLVM_DEBUG(dbgs() << "  [" << Offset << ", " << Offset + Size
256                       << ") undef\n");
257     Value *Ptr = BasePtr;
258     if (Offset)
259       Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset);
260     IRB.CreateCall(SetTagFn, {Ptr, ConstantInt::get(IRB.getInt64Ty(), Size)});
261   }
262 
263   void emitPair(IRBuilder<> &IRB, uint64_t Offset, Value *A, Value *B) {
264     LLVM_DEBUG(dbgs() << "  [" << Offset << ", " << Offset + 16 << "):\n");
265     LLVM_DEBUG(dbgs() << "    " << *A << "\n    " << *B << "\n");
266     Value *Ptr = BasePtr;
267     if (Offset)
268       Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset);
269     IRB.CreateCall(StgpFn, {Ptr, A, B});
270   }
271 
272   Value *flatten(IRBuilder<> &IRB, Value *V) {
273     if (V->getType()->isIntegerTy())
274       return V;
275     // vector of pointers -> vector of ints
276     if (VectorType *VecTy = dyn_cast<VectorType>(V->getType())) {
277       LLVMContext &Ctx = IRB.getContext();
278       Type *EltTy = VecTy->getElementType();
279       if (EltTy->isPointerTy()) {
280         uint32_t EltSize = DL->getTypeSizeInBits(EltTy);
281         auto *NewTy = FixedVectorType::get(
282             IntegerType::get(Ctx, EltSize),
283             cast<FixedVectorType>(VecTy)->getNumElements());
284         V = IRB.CreatePointerCast(V, NewTy);
285       }
286     }
287     return IRB.CreateBitOrPointerCast(
288         V, IRB.getIntNTy(DL->getTypeStoreSize(V->getType()) * 8));
289   }
290 };
291 
292 class AArch64StackTagging : public FunctionPass {
293   const bool MergeInit;
294   const bool UseStackSafety;
295 
296 public:
297   static char ID; // Pass ID, replacement for typeid
298 
299   AArch64StackTagging(bool IsOptNone = false)
300       : FunctionPass(ID),
301         MergeInit(ClMergeInit.getNumOccurrences() ? ClMergeInit : !IsOptNone),
302         UseStackSafety(ClUseStackSafety.getNumOccurrences() ? ClUseStackSafety
303                                                             : !IsOptNone) {
304     initializeAArch64StackTaggingPass(*PassRegistry::getPassRegistry());
305   }
306 
307   void tagAlloca(AllocaInst *AI, Instruction *InsertBefore, Value *Ptr,
308                  uint64_t Size);
309   void untagAlloca(AllocaInst *AI, Instruction *InsertBefore, uint64_t Size);
310 
311   Instruction *collectInitializers(Instruction *StartInst, Value *StartPtr,
312                                    uint64_t Size, InitializerBuilder &IB);
313 
314   Instruction *insertBaseTaggedPointer(
315       const MapVector<AllocaInst *, memtag::AllocaInfo> &Allocas,
316       const DominatorTree *DT);
317   bool runOnFunction(Function &F) override;
318 
319   StringRef getPassName() const override { return "AArch64 Stack Tagging"; }
320 
321 private:
322   Function *F = nullptr;
323   Function *SetTagFunc = nullptr;
324   const DataLayout *DL = nullptr;
325   AAResults *AA = nullptr;
326   const StackSafetyGlobalInfo *SSI = nullptr;
327 
328   void getAnalysisUsage(AnalysisUsage &AU) const override {
329     AU.setPreservesCFG();
330     if (UseStackSafety)
331       AU.addRequired<StackSafetyGlobalInfoWrapperPass>();
332     if (MergeInit)
333       AU.addRequired<AAResultsWrapperPass>();
334   }
335 };
336 
337 } // end anonymous namespace
338 
339 char AArch64StackTagging::ID = 0;
340 
341 INITIALIZE_PASS_BEGIN(AArch64StackTagging, DEBUG_TYPE, "AArch64 Stack Tagging",
342                       false, false)
343 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
344 INITIALIZE_PASS_DEPENDENCY(StackSafetyGlobalInfoWrapperPass)
345 INITIALIZE_PASS_END(AArch64StackTagging, DEBUG_TYPE, "AArch64 Stack Tagging",
346                     false, false)
347 
348 FunctionPass *llvm::createAArch64StackTaggingPass(bool IsOptNone) {
349   return new AArch64StackTagging(IsOptNone);
350 }
351 
352 Instruction *AArch64StackTagging::collectInitializers(Instruction *StartInst,
353                                                       Value *StartPtr,
354                                                       uint64_t Size,
355                                                       InitializerBuilder &IB) {
356   MemoryLocation AllocaLoc{StartPtr, Size};
357   Instruction *LastInst = StartInst;
358   BasicBlock::iterator BI(StartInst);
359 
360   unsigned Count = 0;
361   for (; Count < ClScanLimit && !BI->isTerminator(); ++BI) {
362     if (!isa<DbgInfoIntrinsic>(*BI))
363       ++Count;
364 
365     if (isNoModRef(AA->getModRefInfo(&*BI, AllocaLoc)))
366       continue;
367 
368     if (!isa<StoreInst>(BI) && !isa<MemSetInst>(BI)) {
369       // If the instruction is readnone, ignore it, otherwise bail out.  We
370       // don't even allow readonly here because we don't want something like:
371       // A[1] = 2; strlen(A); A[2] = 2; -> memcpy(A, ...); strlen(A).
372       if (BI->mayWriteToMemory() || BI->mayReadFromMemory())
373         break;
374       continue;
375     }
376 
377     if (StoreInst *NextStore = dyn_cast<StoreInst>(BI)) {
378       if (!NextStore->isSimple())
379         break;
380 
381       // Check to see if this store is to a constant offset from the start ptr.
382       std::optional<int64_t> Offset =
383           NextStore->getPointerOperand()->getPointerOffsetFrom(StartPtr, *DL);
384       if (!Offset)
385         break;
386 
387       if (!IB.addStore(*Offset, NextStore, DL))
388         break;
389       LastInst = NextStore;
390     } else {
391       MemSetInst *MSI = cast<MemSetInst>(BI);
392 
393       if (MSI->isVolatile() || !isa<ConstantInt>(MSI->getLength()))
394         break;
395 
396       if (!isa<ConstantInt>(MSI->getValue()))
397         break;
398 
399       // Check to see if this store is to a constant offset from the start ptr.
400       std::optional<int64_t> Offset =
401           MSI->getDest()->getPointerOffsetFrom(StartPtr, *DL);
402       if (!Offset)
403         break;
404 
405       if (!IB.addMemSet(*Offset, MSI))
406         break;
407       LastInst = MSI;
408     }
409   }
410   return LastInst;
411 }
412 
413 void AArch64StackTagging::tagAlloca(AllocaInst *AI, Instruction *InsertBefore,
414                                     Value *Ptr, uint64_t Size) {
415   auto SetTagZeroFunc =
416       Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_settag_zero);
417   auto StgpFunc =
418       Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_stgp);
419 
420   InitializerBuilder IB(Size, DL, Ptr, SetTagFunc, SetTagZeroFunc, StgpFunc);
421   bool LittleEndian =
422       Triple(AI->getModule()->getTargetTriple()).isLittleEndian();
423   // Current implementation of initializer merging assumes little endianness.
424   if (MergeInit && !F->hasOptNone() && LittleEndian &&
425       Size < ClMergeInitSizeLimit) {
426     LLVM_DEBUG(dbgs() << "collecting initializers for " << *AI
427                       << ", size = " << Size << "\n");
428     InsertBefore = collectInitializers(InsertBefore, Ptr, Size, IB);
429   }
430 
431   IRBuilder<> IRB(InsertBefore);
432   IB.generate(IRB);
433 }
434 
435 void AArch64StackTagging::untagAlloca(AllocaInst *AI, Instruction *InsertBefore,
436                                       uint64_t Size) {
437   IRBuilder<> IRB(InsertBefore);
438   IRB.CreateCall(SetTagFunc, {IRB.CreatePointerCast(AI, IRB.getInt8PtrTy()),
439                               ConstantInt::get(IRB.getInt64Ty(), Size)});
440 }
441 
442 Instruction *AArch64StackTagging::insertBaseTaggedPointer(
443     const MapVector<AllocaInst *, memtag::AllocaInfo> &AllocasToInstrument,
444     const DominatorTree *DT) {
445   BasicBlock *PrologueBB = nullptr;
446   // Try sinking IRG as deep as possible to avoid hurting shrink wrap.
447   for (auto &I : AllocasToInstrument) {
448     const memtag::AllocaInfo &Info = I.second;
449     AllocaInst *AI = Info.AI;
450     if (!PrologueBB) {
451       PrologueBB = AI->getParent();
452       continue;
453     }
454     PrologueBB = DT->findNearestCommonDominator(PrologueBB, AI->getParent());
455   }
456   assert(PrologueBB);
457 
458   IRBuilder<> IRB(&PrologueBB->front());
459   Function *IRG_SP =
460       Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_irg_sp);
461   Instruction *Base =
462       IRB.CreateCall(IRG_SP, {Constant::getNullValue(IRB.getInt64Ty())});
463   Base->setName("basetag");
464   return Base;
465 }
466 
467 // FIXME: check for MTE extension
468 bool AArch64StackTagging::runOnFunction(Function &Fn) {
469   if (!Fn.hasFnAttribute(Attribute::SanitizeMemTag))
470     return false;
471 
472   if (UseStackSafety)
473     SSI = &getAnalysis<StackSafetyGlobalInfoWrapperPass>().getResult();
474   F = &Fn;
475   DL = &Fn.getParent()->getDataLayout();
476   if (MergeInit)
477     AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
478 
479   memtag::StackInfoBuilder SIB(SSI);
480   for (Instruction &I : instructions(F))
481     SIB.visit(I);
482   memtag::StackInfo &SInfo = SIB.get();
483 
484   if (SInfo.AllocasToInstrument.empty())
485     return false;
486 
487   std::unique_ptr<DominatorTree> DeleteDT;
488   DominatorTree *DT = nullptr;
489   if (auto *P = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
490     DT = &P->getDomTree();
491 
492   if (DT == nullptr) {
493     DeleteDT = std::make_unique<DominatorTree>(*F);
494     DT = DeleteDT.get();
495   }
496 
497   std::unique_ptr<PostDominatorTree> DeletePDT;
498   PostDominatorTree *PDT = nullptr;
499   if (auto *P = getAnalysisIfAvailable<PostDominatorTreeWrapperPass>())
500     PDT = &P->getPostDomTree();
501 
502   if (PDT == nullptr) {
503     DeletePDT = std::make_unique<PostDominatorTree>(*F);
504     PDT = DeletePDT.get();
505   }
506 
507   std::unique_ptr<LoopInfo> DeleteLI;
508   LoopInfo *LI = nullptr;
509   if (auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>()) {
510     LI = &LIWP->getLoopInfo();
511   } else {
512     DeleteLI = std::make_unique<LoopInfo>(*DT);
513     LI = DeleteLI.get();
514   }
515 
516   SetTagFunc =
517       Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_settag);
518 
519   Instruction *Base = insertBaseTaggedPointer(SInfo.AllocasToInstrument, DT);
520 
521   int NextTag = 0;
522   for (auto &I : SInfo.AllocasToInstrument) {
523     memtag::AllocaInfo &Info = I.second;
524     assert(Info.AI && SIB.isInterestingAlloca(*Info.AI));
525     TrackingVH<Instruction> OldAI = Info.AI;
526     memtag::alignAndPadAlloca(Info, kTagGranuleSize);
527     AllocaInst *AI = Info.AI;
528     int Tag = NextTag;
529     NextTag = (NextTag + 1) % 16;
530     // Replace alloca with tagp(alloca).
531     IRBuilder<> IRB(Info.AI->getNextNode());
532     Function *TagP = Intrinsic::getDeclaration(
533         F->getParent(), Intrinsic::aarch64_tagp, {Info.AI->getType()});
534     Instruction *TagPCall =
535         IRB.CreateCall(TagP, {Constant::getNullValue(Info.AI->getType()), Base,
536                               ConstantInt::get(IRB.getInt64Ty(), Tag)});
537     if (Info.AI->hasName())
538       TagPCall->setName(Info.AI->getName() + ".tag");
539     Info.AI->replaceAllUsesWith(TagPCall);
540     TagPCall->setOperand(0, Info.AI);
541 
542     // Calls to functions that may return twice (e.g. setjmp) confuse the
543     // postdominator analysis, and will leave us to keep memory tagged after
544     // function return. Work around this by always untagging at every return
545     // statement if return_twice functions are called.
546     bool StandardLifetime =
547         SInfo.UnrecognizedLifetimes.empty() &&
548         memtag::isStandardLifetime(Info.LifetimeStart, Info.LifetimeEnd, DT, LI,
549                                    ClMaxLifetimes) &&
550         !SInfo.CallsReturnTwice;
551     if (StandardLifetime) {
552       IntrinsicInst *Start = Info.LifetimeStart[0];
553       uint64_t Size =
554           cast<ConstantInt>(Start->getArgOperand(0))->getZExtValue();
555       Size = alignTo(Size, kTagGranuleSize);
556       tagAlloca(AI, Start->getNextNode(), Start->getArgOperand(1), Size);
557 
558       auto TagEnd = [&](Instruction *Node) { untagAlloca(AI, Node, Size); };
559       if (!DT || !PDT ||
560           !memtag::forAllReachableExits(*DT, *PDT, *LI, Start, Info.LifetimeEnd,
561                                         SInfo.RetVec, TagEnd)) {
562         for (auto *End : Info.LifetimeEnd)
563           End->eraseFromParent();
564       }
565     } else {
566       uint64_t Size = *Info.AI->getAllocationSize(*DL);
567       Value *Ptr = IRB.CreatePointerCast(TagPCall, IRB.getInt8PtrTy());
568       tagAlloca(AI, &*IRB.GetInsertPoint(), Ptr, Size);
569       for (auto *RI : SInfo.RetVec) {
570         untagAlloca(AI, RI, Size);
571       }
572       // We may have inserted tag/untag outside of any lifetime interval.
573       // Remove all lifetime intrinsics for this alloca.
574       for (auto *II : Info.LifetimeStart)
575         II->eraseFromParent();
576       for (auto *II : Info.LifetimeEnd)
577         II->eraseFromParent();
578     }
579 
580     // Fixup debug intrinsics to point to the new alloca.
581     for (auto *DVI : Info.DbgVariableIntrinsics)
582       DVI->replaceVariableLocationOp(OldAI, Info.AI);
583   }
584 
585   // If we have instrumented at least one alloca, all unrecognized lifetime
586   // intrinsics have to go.
587   for (auto *I : SInfo.UnrecognizedLifetimes)
588     I->eraseFromParent();
589 
590   return true;
591 }
592