1 //===- HWAddressSanitizer.cpp - memory access error detector --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file is a part of HWAddressSanitizer, an address basic correctness
11 /// checker based on tagged addressing.
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
15 #include "llvm/ADT/MapVector.h"
16 #include "llvm/ADT/STLExtras.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/Analysis/GlobalsModRef.h"
21 #include "llvm/Analysis/PostDominators.h"
22 #include "llvm/Analysis/StackSafetyAnalysis.h"
23 #include "llvm/Analysis/ValueTracking.h"
24 #include "llvm/BinaryFormat/Dwarf.h"
25 #include "llvm/BinaryFormat/ELF.h"
26 #include "llvm/IR/Attributes.h"
27 #include "llvm/IR/BasicBlock.h"
28 #include "llvm/IR/Constant.h"
29 #include "llvm/IR/Constants.h"
30 #include "llvm/IR/DataLayout.h"
31 #include "llvm/IR/DebugInfoMetadata.h"
32 #include "llvm/IR/DerivedTypes.h"
33 #include "llvm/IR/Dominators.h"
34 #include "llvm/IR/Function.h"
35 #include "llvm/IR/IRBuilder.h"
36 #include "llvm/IR/InlineAsm.h"
37 #include "llvm/IR/InstIterator.h"
38 #include "llvm/IR/Instruction.h"
39 #include "llvm/IR/Instructions.h"
40 #include "llvm/IR/IntrinsicInst.h"
41 #include "llvm/IR/Intrinsics.h"
42 #include "llvm/IR/LLVMContext.h"
43 #include "llvm/IR/MDBuilder.h"
44 #include "llvm/IR/Module.h"
45 #include "llvm/IR/NoFolder.h"
46 #include "llvm/IR/Type.h"
47 #include "llvm/IR/Value.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/CommandLine.h"
50 #include "llvm/Support/Debug.h"
51 #include "llvm/Support/raw_ostream.h"
52 #include "llvm/TargetParser/Triple.h"
53 #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
54 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
55 #include "llvm/Transforms/Utils/MemoryTaggingSupport.h"
56 #include "llvm/Transforms/Utils/ModuleUtils.h"
57 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
58 #include <optional>
59 
60 using namespace llvm;
61 
62 #define DEBUG_TYPE "hwasan"
63 
64 const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
65 const char kHwasanNoteName[] = "hwasan.note";
66 const char kHwasanInitName[] = "__hwasan_init";
67 const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
68 
69 const char kHwasanShadowMemoryDynamicAddress[] =
70     "__hwasan_shadow_memory_dynamic_address";
71 
72 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
73 static const size_t kNumberOfAccessSizes = 5;
74 
75 static const size_t kDefaultShadowScale = 4;
76 static const uint64_t kDynamicShadowSentinel =
77     std::numeric_limits<uint64_t>::max();
78 
79 static const unsigned kShadowBaseAlignment = 32;
80 
81 static cl::opt<std::string>
82     ClMemoryAccessCallbackPrefix("hwasan-memory-access-callback-prefix",
83                                  cl::desc("Prefix for memory access callbacks"),
84                                  cl::Hidden, cl::init("__hwasan_"));
85 
86 static cl::opt<bool> ClKasanMemIntrinCallbackPrefix(
87     "hwasan-kernel-mem-intrinsic-prefix",
88     cl::desc("Use prefix for memory intrinsics in KASAN mode"), cl::Hidden,
89     cl::init(false));
90 
91 static cl::opt<bool> ClInstrumentWithCalls(
92     "hwasan-instrument-with-calls",
93     cl::desc("instrument reads and writes with callbacks"), cl::Hidden,
94     cl::init(false));
95 
96 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
97                                        cl::desc("instrument read instructions"),
98                                        cl::Hidden, cl::init(true));
99 
100 static cl::opt<bool>
101     ClInstrumentWrites("hwasan-instrument-writes",
102                        cl::desc("instrument write instructions"), cl::Hidden,
103                        cl::init(true));
104 
105 static cl::opt<bool> ClInstrumentAtomics(
106     "hwasan-instrument-atomics",
107     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
108     cl::init(true));
109 
110 static cl::opt<bool> ClInstrumentByval("hwasan-instrument-byval",
111                                        cl::desc("instrument byval arguments"),
112                                        cl::Hidden, cl::init(true));
113 
114 static cl::opt<bool>
115     ClRecover("hwasan-recover",
116               cl::desc("Enable recovery mode (continue-after-error)."),
117               cl::Hidden, cl::init(false));
118 
119 static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
120                                        cl::desc("instrument stack (allocas)"),
121                                        cl::Hidden, cl::init(true));
122 
123 static cl::opt<bool>
124     ClUseStackSafety("hwasan-use-stack-safety", cl::Hidden, cl::init(true),
125                      cl::Hidden, cl::desc("Use Stack Safety analysis results"),
126                      cl::Optional);
127 
128 static cl::opt<size_t> ClMaxLifetimes(
129     "hwasan-max-lifetimes-for-alloca", cl::Hidden, cl::init(3),
130     cl::ReallyHidden,
131     cl::desc("How many lifetime ends to handle for a single alloca."),
132     cl::Optional);
133 
134 static cl::opt<bool>
135     ClUseAfterScope("hwasan-use-after-scope",
136                     cl::desc("detect use after scope within function"),
137                     cl::Hidden, cl::init(false));
138 
139 static cl::opt<bool> ClGenerateTagsWithCalls(
140     "hwasan-generate-tags-with-calls",
141     cl::desc("generate new tags with runtime library calls"), cl::Hidden,
142     cl::init(false));
143 
144 static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
145                                cl::Hidden, cl::init(false));
146 
147 static cl::opt<int> ClMatchAllTag(
148     "hwasan-match-all-tag",
149     cl::desc("don't report bad accesses via pointers with this tag"),
150     cl::Hidden, cl::init(-1));
151 
152 static cl::opt<bool>
153     ClEnableKhwasan("hwasan-kernel",
154                     cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
155                     cl::Hidden, cl::init(false));
156 
157 // These flags allow to change the shadow mapping and control how shadow memory
158 // is accessed. The shadow mapping looks like:
159 //    Shadow = (Mem >> scale) + offset
160 
161 static cl::opt<uint64_t>
162     ClMappingOffset("hwasan-mapping-offset",
163                     cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
164                     cl::Hidden, cl::init(0));
165 
166 static cl::opt<bool>
167     ClWithIfunc("hwasan-with-ifunc",
168                 cl::desc("Access dynamic shadow through an ifunc global on "
169                          "platforms that support this"),
170                 cl::Hidden, cl::init(false));
171 
172 static cl::opt<bool> ClWithTls(
173     "hwasan-with-tls",
174     cl::desc("Access dynamic shadow through an thread-local pointer on "
175              "platforms that support this"),
176     cl::Hidden, cl::init(true));
177 
178 // Mode for selecting how to insert frame record info into the stack ring
179 // buffer.
180 enum RecordStackHistoryMode {
181   // Do not record frame record info.
182   none,
183 
184   // Insert instructions into the prologue for storing into the stack ring
185   // buffer directly.
186   instr,
187 
188   // Add a call to __hwasan_add_frame_record in the runtime.
189   libcall,
190 };
191 
192 static cl::opt<RecordStackHistoryMode> ClRecordStackHistory(
193     "hwasan-record-stack-history",
194     cl::desc("Record stack frames with tagged allocations in a thread-local "
195              "ring buffer"),
196     cl::values(clEnumVal(none, "Do not record stack ring history"),
197                clEnumVal(instr, "Insert instructions into the prologue for "
198                                 "storing into the stack ring buffer directly"),
199                clEnumVal(libcall, "Add a call to __hwasan_add_frame_record for "
200                                   "storing into the stack ring buffer")),
201     cl::Hidden, cl::init(instr));
202 
203 static cl::opt<bool>
204     ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
205                               cl::desc("instrument memory intrinsics"),
206                               cl::Hidden, cl::init(true));
207 
208 static cl::opt<bool>
209     ClInstrumentLandingPads("hwasan-instrument-landing-pads",
210                             cl::desc("instrument landing pads"), cl::Hidden,
211                             cl::init(false));
212 
213 static cl::opt<bool> ClUseShortGranules(
214     "hwasan-use-short-granules",
215     cl::desc("use short granules in allocas and outlined checks"), cl::Hidden,
216     cl::init(false));
217 
218 static cl::opt<bool> ClInstrumentPersonalityFunctions(
219     "hwasan-instrument-personality-functions",
220     cl::desc("instrument personality functions"), cl::Hidden);
221 
222 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
223                                        cl::desc("inline all checks"),
224                                        cl::Hidden, cl::init(false));
225 
226 // Enabled from clang by "-fsanitize-hwaddress-experimental-aliasing".
227 static cl::opt<bool> ClUsePageAliases("hwasan-experimental-use-page-aliases",
228                                       cl::desc("Use page aliasing in HWASan"),
229                                       cl::Hidden, cl::init(false));
230 
231 namespace {
232 
233 bool shouldUsePageAliases(const Triple &TargetTriple) {
234   return ClUsePageAliases && TargetTriple.getArch() == Triple::x86_64;
235 }
236 
237 bool shouldInstrumentStack(const Triple &TargetTriple) {
238   return !shouldUsePageAliases(TargetTriple) && ClInstrumentStack;
239 }
240 
241 bool shouldInstrumentWithCalls(const Triple &TargetTriple) {
242   return ClInstrumentWithCalls.getNumOccurrences()
243              ? ClInstrumentWithCalls
244              : TargetTriple.getArch() == Triple::x86_64;
245 }
246 
247 bool mightUseStackSafetyAnalysis(bool DisableOptimization) {
248   return ClUseStackSafety.getNumOccurrences() ? ClUseStackSafety
249                                               : !DisableOptimization;
250 }
251 
252 bool shouldUseStackSafetyAnalysis(const Triple &TargetTriple,
253                                   bool DisableOptimization) {
254   return shouldInstrumentStack(TargetTriple) &&
255          mightUseStackSafetyAnalysis(DisableOptimization);
256 }
257 
258 bool shouldDetectUseAfterScope(const Triple &TargetTriple) {
259   return ClUseAfterScope && shouldInstrumentStack(TargetTriple);
260 }
261 
262 /// An instrumentation pass implementing detection of addressability bugs
263 /// using tagged pointers.
264 class HWAddressSanitizer {
265 public:
266   HWAddressSanitizer(Module &M, bool CompileKernel, bool Recover,
267                      const StackSafetyGlobalInfo *SSI)
268       : M(M), SSI(SSI) {
269     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
270     this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0
271                               ? ClEnableKhwasan
272                               : CompileKernel;
273 
274     initializeModule();
275   }
276 
277   void setSSI(const StackSafetyGlobalInfo *S) { SSI = S; }
278 
279   void sanitizeFunction(Function &F, FunctionAnalysisManager &FAM);
280   void initializeModule();
281   void createHwasanCtorComdat();
282 
283   void initializeCallbacks(Module &M);
284 
285   Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
286 
287   Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
288   Value *getShadowNonTls(IRBuilder<> &IRB);
289 
290   void untagPointerOperand(Instruction *I, Value *Addr);
291   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
292 
293   int64_t getAccessInfo(bool IsWrite, unsigned AccessSizeIndex);
294   void instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
295                                   unsigned AccessSizeIndex,
296                                   Instruction *InsertBefore);
297   void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
298                                  unsigned AccessSizeIndex,
299                                  Instruction *InsertBefore);
300   bool ignoreMemIntrinsic(MemIntrinsic *MI);
301   void instrumentMemIntrinsic(MemIntrinsic *MI);
302   bool instrumentMemAccess(InterestingMemoryOperand &O);
303   bool ignoreAccess(Instruction *Inst, Value *Ptr);
304   void getInterestingMemoryOperands(
305       Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
306 
307   void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
308   Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
309   Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
310   bool instrumentStack(memtag::StackInfo &Info, Value *StackTag, Value *UARTag,
311                        const DominatorTree &DT, const PostDominatorTree &PDT,
312                        const LoopInfo &LI);
313   Value *readRegister(IRBuilder<> &IRB, StringRef Name);
314   bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
315   Value *getNextTagWithCall(IRBuilder<> &IRB);
316   Value *getStackBaseTag(IRBuilder<> &IRB);
317   Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, unsigned AllocaNo);
318   Value *getUARTag(IRBuilder<> &IRB);
319 
320   Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
321   Value *applyTagMask(IRBuilder<> &IRB, Value *OldTag);
322   unsigned retagMask(unsigned AllocaNo);
323 
324   void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
325 
326   void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
327   void instrumentGlobals();
328 
329   Value *getPC(IRBuilder<> &IRB);
330   Value *getSP(IRBuilder<> &IRB);
331   Value *getFrameRecordInfo(IRBuilder<> &IRB);
332 
333   void instrumentPersonalityFunctions();
334 
335 private:
336   LLVMContext *C;
337   Module &M;
338   const StackSafetyGlobalInfo *SSI;
339   Triple TargetTriple;
340 
341   /// This struct defines the shadow mapping using the rule:
342   ///   shadow = (mem >> Scale) + Offset.
343   /// If InGlobal is true, then
344   ///   extern char __hwasan_shadow[];
345   ///   shadow = (mem >> Scale) + &__hwasan_shadow
346   /// If InTls is true, then
347   ///   extern char *__hwasan_tls;
348   ///   shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
349   ///
350   /// If WithFrameRecord is true, then __hwasan_tls will be used to access the
351   /// ring buffer for storing stack allocations on targets that support it.
352   struct ShadowMapping {
353     uint8_t Scale;
354     uint64_t Offset;
355     bool InGlobal;
356     bool InTls;
357     bool WithFrameRecord;
358 
359     void init(Triple &TargetTriple, bool InstrumentWithCalls);
360     Align getObjectAlignment() const { return Align(1ULL << Scale); }
361   };
362 
363   ShadowMapping Mapping;
364 
365   Type *VoidTy = Type::getVoidTy(M.getContext());
366   Type *IntptrTy;
367   Type *Int8PtrTy;
368   Type *Int8Ty;
369   Type *Int32Ty;
370   Type *Int64Ty = Type::getInt64Ty(M.getContext());
371 
372   bool CompileKernel;
373   bool Recover;
374   bool OutlinedChecks;
375   bool UseShortGranules;
376   bool InstrumentLandingPads;
377   bool InstrumentWithCalls;
378   bool InstrumentStack;
379   bool DetectUseAfterScope;
380   bool UsePageAliases;
381   bool UseMatchAllCallback;
382 
383   std::optional<uint8_t> MatchAllTag;
384 
385   unsigned PointerTagShift;
386   uint64_t TagMaskByte;
387 
388   Function *HwasanCtorFunction;
389 
390   FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
391   FunctionCallee HwasanMemoryAccessCallbackSized[2];
392 
393   FunctionCallee HwasanMemmove, HwasanMemcpy, HwasanMemset;
394   FunctionCallee HwasanHandleVfork;
395 
396   FunctionCallee HwasanTagMemoryFunc;
397   FunctionCallee HwasanGenerateTagFunc;
398   FunctionCallee HwasanRecordFrameRecordFunc;
399 
400   Constant *ShadowGlobal;
401 
402   Value *ShadowBase = nullptr;
403   Value *StackBaseTag = nullptr;
404   Value *CachedSP = nullptr;
405   GlobalValue *ThreadPtrGlobal = nullptr;
406 };
407 
408 } // end anonymous namespace
409 
410 PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
411                                               ModuleAnalysisManager &MAM) {
412   const StackSafetyGlobalInfo *SSI = nullptr;
413   auto TargetTriple = llvm::Triple(M.getTargetTriple());
414   if (shouldUseStackSafetyAnalysis(TargetTriple, Options.DisableOptimization))
415     SSI = &MAM.getResult<StackSafetyGlobalAnalysis>(M);
416 
417   HWAddressSanitizer HWASan(M, Options.CompileKernel, Options.Recover, SSI);
418   auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
419   for (Function &F : M)
420     HWASan.sanitizeFunction(F, FAM);
421 
422   PreservedAnalyses PA = PreservedAnalyses::none();
423   // GlobalsAA is considered stateless and does not get invalidated unless
424   // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
425   // make changes that require GlobalsAA to be invalidated.
426   PA.abandon<GlobalsAA>();
427   return PA;
428 }
429 void HWAddressSanitizerPass::printPipeline(
430     raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
431   static_cast<PassInfoMixin<HWAddressSanitizerPass> *>(this)->printPipeline(
432       OS, MapClassName2PassName);
433   OS << '<';
434   if (Options.CompileKernel)
435     OS << "kernel;";
436   if (Options.Recover)
437     OS << "recover";
438   OS << '>';
439 }
440 
441 void HWAddressSanitizer::createHwasanCtorComdat() {
442   std::tie(HwasanCtorFunction, std::ignore) =
443       getOrCreateSanitizerCtorAndInitFunctions(
444           M, kHwasanModuleCtorName, kHwasanInitName,
445           /*InitArgTypes=*/{},
446           /*InitArgs=*/{},
447           // This callback is invoked when the functions are created the first
448           // time. Hook them into the global ctors list in that case:
449           [&](Function *Ctor, FunctionCallee) {
450             Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
451             Ctor->setComdat(CtorComdat);
452             appendToGlobalCtors(M, Ctor, 0, Ctor);
453           });
454 
455   // Create a note that contains pointers to the list of global
456   // descriptors. Adding a note to the output file will cause the linker to
457   // create a PT_NOTE program header pointing to the note that we can use to
458   // find the descriptor list starting from the program headers. A function
459   // provided by the runtime initializes the shadow memory for the globals by
460   // accessing the descriptor list via the note. The dynamic loader needs to
461   // call this function whenever a library is loaded.
462   //
463   // The reason why we use a note for this instead of a more conventional
464   // approach of having a global constructor pass a descriptor list pointer to
465   // the runtime is because of an order of initialization problem. With
466   // constructors we can encounter the following problematic scenario:
467   //
468   // 1) library A depends on library B and also interposes one of B's symbols
469   // 2) B's constructors are called before A's (as required for correctness)
470   // 3) during construction, B accesses one of its "own" globals (actually
471   //    interposed by A) and triggers a HWASAN failure due to the initialization
472   //    for A not having happened yet
473   //
474   // Even without interposition it is possible to run into similar situations in
475   // cases where two libraries mutually depend on each other.
476   //
477   // We only need one note per binary, so put everything for the note in a
478   // comdat. This needs to be a comdat with an .init_array section to prevent
479   // newer versions of lld from discarding the note.
480   //
481   // Create the note even if we aren't instrumenting globals. This ensures that
482   // binaries linked from object files with both instrumented and
483   // non-instrumented globals will end up with a note, even if a comdat from an
484   // object file with non-instrumented globals is selected. The note is harmless
485   // if the runtime doesn't support it, since it will just be ignored.
486   Comdat *NoteComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
487 
488   Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
489   auto *Start =
490       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
491                          nullptr, "__start_hwasan_globals");
492   Start->setVisibility(GlobalValue::HiddenVisibility);
493   auto *Stop =
494       new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
495                          nullptr, "__stop_hwasan_globals");
496   Stop->setVisibility(GlobalValue::HiddenVisibility);
497 
498   // Null-terminated so actually 8 bytes, which are required in order to align
499   // the note properly.
500   auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
501 
502   auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
503                                  Int32Ty, Int32Ty);
504   auto *Note =
505       new GlobalVariable(M, NoteTy, /*isConstant=*/true,
506                          GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
507   Note->setSection(".note.hwasan.globals");
508   Note->setComdat(NoteComdat);
509   Note->setAlignment(Align(4));
510 
511   // The pointers in the note need to be relative so that the note ends up being
512   // placed in rodata, which is the standard location for notes.
513   auto CreateRelPtr = [&](Constant *Ptr) {
514     return ConstantExpr::getTrunc(
515         ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
516                              ConstantExpr::getPtrToInt(Note, Int64Ty)),
517         Int32Ty);
518   };
519   Note->setInitializer(ConstantStruct::getAnon(
520       {ConstantInt::get(Int32Ty, 8),                           // n_namesz
521        ConstantInt::get(Int32Ty, 8),                           // n_descsz
522        ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
523        Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
524   appendToCompilerUsed(M, Note);
525 
526   // Create a zero-length global in hwasan_globals so that the linker will
527   // always create start and stop symbols.
528   auto *Dummy = new GlobalVariable(
529       M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
530       Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
531   Dummy->setSection("hwasan_globals");
532   Dummy->setComdat(NoteComdat);
533   Dummy->setMetadata(LLVMContext::MD_associated,
534                      MDNode::get(*C, ValueAsMetadata::get(Note)));
535   appendToCompilerUsed(M, Dummy);
536 }
537 
538 /// Module-level initialization.
539 ///
540 /// inserts a call to __hwasan_init to the module's constructor list.
541 void HWAddressSanitizer::initializeModule() {
542   LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
543   auto &DL = M.getDataLayout();
544 
545   TargetTriple = Triple(M.getTargetTriple());
546 
547   // x86_64 currently has two modes:
548   // - Intel LAM (default)
549   // - pointer aliasing (heap only)
550   bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
551   UsePageAliases = shouldUsePageAliases(TargetTriple);
552   InstrumentWithCalls = shouldInstrumentWithCalls(TargetTriple);
553   InstrumentStack = shouldInstrumentStack(TargetTriple);
554   DetectUseAfterScope = shouldDetectUseAfterScope(TargetTriple);
555   PointerTagShift = IsX86_64 ? 57 : 56;
556   TagMaskByte = IsX86_64 ? 0x3F : 0xFF;
557 
558   Mapping.init(TargetTriple, InstrumentWithCalls);
559 
560   C = &(M.getContext());
561   IRBuilder<> IRB(*C);
562   IntptrTy = IRB.getIntPtrTy(DL);
563   Int8PtrTy = IRB.getInt8PtrTy();
564   Int8Ty = IRB.getInt8Ty();
565   Int32Ty = IRB.getInt32Ty();
566 
567   HwasanCtorFunction = nullptr;
568 
569   // Older versions of Android do not have the required runtime support for
570   // short granules, global or personality function instrumentation. On other
571   // platforms we currently require using the latest version of the runtime.
572   bool NewRuntime =
573       !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30);
574 
575   UseShortGranules =
576       ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
577   OutlinedChecks =
578       (TargetTriple.isAArch64() || TargetTriple.isRISCV64()) &&
579       TargetTriple.isOSBinFormatELF() &&
580       (ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover);
581 
582   if (ClMatchAllTag.getNumOccurrences()) {
583     if (ClMatchAllTag != -1) {
584       MatchAllTag = ClMatchAllTag & 0xFF;
585     }
586   } else if (CompileKernel) {
587     MatchAllTag = 0xFF;
588   }
589   UseMatchAllCallback = !CompileKernel && MatchAllTag.has_value();
590 
591   // If we don't have personality function support, fall back to landing pads.
592   InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
593                               ? ClInstrumentLandingPads
594                               : !NewRuntime;
595 
596   if (!CompileKernel) {
597     createHwasanCtorComdat();
598     bool InstrumentGlobals =
599         ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime;
600 
601     if (InstrumentGlobals && !UsePageAliases)
602       instrumentGlobals();
603 
604     bool InstrumentPersonalityFunctions =
605         ClInstrumentPersonalityFunctions.getNumOccurrences()
606             ? ClInstrumentPersonalityFunctions
607             : NewRuntime;
608     if (InstrumentPersonalityFunctions)
609       instrumentPersonalityFunctions();
610   }
611 
612   if (!TargetTriple.isAndroid()) {
613     Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] {
614       auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
615                                     GlobalValue::ExternalLinkage, nullptr,
616                                     "__hwasan_tls", nullptr,
617                                     GlobalVariable::InitialExecTLSModel);
618       appendToCompilerUsed(M, GV);
619       return GV;
620     });
621     ThreadPtrGlobal = cast<GlobalVariable>(C);
622   }
623 }
624 
625 void HWAddressSanitizer::initializeCallbacks(Module &M) {
626   IRBuilder<> IRB(*C);
627   const std::string MatchAllStr = UseMatchAllCallback ? "_match_all" : "";
628   FunctionType *HwasanMemoryAccessCallbackSizedFnTy,
629       *HwasanMemoryAccessCallbackFnTy, *HwasanMemTransferFnTy,
630       *HwasanMemsetFnTy;
631   if (UseMatchAllCallback) {
632     HwasanMemoryAccessCallbackSizedFnTy =
633         FunctionType::get(VoidTy, {IntptrTy, IntptrTy, Int8Ty}, false);
634     HwasanMemoryAccessCallbackFnTy =
635         FunctionType::get(VoidTy, {IntptrTy, Int8Ty}, false);
636     HwasanMemTransferFnTy = FunctionType::get(
637         Int8PtrTy, {Int8PtrTy, Int8PtrTy, IntptrTy, Int8Ty}, false);
638     HwasanMemsetFnTy = FunctionType::get(
639         Int8PtrTy, {Int8PtrTy, Int32Ty, IntptrTy, Int8Ty}, false);
640   } else {
641     HwasanMemoryAccessCallbackSizedFnTy =
642         FunctionType::get(VoidTy, {IntptrTy, IntptrTy}, false);
643     HwasanMemoryAccessCallbackFnTy =
644         FunctionType::get(VoidTy, {IntptrTy}, false);
645     HwasanMemTransferFnTy =
646         FunctionType::get(Int8PtrTy, {Int8PtrTy, Int8PtrTy, IntptrTy}, false);
647     HwasanMemsetFnTy =
648         FunctionType::get(Int8PtrTy, {Int8PtrTy, Int32Ty, IntptrTy}, false);
649   }
650 
651   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
652     const std::string TypeStr = AccessIsWrite ? "store" : "load";
653     const std::string EndingStr = Recover ? "_noabort" : "";
654 
655     HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
656         ClMemoryAccessCallbackPrefix + TypeStr + "N" + MatchAllStr + EndingStr,
657         HwasanMemoryAccessCallbackSizedFnTy);
658 
659     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
660          AccessSizeIndex++) {
661       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
662           M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr +
663                                     itostr(1ULL << AccessSizeIndex) +
664                                     MatchAllStr + EndingStr,
665                                 HwasanMemoryAccessCallbackFnTy);
666     }
667   }
668 
669   const std::string MemIntrinCallbackPrefix =
670       (CompileKernel && !ClKasanMemIntrinCallbackPrefix)
671           ? std::string("")
672           : ClMemoryAccessCallbackPrefix;
673 
674   HwasanMemmove = M.getOrInsertFunction(
675       MemIntrinCallbackPrefix + "memmove" + MatchAllStr, HwasanMemTransferFnTy);
676   HwasanMemcpy = M.getOrInsertFunction(
677       MemIntrinCallbackPrefix + "memcpy" + MatchAllStr, HwasanMemTransferFnTy);
678   HwasanMemset = M.getOrInsertFunction(
679       MemIntrinCallbackPrefix + "memset" + MatchAllStr, HwasanMemsetFnTy);
680 
681   HwasanTagMemoryFunc = M.getOrInsertFunction("__hwasan_tag_memory", VoidTy,
682                                               Int8PtrTy, Int8Ty, IntptrTy);
683   HwasanGenerateTagFunc =
684       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
685 
686   HwasanRecordFrameRecordFunc =
687       M.getOrInsertFunction("__hwasan_add_frame_record", VoidTy, Int64Ty);
688 
689   ShadowGlobal =
690       M.getOrInsertGlobal("__hwasan_shadow", ArrayType::get(Int8Ty, 0));
691 
692   HwasanHandleVfork =
693       M.getOrInsertFunction("__hwasan_handle_vfork", VoidTy, IntptrTy);
694 }
695 
696 Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
697   // An empty inline asm with input reg == output reg.
698   // An opaque no-op cast, basically.
699   // This prevents code bloat as a result of rematerializing trivial definitions
700   // such as constants or global addresses at every load and store.
701   InlineAsm *Asm =
702       InlineAsm::get(FunctionType::get(Int8PtrTy, {Val->getType()}, false),
703                      StringRef(""), StringRef("=r,0"),
704                      /*hasSideEffects=*/false);
705   return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");
706 }
707 
708 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
709   return getOpaqueNoopCast(IRB, ShadowGlobal);
710 }
711 
712 Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
713   if (Mapping.Offset != kDynamicShadowSentinel)
714     return getOpaqueNoopCast(
715         IRB, ConstantExpr::getIntToPtr(
716                  ConstantInt::get(IntptrTy, Mapping.Offset), Int8PtrTy));
717 
718   if (Mapping.InGlobal)
719     return getDynamicShadowIfunc(IRB);
720 
721   Value *GlobalDynamicAddress =
722       IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
723           kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
724   return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);
725 }
726 
727 bool HWAddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
728   // Do not instrument accesses from different address spaces; we cannot deal
729   // with them.
730   Type *PtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
731   if (PtrTy->getPointerAddressSpace() != 0)
732     return true;
733 
734   // Ignore swifterror addresses.
735   // swifterror memory addresses are mem2reg promoted by instruction
736   // selection. As such they cannot have regular uses like an instrumentation
737   // function and it makes no sense to track them as memory.
738   if (Ptr->isSwiftError())
739     return true;
740 
741   if (findAllocaForValue(Ptr)) {
742     if (!InstrumentStack)
743       return true;
744     if (SSI && SSI->stackAccessIsSafe(*Inst))
745       return true;
746   }
747   return false;
748 }
749 
750 void HWAddressSanitizer::getInterestingMemoryOperands(
751     Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
752   // Skip memory accesses inserted by another instrumentation.
753   if (I->hasMetadata(LLVMContext::MD_nosanitize))
754     return;
755 
756   // Do not instrument the load fetching the dynamic shadow address.
757   if (ShadowBase == I)
758     return;
759 
760   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
761     if (!ClInstrumentReads || ignoreAccess(I, LI->getPointerOperand()))
762       return;
763     Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
764                              LI->getType(), LI->getAlign());
765   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
766     if (!ClInstrumentWrites || ignoreAccess(I, SI->getPointerOperand()))
767       return;
768     Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
769                              SI->getValueOperand()->getType(), SI->getAlign());
770   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
771     if (!ClInstrumentAtomics || ignoreAccess(I, RMW->getPointerOperand()))
772       return;
773     Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
774                              RMW->getValOperand()->getType(), std::nullopt);
775   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
776     if (!ClInstrumentAtomics || ignoreAccess(I, XCHG->getPointerOperand()))
777       return;
778     Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
779                              XCHG->getCompareOperand()->getType(),
780                              std::nullopt);
781   } else if (auto *CI = dyn_cast<CallInst>(I)) {
782     for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
783       if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
784           ignoreAccess(I, CI->getArgOperand(ArgNo)))
785         continue;
786       Type *Ty = CI->getParamByValType(ArgNo);
787       Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
788     }
789   }
790 }
791 
792 static unsigned getPointerOperandIndex(Instruction *I) {
793   if (LoadInst *LI = dyn_cast<LoadInst>(I))
794     return LI->getPointerOperandIndex();
795   if (StoreInst *SI = dyn_cast<StoreInst>(I))
796     return SI->getPointerOperandIndex();
797   if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
798     return RMW->getPointerOperandIndex();
799   if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
800     return XCHG->getPointerOperandIndex();
801   report_fatal_error("Unexpected instruction");
802   return -1;
803 }
804 
805 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
806   size_t Res = llvm::countr_zero(TypeSize / 8);
807   assert(Res < kNumberOfAccessSizes);
808   return Res;
809 }
810 
811 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
812   if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64 ||
813       TargetTriple.isRISCV64())
814     return;
815 
816   IRBuilder<> IRB(I);
817   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
818   Value *UntaggedPtr =
819       IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
820   I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
821 }
822 
823 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
824   // Mem >> Scale
825   Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
826   if (Mapping.Offset == 0)
827     return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
828   // (Mem >> Scale) + Offset
829   return IRB.CreateGEP(Int8Ty, ShadowBase, Shadow);
830 }
831 
832 int64_t HWAddressSanitizer::getAccessInfo(bool IsWrite,
833                                           unsigned AccessSizeIndex) {
834   return (CompileKernel << HWASanAccessInfo::CompileKernelShift) |
835          (MatchAllTag.has_value() << HWASanAccessInfo::HasMatchAllShift) |
836          (MatchAllTag.value_or(0) << HWASanAccessInfo::MatchAllShift) |
837          (Recover << HWASanAccessInfo::RecoverShift) |
838          (IsWrite << HWASanAccessInfo::IsWriteShift) |
839          (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
840 }
841 
842 void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
843                                                     unsigned AccessSizeIndex,
844                                                     Instruction *InsertBefore) {
845   assert(!UsePageAliases);
846   const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
847   IRBuilder<> IRB(InsertBefore);
848   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
849   Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
850   IRB.CreateCall(Intrinsic::getDeclaration(
851                      M, UseShortGranules
852                             ? Intrinsic::hwasan_check_memaccess_shortgranules
853                             : Intrinsic::hwasan_check_memaccess),
854                  {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
855 }
856 
857 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
858                                                    unsigned AccessSizeIndex,
859                                                    Instruction *InsertBefore) {
860   assert(!UsePageAliases);
861   const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
862   IRBuilder<> IRB(InsertBefore);
863 
864   Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
865   Value *PtrTag =
866       IRB.CreateTrunc(IRB.CreateLShr(PtrLong, PointerTagShift), Int8Ty);
867   Value *AddrLong = untagPointer(IRB, PtrLong);
868   Value *Shadow = memToShadow(AddrLong, IRB);
869   Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
870   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
871 
872   if (MatchAllTag.has_value()) {
873     Value *TagNotIgnored = IRB.CreateICmpNE(
874         PtrTag, ConstantInt::get(PtrTag->getType(), *MatchAllTag));
875     TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
876   }
877 
878   Instruction *CheckTerm =
879       SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false,
880                                 MDBuilder(*C).createBranchWeights(1, 100000));
881 
882   IRB.SetInsertPoint(CheckTerm);
883   Value *OutOfShortGranuleTagRange =
884       IRB.CreateICmpUGT(MemTag, ConstantInt::get(Int8Ty, 15));
885   Instruction *CheckFailTerm =
886       SplitBlockAndInsertIfThen(OutOfShortGranuleTagRange, CheckTerm, !Recover,
887                                 MDBuilder(*C).createBranchWeights(1, 100000));
888 
889   IRB.SetInsertPoint(CheckTerm);
890   Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(PtrLong, 15), Int8Ty);
891   PtrLowBits = IRB.CreateAdd(
892       PtrLowBits, ConstantInt::get(Int8Ty, (1 << AccessSizeIndex) - 1));
893   Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, MemTag);
894   SplitBlockAndInsertIfThen(PtrLowBitsOOB, CheckTerm, false,
895                             MDBuilder(*C).createBranchWeights(1, 100000),
896                             (DomTreeUpdater *)nullptr, nullptr,
897                             CheckFailTerm->getParent());
898 
899   IRB.SetInsertPoint(CheckTerm);
900   Value *InlineTagAddr = IRB.CreateOr(AddrLong, 15);
901   InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, Int8PtrTy);
902   Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr);
903   Value *InlineTagMismatch = IRB.CreateICmpNE(PtrTag, InlineTag);
904   SplitBlockAndInsertIfThen(InlineTagMismatch, CheckTerm, false,
905                             MDBuilder(*C).createBranchWeights(1, 100000),
906                             (DomTreeUpdater *)nullptr, nullptr,
907                             CheckFailTerm->getParent());
908 
909   IRB.SetInsertPoint(CheckFailTerm);
910   InlineAsm *Asm;
911   switch (TargetTriple.getArch()) {
912   case Triple::x86_64:
913     // The signal handler will find the data address in rdi.
914     Asm = InlineAsm::get(
915         FunctionType::get(VoidTy, {PtrLong->getType()}, false),
916         "int3\nnopl " +
917             itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
918             "(%rax)",
919         "{rdi}",
920         /*hasSideEffects=*/true);
921     break;
922   case Triple::aarch64:
923   case Triple::aarch64_be:
924     // The signal handler will find the data address in x0.
925     Asm = InlineAsm::get(
926         FunctionType::get(VoidTy, {PtrLong->getType()}, false),
927         "brk #" + itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
928         "{x0}",
929         /*hasSideEffects=*/true);
930     break;
931   case Triple::riscv64:
932     // The signal handler will find the data address in x10.
933     Asm = InlineAsm::get(
934         FunctionType::get(VoidTy, {PtrLong->getType()}, false),
935         "ebreak\naddiw x0, x11, " +
936             itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
937         "{x10}",
938         /*hasSideEffects=*/true);
939     break;
940   default:
941     report_fatal_error("unsupported architecture");
942   }
943   IRB.CreateCall(Asm, PtrLong);
944   if (Recover)
945     cast<BranchInst>(CheckFailTerm)->setSuccessor(0, CheckTerm->getParent());
946 }
947 
948 bool HWAddressSanitizer::ignoreMemIntrinsic(MemIntrinsic *MI) {
949   if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI)) {
950     return (!ClInstrumentWrites || ignoreAccess(MTI, MTI->getDest())) &&
951            (!ClInstrumentReads || ignoreAccess(MTI, MTI->getSource()));
952   }
953   if (isa<MemSetInst>(MI))
954     return !ClInstrumentWrites || ignoreAccess(MI, MI->getDest());
955   return false;
956 }
957 
958 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
959   IRBuilder<> IRB(MI);
960   if (isa<MemTransferInst>(MI)) {
961     if (UseMatchAllCallback) {
962       IRB.CreateCall(
963           isa<MemMoveInst>(MI) ? HwasanMemmove : HwasanMemcpy,
964           {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
965            IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
966            IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false),
967            ConstantInt::get(Int8Ty, *MatchAllTag)});
968     } else {
969       IRB.CreateCall(
970           isa<MemMoveInst>(MI) ? HwasanMemmove : HwasanMemcpy,
971           {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
972            IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
973            IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
974     }
975   } else if (isa<MemSetInst>(MI)) {
976     if (UseMatchAllCallback) {
977       IRB.CreateCall(
978           HwasanMemset,
979           {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
980            IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
981            IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false),
982            ConstantInt::get(Int8Ty, *MatchAllTag)});
983     } else {
984       IRB.CreateCall(
985           HwasanMemset,
986           {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
987            IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
988            IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
989     }
990   }
991   MI->eraseFromParent();
992 }
993 
994 bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O) {
995   Value *Addr = O.getPtr();
996 
997   LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
998 
999   if (O.MaybeMask)
1000     return false; // FIXME
1001 
1002   IRBuilder<> IRB(O.getInsn());
1003   if (!O.TypeStoreSize.isScalable() && isPowerOf2_64(O.TypeStoreSize) &&
1004       (O.TypeStoreSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
1005       (!O.Alignment || *O.Alignment >= Mapping.getObjectAlignment() ||
1006        *O.Alignment >= O.TypeStoreSize / 8)) {
1007     size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeStoreSize);
1008     if (InstrumentWithCalls) {
1009       if (UseMatchAllCallback) {
1010         IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
1011                        {IRB.CreatePointerCast(Addr, IntptrTy),
1012                         ConstantInt::get(Int8Ty, *MatchAllTag)});
1013       } else {
1014         IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
1015                        IRB.CreatePointerCast(Addr, IntptrTy));
1016       }
1017     } else if (OutlinedChecks) {
1018       instrumentMemAccessOutline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
1019     } else {
1020       instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
1021     }
1022   } else {
1023     if (UseMatchAllCallback) {
1024       IRB.CreateCall(
1025           HwasanMemoryAccessCallbackSized[O.IsWrite],
1026           {IRB.CreatePointerCast(Addr, IntptrTy),
1027            IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize),
1028                           ConstantInt::get(IntptrTy, 8)),
1029            ConstantInt::get(Int8Ty, *MatchAllTag)});
1030     } else {
1031       IRB.CreateCall(
1032           HwasanMemoryAccessCallbackSized[O.IsWrite],
1033           {IRB.CreatePointerCast(Addr, IntptrTy),
1034            IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize),
1035                           ConstantInt::get(IntptrTy, 8))});
1036     }
1037   }
1038   untagPointerOperand(O.getInsn(), Addr);
1039 
1040   return true;
1041 }
1042 
1043 void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
1044                                    size_t Size) {
1045   size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
1046   if (!UseShortGranules)
1047     Size = AlignedSize;
1048 
1049   Tag = IRB.CreateTrunc(Tag, Int8Ty);
1050   if (InstrumentWithCalls) {
1051     IRB.CreateCall(HwasanTagMemoryFunc,
1052                    {IRB.CreatePointerCast(AI, Int8PtrTy), Tag,
1053                     ConstantInt::get(IntptrTy, AlignedSize)});
1054   } else {
1055     size_t ShadowSize = Size >> Mapping.Scale;
1056     Value *AddrLong = untagPointer(IRB, IRB.CreatePointerCast(AI, IntptrTy));
1057     Value *ShadowPtr = memToShadow(AddrLong, IRB);
1058     // If this memset is not inlined, it will be intercepted in the hwasan
1059     // runtime library. That's OK, because the interceptor skips the checks if
1060     // the address is in the shadow region.
1061     // FIXME: the interceptor is not as fast as real memset. Consider lowering
1062     // llvm.memset right here into either a sequence of stores, or a call to
1063     // hwasan_tag_memory.
1064     if (ShadowSize)
1065       IRB.CreateMemSet(ShadowPtr, Tag, ShadowSize, Align(1));
1066     if (Size != AlignedSize) {
1067       const uint8_t SizeRemainder = Size % Mapping.getObjectAlignment().value();
1068       IRB.CreateStore(ConstantInt::get(Int8Ty, SizeRemainder),
1069                       IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));
1070       IRB.CreateStore(Tag, IRB.CreateConstGEP1_32(
1071                                Int8Ty, IRB.CreatePointerCast(AI, Int8PtrTy),
1072                                AlignedSize - 1));
1073     }
1074   }
1075 }
1076 
1077 unsigned HWAddressSanitizer::retagMask(unsigned AllocaNo) {
1078   if (TargetTriple.getArch() == Triple::x86_64)
1079     return AllocaNo & TagMaskByte;
1080 
1081   // A list of 8-bit numbers that have at most one run of non-zero bits.
1082   // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
1083   // masks.
1084   // The list does not include the value 255, which is used for UAR.
1085   //
1086   // Because we are more likely to use earlier elements of this list than later
1087   // ones, it is sorted in increasing order of probability of collision with a
1088   // mask allocated (temporally) nearby. The program that generated this list
1089   // can be found at:
1090   // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
1091   static const unsigned FastMasks[] = {
1092       0,   128, 64, 192, 32,  96,  224, 112, 240, 48, 16,  120,
1093       248, 56,  24, 8,   124, 252, 60,  28,  12,  4,  126, 254,
1094       62,  30,  14, 6,   2,   127, 63,  31,  15,  7,  3,   1};
1095   return FastMasks[AllocaNo % std::size(FastMasks)];
1096 }
1097 
1098 Value *HWAddressSanitizer::applyTagMask(IRBuilder<> &IRB, Value *OldTag) {
1099   if (TagMaskByte == 0xFF)
1100     return OldTag; // No need to clear the tag byte.
1101   return IRB.CreateAnd(OldTag,
1102                        ConstantInt::get(OldTag->getType(), TagMaskByte));
1103 }
1104 
1105 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
1106   return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
1107 }
1108 
1109 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
1110   if (ClGenerateTagsWithCalls)
1111     return nullptr;
1112   if (StackBaseTag)
1113     return StackBaseTag;
1114   // Extract some entropy from the stack pointer for the tags.
1115   // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
1116   // between functions).
1117   Value *StackPointerLong = getSP(IRB);
1118   Value *StackTag =
1119       applyTagMask(IRB, IRB.CreateXor(StackPointerLong,
1120                                       IRB.CreateLShr(StackPointerLong, 20)));
1121   StackTag->setName("hwasan.stack.base.tag");
1122   return StackTag;
1123 }
1124 
1125 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
1126                                         unsigned AllocaNo) {
1127   if (ClGenerateTagsWithCalls)
1128     return getNextTagWithCall(IRB);
1129   return IRB.CreateXor(
1130       StackTag, ConstantInt::get(StackTag->getType(), retagMask(AllocaNo)));
1131 }
1132 
1133 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB) {
1134   Value *StackPointerLong = getSP(IRB);
1135   Value *UARTag =
1136       applyTagMask(IRB, IRB.CreateLShr(StackPointerLong, PointerTagShift));
1137 
1138   UARTag->setName("hwasan.uar.tag");
1139   return UARTag;
1140 }
1141 
1142 // Add a tag to an address.
1143 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
1144                                       Value *PtrLong, Value *Tag) {
1145   assert(!UsePageAliases);
1146   Value *TaggedPtrLong;
1147   if (CompileKernel) {
1148     // Kernel addresses have 0xFF in the most significant byte.
1149     Value *ShiftedTag =
1150         IRB.CreateOr(IRB.CreateShl(Tag, PointerTagShift),
1151                      ConstantInt::get(IntptrTy, (1ULL << PointerTagShift) - 1));
1152     TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
1153   } else {
1154     // Userspace can simply do OR (tag << PointerTagShift);
1155     Value *ShiftedTag = IRB.CreateShl(Tag, PointerTagShift);
1156     TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
1157   }
1158   return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
1159 }
1160 
1161 // Remove tag from an address.
1162 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
1163   assert(!UsePageAliases);
1164   Value *UntaggedPtrLong;
1165   if (CompileKernel) {
1166     // Kernel addresses have 0xFF in the most significant byte.
1167     UntaggedPtrLong =
1168         IRB.CreateOr(PtrLong, ConstantInt::get(PtrLong->getType(),
1169                                                TagMaskByte << PointerTagShift));
1170   } else {
1171     // Userspace addresses have 0x00.
1172     UntaggedPtrLong = IRB.CreateAnd(
1173         PtrLong, ConstantInt::get(PtrLong->getType(),
1174                                   ~(TagMaskByte << PointerTagShift)));
1175   }
1176   return UntaggedPtrLong;
1177 }
1178 
1179 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
1180   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1181   if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
1182     // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
1183     // in Bionic's libc/private/bionic_tls.h.
1184     Function *ThreadPointerFunc =
1185         Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
1186     Value *SlotPtr = IRB.CreatePointerCast(
1187         IRB.CreateConstGEP1_32(Int8Ty, IRB.CreateCall(ThreadPointerFunc), 0x30),
1188         Ty->getPointerTo(0));
1189     return SlotPtr;
1190   }
1191   if (ThreadPtrGlobal)
1192     return ThreadPtrGlobal;
1193 
1194   return nullptr;
1195 }
1196 
1197 Value *HWAddressSanitizer::getPC(IRBuilder<> &IRB) {
1198   if (TargetTriple.getArch() == Triple::aarch64)
1199     return readRegister(IRB, "pc");
1200   return IRB.CreatePtrToInt(IRB.GetInsertBlock()->getParent(), IntptrTy);
1201 }
1202 
1203 Value *HWAddressSanitizer::getSP(IRBuilder<> &IRB) {
1204   if (!CachedSP) {
1205     // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
1206     // first).
1207     Function *F = IRB.GetInsertBlock()->getParent();
1208     Module *M = F->getParent();
1209     auto *GetStackPointerFn = Intrinsic::getDeclaration(
1210         M, Intrinsic::frameaddress,
1211         IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
1212     CachedSP = IRB.CreatePtrToInt(
1213         IRB.CreateCall(GetStackPointerFn, {Constant::getNullValue(Int32Ty)}),
1214         IntptrTy);
1215   }
1216   return CachedSP;
1217 }
1218 
1219 Value *HWAddressSanitizer::getFrameRecordInfo(IRBuilder<> &IRB) {
1220   // Prepare ring buffer data.
1221   Value *PC = getPC(IRB);
1222   Value *SP = getSP(IRB);
1223 
1224   // Mix SP and PC.
1225   // Assumptions:
1226   // PC is 0x0000PPPPPPPPPPPP  (48 bits are meaningful, others are zero)
1227   // SP is 0xsssssssssssSSSS0  (4 lower bits are zero)
1228   // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
1229   //       0xSSSSPPPPPPPPPPPP
1230   SP = IRB.CreateShl(SP, 44);
1231   return IRB.CreateOr(PC, SP);
1232 }
1233 
1234 void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
1235   if (!Mapping.InTls)
1236     ShadowBase = getShadowNonTls(IRB);
1237   else if (!WithFrameRecord && TargetTriple.isAndroid())
1238     ShadowBase = getDynamicShadowIfunc(IRB);
1239 
1240   if (!WithFrameRecord && ShadowBase)
1241     return;
1242 
1243   Value *SlotPtr = nullptr;
1244   Value *ThreadLong = nullptr;
1245   Value *ThreadLongMaybeUntagged = nullptr;
1246 
1247   auto getThreadLongMaybeUntagged = [&]() {
1248     if (!SlotPtr)
1249       SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
1250     if (!ThreadLong)
1251       ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
1252     // Extract the address field from ThreadLong. Unnecessary on AArch64 with
1253     // TBI.
1254     return TargetTriple.isAArch64() ? ThreadLong
1255                                     : untagPointer(IRB, ThreadLong);
1256   };
1257 
1258   if (WithFrameRecord) {
1259     switch (ClRecordStackHistory) {
1260     case libcall: {
1261       // Emit a runtime call into hwasan rather than emitting instructions for
1262       // recording stack history.
1263       Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1264       IRB.CreateCall(HwasanRecordFrameRecordFunc, {FrameRecordInfo});
1265       break;
1266     }
1267     case instr: {
1268       ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1269 
1270       StackBaseTag = IRB.CreateAShr(ThreadLong, 3);
1271 
1272       // Store data to ring buffer.
1273       Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1274       Value *RecordPtr = IRB.CreateIntToPtr(ThreadLongMaybeUntagged,
1275                                             IntptrTy->getPointerTo(0));
1276       IRB.CreateStore(FrameRecordInfo, RecordPtr);
1277 
1278       // Update the ring buffer. Top byte of ThreadLong defines the size of the
1279       // buffer in pages, it must be a power of two, and the start of the buffer
1280       // must be aligned by twice that much. Therefore wrap around of the ring
1281       // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
1282       // The use of AShr instead of LShr is due to
1283       //   https://bugs.llvm.org/show_bug.cgi?id=39030
1284       // Runtime library makes sure not to use the highest bit.
1285       Value *WrapMask = IRB.CreateXor(
1286           IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),
1287           ConstantInt::get(IntptrTy, (uint64_t)-1));
1288       Value *ThreadLongNew = IRB.CreateAnd(
1289           IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
1290       IRB.CreateStore(ThreadLongNew, SlotPtr);
1291       break;
1292     }
1293     case none: {
1294       llvm_unreachable(
1295           "A stack history recording mode should've been selected.");
1296     }
1297     }
1298   }
1299 
1300   if (!ShadowBase) {
1301     if (!ThreadLongMaybeUntagged)
1302       ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1303 
1304     // Get shadow base address by aligning RecordPtr up.
1305     // Note: this is not correct if the pointer is already aligned.
1306     // Runtime library will make sure this never happens.
1307     ShadowBase = IRB.CreateAdd(
1308         IRB.CreateOr(
1309             ThreadLongMaybeUntagged,
1310             ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
1311         ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
1312     ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
1313   }
1314 }
1315 
1316 Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {
1317   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1318   Function *ReadRegister =
1319       Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy);
1320   MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)});
1321   Value *Args[] = {MetadataAsValue::get(*C, MD)};
1322   return IRB.CreateCall(ReadRegister, Args);
1323 }
1324 
1325 bool HWAddressSanitizer::instrumentLandingPads(
1326     SmallVectorImpl<Instruction *> &LandingPadVec) {
1327   for (auto *LP : LandingPadVec) {
1328     IRBuilder<> IRB(LP->getNextNode());
1329     IRB.CreateCall(
1330         HwasanHandleVfork,
1331         {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp"
1332                                                                       : "sp")});
1333   }
1334   return true;
1335 }
1336 
1337 static bool isLifetimeIntrinsic(Value *V) {
1338   auto *II = dyn_cast<IntrinsicInst>(V);
1339   return II && II->isLifetimeStartOrEnd();
1340 }
1341 
1342 bool HWAddressSanitizer::instrumentStack(memtag::StackInfo &SInfo,
1343                                          Value *StackTag, Value *UARTag,
1344                                          const DominatorTree &DT,
1345                                          const PostDominatorTree &PDT,
1346                                          const LoopInfo &LI) {
1347   // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1348   // alloca addresses using that. Unfortunately, offsets are not known yet
1349   // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1350   // temp, shift-OR it into each alloca address and xor with the retag mask.
1351   // This generates one extra instruction per alloca use.
1352   unsigned int I = 0;
1353 
1354   for (auto &KV : SInfo.AllocasToInstrument) {
1355     auto N = I++;
1356     auto *AI = KV.first;
1357     memtag::AllocaInfo &Info = KV.second;
1358     IRBuilder<> IRB(AI->getNextNode());
1359 
1360     // Replace uses of the alloca with tagged address.
1361     Value *Tag = getAllocaTag(IRB, StackTag, N);
1362     Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
1363     Value *AINoTagLong = untagPointer(IRB, AILong);
1364     Value *Replacement = tagPointer(IRB, AI->getType(), AINoTagLong, Tag);
1365     std::string Name =
1366         AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
1367     Replacement->setName(Name + ".hwasan");
1368 
1369     size_t Size = memtag::getAllocaSizeInBytes(*AI);
1370     size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
1371 
1372     Value *AICast = IRB.CreatePointerCast(AI, Int8PtrTy);
1373 
1374     auto HandleLifetime = [&](IntrinsicInst *II) {
1375       // Set the lifetime intrinsic to cover the whole alloca. This reduces the
1376       // set of assumptions we need to make about the lifetime. Without this we
1377       // would need to ensure that we can track the lifetime pointer to a
1378       // constant offset from the alloca, and would still need to change the
1379       // size to include the extra alignment we use for the untagging to make
1380       // the size consistent.
1381       //
1382       // The check for standard lifetime below makes sure that we have exactly
1383       // one set of start / end in any execution (i.e. the ends are not
1384       // reachable from each other), so this will not cause any problems.
1385       II->setArgOperand(0, ConstantInt::get(Int64Ty, AlignedSize));
1386       II->setArgOperand(1, AICast);
1387     };
1388     llvm::for_each(Info.LifetimeStart, HandleLifetime);
1389     llvm::for_each(Info.LifetimeEnd, HandleLifetime);
1390 
1391     AI->replaceUsesWithIf(Replacement, [AICast, AILong](const Use &U) {
1392       auto *User = U.getUser();
1393       return User != AILong && User != AICast && !isLifetimeIntrinsic(User);
1394     });
1395 
1396     for (auto *DDI : Info.DbgVariableIntrinsics) {
1397       // Prepend "tag_offset, N" to the dwarf expression.
1398       // Tag offset logically applies to the alloca pointer, and it makes sense
1399       // to put it at the beginning of the expression.
1400       SmallVector<uint64_t, 8> NewOps = {dwarf::DW_OP_LLVM_tag_offset,
1401                                          retagMask(N)};
1402       for (size_t LocNo = 0; LocNo < DDI->getNumVariableLocationOps(); ++LocNo)
1403         if (DDI->getVariableLocationOp(LocNo) == AI)
1404           DDI->setExpression(DIExpression::appendOpsToArg(DDI->getExpression(),
1405                                                           NewOps, LocNo));
1406     }
1407 
1408     auto TagEnd = [&](Instruction *Node) {
1409       IRB.SetInsertPoint(Node);
1410       // When untagging, use the `AlignedSize` because we need to set the tags
1411       // for the entire alloca to original. If we used `Size` here, we would
1412       // keep the last granule tagged, and store zero in the last byte of the
1413       // last granule, due to how short granules are implemented.
1414       tagAlloca(IRB, AI, UARTag, AlignedSize);
1415     };
1416     // Calls to functions that may return twice (e.g. setjmp) confuse the
1417     // postdominator analysis, and will leave us to keep memory tagged after
1418     // function return. Work around this by always untagging at every return
1419     // statement if return_twice functions are called.
1420     bool StandardLifetime =
1421         SInfo.UnrecognizedLifetimes.empty() &&
1422         memtag::isStandardLifetime(Info.LifetimeStart, Info.LifetimeEnd, &DT,
1423                                    &LI, ClMaxLifetimes) &&
1424         !SInfo.CallsReturnTwice;
1425     if (DetectUseAfterScope && StandardLifetime) {
1426       IntrinsicInst *Start = Info.LifetimeStart[0];
1427       IRB.SetInsertPoint(Start->getNextNode());
1428       tagAlloca(IRB, AI, Tag, Size);
1429       if (!memtag::forAllReachableExits(DT, PDT, LI, Start, Info.LifetimeEnd,
1430                                         SInfo.RetVec, TagEnd)) {
1431         for (auto *End : Info.LifetimeEnd)
1432           End->eraseFromParent();
1433       }
1434     } else {
1435       tagAlloca(IRB, AI, Tag, Size);
1436       for (auto *RI : SInfo.RetVec)
1437         TagEnd(RI);
1438       // We inserted tagging outside of the lifetimes, so we have to remove
1439       // them.
1440       for (auto &II : Info.LifetimeStart)
1441         II->eraseFromParent();
1442       for (auto &II : Info.LifetimeEnd)
1443         II->eraseFromParent();
1444     }
1445     memtag::alignAndPadAlloca(Info, Mapping.getObjectAlignment());
1446   }
1447   for (auto &I : SInfo.UnrecognizedLifetimes)
1448     I->eraseFromParent();
1449   return true;
1450 }
1451 
1452 void HWAddressSanitizer::sanitizeFunction(Function &F,
1453                                           FunctionAnalysisManager &FAM) {
1454   if (&F == HwasanCtorFunction)
1455     return;
1456 
1457   if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
1458     return;
1459 
1460   LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1461 
1462   SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
1463   SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
1464   SmallVector<Instruction *, 8> LandingPadVec;
1465 
1466   memtag::StackInfoBuilder SIB(SSI);
1467   for (auto &Inst : instructions(F)) {
1468     if (InstrumentStack) {
1469       SIB.visit(Inst);
1470     }
1471 
1472     if (InstrumentLandingPads && isa<LandingPadInst>(Inst))
1473       LandingPadVec.push_back(&Inst);
1474 
1475     getInterestingMemoryOperands(&Inst, OperandsToInstrument);
1476 
1477     if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst))
1478       if (!ignoreMemIntrinsic(MI))
1479         IntrinToInstrument.push_back(MI);
1480   }
1481 
1482   memtag::StackInfo &SInfo = SIB.get();
1483 
1484   initializeCallbacks(*F.getParent());
1485 
1486   if (!LandingPadVec.empty())
1487     instrumentLandingPads(LandingPadVec);
1488 
1489   if (SInfo.AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1490       F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1491     // __hwasan_personality_thunk is a no-op for functions without an
1492     // instrumented stack, so we can drop it.
1493     F.setPersonalityFn(nullptr);
1494   }
1495 
1496   if (SInfo.AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
1497       IntrinToInstrument.empty())
1498     return;
1499 
1500   assert(!ShadowBase);
1501 
1502   Instruction *InsertPt = &*F.getEntryBlock().begin();
1503   IRBuilder<> EntryIRB(InsertPt);
1504   emitPrologue(EntryIRB,
1505                /*WithFrameRecord*/ ClRecordStackHistory != none &&
1506                    Mapping.WithFrameRecord &&
1507                    !SInfo.AllocasToInstrument.empty());
1508 
1509   if (!SInfo.AllocasToInstrument.empty()) {
1510     const DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F);
1511     const PostDominatorTree &PDT = FAM.getResult<PostDominatorTreeAnalysis>(F);
1512     const LoopInfo &LI = FAM.getResult<LoopAnalysis>(F);
1513     Value *StackTag = getStackBaseTag(EntryIRB);
1514     Value *UARTag = getUARTag(EntryIRB);
1515     instrumentStack(SInfo, StackTag, UARTag, DT, PDT, LI);
1516   }
1517 
1518   // If we split the entry block, move any allocas that were originally in the
1519   // entry block back into the entry block so that they aren't treated as
1520   // dynamic allocas.
1521   if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1522     InsertPt = &*F.getEntryBlock().begin();
1523     for (Instruction &I :
1524          llvm::make_early_inc_range(*EntryIRB.GetInsertBlock())) {
1525       if (auto *AI = dyn_cast<AllocaInst>(&I))
1526         if (isa<ConstantInt>(AI->getArraySize()))
1527           I.moveBefore(InsertPt);
1528     }
1529   }
1530 
1531   for (auto &Operand : OperandsToInstrument)
1532     instrumentMemAccess(Operand);
1533 
1534   if (ClInstrumentMemIntrinsics && !IntrinToInstrument.empty()) {
1535     for (auto *Inst : IntrinToInstrument)
1536       instrumentMemIntrinsic(Inst);
1537   }
1538 
1539   ShadowBase = nullptr;
1540   StackBaseTag = nullptr;
1541   CachedSP = nullptr;
1542 }
1543 
1544 void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1545   assert(!UsePageAliases);
1546   Constant *Initializer = GV->getInitializer();
1547   uint64_t SizeInBytes =
1548       M.getDataLayout().getTypeAllocSize(Initializer->getType());
1549   uint64_t NewSize = alignTo(SizeInBytes, Mapping.getObjectAlignment());
1550   if (SizeInBytes != NewSize) {
1551     // Pad the initializer out to the next multiple of 16 bytes and add the
1552     // required short granule tag.
1553     std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1554     Init.back() = Tag;
1555     Constant *Padding = ConstantDataArray::get(*C, Init);
1556     Initializer = ConstantStruct::getAnon({Initializer, Padding});
1557   }
1558 
1559   auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1560                                    GlobalValue::ExternalLinkage, Initializer,
1561                                    GV->getName() + ".hwasan");
1562   NewGV->copyAttributesFrom(GV);
1563   NewGV->setLinkage(GlobalValue::PrivateLinkage);
1564   NewGV->copyMetadata(GV, 0);
1565   NewGV->setAlignment(
1566       std::max(GV->getAlign().valueOrOne(), Mapping.getObjectAlignment()));
1567 
1568   // It is invalid to ICF two globals that have different tags. In the case
1569   // where the size of the global is a multiple of the tag granularity the
1570   // contents of the globals may be the same but the tags (i.e. symbol values)
1571   // may be different, and the symbols are not considered during ICF. In the
1572   // case where the size is not a multiple of the granularity, the short granule
1573   // tags would discriminate two globals with different tags, but there would
1574   // otherwise be nothing stopping such a global from being incorrectly ICF'd
1575   // with an uninstrumented (i.e. tag 0) global that happened to have the short
1576   // granule tag in the last byte.
1577   NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1578 
1579   // Descriptor format (assuming little-endian):
1580   // bytes 0-3: relative address of global
1581   // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1582   // it isn't, we create multiple descriptors)
1583   // byte 7: tag
1584   auto *DescriptorTy = StructType::get(Int32Ty, Int32Ty);
1585   const uint64_t MaxDescriptorSize = 0xfffff0;
1586   for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1587        DescriptorPos += MaxDescriptorSize) {
1588     auto *Descriptor =
1589         new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1590                            nullptr, GV->getName() + ".hwasan.descriptor");
1591     auto *GVRelPtr = ConstantExpr::getTrunc(
1592         ConstantExpr::getAdd(
1593             ConstantExpr::getSub(
1594                 ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1595                 ConstantExpr::getPtrToInt(Descriptor, Int64Ty)),
1596             ConstantInt::get(Int64Ty, DescriptorPos)),
1597         Int32Ty);
1598     uint32_t Size = std::min(SizeInBytes - DescriptorPos, MaxDescriptorSize);
1599     auto *SizeAndTag = ConstantInt::get(Int32Ty, Size | (uint32_t(Tag) << 24));
1600     Descriptor->setComdat(NewGV->getComdat());
1601     Descriptor->setInitializer(ConstantStruct::getAnon({GVRelPtr, SizeAndTag}));
1602     Descriptor->setSection("hwasan_globals");
1603     Descriptor->setMetadata(LLVMContext::MD_associated,
1604                             MDNode::get(*C, ValueAsMetadata::get(NewGV)));
1605     appendToCompilerUsed(M, Descriptor);
1606   }
1607 
1608   Constant *Aliasee = ConstantExpr::getIntToPtr(
1609       ConstantExpr::getAdd(
1610           ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1611           ConstantInt::get(Int64Ty, uint64_t(Tag) << PointerTagShift)),
1612       GV->getType());
1613   auto *Alias = GlobalAlias::create(GV->getValueType(), GV->getAddressSpace(),
1614                                     GV->getLinkage(), "", Aliasee, &M);
1615   Alias->setVisibility(GV->getVisibility());
1616   Alias->takeName(GV);
1617   GV->replaceAllUsesWith(Alias);
1618   GV->eraseFromParent();
1619 }
1620 
1621 void HWAddressSanitizer::instrumentGlobals() {
1622   std::vector<GlobalVariable *> Globals;
1623   for (GlobalVariable &GV : M.globals()) {
1624     if (GV.hasSanitizerMetadata() && GV.getSanitizerMetadata().NoHWAddress)
1625       continue;
1626 
1627     if (GV.isDeclarationForLinker() || GV.getName().startswith("llvm.") ||
1628         GV.isThreadLocal())
1629       continue;
1630 
1631     // Common symbols can't have aliases point to them, so they can't be tagged.
1632     if (GV.hasCommonLinkage())
1633       continue;
1634 
1635     // Globals with custom sections may be used in __start_/__stop_ enumeration,
1636     // which would be broken both by adding tags and potentially by the extra
1637     // padding/alignment that we insert.
1638     if (GV.hasSection())
1639       continue;
1640 
1641     Globals.push_back(&GV);
1642   }
1643 
1644   MD5 Hasher;
1645   Hasher.update(M.getSourceFileName());
1646   MD5::MD5Result Hash;
1647   Hasher.final(Hash);
1648   uint8_t Tag = Hash[0];
1649 
1650   assert(TagMaskByte >= 16);
1651 
1652   for (GlobalVariable *GV : Globals) {
1653     // Don't allow globals to be tagged with something that looks like a
1654     // short-granule tag, otherwise we lose inter-granule overflow detection, as
1655     // the fast path shadow-vs-address check succeeds.
1656     if (Tag < 16 || Tag > TagMaskByte)
1657       Tag = 16;
1658     instrumentGlobal(GV, Tag++);
1659   }
1660 }
1661 
1662 void HWAddressSanitizer::instrumentPersonalityFunctions() {
1663   // We need to untag stack frames as we unwind past them. That is the job of
1664   // the personality function wrapper, which either wraps an existing
1665   // personality function or acts as a personality function on its own. Each
1666   // function that has a personality function or that can be unwound past has
1667   // its personality function changed to a thunk that calls the personality
1668   // function wrapper in the runtime.
1669   MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1670   for (Function &F : M) {
1671     if (F.isDeclaration() || !F.hasFnAttribute(Attribute::SanitizeHWAddress))
1672       continue;
1673 
1674     if (F.hasPersonalityFn()) {
1675       PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(&F);
1676     } else if (!F.hasFnAttribute(Attribute::NoUnwind)) {
1677       PersonalityFns[nullptr].push_back(&F);
1678     }
1679   }
1680 
1681   if (PersonalityFns.empty())
1682     return;
1683 
1684   FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1685       "__hwasan_personality_wrapper", Int32Ty, Int32Ty, Int32Ty, Int64Ty,
1686       Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy);
1687   FunctionCallee UnwindGetGR = M.getOrInsertFunction("_Unwind_GetGR", VoidTy);
1688   FunctionCallee UnwindGetCFA = M.getOrInsertFunction("_Unwind_GetCFA", VoidTy);
1689 
1690   for (auto &P : PersonalityFns) {
1691     std::string ThunkName = kHwasanPersonalityThunkName;
1692     if (P.first)
1693       ThunkName += ("." + P.first->getName()).str();
1694     FunctionType *ThunkFnTy = FunctionType::get(
1695         Int32Ty, {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int8PtrTy}, false);
1696     bool IsLocal = P.first && (!isa<GlobalValue>(P.first) ||
1697                                cast<GlobalValue>(P.first)->hasLocalLinkage());
1698     auto *ThunkFn = Function::Create(ThunkFnTy,
1699                                      IsLocal ? GlobalValue::InternalLinkage
1700                                              : GlobalValue::LinkOnceODRLinkage,
1701                                      ThunkName, &M);
1702     if (!IsLocal) {
1703       ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1704       ThunkFn->setComdat(M.getOrInsertComdat(ThunkName));
1705     }
1706 
1707     auto *BB = BasicBlock::Create(*C, "entry", ThunkFn);
1708     IRBuilder<> IRB(BB);
1709     CallInst *WrapperCall = IRB.CreateCall(
1710         HwasanPersonalityWrapper,
1711         {ThunkFn->getArg(0), ThunkFn->getArg(1), ThunkFn->getArg(2),
1712          ThunkFn->getArg(3), ThunkFn->getArg(4),
1713          P.first ? IRB.CreateBitCast(P.first, Int8PtrTy)
1714                  : Constant::getNullValue(Int8PtrTy),
1715          IRB.CreateBitCast(UnwindGetGR.getCallee(), Int8PtrTy),
1716          IRB.CreateBitCast(UnwindGetCFA.getCallee(), Int8PtrTy)});
1717     WrapperCall->setTailCall();
1718     IRB.CreateRet(WrapperCall);
1719 
1720     for (Function *F : P.second)
1721       F->setPersonalityFn(ThunkFn);
1722   }
1723 }
1724 
1725 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple,
1726                                              bool InstrumentWithCalls) {
1727   Scale = kDefaultShadowScale;
1728   if (TargetTriple.isOSFuchsia()) {
1729     // Fuchsia is always PIE, which means that the beginning of the address
1730     // space is always available.
1731     InGlobal = false;
1732     InTls = false;
1733     Offset = 0;
1734     WithFrameRecord = true;
1735   } else if (ClMappingOffset.getNumOccurrences() > 0) {
1736     InGlobal = false;
1737     InTls = false;
1738     Offset = ClMappingOffset;
1739     WithFrameRecord = false;
1740   } else if (ClEnableKhwasan || InstrumentWithCalls) {
1741     InGlobal = false;
1742     InTls = false;
1743     Offset = 0;
1744     WithFrameRecord = false;
1745   } else if (ClWithIfunc) {
1746     InGlobal = true;
1747     InTls = false;
1748     Offset = kDynamicShadowSentinel;
1749     WithFrameRecord = false;
1750   } else if (ClWithTls) {
1751     InGlobal = false;
1752     InTls = true;
1753     Offset = kDynamicShadowSentinel;
1754     WithFrameRecord = true;
1755   } else {
1756     InGlobal = false;
1757     InTls = false;
1758     Offset = kDynamicShadowSentinel;
1759     WithFrameRecord = false;
1760   }
1761 }
1762