1 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file is a part of HWAddressSanitizer, an address sanity checker
11 /// based on tagged addressing.
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
15 #include "llvm/ADT/MapVector.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/StringExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/BinaryFormat/ELF.h"
21 #include "llvm/IR/Attributes.h"
22 #include "llvm/IR/BasicBlock.h"
23 #include "llvm/IR/Constant.h"
24 #include "llvm/IR/Constants.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DebugInfoMetadata.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/IRBuilder.h"
30 #include "llvm/IR/InlineAsm.h"
31 #include "llvm/IR/InstVisitor.h"
32 #include "llvm/IR/Instruction.h"
33 #include "llvm/IR/Instructions.h"
34 #include "llvm/IR/IntrinsicInst.h"
35 #include "llvm/IR/Intrinsics.h"
36 #include "llvm/IR/LLVMContext.h"
37 #include "llvm/IR/MDBuilder.h"
38 #include "llvm/IR/Module.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/IR/Value.h"
41 #include "llvm/InitializePasses.h"
42 #include "llvm/Pass.h"
43 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/CommandLine.h"
45 #include "llvm/Support/Debug.h"
46 #include "llvm/Support/raw_ostream.h"
47 #include "llvm/Transforms/Instrumentation.h"
48 #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
49 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
50 #include "llvm/Transforms/Utils/ModuleUtils.h"
51 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
52 #include <sstream>
53
54 using namespace llvm;
55
56 #define DEBUG_TYPE "hwasan"
57
58 const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
59 const char kHwasanNoteName[] = "hwasan.note";
60 const char kHwasanInitName[] = "__hwasan_init";
61 const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
62
63 const char kHwasanShadowMemoryDynamicAddress[] =
64 "__hwasan_shadow_memory_dynamic_address";
65
66 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
67 static const size_t kNumberOfAccessSizes = 5;
68
69 static const size_t kDefaultShadowScale = 4;
70 static const uint64_t kDynamicShadowSentinel =
71 std::numeric_limits<uint64_t>::max();
72 static const unsigned kPointerTagShift = 56;
73
74 static const unsigned kShadowBaseAlignment = 32;
75
76 static cl::opt<std::string>
77 ClMemoryAccessCallbackPrefix("hwasan-memory-access-callback-prefix",
78 cl::desc("Prefix for memory access callbacks"),
79 cl::Hidden, cl::init("__hwasan_"));
80
81 static cl::opt<bool> ClInstrumentWithCalls(
82 "hwasan-instrument-with-calls",
83 cl::desc("instrument reads and writes with callbacks"), cl::Hidden,
84 cl::init(false));
85
86 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
87 cl::desc("instrument read instructions"),
88 cl::Hidden, cl::init(true));
89
90 static cl::opt<bool>
91 ClInstrumentWrites("hwasan-instrument-writes",
92 cl::desc("instrument write instructions"), cl::Hidden,
93 cl::init(true));
94
95 static cl::opt<bool> ClInstrumentAtomics(
96 "hwasan-instrument-atomics",
97 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
98 cl::init(true));
99
100 static cl::opt<bool> ClInstrumentByval("hwasan-instrument-byval",
101 cl::desc("instrument byval arguments"),
102 cl::Hidden, cl::init(true));
103
104 static cl::opt<bool>
105 ClRecover("hwasan-recover",
106 cl::desc("Enable recovery mode (continue-after-error)."),
107 cl::Hidden, cl::init(false));
108
109 static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
110 cl::desc("instrument stack (allocas)"),
111 cl::Hidden, cl::init(true));
112
113 static cl::opt<bool> ClUARRetagToZero(
114 "hwasan-uar-retag-to-zero",
115 cl::desc("Clear alloca tags before returning from the function to allow "
116 "non-instrumented and instrumented function calls mix. When set "
117 "to false, allocas are retagged before returning from the "
118 "function to detect use after return."),
119 cl::Hidden, cl::init(true));
120
121 static cl::opt<bool> ClGenerateTagsWithCalls(
122 "hwasan-generate-tags-with-calls",
123 cl::desc("generate new tags with runtime library calls"), cl::Hidden,
124 cl::init(false));
125
126 static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
127 cl::Hidden, cl::init(false), cl::ZeroOrMore);
128
129 static cl::opt<int> ClMatchAllTag(
130 "hwasan-match-all-tag",
131 cl::desc("don't report bad accesses via pointers with this tag"),
132 cl::Hidden, cl::init(-1));
133
134 static cl::opt<bool>
135 ClEnableKhwasan("hwasan-kernel",
136 cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
137 cl::Hidden, cl::init(false));
138
139 // These flags allow to change the shadow mapping and control how shadow memory
140 // is accessed. The shadow mapping looks like:
141 // Shadow = (Mem >> scale) + offset
142
143 static cl::opt<uint64_t>
144 ClMappingOffset("hwasan-mapping-offset",
145 cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
146 cl::Hidden, cl::init(0));
147
148 static cl::opt<bool>
149 ClWithIfunc("hwasan-with-ifunc",
150 cl::desc("Access dynamic shadow through an ifunc global on "
151 "platforms that support this"),
152 cl::Hidden, cl::init(false));
153
154 static cl::opt<bool> ClWithTls(
155 "hwasan-with-tls",
156 cl::desc("Access dynamic shadow through an thread-local pointer on "
157 "platforms that support this"),
158 cl::Hidden, cl::init(true));
159
160 static cl::opt<bool>
161 ClRecordStackHistory("hwasan-record-stack-history",
162 cl::desc("Record stack frames with tagged allocations "
163 "in a thread-local ring buffer"),
164 cl::Hidden, cl::init(true));
165 static cl::opt<bool>
166 ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
167 cl::desc("instrument memory intrinsics"),
168 cl::Hidden, cl::init(true));
169
170 static cl::opt<bool>
171 ClInstrumentLandingPads("hwasan-instrument-landing-pads",
172 cl::desc("instrument landing pads"), cl::Hidden,
173 cl::init(false), cl::ZeroOrMore);
174
175 static cl::opt<bool> ClUseShortGranules(
176 "hwasan-use-short-granules",
177 cl::desc("use short granules in allocas and outlined checks"), cl::Hidden,
178 cl::init(false), cl::ZeroOrMore);
179
180 static cl::opt<bool> ClInstrumentPersonalityFunctions(
181 "hwasan-instrument-personality-functions",
182 cl::desc("instrument personality functions"), cl::Hidden, cl::init(false),
183 cl::ZeroOrMore);
184
185 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
186 cl::desc("inline all checks"),
187 cl::Hidden, cl::init(false));
188
189 namespace {
190
191 /// An instrumentation pass implementing detection of addressability bugs
192 /// using tagged pointers.
193 class HWAddressSanitizer {
194 public:
HWAddressSanitizer(Module & M,bool CompileKernel=false,bool Recover=false)195 explicit HWAddressSanitizer(Module &M, bool CompileKernel = false,
196 bool Recover = false)
197 : M(M) {
198 this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
199 this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0
200 ? ClEnableKhwasan
201 : CompileKernel;
202
203 initializeModule();
204 }
205
206 bool sanitizeFunction(Function &F);
207 void initializeModule();
208 void createHwasanCtorComdat();
209
210 void initializeCallbacks(Module &M);
211
212 Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
213
214 Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
215 Value *getShadowNonTls(IRBuilder<> &IRB);
216
217 void untagPointerOperand(Instruction *I, Value *Addr);
218 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
219 void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
220 unsigned AccessSizeIndex,
221 Instruction *InsertBefore);
222 void instrumentMemIntrinsic(MemIntrinsic *MI);
223 bool instrumentMemAccess(InterestingMemoryOperand &O);
224 bool ignoreAccess(Value *Ptr);
225 void getInterestingMemoryOperands(
226 Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting);
227
228 bool isInterestingAlloca(const AllocaInst &AI);
229 bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
230 Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
231 Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
232 bool instrumentStack(
233 SmallVectorImpl<AllocaInst *> &Allocas,
234 DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap,
235 SmallVectorImpl<Instruction *> &RetVec, Value *StackTag);
236 Value *readRegister(IRBuilder<> &IRB, StringRef Name);
237 bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
238 Value *getNextTagWithCall(IRBuilder<> &IRB);
239 Value *getStackBaseTag(IRBuilder<> &IRB);
240 Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI,
241 unsigned AllocaNo);
242 Value *getUARTag(IRBuilder<> &IRB, Value *StackTag);
243
244 Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
245 void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
246
247 void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
248 void instrumentGlobals();
249
250 void instrumentPersonalityFunctions();
251
252 private:
253 LLVMContext *C;
254 Module &M;
255 Triple TargetTriple;
256 FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset;
257 FunctionCallee HWAsanHandleVfork;
258
259 /// This struct defines the shadow mapping using the rule:
260 /// shadow = (mem >> Scale) + Offset.
261 /// If InGlobal is true, then
262 /// extern char __hwasan_shadow[];
263 /// shadow = (mem >> Scale) + &__hwasan_shadow
264 /// If InTls is true, then
265 /// extern char *__hwasan_tls;
266 /// shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
267 struct ShadowMapping {
268 int Scale;
269 uint64_t Offset;
270 bool InGlobal;
271 bool InTls;
272
273 void init(Triple &TargetTriple, bool InstrumentWithCalls);
getObjectAlignment__anonff39e3700111::HWAddressSanitizer::ShadowMapping274 unsigned getObjectAlignment() const { return 1U << Scale; }
275 };
276 ShadowMapping Mapping;
277
278 Type *VoidTy = Type::getVoidTy(M.getContext());
279 Type *IntptrTy;
280 Type *Int8PtrTy;
281 Type *Int8Ty;
282 Type *Int32Ty;
283 Type *Int64Ty = Type::getInt64Ty(M.getContext());
284
285 bool CompileKernel;
286 bool Recover;
287 bool OutlinedChecks;
288 bool UseShortGranules;
289 bool InstrumentLandingPads;
290 bool InstrumentWithCalls;
291 bool InstrumentStack;
292 bool UsePageAliases;
293
294 bool HasMatchAllTag = false;
295 uint8_t MatchAllTag = 0;
296
297 Function *HwasanCtorFunction;
298
299 FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
300 FunctionCallee HwasanMemoryAccessCallbackSized[2];
301
302 FunctionCallee HwasanTagMemoryFunc;
303 FunctionCallee HwasanGenerateTagFunc;
304
305 Constant *ShadowGlobal;
306
307 Value *ShadowBase = nullptr;
308 Value *StackBaseTag = nullptr;
309 GlobalValue *ThreadPtrGlobal = nullptr;
310 };
311
312 class HWAddressSanitizerLegacyPass : public FunctionPass {
313 public:
314 // Pass identification, replacement for typeid.
315 static char ID;
316
HWAddressSanitizerLegacyPass(bool CompileKernel=false,bool Recover=false)317 explicit HWAddressSanitizerLegacyPass(bool CompileKernel = false,
318 bool Recover = false)
319 : FunctionPass(ID), CompileKernel(CompileKernel), Recover(Recover) {
320 initializeHWAddressSanitizerLegacyPassPass(
321 *PassRegistry::getPassRegistry());
322 }
323
getPassName() const324 StringRef getPassName() const override { return "HWAddressSanitizer"; }
325
doInitialization(Module & M)326 bool doInitialization(Module &M) override {
327 HWASan = std::make_unique<HWAddressSanitizer>(M, CompileKernel, Recover);
328 return true;
329 }
330
runOnFunction(Function & F)331 bool runOnFunction(Function &F) override {
332 return HWASan->sanitizeFunction(F);
333 }
334
doFinalization(Module & M)335 bool doFinalization(Module &M) override {
336 HWASan.reset();
337 return false;
338 }
339
340 private:
341 std::unique_ptr<HWAddressSanitizer> HWASan;
342 bool CompileKernel;
343 bool Recover;
344 };
345
346 } // end anonymous namespace
347
348 char HWAddressSanitizerLegacyPass::ID = 0;
349
350 INITIALIZE_PASS_BEGIN(
351 HWAddressSanitizerLegacyPass, "hwasan",
352 "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
353 false)
354 INITIALIZE_PASS_END(
355 HWAddressSanitizerLegacyPass, "hwasan",
356 "HWAddressSanitizer: detect memory bugs using tagged addressing.", false,
357 false)
358
createHWAddressSanitizerLegacyPassPass(bool CompileKernel,bool Recover)359 FunctionPass *llvm::createHWAddressSanitizerLegacyPassPass(bool CompileKernel,
360 bool Recover) {
361 assert(!CompileKernel || Recover);
362 return new HWAddressSanitizerLegacyPass(CompileKernel, Recover);
363 }
364
HWAddressSanitizerPass(bool CompileKernel,bool Recover)365 HWAddressSanitizerPass::HWAddressSanitizerPass(bool CompileKernel, bool Recover)
366 : CompileKernel(CompileKernel), Recover(Recover) {}
367
run(Module & M,ModuleAnalysisManager & MAM)368 PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
369 ModuleAnalysisManager &MAM) {
370 HWAddressSanitizer HWASan(M, CompileKernel, Recover);
371 bool Modified = false;
372 for (Function &F : M)
373 Modified |= HWASan.sanitizeFunction(F);
374 if (Modified)
375 return PreservedAnalyses::none();
376 return PreservedAnalyses::all();
377 }
378
createHwasanCtorComdat()379 void HWAddressSanitizer::createHwasanCtorComdat() {
380 std::tie(HwasanCtorFunction, std::ignore) =
381 getOrCreateSanitizerCtorAndInitFunctions(
382 M, kHwasanModuleCtorName, kHwasanInitName,
383 /*InitArgTypes=*/{},
384 /*InitArgs=*/{},
385 // This callback is invoked when the functions are created the first
386 // time. Hook them into the global ctors list in that case:
387 [&](Function *Ctor, FunctionCallee) {
388 Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
389 Ctor->setComdat(CtorComdat);
390 appendToGlobalCtors(M, Ctor, 0, Ctor);
391 });
392
393 // Create a note that contains pointers to the list of global
394 // descriptors. Adding a note to the output file will cause the linker to
395 // create a PT_NOTE program header pointing to the note that we can use to
396 // find the descriptor list starting from the program headers. A function
397 // provided by the runtime initializes the shadow memory for the globals by
398 // accessing the descriptor list via the note. The dynamic loader needs to
399 // call this function whenever a library is loaded.
400 //
401 // The reason why we use a note for this instead of a more conventional
402 // approach of having a global constructor pass a descriptor list pointer to
403 // the runtime is because of an order of initialization problem. With
404 // constructors we can encounter the following problematic scenario:
405 //
406 // 1) library A depends on library B and also interposes one of B's symbols
407 // 2) B's constructors are called before A's (as required for correctness)
408 // 3) during construction, B accesses one of its "own" globals (actually
409 // interposed by A) and triggers a HWASAN failure due to the initialization
410 // for A not having happened yet
411 //
412 // Even without interposition it is possible to run into similar situations in
413 // cases where two libraries mutually depend on each other.
414 //
415 // We only need one note per binary, so put everything for the note in a
416 // comdat. This needs to be a comdat with an .init_array section to prevent
417 // newer versions of lld from discarding the note.
418 //
419 // Create the note even if we aren't instrumenting globals. This ensures that
420 // binaries linked from object files with both instrumented and
421 // non-instrumented globals will end up with a note, even if a comdat from an
422 // object file with non-instrumented globals is selected. The note is harmless
423 // if the runtime doesn't support it, since it will just be ignored.
424 Comdat *NoteComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
425
426 Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
427 auto Start =
428 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
429 nullptr, "__start_hwasan_globals");
430 Start->setVisibility(GlobalValue::HiddenVisibility);
431 Start->setDSOLocal(true);
432 auto Stop =
433 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
434 nullptr, "__stop_hwasan_globals");
435 Stop->setVisibility(GlobalValue::HiddenVisibility);
436 Stop->setDSOLocal(true);
437
438 // Null-terminated so actually 8 bytes, which are required in order to align
439 // the note properly.
440 auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
441
442 auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
443 Int32Ty, Int32Ty);
444 auto *Note =
445 new GlobalVariable(M, NoteTy, /*isConstant=*/true,
446 GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
447 Note->setSection(".note.hwasan.globals");
448 Note->setComdat(NoteComdat);
449 Note->setAlignment(Align(4));
450 Note->setDSOLocal(true);
451
452 // The pointers in the note need to be relative so that the note ends up being
453 // placed in rodata, which is the standard location for notes.
454 auto CreateRelPtr = [&](Constant *Ptr) {
455 return ConstantExpr::getTrunc(
456 ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
457 ConstantExpr::getPtrToInt(Note, Int64Ty)),
458 Int32Ty);
459 };
460 Note->setInitializer(ConstantStruct::getAnon(
461 {ConstantInt::get(Int32Ty, 8), // n_namesz
462 ConstantInt::get(Int32Ty, 8), // n_descsz
463 ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
464 Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
465 appendToCompilerUsed(M, Note);
466
467 // Create a zero-length global in hwasan_globals so that the linker will
468 // always create start and stop symbols.
469 auto Dummy = new GlobalVariable(
470 M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
471 Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
472 Dummy->setSection("hwasan_globals");
473 Dummy->setComdat(NoteComdat);
474 Dummy->setMetadata(LLVMContext::MD_associated,
475 MDNode::get(*C, ValueAsMetadata::get(Note)));
476 appendToCompilerUsed(M, Dummy);
477 }
478
479 /// Module-level initialization.
480 ///
481 /// inserts a call to __hwasan_init to the module's constructor list.
initializeModule()482 void HWAddressSanitizer::initializeModule() {
483 LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
484 auto &DL = M.getDataLayout();
485
486 TargetTriple = Triple(M.getTargetTriple());
487
488 // x86_64 uses userspace pointer aliases, currently heap-only with callback
489 // instrumentation only.
490 UsePageAliases = TargetTriple.getArch() == Triple::x86_64;
491 InstrumentWithCalls = UsePageAliases ? true : ClInstrumentWithCalls;
492 InstrumentStack = UsePageAliases ? false : ClInstrumentStack;
493
494 Mapping.init(TargetTriple, InstrumentWithCalls);
495
496 C = &(M.getContext());
497 IRBuilder<> IRB(*C);
498 IntptrTy = IRB.getIntPtrTy(DL);
499 Int8PtrTy = IRB.getInt8PtrTy();
500 Int8Ty = IRB.getInt8Ty();
501 Int32Ty = IRB.getInt32Ty();
502
503 HwasanCtorFunction = nullptr;
504
505 // Older versions of Android do not have the required runtime support for
506 // short granules, global or personality function instrumentation. On other
507 // platforms we currently require using the latest version of the runtime.
508 bool NewRuntime =
509 !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30);
510
511 UseShortGranules =
512 ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
513 OutlinedChecks =
514 TargetTriple.isAArch64() && TargetTriple.isOSBinFormatELF() &&
515 (ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover);
516
517 if (ClMatchAllTag.getNumOccurrences()) {
518 if (ClMatchAllTag != -1) {
519 HasMatchAllTag = true;
520 MatchAllTag = ClMatchAllTag & 0xFF;
521 }
522 } else if (CompileKernel) {
523 HasMatchAllTag = true;
524 MatchAllTag = 0xFF;
525 }
526
527 // If we don't have personality function support, fall back to landing pads.
528 InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
529 ? ClInstrumentLandingPads
530 : !NewRuntime;
531
532 if (!CompileKernel) {
533 createHwasanCtorComdat();
534 bool InstrumentGlobals =
535 ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime;
536 if (InstrumentGlobals && !UsePageAliases)
537 instrumentGlobals();
538
539 bool InstrumentPersonalityFunctions =
540 ClInstrumentPersonalityFunctions.getNumOccurrences()
541 ? ClInstrumentPersonalityFunctions
542 : NewRuntime;
543 if (InstrumentPersonalityFunctions)
544 instrumentPersonalityFunctions();
545 }
546
547 if (!TargetTriple.isAndroid()) {
548 Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] {
549 auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
550 GlobalValue::ExternalLinkage, nullptr,
551 "__hwasan_tls", nullptr,
552 GlobalVariable::InitialExecTLSModel);
553 appendToCompilerUsed(M, GV);
554 return GV;
555 });
556 ThreadPtrGlobal = cast<GlobalVariable>(C);
557 }
558 }
559
initializeCallbacks(Module & M)560 void HWAddressSanitizer::initializeCallbacks(Module &M) {
561 IRBuilder<> IRB(*C);
562 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
563 const std::string TypeStr = AccessIsWrite ? "store" : "load";
564 const std::string EndingStr = Recover ? "_noabort" : "";
565
566 HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
567 ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
568 FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false));
569
570 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
571 AccessSizeIndex++) {
572 HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
573 M.getOrInsertFunction(
574 ClMemoryAccessCallbackPrefix + TypeStr +
575 itostr(1ULL << AccessSizeIndex) + EndingStr,
576 FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false));
577 }
578 }
579
580 HwasanTagMemoryFunc = M.getOrInsertFunction(
581 "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy);
582 HwasanGenerateTagFunc =
583 M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
584
585 ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
586 ArrayType::get(IRB.getInt8Ty(), 0));
587
588 const std::string MemIntrinCallbackPrefix =
589 CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
590 HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
591 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
592 IRB.getInt8PtrTy(), IntptrTy);
593 HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy",
594 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
595 IRB.getInt8PtrTy(), IntptrTy);
596 HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
597 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
598 IRB.getInt32Ty(), IntptrTy);
599
600 HWAsanHandleVfork =
601 M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy);
602 }
603
getOpaqueNoopCast(IRBuilder<> & IRB,Value * Val)604 Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
605 // An empty inline asm with input reg == output reg.
606 // An opaque no-op cast, basically.
607 // This prevents code bloat as a result of rematerializing trivial definitions
608 // such as constants or global addresses at every load and store.
609 InlineAsm *Asm =
610 InlineAsm::get(FunctionType::get(Int8PtrTy, {Val->getType()}, false),
611 StringRef(""), StringRef("=r,0"),
612 /*hasSideEffects=*/false);
613 return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");
614 }
615
getDynamicShadowIfunc(IRBuilder<> & IRB)616 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
617 return getOpaqueNoopCast(IRB, ShadowGlobal);
618 }
619
getShadowNonTls(IRBuilder<> & IRB)620 Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
621 if (Mapping.Offset != kDynamicShadowSentinel)
622 return getOpaqueNoopCast(
623 IRB, ConstantExpr::getIntToPtr(
624 ConstantInt::get(IntptrTy, Mapping.Offset), Int8PtrTy));
625
626 if (Mapping.InGlobal) {
627 return getDynamicShadowIfunc(IRB);
628 } else {
629 Value *GlobalDynamicAddress =
630 IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
631 kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
632 return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);
633 }
634 }
635
ignoreAccess(Value * Ptr)636 bool HWAddressSanitizer::ignoreAccess(Value *Ptr) {
637 // Do not instrument acesses from different address spaces; we cannot deal
638 // with them.
639 Type *PtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
640 if (PtrTy->getPointerAddressSpace() != 0)
641 return true;
642
643 // Ignore swifterror addresses.
644 // swifterror memory addresses are mem2reg promoted by instruction
645 // selection. As such they cannot have regular uses like an instrumentation
646 // function and it makes no sense to track them as memory.
647 if (Ptr->isSwiftError())
648 return true;
649
650 return false;
651 }
652
getInterestingMemoryOperands(Instruction * I,SmallVectorImpl<InterestingMemoryOperand> & Interesting)653 void HWAddressSanitizer::getInterestingMemoryOperands(
654 Instruction *I, SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
655 // Skip memory accesses inserted by another instrumentation.
656 if (I->hasMetadata("nosanitize"))
657 return;
658
659 // Do not instrument the load fetching the dynamic shadow address.
660 if (ShadowBase == I)
661 return;
662
663 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
664 if (!ClInstrumentReads || ignoreAccess(LI->getPointerOperand()))
665 return;
666 Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
667 LI->getType(), LI->getAlign());
668 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
669 if (!ClInstrumentWrites || ignoreAccess(SI->getPointerOperand()))
670 return;
671 Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
672 SI->getValueOperand()->getType(), SI->getAlign());
673 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
674 if (!ClInstrumentAtomics || ignoreAccess(RMW->getPointerOperand()))
675 return;
676 Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
677 RMW->getValOperand()->getType(), None);
678 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
679 if (!ClInstrumentAtomics || ignoreAccess(XCHG->getPointerOperand()))
680 return;
681 Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
682 XCHG->getCompareOperand()->getType(), None);
683 } else if (auto CI = dyn_cast<CallInst>(I)) {
684 for (unsigned ArgNo = 0; ArgNo < CI->getNumArgOperands(); ArgNo++) {
685 if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
686 ignoreAccess(CI->getArgOperand(ArgNo)))
687 continue;
688 Type *Ty = CI->getParamByValType(ArgNo);
689 Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
690 }
691 }
692 }
693
getPointerOperandIndex(Instruction * I)694 static unsigned getPointerOperandIndex(Instruction *I) {
695 if (LoadInst *LI = dyn_cast<LoadInst>(I))
696 return LI->getPointerOperandIndex();
697 if (StoreInst *SI = dyn_cast<StoreInst>(I))
698 return SI->getPointerOperandIndex();
699 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
700 return RMW->getPointerOperandIndex();
701 if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
702 return XCHG->getPointerOperandIndex();
703 report_fatal_error("Unexpected instruction");
704 return -1;
705 }
706
TypeSizeToSizeIndex(uint32_t TypeSize)707 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
708 size_t Res = countTrailingZeros(TypeSize / 8);
709 assert(Res < kNumberOfAccessSizes);
710 return Res;
711 }
712
untagPointerOperand(Instruction * I,Value * Addr)713 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
714 if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64)
715 return;
716
717 IRBuilder<> IRB(I);
718 Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
719 Value *UntaggedPtr =
720 IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
721 I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
722 }
723
memToShadow(Value * Mem,IRBuilder<> & IRB)724 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
725 // Mem >> Scale
726 Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
727 if (Mapping.Offset == 0)
728 return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
729 // (Mem >> Scale) + Offset
730 return IRB.CreateGEP(Int8Ty, ShadowBase, Shadow);
731 }
732
instrumentMemAccessInline(Value * Ptr,bool IsWrite,unsigned AccessSizeIndex,Instruction * InsertBefore)733 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
734 unsigned AccessSizeIndex,
735 Instruction *InsertBefore) {
736 assert(!UsePageAliases);
737 const int64_t AccessInfo =
738 (CompileKernel << HWASanAccessInfo::CompileKernelShift) +
739 (HasMatchAllTag << HWASanAccessInfo::HasMatchAllShift) +
740 (MatchAllTag << HWASanAccessInfo::MatchAllShift) +
741 (Recover << HWASanAccessInfo::RecoverShift) +
742 (IsWrite << HWASanAccessInfo::IsWriteShift) +
743 (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
744 IRBuilder<> IRB(InsertBefore);
745
746 if (OutlinedChecks) {
747 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
748 Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
749 IRB.CreateCall(Intrinsic::getDeclaration(
750 M, UseShortGranules
751 ? Intrinsic::hwasan_check_memaccess_shortgranules
752 : Intrinsic::hwasan_check_memaccess),
753 {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
754 return;
755 }
756
757 Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
758 Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
759 IRB.getInt8Ty());
760 Value *AddrLong = untagPointer(IRB, PtrLong);
761 Value *Shadow = memToShadow(AddrLong, IRB);
762 Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
763 Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
764
765 if (HasMatchAllTag) {
766 Value *TagNotIgnored = IRB.CreateICmpNE(
767 PtrTag, ConstantInt::get(PtrTag->getType(), MatchAllTag));
768 TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
769 }
770
771 Instruction *CheckTerm =
772 SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false,
773 MDBuilder(*C).createBranchWeights(1, 100000));
774
775 IRB.SetInsertPoint(CheckTerm);
776 Value *OutOfShortGranuleTagRange =
777 IRB.CreateICmpUGT(MemTag, ConstantInt::get(Int8Ty, 15));
778 Instruction *CheckFailTerm =
779 SplitBlockAndInsertIfThen(OutOfShortGranuleTagRange, CheckTerm, !Recover,
780 MDBuilder(*C).createBranchWeights(1, 100000));
781
782 IRB.SetInsertPoint(CheckTerm);
783 Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(PtrLong, 15), Int8Ty);
784 PtrLowBits = IRB.CreateAdd(
785 PtrLowBits, ConstantInt::get(Int8Ty, (1 << AccessSizeIndex) - 1));
786 Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, MemTag);
787 SplitBlockAndInsertIfThen(PtrLowBitsOOB, CheckTerm, false,
788 MDBuilder(*C).createBranchWeights(1, 100000),
789 (DomTreeUpdater *)nullptr, nullptr,
790 CheckFailTerm->getParent());
791
792 IRB.SetInsertPoint(CheckTerm);
793 Value *InlineTagAddr = IRB.CreateOr(AddrLong, 15);
794 InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, Int8PtrTy);
795 Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr);
796 Value *InlineTagMismatch = IRB.CreateICmpNE(PtrTag, InlineTag);
797 SplitBlockAndInsertIfThen(InlineTagMismatch, CheckTerm, false,
798 MDBuilder(*C).createBranchWeights(1, 100000),
799 (DomTreeUpdater *)nullptr, nullptr,
800 CheckFailTerm->getParent());
801
802 IRB.SetInsertPoint(CheckFailTerm);
803 InlineAsm *Asm;
804 switch (TargetTriple.getArch()) {
805 case Triple::x86_64:
806 // The signal handler will find the data address in rdi.
807 Asm = InlineAsm::get(
808 FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
809 "int3\nnopl " +
810 itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
811 "(%rax)",
812 "{rdi}",
813 /*hasSideEffects=*/true);
814 break;
815 case Triple::aarch64:
816 case Triple::aarch64_be:
817 // The signal handler will find the data address in x0.
818 Asm = InlineAsm::get(
819 FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
820 "brk #" + itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
821 "{x0}",
822 /*hasSideEffects=*/true);
823 break;
824 default:
825 report_fatal_error("unsupported architecture");
826 }
827 IRB.CreateCall(Asm, PtrLong);
828 if (Recover)
829 cast<BranchInst>(CheckFailTerm)->setSuccessor(0, CheckTerm->getParent());
830 }
831
instrumentMemIntrinsic(MemIntrinsic * MI)832 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
833 IRBuilder<> IRB(MI);
834 if (isa<MemTransferInst>(MI)) {
835 IRB.CreateCall(
836 isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
837 {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
838 IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
839 IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
840 } else if (isa<MemSetInst>(MI)) {
841 IRB.CreateCall(
842 HWAsanMemset,
843 {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
844 IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
845 IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
846 }
847 MI->eraseFromParent();
848 }
849
instrumentMemAccess(InterestingMemoryOperand & O)850 bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O) {
851 Value *Addr = O.getPtr();
852
853 LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
854
855 if (O.MaybeMask)
856 return false; // FIXME
857
858 IRBuilder<> IRB(O.getInsn());
859 if (isPowerOf2_64(O.TypeSize) &&
860 (O.TypeSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
861 (!O.Alignment || *O.Alignment >= (1ULL << Mapping.Scale) ||
862 *O.Alignment >= O.TypeSize / 8)) {
863 size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeSize);
864 if (InstrumentWithCalls) {
865 IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
866 IRB.CreatePointerCast(Addr, IntptrTy));
867 } else {
868 instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
869 }
870 } else {
871 IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite],
872 {IRB.CreatePointerCast(Addr, IntptrTy),
873 ConstantInt::get(IntptrTy, O.TypeSize / 8)});
874 }
875 untagPointerOperand(O.getInsn(), Addr);
876
877 return true;
878 }
879
getAllocaSizeInBytes(const AllocaInst & AI)880 static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
881 uint64_t ArraySize = 1;
882 if (AI.isArrayAllocation()) {
883 const ConstantInt *CI = dyn_cast<ConstantInt>(AI.getArraySize());
884 assert(CI && "non-constant array size");
885 ArraySize = CI->getZExtValue();
886 }
887 Type *Ty = AI.getAllocatedType();
888 uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty);
889 return SizeInBytes * ArraySize;
890 }
891
tagAlloca(IRBuilder<> & IRB,AllocaInst * AI,Value * Tag,size_t Size)892 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
893 size_t Size) {
894 size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
895 if (!UseShortGranules)
896 Size = AlignedSize;
897
898 Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
899 if (InstrumentWithCalls) {
900 IRB.CreateCall(HwasanTagMemoryFunc,
901 {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag,
902 ConstantInt::get(IntptrTy, AlignedSize)});
903 } else {
904 size_t ShadowSize = Size >> Mapping.Scale;
905 Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
906 // If this memset is not inlined, it will be intercepted in the hwasan
907 // runtime library. That's OK, because the interceptor skips the checks if
908 // the address is in the shadow region.
909 // FIXME: the interceptor is not as fast as real memset. Consider lowering
910 // llvm.memset right here into either a sequence of stores, or a call to
911 // hwasan_tag_memory.
912 if (ShadowSize)
913 IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, Align(1));
914 if (Size != AlignedSize) {
915 IRB.CreateStore(
916 ConstantInt::get(Int8Ty, Size % Mapping.getObjectAlignment()),
917 IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));
918 IRB.CreateStore(JustTag, IRB.CreateConstGEP1_32(
919 Int8Ty, IRB.CreateBitCast(AI, Int8PtrTy),
920 AlignedSize - 1));
921 }
922 }
923 return true;
924 }
925
RetagMask(unsigned AllocaNo)926 static unsigned RetagMask(unsigned AllocaNo) {
927 // A list of 8-bit numbers that have at most one run of non-zero bits.
928 // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
929 // masks.
930 // The list does not include the value 255, which is used for UAR.
931 //
932 // Because we are more likely to use earlier elements of this list than later
933 // ones, it is sorted in increasing order of probability of collision with a
934 // mask allocated (temporally) nearby. The program that generated this list
935 // can be found at:
936 // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
937 static unsigned FastMasks[] = {0, 128, 64, 192, 32, 96, 224, 112, 240,
938 48, 16, 120, 248, 56, 24, 8, 124, 252,
939 60, 28, 12, 4, 126, 254, 62, 30, 14,
940 6, 2, 127, 63, 31, 15, 7, 3, 1};
941 return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))];
942 }
943
getNextTagWithCall(IRBuilder<> & IRB)944 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
945 return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
946 }
947
getStackBaseTag(IRBuilder<> & IRB)948 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
949 if (ClGenerateTagsWithCalls)
950 return getNextTagWithCall(IRB);
951 if (StackBaseTag)
952 return StackBaseTag;
953 // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
954 // first).
955 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
956 auto GetStackPointerFn = Intrinsic::getDeclaration(
957 M, Intrinsic::frameaddress,
958 IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
959 Value *StackPointer = IRB.CreateCall(
960 GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())});
961
962 // Extract some entropy from the stack pointer for the tags.
963 // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
964 // between functions).
965 Value *StackPointerLong = IRB.CreatePointerCast(StackPointer, IntptrTy);
966 Value *StackTag =
967 IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20),
968 "hwasan.stack.base.tag");
969 return StackTag;
970 }
971
getAllocaTag(IRBuilder<> & IRB,Value * StackTag,AllocaInst * AI,unsigned AllocaNo)972 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
973 AllocaInst *AI, unsigned AllocaNo) {
974 if (ClGenerateTagsWithCalls)
975 return getNextTagWithCall(IRB);
976 return IRB.CreateXor(StackTag,
977 ConstantInt::get(IntptrTy, RetagMask(AllocaNo)));
978 }
979
getUARTag(IRBuilder<> & IRB,Value * StackTag)980 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) {
981 if (ClUARRetagToZero)
982 return ConstantInt::get(IntptrTy, 0);
983 if (ClGenerateTagsWithCalls)
984 return getNextTagWithCall(IRB);
985 return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU));
986 }
987
988 // Add a tag to an address.
tagPointer(IRBuilder<> & IRB,Type * Ty,Value * PtrLong,Value * Tag)989 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
990 Value *PtrLong, Value *Tag) {
991 assert(!UsePageAliases);
992 Value *TaggedPtrLong;
993 if (CompileKernel) {
994 // Kernel addresses have 0xFF in the most significant byte.
995 Value *ShiftedTag = IRB.CreateOr(
996 IRB.CreateShl(Tag, kPointerTagShift),
997 ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1));
998 TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
999 } else {
1000 // Userspace can simply do OR (tag << 56);
1001 Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift);
1002 TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
1003 }
1004 return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
1005 }
1006
1007 // Remove tag from an address.
untagPointer(IRBuilder<> & IRB,Value * PtrLong)1008 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
1009 assert(!UsePageAliases);
1010 Value *UntaggedPtrLong;
1011 if (CompileKernel) {
1012 // Kernel addresses have 0xFF in the most significant byte.
1013 UntaggedPtrLong =
1014 IRB.CreateOr(PtrLong, ConstantInt::get(PtrLong->getType(),
1015 0xFFULL << kPointerTagShift));
1016 } else {
1017 // Userspace addresses have 0x00.
1018 UntaggedPtrLong = IRB.CreateAnd(
1019 PtrLong,
1020 ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift)));
1021 }
1022 return UntaggedPtrLong;
1023 }
1024
getHwasanThreadSlotPtr(IRBuilder<> & IRB,Type * Ty)1025 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
1026 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1027 if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
1028 // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
1029 // in Bionic's libc/private/bionic_tls.h.
1030 Function *ThreadPointerFunc =
1031 Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
1032 Value *SlotPtr = IRB.CreatePointerCast(
1033 IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
1034 IRB.CreateCall(ThreadPointerFunc), 0x30),
1035 Ty->getPointerTo(0));
1036 return SlotPtr;
1037 }
1038 if (ThreadPtrGlobal)
1039 return ThreadPtrGlobal;
1040
1041 return nullptr;
1042 }
1043
emitPrologue(IRBuilder<> & IRB,bool WithFrameRecord)1044 void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
1045 if (!Mapping.InTls) {
1046 ShadowBase = getShadowNonTls(IRB);
1047 return;
1048 }
1049
1050 if (!WithFrameRecord && TargetTriple.isAndroid()) {
1051 ShadowBase = getDynamicShadowIfunc(IRB);
1052 return;
1053 }
1054
1055 Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
1056 assert(SlotPtr);
1057
1058 Value *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
1059 // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI.
1060 Value *ThreadLongMaybeUntagged =
1061 TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong);
1062
1063 if (WithFrameRecord) {
1064 Function *F = IRB.GetInsertBlock()->getParent();
1065 StackBaseTag = IRB.CreateAShr(ThreadLong, 3);
1066
1067 // Prepare ring buffer data.
1068 Value *PC;
1069 if (TargetTriple.getArch() == Triple::aarch64)
1070 PC = readRegister(IRB, "pc");
1071 else
1072 PC = IRB.CreatePtrToInt(F, IntptrTy);
1073 Module *M = F->getParent();
1074 auto GetStackPointerFn = Intrinsic::getDeclaration(
1075 M, Intrinsic::frameaddress,
1076 IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace()));
1077 Value *SP = IRB.CreatePtrToInt(
1078 IRB.CreateCall(GetStackPointerFn,
1079 {Constant::getNullValue(IRB.getInt32Ty())}),
1080 IntptrTy);
1081 // Mix SP and PC.
1082 // Assumptions:
1083 // PC is 0x0000PPPPPPPPPPPP (48 bits are meaningful, others are zero)
1084 // SP is 0xsssssssssssSSSS0 (4 lower bits are zero)
1085 // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
1086 // 0xSSSSPPPPPPPPPPPP
1087 SP = IRB.CreateShl(SP, 44);
1088
1089 // Store data to ring buffer.
1090 Value *RecordPtr =
1091 IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IntptrTy->getPointerTo(0));
1092 IRB.CreateStore(IRB.CreateOr(PC, SP), RecordPtr);
1093
1094 // Update the ring buffer. Top byte of ThreadLong defines the size of the
1095 // buffer in pages, it must be a power of two, and the start of the buffer
1096 // must be aligned by twice that much. Therefore wrap around of the ring
1097 // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
1098 // The use of AShr instead of LShr is due to
1099 // https://bugs.llvm.org/show_bug.cgi?id=39030
1100 // Runtime library makes sure not to use the highest bit.
1101 Value *WrapMask = IRB.CreateXor(
1102 IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),
1103 ConstantInt::get(IntptrTy, (uint64_t)-1));
1104 Value *ThreadLongNew = IRB.CreateAnd(
1105 IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
1106 IRB.CreateStore(ThreadLongNew, SlotPtr);
1107 }
1108
1109 // Get shadow base address by aligning RecordPtr up.
1110 // Note: this is not correct if the pointer is already aligned.
1111 // Runtime library will make sure this never happens.
1112 ShadowBase = IRB.CreateAdd(
1113 IRB.CreateOr(
1114 ThreadLongMaybeUntagged,
1115 ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
1116 ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
1117 ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
1118 }
1119
readRegister(IRBuilder<> & IRB,StringRef Name)1120 Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {
1121 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
1122 Function *ReadRegister =
1123 Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy);
1124 MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)});
1125 Value *Args[] = {MetadataAsValue::get(*C, MD)};
1126 return IRB.CreateCall(ReadRegister, Args);
1127 }
1128
instrumentLandingPads(SmallVectorImpl<Instruction * > & LandingPadVec)1129 bool HWAddressSanitizer::instrumentLandingPads(
1130 SmallVectorImpl<Instruction *> &LandingPadVec) {
1131 for (auto *LP : LandingPadVec) {
1132 IRBuilder<> IRB(LP->getNextNode());
1133 IRB.CreateCall(
1134 HWAsanHandleVfork,
1135 {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp"
1136 : "sp")});
1137 }
1138 return true;
1139 }
1140
instrumentStack(SmallVectorImpl<AllocaInst * > & Allocas,DenseMap<AllocaInst *,std::vector<DbgVariableIntrinsic * >> & AllocaDbgMap,SmallVectorImpl<Instruction * > & RetVec,Value * StackTag)1141 bool HWAddressSanitizer::instrumentStack(
1142 SmallVectorImpl<AllocaInst *> &Allocas,
1143 DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap,
1144 SmallVectorImpl<Instruction *> &RetVec, Value *StackTag) {
1145 // Ideally, we want to calculate tagged stack base pointer, and rewrite all
1146 // alloca addresses using that. Unfortunately, offsets are not known yet
1147 // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
1148 // temp, shift-OR it into each alloca address and xor with the retag mask.
1149 // This generates one extra instruction per alloca use.
1150 for (unsigned N = 0; N < Allocas.size(); ++N) {
1151 auto *AI = Allocas[N];
1152 IRBuilder<> IRB(AI->getNextNode());
1153
1154 // Replace uses of the alloca with tagged address.
1155 Value *Tag = getAllocaTag(IRB, StackTag, AI, N);
1156 Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
1157 Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag);
1158 std::string Name =
1159 AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
1160 Replacement->setName(Name + ".hwasan");
1161
1162 AI->replaceUsesWithIf(Replacement,
1163 [AILong](Use &U) { return U.getUser() != AILong; });
1164
1165 for (auto *DDI : AllocaDbgMap.lookup(AI)) {
1166 // Prepend "tag_offset, N" to the dwarf expression.
1167 // Tag offset logically applies to the alloca pointer, and it makes sense
1168 // to put it at the beginning of the expression.
1169 SmallVector<uint64_t, 8> NewOps = {dwarf::DW_OP_LLVM_tag_offset,
1170 RetagMask(N)};
1171 auto Locations = DDI->location_ops();
1172 unsigned LocNo = std::distance(Locations.begin(), find(Locations, AI));
1173 DDI->setExpression(
1174 DIExpression::appendOpsToArg(DDI->getExpression(), NewOps, LocNo));
1175 }
1176
1177 size_t Size = getAllocaSizeInBytes(*AI);
1178 tagAlloca(IRB, AI, Tag, Size);
1179
1180 for (auto RI : RetVec) {
1181 IRB.SetInsertPoint(RI);
1182
1183 // Re-tag alloca memory with the special UAR tag.
1184 Value *Tag = getUARTag(IRB, StackTag);
1185 tagAlloca(IRB, AI, Tag, alignTo(Size, Mapping.getObjectAlignment()));
1186 }
1187 }
1188
1189 return true;
1190 }
1191
isInterestingAlloca(const AllocaInst & AI)1192 bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) {
1193 return (AI.getAllocatedType()->isSized() &&
1194 // FIXME: instrument dynamic allocas, too
1195 AI.isStaticAlloca() &&
1196 // alloca() may be called with 0 size, ignore it.
1197 getAllocaSizeInBytes(AI) > 0 &&
1198 // We are only interested in allocas not promotable to registers.
1199 // Promotable allocas are common under -O0.
1200 !isAllocaPromotable(&AI) &&
1201 // inalloca allocas are not treated as static, and we don't want
1202 // dynamic alloca instrumentation for them as well.
1203 !AI.isUsedWithInAlloca() &&
1204 // swifterror allocas are register promoted by ISel
1205 !AI.isSwiftError());
1206 }
1207
sanitizeFunction(Function & F)1208 bool HWAddressSanitizer::sanitizeFunction(Function &F) {
1209 if (&F == HwasanCtorFunction)
1210 return false;
1211
1212 if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
1213 return false;
1214
1215 LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
1216
1217 SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
1218 SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
1219 SmallVector<AllocaInst *, 8> AllocasToInstrument;
1220 SmallVector<Instruction *, 8> RetVec;
1221 SmallVector<Instruction *, 8> LandingPadVec;
1222 DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> AllocaDbgMap;
1223 for (auto &BB : F) {
1224 for (auto &Inst : BB) {
1225 if (InstrumentStack)
1226 if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
1227 if (isInterestingAlloca(*AI))
1228 AllocasToInstrument.push_back(AI);
1229 continue;
1230 }
1231
1232 if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) ||
1233 isa<CleanupReturnInst>(Inst))
1234 RetVec.push_back(&Inst);
1235
1236 if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&Inst))
1237 for (Value *V : DVI->location_ops())
1238 if (auto *Alloca = dyn_cast_or_null<AllocaInst>(V))
1239 AllocaDbgMap[Alloca].push_back(DVI);
1240
1241 if (InstrumentLandingPads && isa<LandingPadInst>(Inst))
1242 LandingPadVec.push_back(&Inst);
1243
1244 getInterestingMemoryOperands(&Inst, OperandsToInstrument);
1245
1246 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst))
1247 IntrinToInstrument.push_back(MI);
1248 }
1249 }
1250
1251 initializeCallbacks(*F.getParent());
1252
1253 bool Changed = false;
1254
1255 if (!LandingPadVec.empty())
1256 Changed |= instrumentLandingPads(LandingPadVec);
1257
1258 if (AllocasToInstrument.empty() && F.hasPersonalityFn() &&
1259 F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
1260 // __hwasan_personality_thunk is a no-op for functions without an
1261 // instrumented stack, so we can drop it.
1262 F.setPersonalityFn(nullptr);
1263 Changed = true;
1264 }
1265
1266 if (AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
1267 IntrinToInstrument.empty())
1268 return Changed;
1269
1270 assert(!ShadowBase);
1271
1272 Instruction *InsertPt = &*F.getEntryBlock().begin();
1273 IRBuilder<> EntryIRB(InsertPt);
1274 emitPrologue(EntryIRB,
1275 /*WithFrameRecord*/ ClRecordStackHistory &&
1276 !AllocasToInstrument.empty());
1277
1278 if (!AllocasToInstrument.empty()) {
1279 Value *StackTag =
1280 ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB);
1281 instrumentStack(AllocasToInstrument, AllocaDbgMap, RetVec, StackTag);
1282 }
1283 // Pad and align each of the allocas that we instrumented to stop small
1284 // uninteresting allocas from hiding in instrumented alloca's padding and so
1285 // that we have enough space to store real tags for short granules.
1286 DenseMap<AllocaInst *, AllocaInst *> AllocaToPaddedAllocaMap;
1287 for (AllocaInst *AI : AllocasToInstrument) {
1288 uint64_t Size = getAllocaSizeInBytes(*AI);
1289 uint64_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
1290 AI->setAlignment(
1291 Align(std::max(AI->getAlignment(), Mapping.getObjectAlignment())));
1292 if (Size != AlignedSize) {
1293 Type *AllocatedType = AI->getAllocatedType();
1294 if (AI->isArrayAllocation()) {
1295 uint64_t ArraySize =
1296 cast<ConstantInt>(AI->getArraySize())->getZExtValue();
1297 AllocatedType = ArrayType::get(AllocatedType, ArraySize);
1298 }
1299 Type *TypeWithPadding = StructType::get(
1300 AllocatedType, ArrayType::get(Int8Ty, AlignedSize - Size));
1301 auto *NewAI = new AllocaInst(
1302 TypeWithPadding, AI->getType()->getAddressSpace(), nullptr, "", AI);
1303 NewAI->takeName(AI);
1304 NewAI->setAlignment(AI->getAlign());
1305 NewAI->setUsedWithInAlloca(AI->isUsedWithInAlloca());
1306 NewAI->setSwiftError(AI->isSwiftError());
1307 NewAI->copyMetadata(*AI);
1308 auto *Bitcast = new BitCastInst(NewAI, AI->getType(), "", AI);
1309 AI->replaceAllUsesWith(Bitcast);
1310 AllocaToPaddedAllocaMap[AI] = NewAI;
1311 }
1312 }
1313
1314 if (!AllocaToPaddedAllocaMap.empty()) {
1315 for (auto &BB : F) {
1316 for (auto &Inst : BB) {
1317 if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&Inst)) {
1318 for (Value *V : DVI->location_ops()) {
1319 if (auto *AI = dyn_cast_or_null<AllocaInst>(V)) {
1320 if (auto *NewAI = AllocaToPaddedAllocaMap.lookup(AI))
1321 DVI->replaceVariableLocationOp(V, NewAI);
1322 }
1323 }
1324 }
1325 }
1326 }
1327 for (auto &P : AllocaToPaddedAllocaMap)
1328 P.first->eraseFromParent();
1329 }
1330
1331 // If we split the entry block, move any allocas that were originally in the
1332 // entry block back into the entry block so that they aren't treated as
1333 // dynamic allocas.
1334 if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
1335 InsertPt = &*F.getEntryBlock().begin();
1336 for (auto II = EntryIRB.GetInsertBlock()->begin(),
1337 IE = EntryIRB.GetInsertBlock()->end();
1338 II != IE;) {
1339 Instruction *I = &*II++;
1340 if (auto *AI = dyn_cast<AllocaInst>(I))
1341 if (isa<ConstantInt>(AI->getArraySize()))
1342 I->moveBefore(InsertPt);
1343 }
1344 }
1345
1346 for (auto &Operand : OperandsToInstrument)
1347 instrumentMemAccess(Operand);
1348
1349 if (ClInstrumentMemIntrinsics && !IntrinToInstrument.empty()) {
1350 for (auto Inst : IntrinToInstrument)
1351 instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
1352 }
1353
1354 ShadowBase = nullptr;
1355 StackBaseTag = nullptr;
1356
1357 return true;
1358 }
1359
instrumentGlobal(GlobalVariable * GV,uint8_t Tag)1360 void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1361 assert(!UsePageAliases);
1362 Constant *Initializer = GV->getInitializer();
1363 uint64_t SizeInBytes =
1364 M.getDataLayout().getTypeAllocSize(Initializer->getType());
1365 uint64_t NewSize = alignTo(SizeInBytes, Mapping.getObjectAlignment());
1366 if (SizeInBytes != NewSize) {
1367 // Pad the initializer out to the next multiple of 16 bytes and add the
1368 // required short granule tag.
1369 std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
1370 Init.back() = Tag;
1371 Constant *Padding = ConstantDataArray::get(*C, Init);
1372 Initializer = ConstantStruct::getAnon({Initializer, Padding});
1373 }
1374
1375 auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
1376 GlobalValue::ExternalLinkage, Initializer,
1377 GV->getName() + ".hwasan");
1378 NewGV->copyAttributesFrom(GV);
1379 NewGV->setLinkage(GlobalValue::PrivateLinkage);
1380 NewGV->copyMetadata(GV, 0);
1381 NewGV->setAlignment(
1382 MaybeAlign(std::max(GV->getAlignment(), Mapping.getObjectAlignment())));
1383
1384 // It is invalid to ICF two globals that have different tags. In the case
1385 // where the size of the global is a multiple of the tag granularity the
1386 // contents of the globals may be the same but the tags (i.e. symbol values)
1387 // may be different, and the symbols are not considered during ICF. In the
1388 // case where the size is not a multiple of the granularity, the short granule
1389 // tags would discriminate two globals with different tags, but there would
1390 // otherwise be nothing stopping such a global from being incorrectly ICF'd
1391 // with an uninstrumented (i.e. tag 0) global that happened to have the short
1392 // granule tag in the last byte.
1393 NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
1394
1395 // Descriptor format (assuming little-endian):
1396 // bytes 0-3: relative address of global
1397 // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
1398 // it isn't, we create multiple descriptors)
1399 // byte 7: tag
1400 auto *DescriptorTy = StructType::get(Int32Ty, Int32Ty);
1401 const uint64_t MaxDescriptorSize = 0xfffff0;
1402 for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
1403 DescriptorPos += MaxDescriptorSize) {
1404 auto *Descriptor =
1405 new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
1406 nullptr, GV->getName() + ".hwasan.descriptor");
1407 auto *GVRelPtr = ConstantExpr::getTrunc(
1408 ConstantExpr::getAdd(
1409 ConstantExpr::getSub(
1410 ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1411 ConstantExpr::getPtrToInt(Descriptor, Int64Ty)),
1412 ConstantInt::get(Int64Ty, DescriptorPos)),
1413 Int32Ty);
1414 uint32_t Size = std::min(SizeInBytes - DescriptorPos, MaxDescriptorSize);
1415 auto *SizeAndTag = ConstantInt::get(Int32Ty, Size | (uint32_t(Tag) << 24));
1416 Descriptor->setComdat(NewGV->getComdat());
1417 Descriptor->setInitializer(ConstantStruct::getAnon({GVRelPtr, SizeAndTag}));
1418 Descriptor->setSection("hwasan_globals");
1419 Descriptor->setMetadata(LLVMContext::MD_associated,
1420 MDNode::get(*C, ValueAsMetadata::get(NewGV)));
1421 appendToCompilerUsed(M, Descriptor);
1422 }
1423
1424 Constant *Aliasee = ConstantExpr::getIntToPtr(
1425 ConstantExpr::getAdd(
1426 ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1427 ConstantInt::get(Int64Ty, uint64_t(Tag) << kPointerTagShift)),
1428 GV->getType());
1429 auto *Alias = GlobalAlias::create(GV->getValueType(), GV->getAddressSpace(),
1430 GV->getLinkage(), "", Aliasee, &M);
1431 Alias->setVisibility(GV->getVisibility());
1432 Alias->takeName(GV);
1433 GV->replaceAllUsesWith(Alias);
1434 GV->eraseFromParent();
1435 }
1436
instrumentGlobals()1437 void HWAddressSanitizer::instrumentGlobals() {
1438 std::vector<GlobalVariable *> Globals;
1439 for (GlobalVariable &GV : M.globals()) {
1440 if (GV.isDeclarationForLinker() || GV.getName().startswith("llvm.") ||
1441 GV.isThreadLocal())
1442 continue;
1443
1444 // Common symbols can't have aliases point to them, so they can't be tagged.
1445 if (GV.hasCommonLinkage())
1446 continue;
1447
1448 // Globals with custom sections may be used in __start_/__stop_ enumeration,
1449 // which would be broken both by adding tags and potentially by the extra
1450 // padding/alignment that we insert.
1451 if (GV.hasSection())
1452 continue;
1453
1454 Globals.push_back(&GV);
1455 }
1456
1457 MD5 Hasher;
1458 Hasher.update(M.getSourceFileName());
1459 MD5::MD5Result Hash;
1460 Hasher.final(Hash);
1461 uint8_t Tag = Hash[0];
1462
1463 for (GlobalVariable *GV : Globals) {
1464 // Skip tag 0 in order to avoid collisions with untagged memory.
1465 if (Tag == 0)
1466 Tag = 1;
1467 instrumentGlobal(GV, Tag++);
1468 }
1469 }
1470
instrumentPersonalityFunctions()1471 void HWAddressSanitizer::instrumentPersonalityFunctions() {
1472 // We need to untag stack frames as we unwind past them. That is the job of
1473 // the personality function wrapper, which either wraps an existing
1474 // personality function or acts as a personality function on its own. Each
1475 // function that has a personality function or that can be unwound past has
1476 // its personality function changed to a thunk that calls the personality
1477 // function wrapper in the runtime.
1478 MapVector<Constant *, std::vector<Function *>> PersonalityFns;
1479 for (Function &F : M) {
1480 if (F.isDeclaration() || !F.hasFnAttribute(Attribute::SanitizeHWAddress))
1481 continue;
1482
1483 if (F.hasPersonalityFn()) {
1484 PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(&F);
1485 } else if (!F.hasFnAttribute(Attribute::NoUnwind)) {
1486 PersonalityFns[nullptr].push_back(&F);
1487 }
1488 }
1489
1490 if (PersonalityFns.empty())
1491 return;
1492
1493 FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
1494 "__hwasan_personality_wrapper", Int32Ty, Int32Ty, Int32Ty, Int64Ty,
1495 Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy);
1496 FunctionCallee UnwindGetGR = M.getOrInsertFunction("_Unwind_GetGR", VoidTy);
1497 FunctionCallee UnwindGetCFA = M.getOrInsertFunction("_Unwind_GetCFA", VoidTy);
1498
1499 for (auto &P : PersonalityFns) {
1500 std::string ThunkName = kHwasanPersonalityThunkName;
1501 if (P.first)
1502 ThunkName += ("." + P.first->getName()).str();
1503 FunctionType *ThunkFnTy = FunctionType::get(
1504 Int32Ty, {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int8PtrTy}, false);
1505 bool IsLocal = P.first && (!isa<GlobalValue>(P.first) ||
1506 cast<GlobalValue>(P.first)->hasLocalLinkage());
1507 auto *ThunkFn = Function::Create(ThunkFnTy,
1508 IsLocal ? GlobalValue::InternalLinkage
1509 : GlobalValue::LinkOnceODRLinkage,
1510 ThunkName, &M);
1511 if (!IsLocal) {
1512 ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
1513 ThunkFn->setComdat(M.getOrInsertComdat(ThunkName));
1514 }
1515
1516 auto *BB = BasicBlock::Create(*C, "entry", ThunkFn);
1517 IRBuilder<> IRB(BB);
1518 CallInst *WrapperCall = IRB.CreateCall(
1519 HwasanPersonalityWrapper,
1520 {ThunkFn->getArg(0), ThunkFn->getArg(1), ThunkFn->getArg(2),
1521 ThunkFn->getArg(3), ThunkFn->getArg(4),
1522 P.first ? IRB.CreateBitCast(P.first, Int8PtrTy)
1523 : Constant::getNullValue(Int8PtrTy),
1524 IRB.CreateBitCast(UnwindGetGR.getCallee(), Int8PtrTy),
1525 IRB.CreateBitCast(UnwindGetCFA.getCallee(), Int8PtrTy)});
1526 WrapperCall->setTailCall();
1527 IRB.CreateRet(WrapperCall);
1528
1529 for (Function *F : P.second)
1530 F->setPersonalityFn(ThunkFn);
1531 }
1532 }
1533
init(Triple & TargetTriple,bool InstrumentWithCalls)1534 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple,
1535 bool InstrumentWithCalls) {
1536 Scale = kDefaultShadowScale;
1537 if (TargetTriple.isOSFuchsia()) {
1538 // Fuchsia is always PIE, which means that the beginning of the address
1539 // space is always available.
1540 InGlobal = false;
1541 InTls = false;
1542 Offset = 0;
1543 } else if (ClMappingOffset.getNumOccurrences() > 0) {
1544 InGlobal = false;
1545 InTls = false;
1546 Offset = ClMappingOffset;
1547 } else if (ClEnableKhwasan || InstrumentWithCalls) {
1548 InGlobal = false;
1549 InTls = false;
1550 Offset = 0;
1551 } else if (ClWithIfunc) {
1552 InGlobal = true;
1553 InTls = false;
1554 Offset = kDynamicShadowSentinel;
1555 } else if (ClWithTls) {
1556 InGlobal = false;
1557 InTls = true;
1558 Offset = kDynamicShadowSentinel;
1559 } else {
1560 InGlobal = false;
1561 InTls = false;
1562 Offset = kDynamicShadowSentinel;
1563 }
1564 }
1565