106c3fb27SDimitry Andric //===- HWAddressSanitizer.cpp - memory access error detector --------------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric /// \file
104824e7fdSDimitry Andric /// This file is a part of HWAddressSanitizer, an address basic correctness
114824e7fdSDimitry Andric /// checker based on tagged addressing.
120b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
130b57cec5SDimitry Andric
140b57cec5SDimitry Andric #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
158bcb0991SDimitry Andric #include "llvm/ADT/MapVector.h"
1681ad6265SDimitry Andric #include "llvm/ADT/STLExtras.h"
170b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h"
180b57cec5SDimitry Andric #include "llvm/ADT/StringExtras.h"
190b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h"
205f757f3fSDimitry Andric #include "llvm/Analysis/DomTreeUpdater.h"
21bdd1243dSDimitry Andric #include "llvm/Analysis/GlobalsModRef.h"
22349cc55cSDimitry Andric #include "llvm/Analysis/PostDominators.h"
23fe6060f1SDimitry Andric #include "llvm/Analysis/StackSafetyAnalysis.h"
245f757f3fSDimitry Andric #include "llvm/Analysis/TargetLibraryInfo.h"
25349cc55cSDimitry Andric #include "llvm/Analysis/ValueTracking.h"
2681ad6265SDimitry Andric #include "llvm/BinaryFormat/Dwarf.h"
278bcb0991SDimitry Andric #include "llvm/BinaryFormat/ELF.h"
280b57cec5SDimitry Andric #include "llvm/IR/Attributes.h"
290b57cec5SDimitry Andric #include "llvm/IR/BasicBlock.h"
300b57cec5SDimitry Andric #include "llvm/IR/Constant.h"
310b57cec5SDimitry Andric #include "llvm/IR/Constants.h"
320b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h"
330b57cec5SDimitry Andric #include "llvm/IR/DebugInfoMetadata.h"
340b57cec5SDimitry Andric #include "llvm/IR/DerivedTypes.h"
35349cc55cSDimitry Andric #include "llvm/IR/Dominators.h"
360b57cec5SDimitry Andric #include "llvm/IR/Function.h"
370b57cec5SDimitry Andric #include "llvm/IR/IRBuilder.h"
380b57cec5SDimitry Andric #include "llvm/IR/InlineAsm.h"
3981ad6265SDimitry Andric #include "llvm/IR/InstIterator.h"
400b57cec5SDimitry Andric #include "llvm/IR/Instruction.h"
410b57cec5SDimitry Andric #include "llvm/IR/Instructions.h"
420b57cec5SDimitry Andric #include "llvm/IR/IntrinsicInst.h"
430b57cec5SDimitry Andric #include "llvm/IR/Intrinsics.h"
440b57cec5SDimitry Andric #include "llvm/IR/LLVMContext.h"
450b57cec5SDimitry Andric #include "llvm/IR/MDBuilder.h"
460b57cec5SDimitry Andric #include "llvm/IR/Module.h"
470b57cec5SDimitry Andric #include "llvm/IR/Type.h"
480b57cec5SDimitry Andric #include "llvm/IR/Value.h"
490b57cec5SDimitry Andric #include "llvm/Support/Casting.h"
500b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h"
510b57cec5SDimitry Andric #include "llvm/Support/Debug.h"
520b57cec5SDimitry Andric #include "llvm/Support/raw_ostream.h"
5306c3fb27SDimitry Andric #include "llvm/TargetParser/Triple.h"
545ffd83dbSDimitry Andric #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
550b57cec5SDimitry Andric #include "llvm/Transforms/Utils/BasicBlockUtils.h"
565f757f3fSDimitry Andric #include "llvm/Transforms/Utils/Local.h"
5781ad6265SDimitry Andric #include "llvm/Transforms/Utils/MemoryTaggingSupport.h"
580b57cec5SDimitry Andric #include "llvm/Transforms/Utils/ModuleUtils.h"
590b57cec5SDimitry Andric #include "llvm/Transforms/Utils/PromoteMemToReg.h"
60bdd1243dSDimitry Andric #include <optional>
610b57cec5SDimitry Andric
620b57cec5SDimitry Andric using namespace llvm;
630b57cec5SDimitry Andric
640b57cec5SDimitry Andric #define DEBUG_TYPE "hwasan"
650b57cec5SDimitry Andric
66e8d8bef9SDimitry Andric const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
67e8d8bef9SDimitry Andric const char kHwasanNoteName[] = "hwasan.note";
68e8d8bef9SDimitry Andric const char kHwasanInitName[] = "__hwasan_init";
69e8d8bef9SDimitry Andric const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
700b57cec5SDimitry Andric
71e8d8bef9SDimitry Andric const char kHwasanShadowMemoryDynamicAddress[] =
720b57cec5SDimitry Andric "__hwasan_shadow_memory_dynamic_address";
730b57cec5SDimitry Andric
740b57cec5SDimitry Andric // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
750b57cec5SDimitry Andric static const size_t kNumberOfAccessSizes = 5;
760b57cec5SDimitry Andric
770b57cec5SDimitry Andric static const size_t kDefaultShadowScale = 4;
780b57cec5SDimitry Andric static const uint64_t kDynamicShadowSentinel =
790b57cec5SDimitry Andric std::numeric_limits<uint64_t>::max();
800b57cec5SDimitry Andric
810b57cec5SDimitry Andric static const unsigned kShadowBaseAlignment = 32;
820b57cec5SDimitry Andric
83fe6060f1SDimitry Andric static cl::opt<std::string>
84fe6060f1SDimitry Andric ClMemoryAccessCallbackPrefix("hwasan-memory-access-callback-prefix",
85fe6060f1SDimitry Andric cl::desc("Prefix for memory access callbacks"),
86fe6060f1SDimitry Andric cl::Hidden, cl::init("__hwasan_"));
870b57cec5SDimitry Andric
8881ad6265SDimitry Andric static cl::opt<bool> ClKasanMemIntrinCallbackPrefix(
8981ad6265SDimitry Andric "hwasan-kernel-mem-intrinsic-prefix",
9081ad6265SDimitry Andric cl::desc("Use prefix for memory intrinsics in KASAN mode"), cl::Hidden,
9181ad6265SDimitry Andric cl::init(false));
9281ad6265SDimitry Andric
93fe6060f1SDimitry Andric static cl::opt<bool> ClInstrumentWithCalls(
94fe6060f1SDimitry Andric "hwasan-instrument-with-calls",
95fe6060f1SDimitry Andric cl::desc("instrument reads and writes with callbacks"), cl::Hidden,
96fe6060f1SDimitry Andric cl::init(false));
970b57cec5SDimitry Andric
980b57cec5SDimitry Andric static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads",
990b57cec5SDimitry Andric cl::desc("instrument read instructions"),
1000b57cec5SDimitry Andric cl::Hidden, cl::init(true));
1010b57cec5SDimitry Andric
102fe6060f1SDimitry Andric static cl::opt<bool>
103fe6060f1SDimitry Andric ClInstrumentWrites("hwasan-instrument-writes",
104fe6060f1SDimitry Andric cl::desc("instrument write instructions"), cl::Hidden,
105fe6060f1SDimitry Andric cl::init(true));
1060b57cec5SDimitry Andric
1070b57cec5SDimitry Andric static cl::opt<bool> ClInstrumentAtomics(
1080b57cec5SDimitry Andric "hwasan-instrument-atomics",
1090b57cec5SDimitry Andric cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
1100b57cec5SDimitry Andric cl::init(true));
1110b57cec5SDimitry Andric
1125ffd83dbSDimitry Andric static cl::opt<bool> ClInstrumentByval("hwasan-instrument-byval",
1135ffd83dbSDimitry Andric cl::desc("instrument byval arguments"),
1145ffd83dbSDimitry Andric cl::Hidden, cl::init(true));
1155ffd83dbSDimitry Andric
116fe6060f1SDimitry Andric static cl::opt<bool>
117fe6060f1SDimitry Andric ClRecover("hwasan-recover",
1180b57cec5SDimitry Andric cl::desc("Enable recovery mode (continue-after-error)."),
1190b57cec5SDimitry Andric cl::Hidden, cl::init(false));
1200b57cec5SDimitry Andric
1210b57cec5SDimitry Andric static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack",
1220b57cec5SDimitry Andric cl::desc("instrument stack (allocas)"),
1230b57cec5SDimitry Andric cl::Hidden, cl::init(true));
1240b57cec5SDimitry Andric
125fe6060f1SDimitry Andric static cl::opt<bool>
126fe6060f1SDimitry Andric ClUseStackSafety("hwasan-use-stack-safety", cl::Hidden, cl::init(true),
127fe6060f1SDimitry Andric cl::Hidden, cl::desc("Use Stack Safety analysis results"),
128fe6060f1SDimitry Andric cl::Optional);
129fe6060f1SDimitry Andric
130349cc55cSDimitry Andric static cl::opt<size_t> ClMaxLifetimes(
131349cc55cSDimitry Andric "hwasan-max-lifetimes-for-alloca", cl::Hidden, cl::init(3),
132349cc55cSDimitry Andric cl::ReallyHidden,
133349cc55cSDimitry Andric cl::desc("How many lifetime ends to handle for a single alloca."),
134349cc55cSDimitry Andric cl::Optional);
135349cc55cSDimitry Andric
136349cc55cSDimitry Andric static cl::opt<bool>
137349cc55cSDimitry Andric ClUseAfterScope("hwasan-use-after-scope",
138349cc55cSDimitry Andric cl::desc("detect use after scope within function"),
1395f757f3fSDimitry Andric cl::Hidden, cl::init(true));
140349cc55cSDimitry Andric
1410b57cec5SDimitry Andric static cl::opt<bool> ClGenerateTagsWithCalls(
1420b57cec5SDimitry Andric "hwasan-generate-tags-with-calls",
1430b57cec5SDimitry Andric cl::desc("generate new tags with runtime library calls"), cl::Hidden,
1440b57cec5SDimitry Andric cl::init(false));
1450b57cec5SDimitry Andric
1468bcb0991SDimitry Andric static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"),
14781ad6265SDimitry Andric cl::Hidden, cl::init(false));
1488bcb0991SDimitry Andric
1490b57cec5SDimitry Andric static cl::opt<int> ClMatchAllTag(
1500b57cec5SDimitry Andric "hwasan-match-all-tag",
1510b57cec5SDimitry Andric cl::desc("don't report bad accesses via pointers with this tag"),
1520b57cec5SDimitry Andric cl::Hidden, cl::init(-1));
1530b57cec5SDimitry Andric
154fe6060f1SDimitry Andric static cl::opt<bool>
155fe6060f1SDimitry Andric ClEnableKhwasan("hwasan-kernel",
1560b57cec5SDimitry Andric cl::desc("Enable KernelHWAddressSanitizer instrumentation"),
1570b57cec5SDimitry Andric cl::Hidden, cl::init(false));
1580b57cec5SDimitry Andric
1590b57cec5SDimitry Andric // These flags allow to change the shadow mapping and control how shadow memory
1600b57cec5SDimitry Andric // is accessed. The shadow mapping looks like:
1610b57cec5SDimitry Andric // Shadow = (Mem >> scale) + offset
1620b57cec5SDimitry Andric
1630b57cec5SDimitry Andric static cl::opt<uint64_t>
1640b57cec5SDimitry Andric ClMappingOffset("hwasan-mapping-offset",
1650b57cec5SDimitry Andric cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"),
1660b57cec5SDimitry Andric cl::Hidden, cl::init(0));
1670b57cec5SDimitry Andric
1680b57cec5SDimitry Andric static cl::opt<bool>
1690b57cec5SDimitry Andric ClWithIfunc("hwasan-with-ifunc",
1700b57cec5SDimitry Andric cl::desc("Access dynamic shadow through an ifunc global on "
1710b57cec5SDimitry Andric "platforms that support this"),
1720b57cec5SDimitry Andric cl::Hidden, cl::init(false));
1730b57cec5SDimitry Andric
1740b57cec5SDimitry Andric static cl::opt<bool> ClWithTls(
1750b57cec5SDimitry Andric "hwasan-with-tls",
1760b57cec5SDimitry Andric cl::desc("Access dynamic shadow through an thread-local pointer on "
1770b57cec5SDimitry Andric "platforms that support this"),
1780b57cec5SDimitry Andric cl::Hidden, cl::init(true));
1790b57cec5SDimitry Andric
180753f127fSDimitry Andric // Mode for selecting how to insert frame record info into the stack ring
181753f127fSDimitry Andric // buffer.
182753f127fSDimitry Andric enum RecordStackHistoryMode {
183753f127fSDimitry Andric // Do not record frame record info.
184753f127fSDimitry Andric none,
185753f127fSDimitry Andric
186753f127fSDimitry Andric // Insert instructions into the prologue for storing into the stack ring
187753f127fSDimitry Andric // buffer directly.
188753f127fSDimitry Andric instr,
189753f127fSDimitry Andric
190753f127fSDimitry Andric // Add a call to __hwasan_add_frame_record in the runtime.
191753f127fSDimitry Andric libcall,
192753f127fSDimitry Andric };
193753f127fSDimitry Andric
194753f127fSDimitry Andric static cl::opt<RecordStackHistoryMode> ClRecordStackHistory(
195753f127fSDimitry Andric "hwasan-record-stack-history",
196753f127fSDimitry Andric cl::desc("Record stack frames with tagged allocations in a thread-local "
197753f127fSDimitry Andric "ring buffer"),
198753f127fSDimitry Andric cl::values(clEnumVal(none, "Do not record stack ring history"),
199753f127fSDimitry Andric clEnumVal(instr, "Insert instructions into the prologue for "
200753f127fSDimitry Andric "storing into the stack ring buffer directly"),
201753f127fSDimitry Andric clEnumVal(libcall, "Add a call to __hwasan_add_frame_record for "
202753f127fSDimitry Andric "storing into the stack ring buffer")),
203753f127fSDimitry Andric cl::Hidden, cl::init(instr));
204753f127fSDimitry Andric
2050b57cec5SDimitry Andric static cl::opt<bool>
2060b57cec5SDimitry Andric ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
2070b57cec5SDimitry Andric cl::desc("instrument memory intrinsics"),
2080b57cec5SDimitry Andric cl::Hidden, cl::init(true));
2090b57cec5SDimitry Andric
2100b57cec5SDimitry Andric static cl::opt<bool>
2110b57cec5SDimitry Andric ClInstrumentLandingPads("hwasan-instrument-landing-pads",
2120b57cec5SDimitry Andric cl::desc("instrument landing pads"), cl::Hidden,
21381ad6265SDimitry Andric cl::init(false));
2148bcb0991SDimitry Andric
2158bcb0991SDimitry Andric static cl::opt<bool> ClUseShortGranules(
2168bcb0991SDimitry Andric "hwasan-use-short-granules",
2178bcb0991SDimitry Andric cl::desc("use short granules in allocas and outlined checks"), cl::Hidden,
21881ad6265SDimitry Andric cl::init(false));
2198bcb0991SDimitry Andric
2208bcb0991SDimitry Andric static cl::opt<bool> ClInstrumentPersonalityFunctions(
2218bcb0991SDimitry Andric "hwasan-instrument-personality-functions",
22281ad6265SDimitry Andric cl::desc("instrument personality functions"), cl::Hidden);
2230b57cec5SDimitry Andric
2240b57cec5SDimitry Andric static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
2250b57cec5SDimitry Andric cl::desc("inline all checks"),
2260b57cec5SDimitry Andric cl::Hidden, cl::init(false));
2270b57cec5SDimitry Andric
2285f757f3fSDimitry Andric static cl::opt<bool> ClInlineFastPathChecks("hwasan-inline-fast-path-checks",
2295f757f3fSDimitry Andric cl::desc("inline all checks"),
2305f757f3fSDimitry Andric cl::Hidden, cl::init(false));
2315f757f3fSDimitry Andric
232fe6060f1SDimitry Andric // Enabled from clang by "-fsanitize-hwaddress-experimental-aliasing".
233fe6060f1SDimitry Andric static cl::opt<bool> ClUsePageAliases("hwasan-experimental-use-page-aliases",
234fe6060f1SDimitry Andric cl::desc("Use page aliasing in HWASan"),
235fe6060f1SDimitry Andric cl::Hidden, cl::init(false));
236fe6060f1SDimitry Andric
2370b57cec5SDimitry Andric namespace {
2380b57cec5SDimitry Andric
shouldUsePageAliases(const Triple & TargetTriple)239fe6060f1SDimitry Andric bool shouldUsePageAliases(const Triple &TargetTriple) {
240fe6060f1SDimitry Andric return ClUsePageAliases && TargetTriple.getArch() == Triple::x86_64;
241fe6060f1SDimitry Andric }
242fe6060f1SDimitry Andric
shouldInstrumentStack(const Triple & TargetTriple)243fe6060f1SDimitry Andric bool shouldInstrumentStack(const Triple &TargetTriple) {
244fe6060f1SDimitry Andric return !shouldUsePageAliases(TargetTriple) && ClInstrumentStack;
245fe6060f1SDimitry Andric }
246fe6060f1SDimitry Andric
shouldInstrumentWithCalls(const Triple & TargetTriple)247fe6060f1SDimitry Andric bool shouldInstrumentWithCalls(const Triple &TargetTriple) {
24806c3fb27SDimitry Andric return ClInstrumentWithCalls.getNumOccurrences()
24906c3fb27SDimitry Andric ? ClInstrumentWithCalls
25006c3fb27SDimitry Andric : TargetTriple.getArch() == Triple::x86_64;
251fe6060f1SDimitry Andric }
252fe6060f1SDimitry Andric
mightUseStackSafetyAnalysis(bool DisableOptimization)253fe6060f1SDimitry Andric bool mightUseStackSafetyAnalysis(bool DisableOptimization) {
254fe6060f1SDimitry Andric return ClUseStackSafety.getNumOccurrences() ? ClUseStackSafety
255fe6060f1SDimitry Andric : !DisableOptimization;
256fe6060f1SDimitry Andric }
257fe6060f1SDimitry Andric
shouldUseStackSafetyAnalysis(const Triple & TargetTriple,bool DisableOptimization)258fe6060f1SDimitry Andric bool shouldUseStackSafetyAnalysis(const Triple &TargetTriple,
259fe6060f1SDimitry Andric bool DisableOptimization) {
260fe6060f1SDimitry Andric return shouldInstrumentStack(TargetTriple) &&
261fe6060f1SDimitry Andric mightUseStackSafetyAnalysis(DisableOptimization);
262fe6060f1SDimitry Andric }
263349cc55cSDimitry Andric
shouldDetectUseAfterScope(const Triple & TargetTriple)264349cc55cSDimitry Andric bool shouldDetectUseAfterScope(const Triple &TargetTriple) {
265349cc55cSDimitry Andric return ClUseAfterScope && shouldInstrumentStack(TargetTriple);
266349cc55cSDimitry Andric }
267349cc55cSDimitry Andric
2680b57cec5SDimitry Andric /// An instrumentation pass implementing detection of addressability bugs
2690b57cec5SDimitry Andric /// using tagged pointers.
2700b57cec5SDimitry Andric class HWAddressSanitizer {
2710b57cec5SDimitry Andric public:
HWAddressSanitizer(Module & M,bool CompileKernel,bool Recover,const StackSafetyGlobalInfo * SSI)272fe6060f1SDimitry Andric HWAddressSanitizer(Module &M, bool CompileKernel, bool Recover,
273fe6060f1SDimitry Andric const StackSafetyGlobalInfo *SSI)
274fe6060f1SDimitry Andric : M(M), SSI(SSI) {
2750b57cec5SDimitry Andric this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
276fe6060f1SDimitry Andric this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0
277fe6060f1SDimitry Andric ? ClEnableKhwasan
278fe6060f1SDimitry Andric : CompileKernel;
2790b57cec5SDimitry Andric
2808bcb0991SDimitry Andric initializeModule();
2810b57cec5SDimitry Andric }
2820b57cec5SDimitry Andric
2835f757f3fSDimitry Andric void sanitizeFunction(Function &F, FunctionAnalysisManager &FAM);
2845f757f3fSDimitry Andric
2855f757f3fSDimitry Andric private:
2865f757f3fSDimitry Andric struct ShadowTagCheckInfo {
2875f757f3fSDimitry Andric Instruction *TagMismatchTerm = nullptr;
2885f757f3fSDimitry Andric Value *PtrLong = nullptr;
2895f757f3fSDimitry Andric Value *AddrLong = nullptr;
2905f757f3fSDimitry Andric Value *PtrTag = nullptr;
2915f757f3fSDimitry Andric Value *MemTag = nullptr;
2925f757f3fSDimitry Andric };
setSSI(const StackSafetyGlobalInfo * S)293fe6060f1SDimitry Andric void setSSI(const StackSafetyGlobalInfo *S) { SSI = S; }
294fe6060f1SDimitry Andric
2958bcb0991SDimitry Andric void initializeModule();
296e8d8bef9SDimitry Andric void createHwasanCtorComdat();
2970b57cec5SDimitry Andric
2980b57cec5SDimitry Andric void initializeCallbacks(Module &M);
2990b57cec5SDimitry Andric
300e8d8bef9SDimitry Andric Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
301e8d8bef9SDimitry Andric
3020b57cec5SDimitry Andric Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
303e8d8bef9SDimitry Andric Value *getShadowNonTls(IRBuilder<> &IRB);
3040b57cec5SDimitry Andric
3050b57cec5SDimitry Andric void untagPointerOperand(Instruction *I, Value *Addr);
3060b57cec5SDimitry Andric Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
307349cc55cSDimitry Andric
308349cc55cSDimitry Andric int64_t getAccessInfo(bool IsWrite, unsigned AccessSizeIndex);
3095f757f3fSDimitry Andric ShadowTagCheckInfo insertShadowTagCheck(Value *Ptr, Instruction *InsertBefore,
3105f757f3fSDimitry Andric DomTreeUpdater &DTU, LoopInfo *LI);
311349cc55cSDimitry Andric void instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
312349cc55cSDimitry Andric unsigned AccessSizeIndex,
3135f757f3fSDimitry Andric Instruction *InsertBefore,
3145f757f3fSDimitry Andric DomTreeUpdater &DTU, LoopInfo *LI);
3150b57cec5SDimitry Andric void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
3160b57cec5SDimitry Andric unsigned AccessSizeIndex,
3175f757f3fSDimitry Andric Instruction *InsertBefore, DomTreeUpdater &DTU,
3185f757f3fSDimitry Andric LoopInfo *LI);
319349cc55cSDimitry Andric bool ignoreMemIntrinsic(MemIntrinsic *MI);
3200b57cec5SDimitry Andric void instrumentMemIntrinsic(MemIntrinsic *MI);
3215f757f3fSDimitry Andric bool instrumentMemAccess(InterestingMemoryOperand &O, DomTreeUpdater &DTU,
3225f757f3fSDimitry Andric LoopInfo *LI);
323349cc55cSDimitry Andric bool ignoreAccess(Instruction *Inst, Value *Ptr);
3245ffd83dbSDimitry Andric void getInterestingMemoryOperands(
3255f757f3fSDimitry Andric Instruction *I, const TargetLibraryInfo &TLI,
3265f757f3fSDimitry Andric SmallVectorImpl<InterestingMemoryOperand> &Interesting);
3270b57cec5SDimitry Andric
328349cc55cSDimitry Andric void tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size);
3290b57cec5SDimitry Andric Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag);
3300b57cec5SDimitry Andric Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong);
33106c3fb27SDimitry Andric bool instrumentStack(memtag::StackInfo &Info, Value *StackTag, Value *UARTag,
33281ad6265SDimitry Andric const DominatorTree &DT, const PostDominatorTree &PDT,
33381ad6265SDimitry Andric const LoopInfo &LI);
3340b57cec5SDimitry Andric Value *readRegister(IRBuilder<> &IRB, StringRef Name);
3350b57cec5SDimitry Andric bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec);
3360b57cec5SDimitry Andric Value *getNextTagWithCall(IRBuilder<> &IRB);
3370b57cec5SDimitry Andric Value *getStackBaseTag(IRBuilder<> &IRB);
33806c3fb27SDimitry Andric Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, unsigned AllocaNo);
33906c3fb27SDimitry Andric Value *getUARTag(IRBuilder<> &IRB);
3400b57cec5SDimitry Andric
3410b57cec5SDimitry Andric Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty);
342fe6060f1SDimitry Andric Value *applyTagMask(IRBuilder<> &IRB, Value *OldTag);
343fe6060f1SDimitry Andric unsigned retagMask(unsigned AllocaNo);
344fe6060f1SDimitry Andric
3450b57cec5SDimitry Andric void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord);
3460b57cec5SDimitry Andric
3478bcb0991SDimitry Andric void instrumentGlobal(GlobalVariable *GV, uint8_t Tag);
3488bcb0991SDimitry Andric void instrumentGlobals();
3498bcb0991SDimitry Andric
35081ad6265SDimitry Andric Value *getPC(IRBuilder<> &IRB);
35181ad6265SDimitry Andric Value *getSP(IRBuilder<> &IRB);
352753f127fSDimitry Andric Value *getFrameRecordInfo(IRBuilder<> &IRB);
35381ad6265SDimitry Andric
3548bcb0991SDimitry Andric void instrumentPersonalityFunctions();
3558bcb0991SDimitry Andric
3560b57cec5SDimitry Andric LLVMContext *C;
3578bcb0991SDimitry Andric Module &M;
358fe6060f1SDimitry Andric const StackSafetyGlobalInfo *SSI;
3590b57cec5SDimitry Andric Triple TargetTriple;
3600b57cec5SDimitry Andric
3610b57cec5SDimitry Andric /// This struct defines the shadow mapping using the rule:
3620b57cec5SDimitry Andric /// shadow = (mem >> Scale) + Offset.
3630b57cec5SDimitry Andric /// If InGlobal is true, then
3640b57cec5SDimitry Andric /// extern char __hwasan_shadow[];
3650b57cec5SDimitry Andric /// shadow = (mem >> Scale) + &__hwasan_shadow
3660b57cec5SDimitry Andric /// If InTls is true, then
3670b57cec5SDimitry Andric /// extern char *__hwasan_tls;
3680b57cec5SDimitry Andric /// shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment)
369fe6060f1SDimitry Andric ///
370fe6060f1SDimitry Andric /// If WithFrameRecord is true, then __hwasan_tls will be used to access the
371fe6060f1SDimitry Andric /// ring buffer for storing stack allocations on targets that support it.
3720b57cec5SDimitry Andric struct ShadowMapping {
373bdd1243dSDimitry Andric uint8_t Scale;
3740b57cec5SDimitry Andric uint64_t Offset;
3750b57cec5SDimitry Andric bool InGlobal;
3760b57cec5SDimitry Andric bool InTls;
377fe6060f1SDimitry Andric bool WithFrameRecord;
3780b57cec5SDimitry Andric
379fe6060f1SDimitry Andric void init(Triple &TargetTriple, bool InstrumentWithCalls);
getObjectAlignment__anon368fb5b70111::HWAddressSanitizer::ShadowMapping380bdd1243dSDimitry Andric Align getObjectAlignment() const { return Align(1ULL << Scale); }
3810b57cec5SDimitry Andric };
382349cc55cSDimitry Andric
3830b57cec5SDimitry Andric ShadowMapping Mapping;
3840b57cec5SDimitry Andric
3858bcb0991SDimitry Andric Type *VoidTy = Type::getVoidTy(M.getContext());
3860b57cec5SDimitry Andric Type *IntptrTy;
3875f757f3fSDimitry Andric PointerType *PtrTy;
3880b57cec5SDimitry Andric Type *Int8Ty;
3890b57cec5SDimitry Andric Type *Int32Ty;
3908bcb0991SDimitry Andric Type *Int64Ty = Type::getInt64Ty(M.getContext());
3910b57cec5SDimitry Andric
3920b57cec5SDimitry Andric bool CompileKernel;
3930b57cec5SDimitry Andric bool Recover;
394e8d8bef9SDimitry Andric bool OutlinedChecks;
3955f757f3fSDimitry Andric bool InlineFastPath;
3968bcb0991SDimitry Andric bool UseShortGranules;
3978bcb0991SDimitry Andric bool InstrumentLandingPads;
398fe6060f1SDimitry Andric bool InstrumentWithCalls;
399fe6060f1SDimitry Andric bool InstrumentStack;
400349cc55cSDimitry Andric bool DetectUseAfterScope;
401fe6060f1SDimitry Andric bool UsePageAliases;
40206c3fb27SDimitry Andric bool UseMatchAllCallback;
4030b57cec5SDimitry Andric
404bdd1243dSDimitry Andric std::optional<uint8_t> MatchAllTag;
405e8d8bef9SDimitry Andric
406fe6060f1SDimitry Andric unsigned PointerTagShift;
407fe6060f1SDimitry Andric uint64_t TagMaskByte;
408fe6060f1SDimitry Andric
4090b57cec5SDimitry Andric Function *HwasanCtorFunction;
4100b57cec5SDimitry Andric
4110b57cec5SDimitry Andric FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
4120b57cec5SDimitry Andric FunctionCallee HwasanMemoryAccessCallbackSized[2];
4130b57cec5SDimitry Andric
41406c3fb27SDimitry Andric FunctionCallee HwasanMemmove, HwasanMemcpy, HwasanMemset;
41506c3fb27SDimitry Andric FunctionCallee HwasanHandleVfork;
41606c3fb27SDimitry Andric
4170b57cec5SDimitry Andric FunctionCallee HwasanTagMemoryFunc;
4180b57cec5SDimitry Andric FunctionCallee HwasanGenerateTagFunc;
419753f127fSDimitry Andric FunctionCallee HwasanRecordFrameRecordFunc;
4200b57cec5SDimitry Andric
4210b57cec5SDimitry Andric Constant *ShadowGlobal;
4220b57cec5SDimitry Andric
423e8d8bef9SDimitry Andric Value *ShadowBase = nullptr;
4240b57cec5SDimitry Andric Value *StackBaseTag = nullptr;
42581ad6265SDimitry Andric Value *CachedSP = nullptr;
4260b57cec5SDimitry Andric GlobalValue *ThreadPtrGlobal = nullptr;
4270b57cec5SDimitry Andric };
4280b57cec5SDimitry Andric
4290b57cec5SDimitry Andric } // end anonymous namespace
4300b57cec5SDimitry Andric
run(Module & M,ModuleAnalysisManager & MAM)4310b57cec5SDimitry Andric PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
4320b57cec5SDimitry Andric ModuleAnalysisManager &MAM) {
433fe6060f1SDimitry Andric const StackSafetyGlobalInfo *SSI = nullptr;
434349cc55cSDimitry Andric auto TargetTriple = llvm::Triple(M.getTargetTriple());
435349cc55cSDimitry Andric if (shouldUseStackSafetyAnalysis(TargetTriple, Options.DisableOptimization))
436fe6060f1SDimitry Andric SSI = &MAM.getResult<StackSafetyGlobalAnalysis>(M);
437349cc55cSDimitry Andric
438349cc55cSDimitry Andric HWAddressSanitizer HWASan(M, Options.CompileKernel, Options.Recover, SSI);
439349cc55cSDimitry Andric auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
44081ad6265SDimitry Andric for (Function &F : M)
44106c3fb27SDimitry Andric HWASan.sanitizeFunction(F, FAM);
442bdd1243dSDimitry Andric
443bdd1243dSDimitry Andric PreservedAnalyses PA = PreservedAnalyses::none();
4445f757f3fSDimitry Andric // DominatorTreeAnalysis, PostDominatorTreeAnalysis, and LoopAnalysis
4455f757f3fSDimitry Andric // are incrementally updated throughout this pass whenever
4465f757f3fSDimitry Andric // SplitBlockAndInsertIfThen is called.
4475f757f3fSDimitry Andric PA.preserve<DominatorTreeAnalysis>();
4485f757f3fSDimitry Andric PA.preserve<PostDominatorTreeAnalysis>();
4495f757f3fSDimitry Andric PA.preserve<LoopAnalysis>();
450bdd1243dSDimitry Andric // GlobalsAA is considered stateless and does not get invalidated unless
451bdd1243dSDimitry Andric // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
452bdd1243dSDimitry Andric // make changes that require GlobalsAA to be invalidated.
453bdd1243dSDimitry Andric PA.abandon<GlobalsAA>();
454bdd1243dSDimitry Andric return PA;
4550b57cec5SDimitry Andric }
printPipeline(raw_ostream & OS,function_ref<StringRef (StringRef)> MapClassName2PassName)456349cc55cSDimitry Andric void HWAddressSanitizerPass::printPipeline(
457349cc55cSDimitry Andric raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
458349cc55cSDimitry Andric static_cast<PassInfoMixin<HWAddressSanitizerPass> *>(this)->printPipeline(
459349cc55cSDimitry Andric OS, MapClassName2PassName);
46006c3fb27SDimitry Andric OS << '<';
461349cc55cSDimitry Andric if (Options.CompileKernel)
462349cc55cSDimitry Andric OS << "kernel;";
463349cc55cSDimitry Andric if (Options.Recover)
464349cc55cSDimitry Andric OS << "recover";
46506c3fb27SDimitry Andric OS << '>';
466349cc55cSDimitry Andric }
4670b57cec5SDimitry Andric
createHwasanCtorComdat()468e8d8bef9SDimitry Andric void HWAddressSanitizer::createHwasanCtorComdat() {
469e8d8bef9SDimitry Andric std::tie(HwasanCtorFunction, std::ignore) =
470e8d8bef9SDimitry Andric getOrCreateSanitizerCtorAndInitFunctions(
471e8d8bef9SDimitry Andric M, kHwasanModuleCtorName, kHwasanInitName,
472e8d8bef9SDimitry Andric /*InitArgTypes=*/{},
473e8d8bef9SDimitry Andric /*InitArgs=*/{},
474e8d8bef9SDimitry Andric // This callback is invoked when the functions are created the first
475e8d8bef9SDimitry Andric // time. Hook them into the global ctors list in that case:
476e8d8bef9SDimitry Andric [&](Function *Ctor, FunctionCallee) {
477e8d8bef9SDimitry Andric Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
478e8d8bef9SDimitry Andric Ctor->setComdat(CtorComdat);
479e8d8bef9SDimitry Andric appendToGlobalCtors(M, Ctor, 0, Ctor);
480e8d8bef9SDimitry Andric });
481e8d8bef9SDimitry Andric
482e8d8bef9SDimitry Andric // Create a note that contains pointers to the list of global
483e8d8bef9SDimitry Andric // descriptors. Adding a note to the output file will cause the linker to
484e8d8bef9SDimitry Andric // create a PT_NOTE program header pointing to the note that we can use to
485e8d8bef9SDimitry Andric // find the descriptor list starting from the program headers. A function
486e8d8bef9SDimitry Andric // provided by the runtime initializes the shadow memory for the globals by
487e8d8bef9SDimitry Andric // accessing the descriptor list via the note. The dynamic loader needs to
488e8d8bef9SDimitry Andric // call this function whenever a library is loaded.
489e8d8bef9SDimitry Andric //
490e8d8bef9SDimitry Andric // The reason why we use a note for this instead of a more conventional
491e8d8bef9SDimitry Andric // approach of having a global constructor pass a descriptor list pointer to
492e8d8bef9SDimitry Andric // the runtime is because of an order of initialization problem. With
493e8d8bef9SDimitry Andric // constructors we can encounter the following problematic scenario:
494e8d8bef9SDimitry Andric //
495e8d8bef9SDimitry Andric // 1) library A depends on library B and also interposes one of B's symbols
496e8d8bef9SDimitry Andric // 2) B's constructors are called before A's (as required for correctness)
497e8d8bef9SDimitry Andric // 3) during construction, B accesses one of its "own" globals (actually
498e8d8bef9SDimitry Andric // interposed by A) and triggers a HWASAN failure due to the initialization
499e8d8bef9SDimitry Andric // for A not having happened yet
500e8d8bef9SDimitry Andric //
501e8d8bef9SDimitry Andric // Even without interposition it is possible to run into similar situations in
502e8d8bef9SDimitry Andric // cases where two libraries mutually depend on each other.
503e8d8bef9SDimitry Andric //
504e8d8bef9SDimitry Andric // We only need one note per binary, so put everything for the note in a
505e8d8bef9SDimitry Andric // comdat. This needs to be a comdat with an .init_array section to prevent
506e8d8bef9SDimitry Andric // newer versions of lld from discarding the note.
507e8d8bef9SDimitry Andric //
508e8d8bef9SDimitry Andric // Create the note even if we aren't instrumenting globals. This ensures that
509e8d8bef9SDimitry Andric // binaries linked from object files with both instrumented and
510e8d8bef9SDimitry Andric // non-instrumented globals will end up with a note, even if a comdat from an
511e8d8bef9SDimitry Andric // object file with non-instrumented globals is selected. The note is harmless
512e8d8bef9SDimitry Andric // if the runtime doesn't support it, since it will just be ignored.
513e8d8bef9SDimitry Andric Comdat *NoteComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
514e8d8bef9SDimitry Andric
515e8d8bef9SDimitry Andric Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
516bdd1243dSDimitry Andric auto *Start =
517e8d8bef9SDimitry Andric new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
518e8d8bef9SDimitry Andric nullptr, "__start_hwasan_globals");
519e8d8bef9SDimitry Andric Start->setVisibility(GlobalValue::HiddenVisibility);
520bdd1243dSDimitry Andric auto *Stop =
521e8d8bef9SDimitry Andric new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
522e8d8bef9SDimitry Andric nullptr, "__stop_hwasan_globals");
523e8d8bef9SDimitry Andric Stop->setVisibility(GlobalValue::HiddenVisibility);
524e8d8bef9SDimitry Andric
525e8d8bef9SDimitry Andric // Null-terminated so actually 8 bytes, which are required in order to align
526e8d8bef9SDimitry Andric // the note properly.
527e8d8bef9SDimitry Andric auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
528e8d8bef9SDimitry Andric
529e8d8bef9SDimitry Andric auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
530e8d8bef9SDimitry Andric Int32Ty, Int32Ty);
531e8d8bef9SDimitry Andric auto *Note =
532e8d8bef9SDimitry Andric new GlobalVariable(M, NoteTy, /*isConstant=*/true,
533e8d8bef9SDimitry Andric GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
534e8d8bef9SDimitry Andric Note->setSection(".note.hwasan.globals");
535e8d8bef9SDimitry Andric Note->setComdat(NoteComdat);
536e8d8bef9SDimitry Andric Note->setAlignment(Align(4));
537e8d8bef9SDimitry Andric
538e8d8bef9SDimitry Andric // The pointers in the note need to be relative so that the note ends up being
539e8d8bef9SDimitry Andric // placed in rodata, which is the standard location for notes.
540e8d8bef9SDimitry Andric auto CreateRelPtr = [&](Constant *Ptr) {
541e8d8bef9SDimitry Andric return ConstantExpr::getTrunc(
542e8d8bef9SDimitry Andric ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
543e8d8bef9SDimitry Andric ConstantExpr::getPtrToInt(Note, Int64Ty)),
544e8d8bef9SDimitry Andric Int32Ty);
545e8d8bef9SDimitry Andric };
546e8d8bef9SDimitry Andric Note->setInitializer(ConstantStruct::getAnon(
547e8d8bef9SDimitry Andric {ConstantInt::get(Int32Ty, 8), // n_namesz
548e8d8bef9SDimitry Andric ConstantInt::get(Int32Ty, 8), // n_descsz
549e8d8bef9SDimitry Andric ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
550e8d8bef9SDimitry Andric Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
551e8d8bef9SDimitry Andric appendToCompilerUsed(M, Note);
552e8d8bef9SDimitry Andric
553e8d8bef9SDimitry Andric // Create a zero-length global in hwasan_globals so that the linker will
554e8d8bef9SDimitry Andric // always create start and stop symbols.
555bdd1243dSDimitry Andric auto *Dummy = new GlobalVariable(
556e8d8bef9SDimitry Andric M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
557e8d8bef9SDimitry Andric Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
558e8d8bef9SDimitry Andric Dummy->setSection("hwasan_globals");
559e8d8bef9SDimitry Andric Dummy->setComdat(NoteComdat);
560e8d8bef9SDimitry Andric Dummy->setMetadata(LLVMContext::MD_associated,
561e8d8bef9SDimitry Andric MDNode::get(*C, ValueAsMetadata::get(Note)));
562e8d8bef9SDimitry Andric appendToCompilerUsed(M, Dummy);
563e8d8bef9SDimitry Andric }
564e8d8bef9SDimitry Andric
5650b57cec5SDimitry Andric /// Module-level initialization.
5660b57cec5SDimitry Andric ///
5670b57cec5SDimitry Andric /// inserts a call to __hwasan_init to the module's constructor list.
initializeModule()5688bcb0991SDimitry Andric void HWAddressSanitizer::initializeModule() {
5690b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n");
5700b57cec5SDimitry Andric auto &DL = M.getDataLayout();
5710b57cec5SDimitry Andric
5720b57cec5SDimitry Andric TargetTriple = Triple(M.getTargetTriple());
5730b57cec5SDimitry Andric
574fe6060f1SDimitry Andric // x86_64 currently has two modes:
575fe6060f1SDimitry Andric // - Intel LAM (default)
576fe6060f1SDimitry Andric // - pointer aliasing (heap only)
577fe6060f1SDimitry Andric bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
578fe6060f1SDimitry Andric UsePageAliases = shouldUsePageAliases(TargetTriple);
579fe6060f1SDimitry Andric InstrumentWithCalls = shouldInstrumentWithCalls(TargetTriple);
580fe6060f1SDimitry Andric InstrumentStack = shouldInstrumentStack(TargetTriple);
581349cc55cSDimitry Andric DetectUseAfterScope = shouldDetectUseAfterScope(TargetTriple);
582fe6060f1SDimitry Andric PointerTagShift = IsX86_64 ? 57 : 56;
583fe6060f1SDimitry Andric TagMaskByte = IsX86_64 ? 0x3F : 0xFF;
584fe6060f1SDimitry Andric
585fe6060f1SDimitry Andric Mapping.init(TargetTriple, InstrumentWithCalls);
5860b57cec5SDimitry Andric
5870b57cec5SDimitry Andric C = &(M.getContext());
5880b57cec5SDimitry Andric IRBuilder<> IRB(*C);
5890b57cec5SDimitry Andric IntptrTy = IRB.getIntPtrTy(DL);
5905f757f3fSDimitry Andric PtrTy = IRB.getPtrTy();
5910b57cec5SDimitry Andric Int8Ty = IRB.getInt8Ty();
5920b57cec5SDimitry Andric Int32Ty = IRB.getInt32Ty();
5930b57cec5SDimitry Andric
5940b57cec5SDimitry Andric HwasanCtorFunction = nullptr;
5958bcb0991SDimitry Andric
5968bcb0991SDimitry Andric // Older versions of Android do not have the required runtime support for
5978bcb0991SDimitry Andric // short granules, global or personality function instrumentation. On other
5988bcb0991SDimitry Andric // platforms we currently require using the latest version of the runtime.
5998bcb0991SDimitry Andric bool NewRuntime =
6008bcb0991SDimitry Andric !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30);
6018bcb0991SDimitry Andric
6028bcb0991SDimitry Andric UseShortGranules =
6038bcb0991SDimitry Andric ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
604e8d8bef9SDimitry Andric OutlinedChecks =
605bdd1243dSDimitry Andric (TargetTriple.isAArch64() || TargetTriple.isRISCV64()) &&
606bdd1243dSDimitry Andric TargetTriple.isOSBinFormatELF() &&
607e8d8bef9SDimitry Andric (ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover);
608e8d8bef9SDimitry Andric
6095f757f3fSDimitry Andric InlineFastPath =
6105f757f3fSDimitry Andric (ClInlineFastPathChecks.getNumOccurrences()
6115f757f3fSDimitry Andric ? ClInlineFastPathChecks
6125f757f3fSDimitry Andric : !(TargetTriple.isAndroid() ||
6135f757f3fSDimitry Andric TargetTriple.isOSFuchsia())); // These platforms may prefer less
6145f757f3fSDimitry Andric // inlining to reduce binary size.
6155f757f3fSDimitry Andric
616e8d8bef9SDimitry Andric if (ClMatchAllTag.getNumOccurrences()) {
617e8d8bef9SDimitry Andric if (ClMatchAllTag != -1) {
618e8d8bef9SDimitry Andric MatchAllTag = ClMatchAllTag & 0xFF;
619e8d8bef9SDimitry Andric }
620e8d8bef9SDimitry Andric } else if (CompileKernel) {
621e8d8bef9SDimitry Andric MatchAllTag = 0xFF;
622e8d8bef9SDimitry Andric }
62306c3fb27SDimitry Andric UseMatchAllCallback = !CompileKernel && MatchAllTag.has_value();
6248bcb0991SDimitry Andric
6258bcb0991SDimitry Andric // If we don't have personality function support, fall back to landing pads.
6268bcb0991SDimitry Andric InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
6278bcb0991SDimitry Andric ? ClInstrumentLandingPads
6288bcb0991SDimitry Andric : !NewRuntime;
6298bcb0991SDimitry Andric
6300b57cec5SDimitry Andric if (!CompileKernel) {
631e8d8bef9SDimitry Andric createHwasanCtorComdat();
6328bcb0991SDimitry Andric bool InstrumentGlobals =
6338bcb0991SDimitry Andric ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime;
634fe6060f1SDimitry Andric
635fe6060f1SDimitry Andric if (InstrumentGlobals && !UsePageAliases)
6368bcb0991SDimitry Andric instrumentGlobals();
6378bcb0991SDimitry Andric
6388bcb0991SDimitry Andric bool InstrumentPersonalityFunctions =
6398bcb0991SDimitry Andric ClInstrumentPersonalityFunctions.getNumOccurrences()
6408bcb0991SDimitry Andric ? ClInstrumentPersonalityFunctions
6418bcb0991SDimitry Andric : NewRuntime;
6428bcb0991SDimitry Andric if (InstrumentPersonalityFunctions)
6438bcb0991SDimitry Andric instrumentPersonalityFunctions();
6440b57cec5SDimitry Andric }
6450b57cec5SDimitry Andric
6460b57cec5SDimitry Andric if (!TargetTriple.isAndroid()) {
6470b57cec5SDimitry Andric Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] {
6480b57cec5SDimitry Andric auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false,
6490b57cec5SDimitry Andric GlobalValue::ExternalLinkage, nullptr,
6500b57cec5SDimitry Andric "__hwasan_tls", nullptr,
6510b57cec5SDimitry Andric GlobalVariable::InitialExecTLSModel);
6520b57cec5SDimitry Andric appendToCompilerUsed(M, GV);
6530b57cec5SDimitry Andric return GV;
6540b57cec5SDimitry Andric });
6550b57cec5SDimitry Andric ThreadPtrGlobal = cast<GlobalVariable>(C);
6560b57cec5SDimitry Andric }
6570b57cec5SDimitry Andric }
6580b57cec5SDimitry Andric
initializeCallbacks(Module & M)6590b57cec5SDimitry Andric void HWAddressSanitizer::initializeCallbacks(Module &M) {
6600b57cec5SDimitry Andric IRBuilder<> IRB(*C);
66106c3fb27SDimitry Andric const std::string MatchAllStr = UseMatchAllCallback ? "_match_all" : "";
66206c3fb27SDimitry Andric FunctionType *HwasanMemoryAccessCallbackSizedFnTy,
66306c3fb27SDimitry Andric *HwasanMemoryAccessCallbackFnTy, *HwasanMemTransferFnTy,
66406c3fb27SDimitry Andric *HwasanMemsetFnTy;
66506c3fb27SDimitry Andric if (UseMatchAllCallback) {
66606c3fb27SDimitry Andric HwasanMemoryAccessCallbackSizedFnTy =
66706c3fb27SDimitry Andric FunctionType::get(VoidTy, {IntptrTy, IntptrTy, Int8Ty}, false);
66806c3fb27SDimitry Andric HwasanMemoryAccessCallbackFnTy =
66906c3fb27SDimitry Andric FunctionType::get(VoidTy, {IntptrTy, Int8Ty}, false);
6705f757f3fSDimitry Andric HwasanMemTransferFnTy =
6715f757f3fSDimitry Andric FunctionType::get(PtrTy, {PtrTy, PtrTy, IntptrTy, Int8Ty}, false);
6725f757f3fSDimitry Andric HwasanMemsetFnTy =
6735f757f3fSDimitry Andric FunctionType::get(PtrTy, {PtrTy, Int32Ty, IntptrTy, Int8Ty}, false);
67406c3fb27SDimitry Andric } else {
67506c3fb27SDimitry Andric HwasanMemoryAccessCallbackSizedFnTy =
67606c3fb27SDimitry Andric FunctionType::get(VoidTy, {IntptrTy, IntptrTy}, false);
67706c3fb27SDimitry Andric HwasanMemoryAccessCallbackFnTy =
67806c3fb27SDimitry Andric FunctionType::get(VoidTy, {IntptrTy}, false);
67906c3fb27SDimitry Andric HwasanMemTransferFnTy =
6805f757f3fSDimitry Andric FunctionType::get(PtrTy, {PtrTy, PtrTy, IntptrTy}, false);
68106c3fb27SDimitry Andric HwasanMemsetFnTy =
6825f757f3fSDimitry Andric FunctionType::get(PtrTy, {PtrTy, Int32Ty, IntptrTy}, false);
68306c3fb27SDimitry Andric }
68406c3fb27SDimitry Andric
6850b57cec5SDimitry Andric for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
6860b57cec5SDimitry Andric const std::string TypeStr = AccessIsWrite ? "store" : "load";
6870b57cec5SDimitry Andric const std::string EndingStr = Recover ? "_noabort" : "";
6880b57cec5SDimitry Andric
6890b57cec5SDimitry Andric HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
69006c3fb27SDimitry Andric ClMemoryAccessCallbackPrefix + TypeStr + "N" + MatchAllStr + EndingStr,
69106c3fb27SDimitry Andric HwasanMemoryAccessCallbackSizedFnTy);
6920b57cec5SDimitry Andric
6930b57cec5SDimitry Andric for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
6940b57cec5SDimitry Andric AccessSizeIndex++) {
6950b57cec5SDimitry Andric HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
69606c3fb27SDimitry Andric M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr +
69706c3fb27SDimitry Andric itostr(1ULL << AccessSizeIndex) +
69806c3fb27SDimitry Andric MatchAllStr + EndingStr,
69906c3fb27SDimitry Andric HwasanMemoryAccessCallbackFnTy);
7000b57cec5SDimitry Andric }
7010b57cec5SDimitry Andric }
7020b57cec5SDimitry Andric
7030b57cec5SDimitry Andric const std::string MemIntrinCallbackPrefix =
70481ad6265SDimitry Andric (CompileKernel && !ClKasanMemIntrinCallbackPrefix)
70581ad6265SDimitry Andric ? std::string("")
70681ad6265SDimitry Andric : ClMemoryAccessCallbackPrefix;
7070b57cec5SDimitry Andric
70806c3fb27SDimitry Andric HwasanMemmove = M.getOrInsertFunction(
70906c3fb27SDimitry Andric MemIntrinCallbackPrefix + "memmove" + MatchAllStr, HwasanMemTransferFnTy);
71006c3fb27SDimitry Andric HwasanMemcpy = M.getOrInsertFunction(
71106c3fb27SDimitry Andric MemIntrinCallbackPrefix + "memcpy" + MatchAllStr, HwasanMemTransferFnTy);
71206c3fb27SDimitry Andric HwasanMemset = M.getOrInsertFunction(
71306c3fb27SDimitry Andric MemIntrinCallbackPrefix + "memset" + MatchAllStr, HwasanMemsetFnTy);
71406c3fb27SDimitry Andric
71506c3fb27SDimitry Andric HwasanTagMemoryFunc = M.getOrInsertFunction("__hwasan_tag_memory", VoidTy,
7165f757f3fSDimitry Andric PtrTy, Int8Ty, IntptrTy);
71706c3fb27SDimitry Andric HwasanGenerateTagFunc =
71806c3fb27SDimitry Andric M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
71906c3fb27SDimitry Andric
72006c3fb27SDimitry Andric HwasanRecordFrameRecordFunc =
72106c3fb27SDimitry Andric M.getOrInsertFunction("__hwasan_add_frame_record", VoidTy, Int64Ty);
72206c3fb27SDimitry Andric
72306c3fb27SDimitry Andric ShadowGlobal =
72406c3fb27SDimitry Andric M.getOrInsertGlobal("__hwasan_shadow", ArrayType::get(Int8Ty, 0));
72506c3fb27SDimitry Andric
72606c3fb27SDimitry Andric HwasanHandleVfork =
72706c3fb27SDimitry Andric M.getOrInsertFunction("__hwasan_handle_vfork", VoidTy, IntptrTy);
7280b57cec5SDimitry Andric }
7290b57cec5SDimitry Andric
getOpaqueNoopCast(IRBuilder<> & IRB,Value * Val)730e8d8bef9SDimitry Andric Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
7310b57cec5SDimitry Andric // An empty inline asm with input reg == output reg.
7320b57cec5SDimitry Andric // An opaque no-op cast, basically.
733e8d8bef9SDimitry Andric // This prevents code bloat as a result of rematerializing trivial definitions
734e8d8bef9SDimitry Andric // such as constants or global addresses at every load and store.
735e8d8bef9SDimitry Andric InlineAsm *Asm =
7365f757f3fSDimitry Andric InlineAsm::get(FunctionType::get(PtrTy, {Val->getType()}, false),
7370b57cec5SDimitry Andric StringRef(""), StringRef("=r,0"),
7380b57cec5SDimitry Andric /*hasSideEffects=*/false);
739e8d8bef9SDimitry Andric return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");
7400b57cec5SDimitry Andric }
7410b57cec5SDimitry Andric
getDynamicShadowIfunc(IRBuilder<> & IRB)742e8d8bef9SDimitry Andric Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
743e8d8bef9SDimitry Andric return getOpaqueNoopCast(IRB, ShadowGlobal);
744e8d8bef9SDimitry Andric }
745e8d8bef9SDimitry Andric
getShadowNonTls(IRBuilder<> & IRB)746e8d8bef9SDimitry Andric Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
7470b57cec5SDimitry Andric if (Mapping.Offset != kDynamicShadowSentinel)
748e8d8bef9SDimitry Andric return getOpaqueNoopCast(
749e8d8bef9SDimitry Andric IRB, ConstantExpr::getIntToPtr(
7505f757f3fSDimitry Andric ConstantInt::get(IntptrTy, Mapping.Offset), PtrTy));
7510b57cec5SDimitry Andric
752bdd1243dSDimitry Andric if (Mapping.InGlobal)
7530b57cec5SDimitry Andric return getDynamicShadowIfunc(IRB);
754bdd1243dSDimitry Andric
7550b57cec5SDimitry Andric Value *GlobalDynamicAddress =
7560b57cec5SDimitry Andric IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
7575f757f3fSDimitry Andric kHwasanShadowMemoryDynamicAddress, PtrTy);
7585f757f3fSDimitry Andric return IRB.CreateLoad(PtrTy, GlobalDynamicAddress);
7590b57cec5SDimitry Andric }
7600b57cec5SDimitry Andric
ignoreAccess(Instruction * Inst,Value * Ptr)761349cc55cSDimitry Andric bool HWAddressSanitizer::ignoreAccess(Instruction *Inst, Value *Ptr) {
762bdd1243dSDimitry Andric // Do not instrument accesses from different address spaces; we cannot deal
7630b57cec5SDimitry Andric // with them.
7645ffd83dbSDimitry Andric Type *PtrTy = cast<PointerType>(Ptr->getType()->getScalarType());
7650b57cec5SDimitry Andric if (PtrTy->getPointerAddressSpace() != 0)
7665ffd83dbSDimitry Andric return true;
7670b57cec5SDimitry Andric
7680b57cec5SDimitry Andric // Ignore swifterror addresses.
7690b57cec5SDimitry Andric // swifterror memory addresses are mem2reg promoted by instruction
7700b57cec5SDimitry Andric // selection. As such they cannot have regular uses like an instrumentation
7710b57cec5SDimitry Andric // function and it makes no sense to track them as memory.
7725ffd83dbSDimitry Andric if (Ptr->isSwiftError())
7735ffd83dbSDimitry Andric return true;
7745ffd83dbSDimitry Andric
775349cc55cSDimitry Andric if (findAllocaForValue(Ptr)) {
776349cc55cSDimitry Andric if (!InstrumentStack)
777349cc55cSDimitry Andric return true;
778349cc55cSDimitry Andric if (SSI && SSI->stackAccessIsSafe(*Inst))
779349cc55cSDimitry Andric return true;
780349cc55cSDimitry Andric }
7815ffd83dbSDimitry Andric return false;
7820b57cec5SDimitry Andric }
7830b57cec5SDimitry Andric
getInterestingMemoryOperands(Instruction * I,const TargetLibraryInfo & TLI,SmallVectorImpl<InterestingMemoryOperand> & Interesting)7845ffd83dbSDimitry Andric void HWAddressSanitizer::getInterestingMemoryOperands(
7855f757f3fSDimitry Andric Instruction *I, const TargetLibraryInfo &TLI,
7865f757f3fSDimitry Andric SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
7875ffd83dbSDimitry Andric // Skip memory accesses inserted by another instrumentation.
78881ad6265SDimitry Andric if (I->hasMetadata(LLVMContext::MD_nosanitize))
7895ffd83dbSDimitry Andric return;
7905ffd83dbSDimitry Andric
7915ffd83dbSDimitry Andric // Do not instrument the load fetching the dynamic shadow address.
792e8d8bef9SDimitry Andric if (ShadowBase == I)
7935ffd83dbSDimitry Andric return;
7945ffd83dbSDimitry Andric
7955ffd83dbSDimitry Andric if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
796349cc55cSDimitry Andric if (!ClInstrumentReads || ignoreAccess(I, LI->getPointerOperand()))
7975ffd83dbSDimitry Andric return;
7985ffd83dbSDimitry Andric Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
7995ffd83dbSDimitry Andric LI->getType(), LI->getAlign());
8005ffd83dbSDimitry Andric } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
801349cc55cSDimitry Andric if (!ClInstrumentWrites || ignoreAccess(I, SI->getPointerOperand()))
8025ffd83dbSDimitry Andric return;
8035ffd83dbSDimitry Andric Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
8045ffd83dbSDimitry Andric SI->getValueOperand()->getType(), SI->getAlign());
8055ffd83dbSDimitry Andric } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
806349cc55cSDimitry Andric if (!ClInstrumentAtomics || ignoreAccess(I, RMW->getPointerOperand()))
8075ffd83dbSDimitry Andric return;
8085ffd83dbSDimitry Andric Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
809bdd1243dSDimitry Andric RMW->getValOperand()->getType(), std::nullopt);
8105ffd83dbSDimitry Andric } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
811349cc55cSDimitry Andric if (!ClInstrumentAtomics || ignoreAccess(I, XCHG->getPointerOperand()))
8125ffd83dbSDimitry Andric return;
8135ffd83dbSDimitry Andric Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
814bdd1243dSDimitry Andric XCHG->getCompareOperand()->getType(),
815bdd1243dSDimitry Andric std::nullopt);
816bdd1243dSDimitry Andric } else if (auto *CI = dyn_cast<CallInst>(I)) {
817349cc55cSDimitry Andric for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
8185ffd83dbSDimitry Andric if (!ClInstrumentByval || !CI->isByValArgument(ArgNo) ||
819349cc55cSDimitry Andric ignoreAccess(I, CI->getArgOperand(ArgNo)))
8205ffd83dbSDimitry Andric continue;
8215ffd83dbSDimitry Andric Type *Ty = CI->getParamByValType(ArgNo);
8225ffd83dbSDimitry Andric Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
8235ffd83dbSDimitry Andric }
8245f757f3fSDimitry Andric maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI);
8255ffd83dbSDimitry Andric }
8260b57cec5SDimitry Andric }
8270b57cec5SDimitry Andric
getPointerOperandIndex(Instruction * I)8280b57cec5SDimitry Andric static unsigned getPointerOperandIndex(Instruction *I) {
8290b57cec5SDimitry Andric if (LoadInst *LI = dyn_cast<LoadInst>(I))
8300b57cec5SDimitry Andric return LI->getPointerOperandIndex();
8310b57cec5SDimitry Andric if (StoreInst *SI = dyn_cast<StoreInst>(I))
8320b57cec5SDimitry Andric return SI->getPointerOperandIndex();
8330b57cec5SDimitry Andric if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
8340b57cec5SDimitry Andric return RMW->getPointerOperandIndex();
8350b57cec5SDimitry Andric if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I))
8360b57cec5SDimitry Andric return XCHG->getPointerOperandIndex();
8370b57cec5SDimitry Andric report_fatal_error("Unexpected instruction");
8380b57cec5SDimitry Andric return -1;
8390b57cec5SDimitry Andric }
8400b57cec5SDimitry Andric
TypeSizeToSizeIndex(uint32_t TypeSize)8410b57cec5SDimitry Andric static size_t TypeSizeToSizeIndex(uint32_t TypeSize) {
84206c3fb27SDimitry Andric size_t Res = llvm::countr_zero(TypeSize / 8);
8430b57cec5SDimitry Andric assert(Res < kNumberOfAccessSizes);
8440b57cec5SDimitry Andric return Res;
8450b57cec5SDimitry Andric }
8460b57cec5SDimitry Andric
untagPointerOperand(Instruction * I,Value * Addr)8470b57cec5SDimitry Andric void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
848bdd1243dSDimitry Andric if (TargetTriple.isAArch64() || TargetTriple.getArch() == Triple::x86_64 ||
849bdd1243dSDimitry Andric TargetTriple.isRISCV64())
8500b57cec5SDimitry Andric return;
8510b57cec5SDimitry Andric
8520b57cec5SDimitry Andric IRBuilder<> IRB(I);
8530b57cec5SDimitry Andric Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
8540b57cec5SDimitry Andric Value *UntaggedPtr =
8550b57cec5SDimitry Andric IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType());
8560b57cec5SDimitry Andric I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
8570b57cec5SDimitry Andric }
8580b57cec5SDimitry Andric
memToShadow(Value * Mem,IRBuilder<> & IRB)8590b57cec5SDimitry Andric Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
8600b57cec5SDimitry Andric // Mem >> Scale
8610b57cec5SDimitry Andric Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
8620b57cec5SDimitry Andric if (Mapping.Offset == 0)
8635f757f3fSDimitry Andric return IRB.CreateIntToPtr(Shadow, PtrTy);
8640b57cec5SDimitry Andric // (Mem >> Scale) + Offset
8657a6dacacSDimitry Andric return IRB.CreatePtrAdd(ShadowBase, Shadow);
8660b57cec5SDimitry Andric }
8670b57cec5SDimitry Andric
getAccessInfo(bool IsWrite,unsigned AccessSizeIndex)868349cc55cSDimitry Andric int64_t HWAddressSanitizer::getAccessInfo(bool IsWrite,
869349cc55cSDimitry Andric unsigned AccessSizeIndex) {
870bdd1243dSDimitry Andric return (CompileKernel << HWASanAccessInfo::CompileKernelShift) |
871bdd1243dSDimitry Andric (MatchAllTag.has_value() << HWASanAccessInfo::HasMatchAllShift) |
872bdd1243dSDimitry Andric (MatchAllTag.value_or(0) << HWASanAccessInfo::MatchAllShift) |
873bdd1243dSDimitry Andric (Recover << HWASanAccessInfo::RecoverShift) |
874bdd1243dSDimitry Andric (IsWrite << HWASanAccessInfo::IsWriteShift) |
875e8d8bef9SDimitry Andric (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
876349cc55cSDimitry Andric }
8770b57cec5SDimitry Andric
8785f757f3fSDimitry Andric HWAddressSanitizer::ShadowTagCheckInfo
insertShadowTagCheck(Value * Ptr,Instruction * InsertBefore,DomTreeUpdater & DTU,LoopInfo * LI)8795f757f3fSDimitry Andric HWAddressSanitizer::insertShadowTagCheck(Value *Ptr, Instruction *InsertBefore,
8805f757f3fSDimitry Andric DomTreeUpdater &DTU, LoopInfo *LI) {
8815f757f3fSDimitry Andric ShadowTagCheckInfo R;
8825f757f3fSDimitry Andric
8835f757f3fSDimitry Andric IRBuilder<> IRB(InsertBefore);
8845f757f3fSDimitry Andric
8855f757f3fSDimitry Andric R.PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
8865f757f3fSDimitry Andric R.PtrTag =
8875f757f3fSDimitry Andric IRB.CreateTrunc(IRB.CreateLShr(R.PtrLong, PointerTagShift), Int8Ty);
8885f757f3fSDimitry Andric R.AddrLong = untagPointer(IRB, R.PtrLong);
8895f757f3fSDimitry Andric Value *Shadow = memToShadow(R.AddrLong, IRB);
8905f757f3fSDimitry Andric R.MemTag = IRB.CreateLoad(Int8Ty, Shadow);
8915f757f3fSDimitry Andric Value *TagMismatch = IRB.CreateICmpNE(R.PtrTag, R.MemTag);
8925f757f3fSDimitry Andric
8935f757f3fSDimitry Andric if (MatchAllTag.has_value()) {
8945f757f3fSDimitry Andric Value *TagNotIgnored = IRB.CreateICmpNE(
8955f757f3fSDimitry Andric R.PtrTag, ConstantInt::get(R.PtrTag->getType(), *MatchAllTag));
8965f757f3fSDimitry Andric TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
8975f757f3fSDimitry Andric }
8985f757f3fSDimitry Andric
8995f757f3fSDimitry Andric R.TagMismatchTerm = SplitBlockAndInsertIfThen(
9005f757f3fSDimitry Andric TagMismatch, InsertBefore, false,
9015f757f3fSDimitry Andric MDBuilder(*C).createBranchWeights(1, 100000), &DTU, LI);
9025f757f3fSDimitry Andric
9035f757f3fSDimitry Andric return R;
9045f757f3fSDimitry Andric }
9055f757f3fSDimitry Andric
instrumentMemAccessOutline(Value * Ptr,bool IsWrite,unsigned AccessSizeIndex,Instruction * InsertBefore,DomTreeUpdater & DTU,LoopInfo * LI)906349cc55cSDimitry Andric void HWAddressSanitizer::instrumentMemAccessOutline(Value *Ptr, bool IsWrite,
907349cc55cSDimitry Andric unsigned AccessSizeIndex,
9085f757f3fSDimitry Andric Instruction *InsertBefore,
9095f757f3fSDimitry Andric DomTreeUpdater &DTU,
9105f757f3fSDimitry Andric LoopInfo *LI) {
911349cc55cSDimitry Andric assert(!UsePageAliases);
912349cc55cSDimitry Andric const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
9135f757f3fSDimitry Andric
9145f757f3fSDimitry Andric if (InlineFastPath)
9155f757f3fSDimitry Andric InsertBefore =
9165f757f3fSDimitry Andric insertShadowTagCheck(Ptr, InsertBefore, DTU, LI).TagMismatchTerm;
9175f757f3fSDimitry Andric
918349cc55cSDimitry Andric IRBuilder<> IRB(InsertBefore);
9190b57cec5SDimitry Andric Module *M = IRB.GetInsertBlock()->getParent()->getParent();
9208bcb0991SDimitry Andric IRB.CreateCall(Intrinsic::getDeclaration(
9218bcb0991SDimitry Andric M, UseShortGranules
9228bcb0991SDimitry Andric ? Intrinsic::hwasan_check_memaccess_shortgranules
9238bcb0991SDimitry Andric : Intrinsic::hwasan_check_memaccess),
924e8d8bef9SDimitry Andric {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
9250b57cec5SDimitry Andric }
9260b57cec5SDimitry Andric
instrumentMemAccessInline(Value * Ptr,bool IsWrite,unsigned AccessSizeIndex,Instruction * InsertBefore,DomTreeUpdater & DTU,LoopInfo * LI)927349cc55cSDimitry Andric void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
928349cc55cSDimitry Andric unsigned AccessSizeIndex,
9295f757f3fSDimitry Andric Instruction *InsertBefore,
9305f757f3fSDimitry Andric DomTreeUpdater &DTU,
9315f757f3fSDimitry Andric LoopInfo *LI) {
932349cc55cSDimitry Andric assert(!UsePageAliases);
933349cc55cSDimitry Andric const int64_t AccessInfo = getAccessInfo(IsWrite, AccessSizeIndex);
934349cc55cSDimitry Andric
9355f757f3fSDimitry Andric ShadowTagCheckInfo TCI = insertShadowTagCheck(Ptr, InsertBefore, DTU, LI);
9360b57cec5SDimitry Andric
9375f757f3fSDimitry Andric IRBuilder<> IRB(TCI.TagMismatchTerm);
9380b57cec5SDimitry Andric Value *OutOfShortGranuleTagRange =
9395f757f3fSDimitry Andric IRB.CreateICmpUGT(TCI.MemTag, ConstantInt::get(Int8Ty, 15));
9405f757f3fSDimitry Andric Instruction *CheckFailTerm = SplitBlockAndInsertIfThen(
9415f757f3fSDimitry Andric OutOfShortGranuleTagRange, TCI.TagMismatchTerm, !Recover,
9425f757f3fSDimitry Andric MDBuilder(*C).createBranchWeights(1, 100000), &DTU, LI);
9430b57cec5SDimitry Andric
9445f757f3fSDimitry Andric IRB.SetInsertPoint(TCI.TagMismatchTerm);
9455f757f3fSDimitry Andric Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(TCI.PtrLong, 15), Int8Ty);
9460b57cec5SDimitry Andric PtrLowBits = IRB.CreateAdd(
9470b57cec5SDimitry Andric PtrLowBits, ConstantInt::get(Int8Ty, (1 << AccessSizeIndex) - 1));
9485f757f3fSDimitry Andric Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, TCI.MemTag);
9495f757f3fSDimitry Andric SplitBlockAndInsertIfThen(PtrLowBitsOOB, TCI.TagMismatchTerm, false,
9505f757f3fSDimitry Andric MDBuilder(*C).createBranchWeights(1, 100000), &DTU,
9515f757f3fSDimitry Andric LI, CheckFailTerm->getParent());
9520b57cec5SDimitry Andric
9535f757f3fSDimitry Andric IRB.SetInsertPoint(TCI.TagMismatchTerm);
9545f757f3fSDimitry Andric Value *InlineTagAddr = IRB.CreateOr(TCI.AddrLong, 15);
9555f757f3fSDimitry Andric InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, PtrTy);
9560b57cec5SDimitry Andric Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr);
9575f757f3fSDimitry Andric Value *InlineTagMismatch = IRB.CreateICmpNE(TCI.PtrTag, InlineTag);
9585f757f3fSDimitry Andric SplitBlockAndInsertIfThen(InlineTagMismatch, TCI.TagMismatchTerm, false,
9595f757f3fSDimitry Andric MDBuilder(*C).createBranchWeights(1, 100000), &DTU,
9605f757f3fSDimitry Andric LI, CheckFailTerm->getParent());
9610b57cec5SDimitry Andric
9620b57cec5SDimitry Andric IRB.SetInsertPoint(CheckFailTerm);
9630b57cec5SDimitry Andric InlineAsm *Asm;
9640b57cec5SDimitry Andric switch (TargetTriple.getArch()) {
9650b57cec5SDimitry Andric case Triple::x86_64:
9660b57cec5SDimitry Andric // The signal handler will find the data address in rdi.
9670b57cec5SDimitry Andric Asm = InlineAsm::get(
9685f757f3fSDimitry Andric FunctionType::get(VoidTy, {TCI.PtrLong->getType()}, false),
969e8d8bef9SDimitry Andric "int3\nnopl " +
970e8d8bef9SDimitry Andric itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
971e8d8bef9SDimitry Andric "(%rax)",
9720b57cec5SDimitry Andric "{rdi}",
9730b57cec5SDimitry Andric /*hasSideEffects=*/true);
9740b57cec5SDimitry Andric break;
9750b57cec5SDimitry Andric case Triple::aarch64:
9760b57cec5SDimitry Andric case Triple::aarch64_be:
9770b57cec5SDimitry Andric // The signal handler will find the data address in x0.
9780b57cec5SDimitry Andric Asm = InlineAsm::get(
9795f757f3fSDimitry Andric FunctionType::get(VoidTy, {TCI.PtrLong->getType()}, false),
980fe6060f1SDimitry Andric "brk #" + itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
9810b57cec5SDimitry Andric "{x0}",
9820b57cec5SDimitry Andric /*hasSideEffects=*/true);
9830b57cec5SDimitry Andric break;
984bdd1243dSDimitry Andric case Triple::riscv64:
985bdd1243dSDimitry Andric // The signal handler will find the data address in x10.
986bdd1243dSDimitry Andric Asm = InlineAsm::get(
9875f757f3fSDimitry Andric FunctionType::get(VoidTy, {TCI.PtrLong->getType()}, false),
988bdd1243dSDimitry Andric "ebreak\naddiw x0, x11, " +
989bdd1243dSDimitry Andric itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
990bdd1243dSDimitry Andric "{x10}",
991bdd1243dSDimitry Andric /*hasSideEffects=*/true);
992bdd1243dSDimitry Andric break;
9930b57cec5SDimitry Andric default:
9940b57cec5SDimitry Andric report_fatal_error("unsupported architecture");
9950b57cec5SDimitry Andric }
9965f757f3fSDimitry Andric IRB.CreateCall(Asm, TCI.PtrLong);
9970b57cec5SDimitry Andric if (Recover)
9985f757f3fSDimitry Andric cast<BranchInst>(CheckFailTerm)
9995f757f3fSDimitry Andric ->setSuccessor(0, TCI.TagMismatchTerm->getParent());
10000b57cec5SDimitry Andric }
10010b57cec5SDimitry Andric
ignoreMemIntrinsic(MemIntrinsic * MI)1002349cc55cSDimitry Andric bool HWAddressSanitizer::ignoreMemIntrinsic(MemIntrinsic *MI) {
1003349cc55cSDimitry Andric if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI)) {
1004349cc55cSDimitry Andric return (!ClInstrumentWrites || ignoreAccess(MTI, MTI->getDest())) &&
1005349cc55cSDimitry Andric (!ClInstrumentReads || ignoreAccess(MTI, MTI->getSource()));
1006349cc55cSDimitry Andric }
1007349cc55cSDimitry Andric if (isa<MemSetInst>(MI))
1008349cc55cSDimitry Andric return !ClInstrumentWrites || ignoreAccess(MI, MI->getDest());
1009349cc55cSDimitry Andric return false;
1010349cc55cSDimitry Andric }
1011349cc55cSDimitry Andric
instrumentMemIntrinsic(MemIntrinsic * MI)10120b57cec5SDimitry Andric void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
10130b57cec5SDimitry Andric IRBuilder<> IRB(MI);
10140b57cec5SDimitry Andric if (isa<MemTransferInst>(MI)) {
10155f757f3fSDimitry Andric SmallVector<Value *, 4> Args{
10165f757f3fSDimitry Andric MI->getOperand(0), MI->getOperand(1),
10175f757f3fSDimitry Andric IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)};
10185f757f3fSDimitry Andric
10195f757f3fSDimitry Andric if (UseMatchAllCallback)
10205f757f3fSDimitry Andric Args.emplace_back(ConstantInt::get(Int8Ty, *MatchAllTag));
10215f757f3fSDimitry Andric IRB.CreateCall(isa<MemMoveInst>(MI) ? HwasanMemmove : HwasanMemcpy, Args);
10220b57cec5SDimitry Andric } else if (isa<MemSetInst>(MI)) {
10235f757f3fSDimitry Andric SmallVector<Value *, 4> Args{
10245f757f3fSDimitry Andric MI->getOperand(0),
102506c3fb27SDimitry Andric IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
10265f757f3fSDimitry Andric IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)};
10275f757f3fSDimitry Andric if (UseMatchAllCallback)
10285f757f3fSDimitry Andric Args.emplace_back(ConstantInt::get(Int8Ty, *MatchAllTag));
10295f757f3fSDimitry Andric IRB.CreateCall(HwasanMemset, Args);
103006c3fb27SDimitry Andric }
10310b57cec5SDimitry Andric MI->eraseFromParent();
10320b57cec5SDimitry Andric }
10330b57cec5SDimitry Andric
instrumentMemAccess(InterestingMemoryOperand & O,DomTreeUpdater & DTU,LoopInfo * LI)10345f757f3fSDimitry Andric bool HWAddressSanitizer::instrumentMemAccess(InterestingMemoryOperand &O,
10355f757f3fSDimitry Andric DomTreeUpdater &DTU,
10365f757f3fSDimitry Andric LoopInfo *LI) {
10375ffd83dbSDimitry Andric Value *Addr = O.getPtr();
10380b57cec5SDimitry Andric
10395ffd83dbSDimitry Andric LLVM_DEBUG(dbgs() << "Instrumenting: " << O.getInsn() << "\n");
10400b57cec5SDimitry Andric
10415ffd83dbSDimitry Andric if (O.MaybeMask)
10420b57cec5SDimitry Andric return false; // FIXME
10430b57cec5SDimitry Andric
10445ffd83dbSDimitry Andric IRBuilder<> IRB(O.getInsn());
104506c3fb27SDimitry Andric if (!O.TypeStoreSize.isScalable() && isPowerOf2_64(O.TypeStoreSize) &&
104606c3fb27SDimitry Andric (O.TypeStoreSize / 8 <= (1ULL << (kNumberOfAccessSizes - 1))) &&
1047bdd1243dSDimitry Andric (!O.Alignment || *O.Alignment >= Mapping.getObjectAlignment() ||
104806c3fb27SDimitry Andric *O.Alignment >= O.TypeStoreSize / 8)) {
104906c3fb27SDimitry Andric size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeStoreSize);
1050fe6060f1SDimitry Andric if (InstrumentWithCalls) {
10515f757f3fSDimitry Andric SmallVector<Value *, 2> Args{IRB.CreatePointerCast(Addr, IntptrTy)};
10525f757f3fSDimitry Andric if (UseMatchAllCallback)
10535f757f3fSDimitry Andric Args.emplace_back(ConstantInt::get(Int8Ty, *MatchAllTag));
105406c3fb27SDimitry Andric IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
10555f757f3fSDimitry Andric Args);
1056349cc55cSDimitry Andric } else if (OutlinedChecks) {
10575f757f3fSDimitry Andric instrumentMemAccessOutline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn(),
10585f757f3fSDimitry Andric DTU, LI);
10590b57cec5SDimitry Andric } else {
10605f757f3fSDimitry Andric instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn(),
10615f757f3fSDimitry Andric DTU, LI);
10620b57cec5SDimitry Andric }
10630b57cec5SDimitry Andric } else {
10645f757f3fSDimitry Andric SmallVector<Value *, 3> Args{
10655f757f3fSDimitry Andric IRB.CreatePointerCast(Addr, IntptrTy),
106606c3fb27SDimitry Andric IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize),
10675f757f3fSDimitry Andric ConstantInt::get(IntptrTy, 8))};
10685f757f3fSDimitry Andric if (UseMatchAllCallback)
10695f757f3fSDimitry Andric Args.emplace_back(ConstantInt::get(Int8Ty, *MatchAllTag));
10705f757f3fSDimitry Andric IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite], Args);
10710b57cec5SDimitry Andric }
10725ffd83dbSDimitry Andric untagPointerOperand(O.getInsn(), Addr);
10730b57cec5SDimitry Andric
10740b57cec5SDimitry Andric return true;
10750b57cec5SDimitry Andric }
10760b57cec5SDimitry Andric
tagAlloca(IRBuilder<> & IRB,AllocaInst * AI,Value * Tag,size_t Size)1077349cc55cSDimitry Andric void HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag,
1078fe6060f1SDimitry Andric size_t Size) {
10798bcb0991SDimitry Andric size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
10808bcb0991SDimitry Andric if (!UseShortGranules)
10818bcb0991SDimitry Andric Size = AlignedSize;
10820b57cec5SDimitry Andric
108306c3fb27SDimitry Andric Tag = IRB.CreateTrunc(Tag, Int8Ty);
1084fe6060f1SDimitry Andric if (InstrumentWithCalls) {
10850b57cec5SDimitry Andric IRB.CreateCall(HwasanTagMemoryFunc,
10865f757f3fSDimitry Andric {IRB.CreatePointerCast(AI, PtrTy), Tag,
10870b57cec5SDimitry Andric ConstantInt::get(IntptrTy, AlignedSize)});
10880b57cec5SDimitry Andric } else {
10890b57cec5SDimitry Andric size_t ShadowSize = Size >> Mapping.Scale;
109006c3fb27SDimitry Andric Value *AddrLong = untagPointer(IRB, IRB.CreatePointerCast(AI, IntptrTy));
109106c3fb27SDimitry Andric Value *ShadowPtr = memToShadow(AddrLong, IRB);
10920b57cec5SDimitry Andric // If this memset is not inlined, it will be intercepted in the hwasan
10930b57cec5SDimitry Andric // runtime library. That's OK, because the interceptor skips the checks if
10940b57cec5SDimitry Andric // the address is in the shadow region.
10950b57cec5SDimitry Andric // FIXME: the interceptor is not as fast as real memset. Consider lowering
10960b57cec5SDimitry Andric // llvm.memset right here into either a sequence of stores, or a call to
10970b57cec5SDimitry Andric // hwasan_tag_memory.
10980b57cec5SDimitry Andric if (ShadowSize)
109906c3fb27SDimitry Andric IRB.CreateMemSet(ShadowPtr, Tag, ShadowSize, Align(1));
11000b57cec5SDimitry Andric if (Size != AlignedSize) {
1101bdd1243dSDimitry Andric const uint8_t SizeRemainder = Size % Mapping.getObjectAlignment().value();
1102bdd1243dSDimitry Andric IRB.CreateStore(ConstantInt::get(Int8Ty, SizeRemainder),
11030b57cec5SDimitry Andric IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize));
11045f757f3fSDimitry Andric IRB.CreateStore(
11055f757f3fSDimitry Andric Tag, IRB.CreateConstGEP1_32(Int8Ty, IRB.CreatePointerCast(AI, PtrTy),
11060b57cec5SDimitry Andric AlignedSize - 1));
11070b57cec5SDimitry Andric }
11080b57cec5SDimitry Andric }
11090b57cec5SDimitry Andric }
11100b57cec5SDimitry Andric
retagMask(unsigned AllocaNo)1111fe6060f1SDimitry Andric unsigned HWAddressSanitizer::retagMask(unsigned AllocaNo) {
1112fe6060f1SDimitry Andric if (TargetTriple.getArch() == Triple::x86_64)
1113fe6060f1SDimitry Andric return AllocaNo & TagMaskByte;
1114fe6060f1SDimitry Andric
11150b57cec5SDimitry Andric // A list of 8-bit numbers that have at most one run of non-zero bits.
11160b57cec5SDimitry Andric // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these
11170b57cec5SDimitry Andric // masks.
11180b57cec5SDimitry Andric // The list does not include the value 255, which is used for UAR.
11190b57cec5SDimitry Andric //
11200b57cec5SDimitry Andric // Because we are more likely to use earlier elements of this list than later
11210b57cec5SDimitry Andric // ones, it is sorted in increasing order of probability of collision with a
11220b57cec5SDimitry Andric // mask allocated (temporally) nearby. The program that generated this list
11230b57cec5SDimitry Andric // can be found at:
11240b57cec5SDimitry Andric // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py
112506c3fb27SDimitry Andric static const unsigned FastMasks[] = {
112606c3fb27SDimitry Andric 0, 128, 64, 192, 32, 96, 224, 112, 240, 48, 16, 120,
112706c3fb27SDimitry Andric 248, 56, 24, 8, 124, 252, 60, 28, 12, 4, 126, 254,
112806c3fb27SDimitry Andric 62, 30, 14, 6, 2, 127, 63, 31, 15, 7, 3, 1};
1129bdd1243dSDimitry Andric return FastMasks[AllocaNo % std::size(FastMasks)];
11300b57cec5SDimitry Andric }
11310b57cec5SDimitry Andric
applyTagMask(IRBuilder<> & IRB,Value * OldTag)1132fe6060f1SDimitry Andric Value *HWAddressSanitizer::applyTagMask(IRBuilder<> &IRB, Value *OldTag) {
113306c3fb27SDimitry Andric if (TagMaskByte == 0xFF)
113406c3fb27SDimitry Andric return OldTag; // No need to clear the tag byte.
113506c3fb27SDimitry Andric return IRB.CreateAnd(OldTag,
113606c3fb27SDimitry Andric ConstantInt::get(OldTag->getType(), TagMaskByte));
1137fe6060f1SDimitry Andric }
1138fe6060f1SDimitry Andric
getNextTagWithCall(IRBuilder<> & IRB)11390b57cec5SDimitry Andric Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) {
11400b57cec5SDimitry Andric return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy);
11410b57cec5SDimitry Andric }
11420b57cec5SDimitry Andric
getStackBaseTag(IRBuilder<> & IRB)11430b57cec5SDimitry Andric Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) {
11440b57cec5SDimitry Andric if (ClGenerateTagsWithCalls)
114506c3fb27SDimitry Andric return nullptr;
11460b57cec5SDimitry Andric if (StackBaseTag)
11470b57cec5SDimitry Andric return StackBaseTag;
11480b57cec5SDimitry Andric // Extract some entropy from the stack pointer for the tags.
11490b57cec5SDimitry Andric // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ
11500b57cec5SDimitry Andric // between functions).
115181ad6265SDimitry Andric Value *StackPointerLong = getSP(IRB);
11520b57cec5SDimitry Andric Value *StackTag =
1153fe6060f1SDimitry Andric applyTagMask(IRB, IRB.CreateXor(StackPointerLong,
1154fe6060f1SDimitry Andric IRB.CreateLShr(StackPointerLong, 20)));
1155fe6060f1SDimitry Andric StackTag->setName("hwasan.stack.base.tag");
11560b57cec5SDimitry Andric return StackTag;
11570b57cec5SDimitry Andric }
11580b57cec5SDimitry Andric
getAllocaTag(IRBuilder<> & IRB,Value * StackTag,unsigned AllocaNo)11590b57cec5SDimitry Andric Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag,
116006c3fb27SDimitry Andric unsigned AllocaNo) {
11610b57cec5SDimitry Andric if (ClGenerateTagsWithCalls)
11620b57cec5SDimitry Andric return getNextTagWithCall(IRB);
116306c3fb27SDimitry Andric return IRB.CreateXor(
116406c3fb27SDimitry Andric StackTag, ConstantInt::get(StackTag->getType(), retagMask(AllocaNo)));
11650b57cec5SDimitry Andric }
11660b57cec5SDimitry Andric
getUARTag(IRBuilder<> & IRB)116706c3fb27SDimitry Andric Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB) {
116806c3fb27SDimitry Andric Value *StackPointerLong = getSP(IRB);
116906c3fb27SDimitry Andric Value *UARTag =
117006c3fb27SDimitry Andric applyTagMask(IRB, IRB.CreateLShr(StackPointerLong, PointerTagShift));
117106c3fb27SDimitry Andric
117206c3fb27SDimitry Andric UARTag->setName("hwasan.uar.tag");
117306c3fb27SDimitry Andric return UARTag;
11740b57cec5SDimitry Andric }
11750b57cec5SDimitry Andric
11760b57cec5SDimitry Andric // Add a tag to an address.
tagPointer(IRBuilder<> & IRB,Type * Ty,Value * PtrLong,Value * Tag)11770b57cec5SDimitry Andric Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty,
11780b57cec5SDimitry Andric Value *PtrLong, Value *Tag) {
1179fe6060f1SDimitry Andric assert(!UsePageAliases);
11800b57cec5SDimitry Andric Value *TaggedPtrLong;
11810b57cec5SDimitry Andric if (CompileKernel) {
11820b57cec5SDimitry Andric // Kernel addresses have 0xFF in the most significant byte.
1183fe6060f1SDimitry Andric Value *ShiftedTag =
1184fe6060f1SDimitry Andric IRB.CreateOr(IRB.CreateShl(Tag, PointerTagShift),
1185fe6060f1SDimitry Andric ConstantInt::get(IntptrTy, (1ULL << PointerTagShift) - 1));
11860b57cec5SDimitry Andric TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag);
11870b57cec5SDimitry Andric } else {
1188fe6060f1SDimitry Andric // Userspace can simply do OR (tag << PointerTagShift);
1189fe6060f1SDimitry Andric Value *ShiftedTag = IRB.CreateShl(Tag, PointerTagShift);
11900b57cec5SDimitry Andric TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag);
11910b57cec5SDimitry Andric }
11920b57cec5SDimitry Andric return IRB.CreateIntToPtr(TaggedPtrLong, Ty);
11930b57cec5SDimitry Andric }
11940b57cec5SDimitry Andric
11950b57cec5SDimitry Andric // Remove tag from an address.
untagPointer(IRBuilder<> & IRB,Value * PtrLong)11960b57cec5SDimitry Andric Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
1197fe6060f1SDimitry Andric assert(!UsePageAliases);
11980b57cec5SDimitry Andric Value *UntaggedPtrLong;
11990b57cec5SDimitry Andric if (CompileKernel) {
12000b57cec5SDimitry Andric // Kernel addresses have 0xFF in the most significant byte.
1201fe6060f1SDimitry Andric UntaggedPtrLong =
1202fe6060f1SDimitry Andric IRB.CreateOr(PtrLong, ConstantInt::get(PtrLong->getType(),
120306c3fb27SDimitry Andric TagMaskByte << PointerTagShift));
12040b57cec5SDimitry Andric } else {
12050b57cec5SDimitry Andric // Userspace addresses have 0x00.
120606c3fb27SDimitry Andric UntaggedPtrLong = IRB.CreateAnd(
120706c3fb27SDimitry Andric PtrLong, ConstantInt::get(PtrLong->getType(),
120806c3fb27SDimitry Andric ~(TagMaskByte << PointerTagShift)));
12090b57cec5SDimitry Andric }
12100b57cec5SDimitry Andric return UntaggedPtrLong;
12110b57cec5SDimitry Andric }
12120b57cec5SDimitry Andric
getHwasanThreadSlotPtr(IRBuilder<> & IRB,Type * Ty)12130b57cec5SDimitry Andric Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
12140b57cec5SDimitry Andric Module *M = IRB.GetInsertBlock()->getParent()->getParent();
12150b57cec5SDimitry Andric if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
12160b57cec5SDimitry Andric // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
12170b57cec5SDimitry Andric // in Bionic's libc/private/bionic_tls.h.
12180b57cec5SDimitry Andric Function *ThreadPointerFunc =
12190b57cec5SDimitry Andric Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
12205f757f3fSDimitry Andric return IRB.CreateConstGEP1_32(Int8Ty, IRB.CreateCall(ThreadPointerFunc),
12215f757f3fSDimitry Andric 0x30);
12220b57cec5SDimitry Andric }
12230b57cec5SDimitry Andric if (ThreadPtrGlobal)
12240b57cec5SDimitry Andric return ThreadPtrGlobal;
12250b57cec5SDimitry Andric
12260b57cec5SDimitry Andric return nullptr;
12270b57cec5SDimitry Andric }
12280b57cec5SDimitry Andric
getPC(IRBuilder<> & IRB)122981ad6265SDimitry Andric Value *HWAddressSanitizer::getPC(IRBuilder<> &IRB) {
123081ad6265SDimitry Andric if (TargetTriple.getArch() == Triple::aarch64)
123181ad6265SDimitry Andric return readRegister(IRB, "pc");
123281ad6265SDimitry Andric return IRB.CreatePtrToInt(IRB.GetInsertBlock()->getParent(), IntptrTy);
123381ad6265SDimitry Andric }
123481ad6265SDimitry Andric
getSP(IRBuilder<> & IRB)123581ad6265SDimitry Andric Value *HWAddressSanitizer::getSP(IRBuilder<> &IRB) {
123681ad6265SDimitry Andric if (!CachedSP) {
123781ad6265SDimitry Andric // FIXME: use addressofreturnaddress (but implement it in aarch64 backend
123881ad6265SDimitry Andric // first).
123981ad6265SDimitry Andric Function *F = IRB.GetInsertBlock()->getParent();
124081ad6265SDimitry Andric Module *M = F->getParent();
1241bdd1243dSDimitry Andric auto *GetStackPointerFn = Intrinsic::getDeclaration(
124281ad6265SDimitry Andric M, Intrinsic::frameaddress,
12435f757f3fSDimitry Andric IRB.getPtrTy(M->getDataLayout().getAllocaAddrSpace()));
124481ad6265SDimitry Andric CachedSP = IRB.CreatePtrToInt(
124506c3fb27SDimitry Andric IRB.CreateCall(GetStackPointerFn, {Constant::getNullValue(Int32Ty)}),
124681ad6265SDimitry Andric IntptrTy);
124781ad6265SDimitry Andric }
124881ad6265SDimitry Andric return CachedSP;
124981ad6265SDimitry Andric }
125081ad6265SDimitry Andric
getFrameRecordInfo(IRBuilder<> & IRB)1251753f127fSDimitry Andric Value *HWAddressSanitizer::getFrameRecordInfo(IRBuilder<> &IRB) {
12520b57cec5SDimitry Andric // Prepare ring buffer data.
125381ad6265SDimitry Andric Value *PC = getPC(IRB);
125481ad6265SDimitry Andric Value *SP = getSP(IRB);
125581ad6265SDimitry Andric
12560b57cec5SDimitry Andric // Mix SP and PC.
12570b57cec5SDimitry Andric // Assumptions:
12580b57cec5SDimitry Andric // PC is 0x0000PPPPPPPPPPPP (48 bits are meaningful, others are zero)
12590b57cec5SDimitry Andric // SP is 0xsssssssssssSSSS0 (4 lower bits are zero)
12600b57cec5SDimitry Andric // We only really need ~20 lower non-zero bits (SSSS), so we mix like this:
12610b57cec5SDimitry Andric // 0xSSSSPPPPPPPPPPPP
12620b57cec5SDimitry Andric SP = IRB.CreateShl(SP, 44);
1263753f127fSDimitry Andric return IRB.CreateOr(PC, SP);
1264753f127fSDimitry Andric }
1265753f127fSDimitry Andric
emitPrologue(IRBuilder<> & IRB,bool WithFrameRecord)1266753f127fSDimitry Andric void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
1267753f127fSDimitry Andric if (!Mapping.InTls)
1268753f127fSDimitry Andric ShadowBase = getShadowNonTls(IRB);
1269753f127fSDimitry Andric else if (!WithFrameRecord && TargetTriple.isAndroid())
1270753f127fSDimitry Andric ShadowBase = getDynamicShadowIfunc(IRB);
1271753f127fSDimitry Andric
1272753f127fSDimitry Andric if (!WithFrameRecord && ShadowBase)
1273753f127fSDimitry Andric return;
1274753f127fSDimitry Andric
1275753f127fSDimitry Andric Value *SlotPtr = nullptr;
1276753f127fSDimitry Andric Value *ThreadLong = nullptr;
1277753f127fSDimitry Andric Value *ThreadLongMaybeUntagged = nullptr;
1278753f127fSDimitry Andric
1279753f127fSDimitry Andric auto getThreadLongMaybeUntagged = [&]() {
1280753f127fSDimitry Andric if (!SlotPtr)
1281753f127fSDimitry Andric SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
1282753f127fSDimitry Andric if (!ThreadLong)
1283753f127fSDimitry Andric ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
1284753f127fSDimitry Andric // Extract the address field from ThreadLong. Unnecessary on AArch64 with
1285753f127fSDimitry Andric // TBI.
1286753f127fSDimitry Andric return TargetTriple.isAArch64() ? ThreadLong
1287753f127fSDimitry Andric : untagPointer(IRB, ThreadLong);
1288753f127fSDimitry Andric };
1289753f127fSDimitry Andric
1290753f127fSDimitry Andric if (WithFrameRecord) {
1291753f127fSDimitry Andric switch (ClRecordStackHistory) {
1292753f127fSDimitry Andric case libcall: {
1293753f127fSDimitry Andric // Emit a runtime call into hwasan rather than emitting instructions for
1294753f127fSDimitry Andric // recording stack history.
1295753f127fSDimitry Andric Value *FrameRecordInfo = getFrameRecordInfo(IRB);
1296753f127fSDimitry Andric IRB.CreateCall(HwasanRecordFrameRecordFunc, {FrameRecordInfo});
1297753f127fSDimitry Andric break;
1298753f127fSDimitry Andric }
1299753f127fSDimitry Andric case instr: {
1300753f127fSDimitry Andric ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1301753f127fSDimitry Andric
1302753f127fSDimitry Andric StackBaseTag = IRB.CreateAShr(ThreadLong, 3);
13030b57cec5SDimitry Andric
13040b57cec5SDimitry Andric // Store data to ring buffer.
1305753f127fSDimitry Andric Value *FrameRecordInfo = getFrameRecordInfo(IRB);
13065f757f3fSDimitry Andric Value *RecordPtr =
13075f757f3fSDimitry Andric IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IRB.getPtrTy(0));
1308753f127fSDimitry Andric IRB.CreateStore(FrameRecordInfo, RecordPtr);
13090b57cec5SDimitry Andric
13100b57cec5SDimitry Andric // Update the ring buffer. Top byte of ThreadLong defines the size of the
13110b57cec5SDimitry Andric // buffer in pages, it must be a power of two, and the start of the buffer
13120b57cec5SDimitry Andric // must be aligned by twice that much. Therefore wrap around of the ring
13130b57cec5SDimitry Andric // buffer is simply Addr &= ~((ThreadLong >> 56) << 12).
13140b57cec5SDimitry Andric // The use of AShr instead of LShr is due to
13150b57cec5SDimitry Andric // https://bugs.llvm.org/show_bug.cgi?id=39030
13160b57cec5SDimitry Andric // Runtime library makes sure not to use the highest bit.
13170b57cec5SDimitry Andric Value *WrapMask = IRB.CreateXor(
13180b57cec5SDimitry Andric IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true),
13190b57cec5SDimitry Andric ConstantInt::get(IntptrTy, (uint64_t)-1));
13200b57cec5SDimitry Andric Value *ThreadLongNew = IRB.CreateAnd(
13210b57cec5SDimitry Andric IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask);
13220b57cec5SDimitry Andric IRB.CreateStore(ThreadLongNew, SlotPtr);
1323753f127fSDimitry Andric break;
1324753f127fSDimitry Andric }
1325753f127fSDimitry Andric case none: {
1326753f127fSDimitry Andric llvm_unreachable(
1327753f127fSDimitry Andric "A stack history recording mode should've been selected.");
1328753f127fSDimitry Andric }
1329753f127fSDimitry Andric }
13300b57cec5SDimitry Andric }
13310b57cec5SDimitry Andric
1332fe6060f1SDimitry Andric if (!ShadowBase) {
1333753f127fSDimitry Andric if (!ThreadLongMaybeUntagged)
1334753f127fSDimitry Andric ThreadLongMaybeUntagged = getThreadLongMaybeUntagged();
1335753f127fSDimitry Andric
13360b57cec5SDimitry Andric // Get shadow base address by aligning RecordPtr up.
13370b57cec5SDimitry Andric // Note: this is not correct if the pointer is already aligned.
13380b57cec5SDimitry Andric // Runtime library will make sure this never happens.
1339e8d8bef9SDimitry Andric ShadowBase = IRB.CreateAdd(
13400b57cec5SDimitry Andric IRB.CreateOr(
13410b57cec5SDimitry Andric ThreadLongMaybeUntagged,
13420b57cec5SDimitry Andric ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
13430b57cec5SDimitry Andric ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
13445f757f3fSDimitry Andric ShadowBase = IRB.CreateIntToPtr(ShadowBase, PtrTy);
13450b57cec5SDimitry Andric }
1346fe6060f1SDimitry Andric }
13470b57cec5SDimitry Andric
readRegister(IRBuilder<> & IRB,StringRef Name)13480b57cec5SDimitry Andric Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {
13490b57cec5SDimitry Andric Module *M = IRB.GetInsertBlock()->getParent()->getParent();
13500b57cec5SDimitry Andric Function *ReadRegister =
13510b57cec5SDimitry Andric Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy);
13520b57cec5SDimitry Andric MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)});
13530b57cec5SDimitry Andric Value *Args[] = {MetadataAsValue::get(*C, MD)};
13540b57cec5SDimitry Andric return IRB.CreateCall(ReadRegister, Args);
13550b57cec5SDimitry Andric }
13560b57cec5SDimitry Andric
instrumentLandingPads(SmallVectorImpl<Instruction * > & LandingPadVec)13570b57cec5SDimitry Andric bool HWAddressSanitizer::instrumentLandingPads(
13580b57cec5SDimitry Andric SmallVectorImpl<Instruction *> &LandingPadVec) {
13590b57cec5SDimitry Andric for (auto *LP : LandingPadVec) {
13600b57cec5SDimitry Andric IRBuilder<> IRB(LP->getNextNode());
13610b57cec5SDimitry Andric IRB.CreateCall(
136206c3fb27SDimitry Andric HwasanHandleVfork,
13630b57cec5SDimitry Andric {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp"
13640b57cec5SDimitry Andric : "sp")});
13650b57cec5SDimitry Andric }
13660b57cec5SDimitry Andric return true;
13670b57cec5SDimitry Andric }
13680b57cec5SDimitry Andric
isLifetimeIntrinsic(Value * V)136981ad6265SDimitry Andric static bool isLifetimeIntrinsic(Value *V) {
137081ad6265SDimitry Andric auto *II = dyn_cast<IntrinsicInst>(V);
137181ad6265SDimitry Andric return II && II->isLifetimeStartOrEnd();
1372349cc55cSDimitry Andric }
1373349cc55cSDimitry Andric
instrumentStack(memtag::StackInfo & SInfo,Value * StackTag,Value * UARTag,const DominatorTree & DT,const PostDominatorTree & PDT,const LoopInfo & LI)137481ad6265SDimitry Andric bool HWAddressSanitizer::instrumentStack(memtag::StackInfo &SInfo,
137506c3fb27SDimitry Andric Value *StackTag, Value *UARTag,
137681ad6265SDimitry Andric const DominatorTree &DT,
137781ad6265SDimitry Andric const PostDominatorTree &PDT,
137881ad6265SDimitry Andric const LoopInfo &LI) {
13790b57cec5SDimitry Andric // Ideally, we want to calculate tagged stack base pointer, and rewrite all
13800b57cec5SDimitry Andric // alloca addresses using that. Unfortunately, offsets are not known yet
13810b57cec5SDimitry Andric // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a
13820b57cec5SDimitry Andric // temp, shift-OR it into each alloca address and xor with the retag mask.
13830b57cec5SDimitry Andric // This generates one extra instruction per alloca use.
1384349cc55cSDimitry Andric unsigned int I = 0;
1385349cc55cSDimitry Andric
138681ad6265SDimitry Andric for (auto &KV : SInfo.AllocasToInstrument) {
1387349cc55cSDimitry Andric auto N = I++;
1388349cc55cSDimitry Andric auto *AI = KV.first;
138981ad6265SDimitry Andric memtag::AllocaInfo &Info = KV.second;
13900b57cec5SDimitry Andric IRBuilder<> IRB(AI->getNextNode());
13910b57cec5SDimitry Andric
13920b57cec5SDimitry Andric // Replace uses of the alloca with tagged address.
139306c3fb27SDimitry Andric Value *Tag = getAllocaTag(IRB, StackTag, N);
13940b57cec5SDimitry Andric Value *AILong = IRB.CreatePointerCast(AI, IntptrTy);
139506c3fb27SDimitry Andric Value *AINoTagLong = untagPointer(IRB, AILong);
139606c3fb27SDimitry Andric Value *Replacement = tagPointer(IRB, AI->getType(), AINoTagLong, Tag);
13970b57cec5SDimitry Andric std::string Name =
13980b57cec5SDimitry Andric AI->hasName() ? AI->getName().str() : "alloca." + itostr(N);
13990b57cec5SDimitry Andric Replacement->setName(Name + ".hwasan");
14000b57cec5SDimitry Andric
140181ad6265SDimitry Andric size_t Size = memtag::getAllocaSizeInBytes(*AI);
140281ad6265SDimitry Andric size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
14030b57cec5SDimitry Andric
14045f757f3fSDimitry Andric Value *AICast = IRB.CreatePointerCast(AI, PtrTy);
140581ad6265SDimitry Andric
140681ad6265SDimitry Andric auto HandleLifetime = [&](IntrinsicInst *II) {
140781ad6265SDimitry Andric // Set the lifetime intrinsic to cover the whole alloca. This reduces the
140881ad6265SDimitry Andric // set of assumptions we need to make about the lifetime. Without this we
140981ad6265SDimitry Andric // would need to ensure that we can track the lifetime pointer to a
141081ad6265SDimitry Andric // constant offset from the alloca, and would still need to change the
141181ad6265SDimitry Andric // size to include the extra alignment we use for the untagging to make
141281ad6265SDimitry Andric // the size consistent.
141381ad6265SDimitry Andric //
141481ad6265SDimitry Andric // The check for standard lifetime below makes sure that we have exactly
141581ad6265SDimitry Andric // one set of start / end in any execution (i.e. the ends are not
141681ad6265SDimitry Andric // reachable from each other), so this will not cause any problems.
141781ad6265SDimitry Andric II->setArgOperand(0, ConstantInt::get(Int64Ty, AlignedSize));
141881ad6265SDimitry Andric II->setArgOperand(1, AICast);
141981ad6265SDimitry Andric };
142081ad6265SDimitry Andric llvm::for_each(Info.LifetimeStart, HandleLifetime);
142181ad6265SDimitry Andric llvm::for_each(Info.LifetimeEnd, HandleLifetime);
142281ad6265SDimitry Andric
142306c3fb27SDimitry Andric AI->replaceUsesWithIf(Replacement, [AICast, AILong](const Use &U) {
142481ad6265SDimitry Andric auto *User = U.getUser();
142581ad6265SDimitry Andric return User != AILong && User != AICast && !isLifetimeIntrinsic(User);
142681ad6265SDimitry Andric });
142781ad6265SDimitry Andric
142881ad6265SDimitry Andric for (auto *DDI : Info.DbgVariableIntrinsics) {
1429480093f4SDimitry Andric // Prepend "tag_offset, N" to the dwarf expression.
1430480093f4SDimitry Andric // Tag offset logically applies to the alloca pointer, and it makes sense
1431480093f4SDimitry Andric // to put it at the beginning of the expression.
1432480093f4SDimitry Andric SmallVector<uint64_t, 8> NewOps = {dwarf::DW_OP_LLVM_tag_offset,
1433fe6060f1SDimitry Andric retagMask(N)};
1434fe6060f1SDimitry Andric for (size_t LocNo = 0; LocNo < DDI->getNumVariableLocationOps(); ++LocNo)
1435fe6060f1SDimitry Andric if (DDI->getVariableLocationOp(LocNo) == AI)
1436fe6060f1SDimitry Andric DDI->setExpression(DIExpression::appendOpsToArg(DDI->getExpression(),
1437fe6060f1SDimitry Andric NewOps, LocNo));
14380b57cec5SDimitry Andric }
14390b57cec5SDimitry Andric
1440349cc55cSDimitry Andric auto TagEnd = [&](Instruction *Node) {
1441349cc55cSDimitry Andric IRB.SetInsertPoint(Node);
144281ad6265SDimitry Andric // When untagging, use the `AlignedSize` because we need to set the tags
144306c3fb27SDimitry Andric // for the entire alloca to original. If we used `Size` here, we would
144481ad6265SDimitry Andric // keep the last granule tagged, and store zero in the last byte of the
144581ad6265SDimitry Andric // last granule, due to how short granules are implemented.
1446349cc55cSDimitry Andric tagAlloca(IRB, AI, UARTag, AlignedSize);
1447349cc55cSDimitry Andric };
144881ad6265SDimitry Andric // Calls to functions that may return twice (e.g. setjmp) confuse the
144981ad6265SDimitry Andric // postdominator analysis, and will leave us to keep memory tagged after
145081ad6265SDimitry Andric // function return. Work around this by always untagging at every return
145181ad6265SDimitry Andric // statement if return_twice functions are called.
145204eeddc0SDimitry Andric bool StandardLifetime =
145381ad6265SDimitry Andric SInfo.UnrecognizedLifetimes.empty() &&
145481ad6265SDimitry Andric memtag::isStandardLifetime(Info.LifetimeStart, Info.LifetimeEnd, &DT,
145581ad6265SDimitry Andric &LI, ClMaxLifetimes) &&
145681ad6265SDimitry Andric !SInfo.CallsReturnTwice;
145781ad6265SDimitry Andric if (DetectUseAfterScope && StandardLifetime) {
145804eeddc0SDimitry Andric IntrinsicInst *Start = Info.LifetimeStart[0];
145904eeddc0SDimitry Andric IRB.SetInsertPoint(Start->getNextNode());
14600b57cec5SDimitry Andric tagAlloca(IRB, AI, Tag, Size);
146181ad6265SDimitry Andric if (!memtag::forAllReachableExits(DT, PDT, LI, Start, Info.LifetimeEnd,
146281ad6265SDimitry Andric SInfo.RetVec, TagEnd)) {
1463349cc55cSDimitry Andric for (auto *End : Info.LifetimeEnd)
1464349cc55cSDimitry Andric End->eraseFromParent();
1465349cc55cSDimitry Andric }
1466349cc55cSDimitry Andric } else {
1467349cc55cSDimitry Andric tagAlloca(IRB, AI, Tag, Size);
146881ad6265SDimitry Andric for (auto *RI : SInfo.RetVec)
146904eeddc0SDimitry Andric TagEnd(RI);
147081ad6265SDimitry Andric // We inserted tagging outside of the lifetimes, so we have to remove
147181ad6265SDimitry Andric // them.
1472349cc55cSDimitry Andric for (auto &II : Info.LifetimeStart)
1473349cc55cSDimitry Andric II->eraseFromParent();
1474349cc55cSDimitry Andric for (auto &II : Info.LifetimeEnd)
1475349cc55cSDimitry Andric II->eraseFromParent();
14760b57cec5SDimitry Andric }
1477bdd1243dSDimitry Andric memtag::alignAndPadAlloca(Info, Mapping.getObjectAlignment());
14780b57cec5SDimitry Andric }
147981ad6265SDimitry Andric for (auto &I : SInfo.UnrecognizedLifetimes)
1480349cc55cSDimitry Andric I->eraseFromParent();
14810b57cec5SDimitry Andric return true;
14820b57cec5SDimitry Andric }
14830b57cec5SDimitry Andric
sanitizeFunction(Function & F,FunctionAnalysisManager & FAM)148406c3fb27SDimitry Andric void HWAddressSanitizer::sanitizeFunction(Function &F,
148581ad6265SDimitry Andric FunctionAnalysisManager &FAM) {
14860b57cec5SDimitry Andric if (&F == HwasanCtorFunction)
148706c3fb27SDimitry Andric return;
14880b57cec5SDimitry Andric
14890b57cec5SDimitry Andric if (!F.hasFnAttribute(Attribute::SanitizeHWAddress))
149006c3fb27SDimitry Andric return;
14910b57cec5SDimitry Andric
14920b57cec5SDimitry Andric LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
14930b57cec5SDimitry Andric
14945ffd83dbSDimitry Andric SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
14955ffd83dbSDimitry Andric SmallVector<MemIntrinsic *, 16> IntrinToInstrument;
14960b57cec5SDimitry Andric SmallVector<Instruction *, 8> LandingPadVec;
14975f757f3fSDimitry Andric const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
149881ad6265SDimitry Andric
1499bdd1243dSDimitry Andric memtag::StackInfoBuilder SIB(SSI);
150081ad6265SDimitry Andric for (auto &Inst : instructions(F)) {
1501349cc55cSDimitry Andric if (InstrumentStack) {
150281ad6265SDimitry Andric SIB.visit(Inst);
1503fe6060f1SDimitry Andric }
15040b57cec5SDimitry Andric
15058bcb0991SDimitry Andric if (InstrumentLandingPads && isa<LandingPadInst>(Inst))
15060b57cec5SDimitry Andric LandingPadVec.push_back(&Inst);
15070b57cec5SDimitry Andric
15085f757f3fSDimitry Andric getInterestingMemoryOperands(&Inst, TLI, OperandsToInstrument);
15095ffd83dbSDimitry Andric
15105ffd83dbSDimitry Andric if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(&Inst))
1511349cc55cSDimitry Andric if (!ignoreMemIntrinsic(MI))
15125ffd83dbSDimitry Andric IntrinToInstrument.push_back(MI);
15130b57cec5SDimitry Andric }
151481ad6265SDimitry Andric
151581ad6265SDimitry Andric memtag::StackInfo &SInfo = SIB.get();
15160b57cec5SDimitry Andric
15170b57cec5SDimitry Andric initializeCallbacks(*F.getParent());
15180b57cec5SDimitry Andric
15190b57cec5SDimitry Andric if (!LandingPadVec.empty())
152006c3fb27SDimitry Andric instrumentLandingPads(LandingPadVec);
15210b57cec5SDimitry Andric
152281ad6265SDimitry Andric if (SInfo.AllocasToInstrument.empty() && F.hasPersonalityFn() &&
15238bcb0991SDimitry Andric F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) {
15248bcb0991SDimitry Andric // __hwasan_personality_thunk is a no-op for functions without an
15258bcb0991SDimitry Andric // instrumented stack, so we can drop it.
15268bcb0991SDimitry Andric F.setPersonalityFn(nullptr);
15278bcb0991SDimitry Andric }
15288bcb0991SDimitry Andric
152981ad6265SDimitry Andric if (SInfo.AllocasToInstrument.empty() && OperandsToInstrument.empty() &&
15305ffd83dbSDimitry Andric IntrinToInstrument.empty())
153106c3fb27SDimitry Andric return;
15320b57cec5SDimitry Andric
1533e8d8bef9SDimitry Andric assert(!ShadowBase);
15340b57cec5SDimitry Andric
15350b57cec5SDimitry Andric Instruction *InsertPt = &*F.getEntryBlock().begin();
15360b57cec5SDimitry Andric IRBuilder<> EntryIRB(InsertPt);
15370b57cec5SDimitry Andric emitPrologue(EntryIRB,
1538753f127fSDimitry Andric /*WithFrameRecord*/ ClRecordStackHistory != none &&
153981ad6265SDimitry Andric Mapping.WithFrameRecord &&
154081ad6265SDimitry Andric !SInfo.AllocasToInstrument.empty());
15410b57cec5SDimitry Andric
154281ad6265SDimitry Andric if (!SInfo.AllocasToInstrument.empty()) {
154381ad6265SDimitry Andric const DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F);
154481ad6265SDimitry Andric const PostDominatorTree &PDT = FAM.getResult<PostDominatorTreeAnalysis>(F);
154581ad6265SDimitry Andric const LoopInfo &LI = FAM.getResult<LoopAnalysis>(F);
154606c3fb27SDimitry Andric Value *StackTag = getStackBaseTag(EntryIRB);
154706c3fb27SDimitry Andric Value *UARTag = getUARTag(EntryIRB);
154806c3fb27SDimitry Andric instrumentStack(SInfo, StackTag, UARTag, DT, PDT, LI);
15490b57cec5SDimitry Andric }
15500b57cec5SDimitry Andric
15510b57cec5SDimitry Andric // If we split the entry block, move any allocas that were originally in the
15520b57cec5SDimitry Andric // entry block back into the entry block so that they aren't treated as
15530b57cec5SDimitry Andric // dynamic allocas.
15540b57cec5SDimitry Andric if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
15550b57cec5SDimitry Andric InsertPt = &*F.getEntryBlock().begin();
1556349cc55cSDimitry Andric for (Instruction &I :
1557349cc55cSDimitry Andric llvm::make_early_inc_range(*EntryIRB.GetInsertBlock())) {
1558349cc55cSDimitry Andric if (auto *AI = dyn_cast<AllocaInst>(&I))
15590b57cec5SDimitry Andric if (isa<ConstantInt>(AI->getArraySize()))
1560349cc55cSDimitry Andric I.moveBefore(InsertPt);
15610b57cec5SDimitry Andric }
15620b57cec5SDimitry Andric }
15630b57cec5SDimitry Andric
15645f757f3fSDimitry Andric DominatorTree *DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
15655f757f3fSDimitry Andric PostDominatorTree *PDT = FAM.getCachedResult<PostDominatorTreeAnalysis>(F);
15665f757f3fSDimitry Andric LoopInfo *LI = FAM.getCachedResult<LoopAnalysis>(F);
15675f757f3fSDimitry Andric DomTreeUpdater DTU(DT, PDT, DomTreeUpdater::UpdateStrategy::Lazy);
15685ffd83dbSDimitry Andric for (auto &Operand : OperandsToInstrument)
15695f757f3fSDimitry Andric instrumentMemAccess(Operand, DTU, LI);
15705f757f3fSDimitry Andric DTU.flush();
15715ffd83dbSDimitry Andric
15725ffd83dbSDimitry Andric if (ClInstrumentMemIntrinsics && !IntrinToInstrument.empty()) {
1573bdd1243dSDimitry Andric for (auto *Inst : IntrinToInstrument)
1574bdd1243dSDimitry Andric instrumentMemIntrinsic(Inst);
15755ffd83dbSDimitry Andric }
15760b57cec5SDimitry Andric
1577e8d8bef9SDimitry Andric ShadowBase = nullptr;
15780b57cec5SDimitry Andric StackBaseTag = nullptr;
157981ad6265SDimitry Andric CachedSP = nullptr;
15800b57cec5SDimitry Andric }
15810b57cec5SDimitry Andric
instrumentGlobal(GlobalVariable * GV,uint8_t Tag)15828bcb0991SDimitry Andric void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) {
1583fe6060f1SDimitry Andric assert(!UsePageAliases);
15848bcb0991SDimitry Andric Constant *Initializer = GV->getInitializer();
15858bcb0991SDimitry Andric uint64_t SizeInBytes =
15868bcb0991SDimitry Andric M.getDataLayout().getTypeAllocSize(Initializer->getType());
15878bcb0991SDimitry Andric uint64_t NewSize = alignTo(SizeInBytes, Mapping.getObjectAlignment());
15888bcb0991SDimitry Andric if (SizeInBytes != NewSize) {
15898bcb0991SDimitry Andric // Pad the initializer out to the next multiple of 16 bytes and add the
15908bcb0991SDimitry Andric // required short granule tag.
15918bcb0991SDimitry Andric std::vector<uint8_t> Init(NewSize - SizeInBytes, 0);
15928bcb0991SDimitry Andric Init.back() = Tag;
15938bcb0991SDimitry Andric Constant *Padding = ConstantDataArray::get(*C, Init);
15948bcb0991SDimitry Andric Initializer = ConstantStruct::getAnon({Initializer, Padding});
15958bcb0991SDimitry Andric }
15968bcb0991SDimitry Andric
15978bcb0991SDimitry Andric auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(),
15988bcb0991SDimitry Andric GlobalValue::ExternalLinkage, Initializer,
15998bcb0991SDimitry Andric GV->getName() + ".hwasan");
16008bcb0991SDimitry Andric NewGV->copyAttributesFrom(GV);
16018bcb0991SDimitry Andric NewGV->setLinkage(GlobalValue::PrivateLinkage);
16028bcb0991SDimitry Andric NewGV->copyMetadata(GV, 0);
16038bcb0991SDimitry Andric NewGV->setAlignment(
1604bdd1243dSDimitry Andric std::max(GV->getAlign().valueOrOne(), Mapping.getObjectAlignment()));
16058bcb0991SDimitry Andric
16068bcb0991SDimitry Andric // It is invalid to ICF two globals that have different tags. In the case
16078bcb0991SDimitry Andric // where the size of the global is a multiple of the tag granularity the
16088bcb0991SDimitry Andric // contents of the globals may be the same but the tags (i.e. symbol values)
16098bcb0991SDimitry Andric // may be different, and the symbols are not considered during ICF. In the
16108bcb0991SDimitry Andric // case where the size is not a multiple of the granularity, the short granule
16118bcb0991SDimitry Andric // tags would discriminate two globals with different tags, but there would
16128bcb0991SDimitry Andric // otherwise be nothing stopping such a global from being incorrectly ICF'd
16138bcb0991SDimitry Andric // with an uninstrumented (i.e. tag 0) global that happened to have the short
16148bcb0991SDimitry Andric // granule tag in the last byte.
16158bcb0991SDimitry Andric NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
16168bcb0991SDimitry Andric
16178bcb0991SDimitry Andric // Descriptor format (assuming little-endian):
16188bcb0991SDimitry Andric // bytes 0-3: relative address of global
16198bcb0991SDimitry Andric // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case
16208bcb0991SDimitry Andric // it isn't, we create multiple descriptors)
16218bcb0991SDimitry Andric // byte 7: tag
16228bcb0991SDimitry Andric auto *DescriptorTy = StructType::get(Int32Ty, Int32Ty);
16238bcb0991SDimitry Andric const uint64_t MaxDescriptorSize = 0xfffff0;
16248bcb0991SDimitry Andric for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes;
16258bcb0991SDimitry Andric DescriptorPos += MaxDescriptorSize) {
16268bcb0991SDimitry Andric auto *Descriptor =
16278bcb0991SDimitry Andric new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage,
16288bcb0991SDimitry Andric nullptr, GV->getName() + ".hwasan.descriptor");
16298bcb0991SDimitry Andric auto *GVRelPtr = ConstantExpr::getTrunc(
16308bcb0991SDimitry Andric ConstantExpr::getAdd(
16318bcb0991SDimitry Andric ConstantExpr::getSub(
16328bcb0991SDimitry Andric ConstantExpr::getPtrToInt(NewGV, Int64Ty),
16338bcb0991SDimitry Andric ConstantExpr::getPtrToInt(Descriptor, Int64Ty)),
16348bcb0991SDimitry Andric ConstantInt::get(Int64Ty, DescriptorPos)),
16358bcb0991SDimitry Andric Int32Ty);
16368bcb0991SDimitry Andric uint32_t Size = std::min(SizeInBytes - DescriptorPos, MaxDescriptorSize);
16378bcb0991SDimitry Andric auto *SizeAndTag = ConstantInt::get(Int32Ty, Size | (uint32_t(Tag) << 24));
16388bcb0991SDimitry Andric Descriptor->setComdat(NewGV->getComdat());
16398bcb0991SDimitry Andric Descriptor->setInitializer(ConstantStruct::getAnon({GVRelPtr, SizeAndTag}));
16408bcb0991SDimitry Andric Descriptor->setSection("hwasan_globals");
16418bcb0991SDimitry Andric Descriptor->setMetadata(LLVMContext::MD_associated,
16428bcb0991SDimitry Andric MDNode::get(*C, ValueAsMetadata::get(NewGV)));
16438bcb0991SDimitry Andric appendToCompilerUsed(M, Descriptor);
16448bcb0991SDimitry Andric }
16458bcb0991SDimitry Andric
16468bcb0991SDimitry Andric Constant *Aliasee = ConstantExpr::getIntToPtr(
16478bcb0991SDimitry Andric ConstantExpr::getAdd(
16488bcb0991SDimitry Andric ConstantExpr::getPtrToInt(NewGV, Int64Ty),
1649fe6060f1SDimitry Andric ConstantInt::get(Int64Ty, uint64_t(Tag) << PointerTagShift)),
16508bcb0991SDimitry Andric GV->getType());
16518bcb0991SDimitry Andric auto *Alias = GlobalAlias::create(GV->getValueType(), GV->getAddressSpace(),
16528bcb0991SDimitry Andric GV->getLinkage(), "", Aliasee, &M);
16538bcb0991SDimitry Andric Alias->setVisibility(GV->getVisibility());
16548bcb0991SDimitry Andric Alias->takeName(GV);
16558bcb0991SDimitry Andric GV->replaceAllUsesWith(Alias);
16568bcb0991SDimitry Andric GV->eraseFromParent();
16578bcb0991SDimitry Andric }
16588bcb0991SDimitry Andric
instrumentGlobals()16598bcb0991SDimitry Andric void HWAddressSanitizer::instrumentGlobals() {
16608bcb0991SDimitry Andric std::vector<GlobalVariable *> Globals;
16618bcb0991SDimitry Andric for (GlobalVariable &GV : M.globals()) {
166281ad6265SDimitry Andric if (GV.hasSanitizerMetadata() && GV.getSanitizerMetadata().NoHWAddress)
1663fe6060f1SDimitry Andric continue;
1664fe6060f1SDimitry Andric
16655f757f3fSDimitry Andric if (GV.isDeclarationForLinker() || GV.getName().starts_with("llvm.") ||
16668bcb0991SDimitry Andric GV.isThreadLocal())
16678bcb0991SDimitry Andric continue;
16688bcb0991SDimitry Andric
16698bcb0991SDimitry Andric // Common symbols can't have aliases point to them, so they can't be tagged.
16708bcb0991SDimitry Andric if (GV.hasCommonLinkage())
16718bcb0991SDimitry Andric continue;
16728bcb0991SDimitry Andric
16738bcb0991SDimitry Andric // Globals with custom sections may be used in __start_/__stop_ enumeration,
16748bcb0991SDimitry Andric // which would be broken both by adding tags and potentially by the extra
16758bcb0991SDimitry Andric // padding/alignment that we insert.
16768bcb0991SDimitry Andric if (GV.hasSection())
16778bcb0991SDimitry Andric continue;
16788bcb0991SDimitry Andric
16798bcb0991SDimitry Andric Globals.push_back(&GV);
16808bcb0991SDimitry Andric }
16818bcb0991SDimitry Andric
16828bcb0991SDimitry Andric MD5 Hasher;
16838bcb0991SDimitry Andric Hasher.update(M.getSourceFileName());
16848bcb0991SDimitry Andric MD5::MD5Result Hash;
16858bcb0991SDimitry Andric Hasher.final(Hash);
1686349cc55cSDimitry Andric uint8_t Tag = Hash[0];
16878bcb0991SDimitry Andric
168806c3fb27SDimitry Andric assert(TagMaskByte >= 16);
168906c3fb27SDimitry Andric
16908bcb0991SDimitry Andric for (GlobalVariable *GV : Globals) {
169106c3fb27SDimitry Andric // Don't allow globals to be tagged with something that looks like a
169206c3fb27SDimitry Andric // short-granule tag, otherwise we lose inter-granule overflow detection, as
169306c3fb27SDimitry Andric // the fast path shadow-vs-address check succeeds.
169406c3fb27SDimitry Andric if (Tag < 16 || Tag > TagMaskByte)
169506c3fb27SDimitry Andric Tag = 16;
16968bcb0991SDimitry Andric instrumentGlobal(GV, Tag++);
16978bcb0991SDimitry Andric }
16988bcb0991SDimitry Andric }
16998bcb0991SDimitry Andric
instrumentPersonalityFunctions()17008bcb0991SDimitry Andric void HWAddressSanitizer::instrumentPersonalityFunctions() {
17018bcb0991SDimitry Andric // We need to untag stack frames as we unwind past them. That is the job of
17028bcb0991SDimitry Andric // the personality function wrapper, which either wraps an existing
17038bcb0991SDimitry Andric // personality function or acts as a personality function on its own. Each
17048bcb0991SDimitry Andric // function that has a personality function or that can be unwound past has
17058bcb0991SDimitry Andric // its personality function changed to a thunk that calls the personality
17068bcb0991SDimitry Andric // function wrapper in the runtime.
17078bcb0991SDimitry Andric MapVector<Constant *, std::vector<Function *>> PersonalityFns;
17088bcb0991SDimitry Andric for (Function &F : M) {
17098bcb0991SDimitry Andric if (F.isDeclaration() || !F.hasFnAttribute(Attribute::SanitizeHWAddress))
17108bcb0991SDimitry Andric continue;
17118bcb0991SDimitry Andric
17128bcb0991SDimitry Andric if (F.hasPersonalityFn()) {
17138bcb0991SDimitry Andric PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(&F);
17148bcb0991SDimitry Andric } else if (!F.hasFnAttribute(Attribute::NoUnwind)) {
17158bcb0991SDimitry Andric PersonalityFns[nullptr].push_back(&F);
17168bcb0991SDimitry Andric }
17178bcb0991SDimitry Andric }
17188bcb0991SDimitry Andric
17198bcb0991SDimitry Andric if (PersonalityFns.empty())
17208bcb0991SDimitry Andric return;
17218bcb0991SDimitry Andric
17228bcb0991SDimitry Andric FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction(
17235f757f3fSDimitry Andric "__hwasan_personality_wrapper", Int32Ty, Int32Ty, Int32Ty, Int64Ty, PtrTy,
17245f757f3fSDimitry Andric PtrTy, PtrTy, PtrTy, PtrTy);
17258bcb0991SDimitry Andric FunctionCallee UnwindGetGR = M.getOrInsertFunction("_Unwind_GetGR", VoidTy);
17268bcb0991SDimitry Andric FunctionCallee UnwindGetCFA = M.getOrInsertFunction("_Unwind_GetCFA", VoidTy);
17278bcb0991SDimitry Andric
17288bcb0991SDimitry Andric for (auto &P : PersonalityFns) {
17298bcb0991SDimitry Andric std::string ThunkName = kHwasanPersonalityThunkName;
17308bcb0991SDimitry Andric if (P.first)
17318bcb0991SDimitry Andric ThunkName += ("." + P.first->getName()).str();
17328bcb0991SDimitry Andric FunctionType *ThunkFnTy = FunctionType::get(
17335f757f3fSDimitry Andric Int32Ty, {Int32Ty, Int32Ty, Int64Ty, PtrTy, PtrTy}, false);
17348bcb0991SDimitry Andric bool IsLocal = P.first && (!isa<GlobalValue>(P.first) ||
17358bcb0991SDimitry Andric cast<GlobalValue>(P.first)->hasLocalLinkage());
17368bcb0991SDimitry Andric auto *ThunkFn = Function::Create(ThunkFnTy,
17378bcb0991SDimitry Andric IsLocal ? GlobalValue::InternalLinkage
17388bcb0991SDimitry Andric : GlobalValue::LinkOnceODRLinkage,
17398bcb0991SDimitry Andric ThunkName, &M);
17408bcb0991SDimitry Andric if (!IsLocal) {
17418bcb0991SDimitry Andric ThunkFn->setVisibility(GlobalValue::HiddenVisibility);
17428bcb0991SDimitry Andric ThunkFn->setComdat(M.getOrInsertComdat(ThunkName));
17438bcb0991SDimitry Andric }
17448bcb0991SDimitry Andric
17458bcb0991SDimitry Andric auto *BB = BasicBlock::Create(*C, "entry", ThunkFn);
17468bcb0991SDimitry Andric IRBuilder<> IRB(BB);
17478bcb0991SDimitry Andric CallInst *WrapperCall = IRB.CreateCall(
17488bcb0991SDimitry Andric HwasanPersonalityWrapper,
17498bcb0991SDimitry Andric {ThunkFn->getArg(0), ThunkFn->getArg(1), ThunkFn->getArg(2),
17508bcb0991SDimitry Andric ThunkFn->getArg(3), ThunkFn->getArg(4),
17515f757f3fSDimitry Andric P.first ? P.first : Constant::getNullValue(PtrTy),
17525f757f3fSDimitry Andric UnwindGetGR.getCallee(), UnwindGetCFA.getCallee()});
17538bcb0991SDimitry Andric WrapperCall->setTailCall();
17548bcb0991SDimitry Andric IRB.CreateRet(WrapperCall);
17558bcb0991SDimitry Andric
17568bcb0991SDimitry Andric for (Function *F : P.second)
17578bcb0991SDimitry Andric F->setPersonalityFn(ThunkFn);
17588bcb0991SDimitry Andric }
17598bcb0991SDimitry Andric }
17608bcb0991SDimitry Andric
init(Triple & TargetTriple,bool InstrumentWithCalls)1761fe6060f1SDimitry Andric void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple,
1762fe6060f1SDimitry Andric bool InstrumentWithCalls) {
17630b57cec5SDimitry Andric Scale = kDefaultShadowScale;
1764fe6060f1SDimitry Andric if (TargetTriple.isOSFuchsia()) {
1765fe6060f1SDimitry Andric // Fuchsia is always PIE, which means that the beginning of the address
1766fe6060f1SDimitry Andric // space is always available.
17670b57cec5SDimitry Andric InGlobal = false;
17680b57cec5SDimitry Andric InTls = false;
17690b57cec5SDimitry Andric Offset = 0;
1770fe6060f1SDimitry Andric WithFrameRecord = true;
1771fe6060f1SDimitry Andric } else if (ClMappingOffset.getNumOccurrences() > 0) {
1772fe6060f1SDimitry Andric InGlobal = false;
1773fe6060f1SDimitry Andric InTls = false;
1774fe6060f1SDimitry Andric Offset = ClMappingOffset;
1775fe6060f1SDimitry Andric WithFrameRecord = false;
1776fe6060f1SDimitry Andric } else if (ClEnableKhwasan || InstrumentWithCalls) {
1777fe6060f1SDimitry Andric InGlobal = false;
1778fe6060f1SDimitry Andric InTls = false;
1779fe6060f1SDimitry Andric Offset = 0;
1780fe6060f1SDimitry Andric WithFrameRecord = false;
17810b57cec5SDimitry Andric } else if (ClWithIfunc) {
17820b57cec5SDimitry Andric InGlobal = true;
17830b57cec5SDimitry Andric InTls = false;
17840b57cec5SDimitry Andric Offset = kDynamicShadowSentinel;
1785fe6060f1SDimitry Andric WithFrameRecord = false;
17860b57cec5SDimitry Andric } else if (ClWithTls) {
17870b57cec5SDimitry Andric InGlobal = false;
17880b57cec5SDimitry Andric InTls = true;
17890b57cec5SDimitry Andric Offset = kDynamicShadowSentinel;
1790fe6060f1SDimitry Andric WithFrameRecord = true;
17910b57cec5SDimitry Andric } else {
17920b57cec5SDimitry Andric InGlobal = false;
17930b57cec5SDimitry Andric InTls = false;
17940b57cec5SDimitry Andric Offset = kDynamicShadowSentinel;
1795fe6060f1SDimitry Andric WithFrameRecord = false;
17960b57cec5SDimitry Andric }
17970b57cec5SDimitry Andric }
1798