1 //===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// This file is a part of MemorySanitizer, a detector of uninitialized
12 /// reads.
13 ///
14 /// The algorithm of the tool is similar to Memcheck
15 /// (http://goo.gl/QKbem). We associate a few shadow bits with every
16 /// byte of the application memory, poison the shadow of the malloc-ed
17 /// or alloca-ed memory, load the shadow bits on every memory read,
18 /// propagate the shadow bits through some of the arithmetic
19 /// instruction (including MOV), store the shadow bits on every memory
20 /// write, report a bug on some other instructions (e.g. JMP) if the
21 /// associated shadow is poisoned.
22 ///
23 /// But there are differences too. The first and the major one:
24 /// compiler instrumentation instead of binary instrumentation. This
25 /// gives us much better register allocation, possible compiler
26 /// optimizations and a fast start-up. But this brings the major issue
27 /// as well: msan needs to see all program events, including system
28 /// calls and reads/writes in system libraries, so we either need to
29 /// compile *everything* with msan or use a binary translation
30 /// component (e.g. DynamoRIO) to instrument pre-built libraries.
31 /// Another difference from Memcheck is that we use 8 shadow bits per
32 /// byte of application memory and use a direct shadow mapping. This
33 /// greatly simplifies the instrumentation code and avoids races on
34 /// shadow updates (Memcheck is single-threaded so races are not a
35 /// concern there. Memcheck uses 2 shadow bits per byte with a slow
36 /// path storage that uses 8 bits per byte).
37 ///
38 /// The default value of shadow is 0, which means "clean" (not poisoned).
39 ///
40 /// Every module initializer should call __msan_init to ensure that the
41 /// shadow memory is ready. On error, __msan_warning is called. Since
42 /// parameters and return values may be passed via registers, we have a
43 /// specialized thread-local shadow for return values
44 /// (__msan_retval_tls) and parameters (__msan_param_tls).
45 ///
46 /// Origin tracking.
47 ///
48 /// MemorySanitizer can track origins (allocation points) of all uninitialized
49 /// values. This behavior is controlled with a flag (msan-track-origins) and is
50 /// disabled by default.
51 ///
52 /// Origins are 4-byte values created and interpreted by the runtime library.
53 /// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
54 /// of application memory. Propagation of origins is basically a bunch of
55 /// "select" instructions that pick the origin of a dirty argument, if an
56 /// instruction has one.
57 ///
58 /// Every 4 aligned, consecutive bytes of application memory have one origin
59 /// value associated with them. If these bytes contain uninitialized data
60 /// coming from 2 different allocations, the last store wins. Because of this,
61 /// MemorySanitizer reports can show unrelated origins, but this is unlikely in
62 /// practice.
63 ///
64 /// Origins are meaningless for fully initialized values, so MemorySanitizer
65 /// avoids storing origin to memory when a fully initialized value is stored.
66 /// This way it avoids needless overwritting origin of the 4-byte region on
67 /// a short (i.e. 1 byte) clean store, and it is also good for performance.
68 ///
69 /// Atomic handling.
70 ///
71 /// Ideally, every atomic store of application value should update the
72 /// corresponding shadow location in an atomic way. Unfortunately, atomic store
73 /// of two disjoint locations can not be done without severe slowdown.
74 ///
75 /// Therefore, we implement an approximation that may err on the safe side.
76 /// In this implementation, every atomically accessed location in the program
77 /// may only change from (partially) uninitialized to fully initialized, but
78 /// not the other way around. We load the shadow _after_ the application load,
79 /// and we store the shadow _before_ the app store. Also, we always store clean
80 /// shadow (if the application store is atomic). This way, if the store-load
81 /// pair constitutes a happens-before arc, shadow store and load are correctly
82 /// ordered such that the load will get either the value that was stored, or
83 /// some later value (which is always clean).
84 ///
85 /// This does not work very well with Compare-And-Swap (CAS) and
86 /// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
87 /// must store the new shadow before the app operation, and load the shadow
88 /// after the app operation. Computers don't work this way. Current
89 /// implementation ignores the load aspect of CAS/RMW, always returning a clean
90 /// value. It implements the store part as a simple atomic store by storing a
91 /// clean shadow.
92 //
93 //===----------------------------------------------------------------------===//
94
95 #include "llvm/ADT/APInt.h"
96 #include "llvm/ADT/ArrayRef.h"
97 #include "llvm/ADT/DepthFirstIterator.h"
98 #include "llvm/ADT/SmallString.h"
99 #include "llvm/ADT/SmallVector.h"
100 #include "llvm/ADT/StringExtras.h"
101 #include "llvm/ADT/StringRef.h"
102 #include "llvm/ADT/Triple.h"
103 #include "llvm/Analysis/TargetLibraryInfo.h"
104 #include "llvm/Transforms/Utils/Local.h"
105 #include "llvm/IR/Argument.h"
106 #include "llvm/IR/Attributes.h"
107 #include "llvm/IR/BasicBlock.h"
108 #include "llvm/IR/CallSite.h"
109 #include "llvm/IR/CallingConv.h"
110 #include "llvm/IR/Constant.h"
111 #include "llvm/IR/Constants.h"
112 #include "llvm/IR/DataLayout.h"
113 #include "llvm/IR/DerivedTypes.h"
114 #include "llvm/IR/Function.h"
115 #include "llvm/IR/GlobalValue.h"
116 #include "llvm/IR/GlobalVariable.h"
117 #include "llvm/IR/IRBuilder.h"
118 #include "llvm/IR/InlineAsm.h"
119 #include "llvm/IR/InstVisitor.h"
120 #include "llvm/IR/InstrTypes.h"
121 #include "llvm/IR/Instruction.h"
122 #include "llvm/IR/Instructions.h"
123 #include "llvm/IR/IntrinsicInst.h"
124 #include "llvm/IR/Intrinsics.h"
125 #include "llvm/IR/LLVMContext.h"
126 #include "llvm/IR/MDBuilder.h"
127 #include "llvm/IR/Module.h"
128 #include "llvm/IR/Type.h"
129 #include "llvm/IR/Value.h"
130 #include "llvm/IR/ValueMap.h"
131 #include "llvm/Pass.h"
132 #include "llvm/Support/AtomicOrdering.h"
133 #include "llvm/Support/Casting.h"
134 #include "llvm/Support/CommandLine.h"
135 #include "llvm/Support/Compiler.h"
136 #include "llvm/Support/Debug.h"
137 #include "llvm/Support/ErrorHandling.h"
138 #include "llvm/Support/MathExtras.h"
139 #include "llvm/Support/raw_ostream.h"
140 #include "llvm/Transforms/Instrumentation.h"
141 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
142 #include "llvm/Transforms/Utils/ModuleUtils.h"
143 #include <algorithm>
144 #include <cassert>
145 #include <cstddef>
146 #include <cstdint>
147 #include <memory>
148 #include <string>
149 #include <tuple>
150
151 using namespace llvm;
152
153 #define DEBUG_TYPE "msan"
154
155 static const unsigned kOriginSize = 4;
156 static const unsigned kMinOriginAlignment = 4;
157 static const unsigned kShadowTLSAlignment = 8;
158
159 // These constants must be kept in sync with the ones in msan.h.
160 static const unsigned kParamTLSSize = 800;
161 static const unsigned kRetvalTLSSize = 800;
162
163 // Accesses sizes are powers of two: 1, 2, 4, 8.
164 static const size_t kNumberOfAccessSizes = 4;
165
166 /// Track origins of uninitialized values.
167 ///
168 /// Adds a section to MemorySanitizer report that points to the allocation
169 /// (stack or heap) the uninitialized bits came from originally.
170 static cl::opt<int> ClTrackOrigins("msan-track-origins",
171 cl::desc("Track origins (allocation sites) of poisoned memory"),
172 cl::Hidden, cl::init(0));
173
174 static cl::opt<bool> ClKeepGoing("msan-keep-going",
175 cl::desc("keep going after reporting a UMR"),
176 cl::Hidden, cl::init(false));
177
178 static cl::opt<bool> ClPoisonStack("msan-poison-stack",
179 cl::desc("poison uninitialized stack variables"),
180 cl::Hidden, cl::init(true));
181
182 static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call",
183 cl::desc("poison uninitialized stack variables with a call"),
184 cl::Hidden, cl::init(false));
185
186 static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern",
187 cl::desc("poison uninitialized stack variables with the given pattern"),
188 cl::Hidden, cl::init(0xff));
189
190 static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
191 cl::desc("poison undef temps"),
192 cl::Hidden, cl::init(true));
193
194 static cl::opt<bool> ClHandleICmp("msan-handle-icmp",
195 cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
196 cl::Hidden, cl::init(true));
197
198 static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact",
199 cl::desc("exact handling of relational integer ICmp"),
200 cl::Hidden, cl::init(false));
201
202 // When compiling the Linux kernel, we sometimes see false positives related to
203 // MSan being unable to understand that inline assembly calls may initialize
204 // local variables.
205 // This flag makes the compiler conservatively unpoison every memory location
206 // passed into an assembly call. Note that this may cause false positives.
207 // Because it's impossible to figure out the array sizes, we can only unpoison
208 // the first sizeof(type) bytes for each type* pointer.
209 static cl::opt<bool> ClHandleAsmConservative(
210 "msan-handle-asm-conservative",
211 cl::desc("conservative handling of inline assembly"), cl::Hidden,
212 cl::init(false));
213
214 // This flag controls whether we check the shadow of the address
215 // operand of load or store. Such bugs are very rare, since load from
216 // a garbage address typically results in SEGV, but still happen
217 // (e.g. only lower bits of address are garbage, or the access happens
218 // early at program startup where malloc-ed memory is more likely to
219 // be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
220 static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address",
221 cl::desc("report accesses through a pointer which has poisoned shadow"),
222 cl::Hidden, cl::init(true));
223
224 static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions",
225 cl::desc("print out instructions with default strict semantics"),
226 cl::Hidden, cl::init(false));
227
228 static cl::opt<int> ClInstrumentationWithCallThreshold(
229 "msan-instrumentation-with-call-threshold",
230 cl::desc(
231 "If the function being instrumented requires more than "
232 "this number of checks and origin stores, use callbacks instead of "
233 "inline checks (-1 means never use callbacks)."),
234 cl::Hidden, cl::init(3500));
235
236 // This is an experiment to enable handling of cases where shadow is a non-zero
237 // compile-time constant. For some unexplainable reason they were silently
238 // ignored in the instrumentation.
239 static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
240 cl::desc("Insert checks for constant shadow values"),
241 cl::Hidden, cl::init(false));
242
243 // This is off by default because of a bug in gold:
244 // https://sourceware.org/bugzilla/show_bug.cgi?id=19002
245 static cl::opt<bool> ClWithComdat("msan-with-comdat",
246 cl::desc("Place MSan constructors in comdat sections"),
247 cl::Hidden, cl::init(false));
248
249 // These options allow to specify custom memory map parameters
250 // See MemoryMapParams for details.
251 static cl::opt<unsigned long long> ClAndMask("msan-and-mask",
252 cl::desc("Define custom MSan AndMask"),
253 cl::Hidden, cl::init(0));
254
255 static cl::opt<unsigned long long> ClXorMask("msan-xor-mask",
256 cl::desc("Define custom MSan XorMask"),
257 cl::Hidden, cl::init(0));
258
259 static cl::opt<unsigned long long> ClShadowBase("msan-shadow-base",
260 cl::desc("Define custom MSan ShadowBase"),
261 cl::Hidden, cl::init(0));
262
263 static cl::opt<unsigned long long> ClOriginBase("msan-origin-base",
264 cl::desc("Define custom MSan OriginBase"),
265 cl::Hidden, cl::init(0));
266
267 static const char *const kMsanModuleCtorName = "msan.module_ctor";
268 static const char *const kMsanInitName = "__msan_init";
269
270 namespace {
271
272 // Memory map parameters used in application-to-shadow address calculation.
273 // Offset = (Addr & ~AndMask) ^ XorMask
274 // Shadow = ShadowBase + Offset
275 // Origin = OriginBase + Offset
276 struct MemoryMapParams {
277 uint64_t AndMask;
278 uint64_t XorMask;
279 uint64_t ShadowBase;
280 uint64_t OriginBase;
281 };
282
283 struct PlatformMemoryMapParams {
284 const MemoryMapParams *bits32;
285 const MemoryMapParams *bits64;
286 };
287
288 } // end anonymous namespace
289
290 // i386 Linux
291 static const MemoryMapParams Linux_I386_MemoryMapParams = {
292 0x000080000000, // AndMask
293 0, // XorMask (not used)
294 0, // ShadowBase (not used)
295 0x000040000000, // OriginBase
296 };
297
298 // x86_64 Linux
299 static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
300 #ifdef MSAN_LINUX_X86_64_OLD_MAPPING
301 0x400000000000, // AndMask
302 0, // XorMask (not used)
303 0, // ShadowBase (not used)
304 0x200000000000, // OriginBase
305 #else
306 0, // AndMask (not used)
307 0x500000000000, // XorMask
308 0, // ShadowBase (not used)
309 0x100000000000, // OriginBase
310 #endif
311 };
312
313 // mips64 Linux
314 static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
315 0, // AndMask (not used)
316 0x008000000000, // XorMask
317 0, // ShadowBase (not used)
318 0x002000000000, // OriginBase
319 };
320
321 // ppc64 Linux
322 static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
323 0xE00000000000, // AndMask
324 0x100000000000, // XorMask
325 0x080000000000, // ShadowBase
326 0x1C0000000000, // OriginBase
327 };
328
329 // aarch64 Linux
330 static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
331 0, // AndMask (not used)
332 0x06000000000, // XorMask
333 0, // ShadowBase (not used)
334 0x01000000000, // OriginBase
335 };
336
337 // i386 FreeBSD
338 static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
339 0x000180000000, // AndMask
340 0x000040000000, // XorMask
341 0x000020000000, // ShadowBase
342 0x000700000000, // OriginBase
343 };
344
345 // x86_64 FreeBSD
346 static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
347 0xc00000000000, // AndMask
348 0x200000000000, // XorMask
349 0x100000000000, // ShadowBase
350 0x380000000000, // OriginBase
351 };
352
353 // x86_64 NetBSD
354 static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
355 0, // AndMask
356 0x500000000000, // XorMask
357 0, // ShadowBase
358 0x100000000000, // OriginBase
359 };
360
361 static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
362 &Linux_I386_MemoryMapParams,
363 &Linux_X86_64_MemoryMapParams,
364 };
365
366 static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
367 nullptr,
368 &Linux_MIPS64_MemoryMapParams,
369 };
370
371 static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
372 nullptr,
373 &Linux_PowerPC64_MemoryMapParams,
374 };
375
376 static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
377 nullptr,
378 &Linux_AArch64_MemoryMapParams,
379 };
380
381 static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
382 &FreeBSD_I386_MemoryMapParams,
383 &FreeBSD_X86_64_MemoryMapParams,
384 };
385
386 static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
387 nullptr,
388 &NetBSD_X86_64_MemoryMapParams,
389 };
390
391 namespace {
392
393 /// An instrumentation pass implementing detection of uninitialized
394 /// reads.
395 ///
396 /// MemorySanitizer: instrument the code in module to find
397 /// uninitialized reads.
398 class MemorySanitizer : public FunctionPass {
399 public:
400 // Pass identification, replacement for typeid.
401 static char ID;
402
MemorySanitizer(int TrackOrigins=0,bool Recover=false)403 MemorySanitizer(int TrackOrigins = 0, bool Recover = false)
404 : FunctionPass(ID),
405 TrackOrigins(std::max(TrackOrigins, (int)ClTrackOrigins)),
406 Recover(Recover || ClKeepGoing) {}
407
getPassName() const408 StringRef getPassName() const override { return "MemorySanitizer"; }
409
getAnalysisUsage(AnalysisUsage & AU) const410 void getAnalysisUsage(AnalysisUsage &AU) const override {
411 AU.addRequired<TargetLibraryInfoWrapperPass>();
412 }
413
414 bool runOnFunction(Function &F) override;
415 bool doInitialization(Module &M) override;
416
417 private:
418 friend struct MemorySanitizerVisitor;
419 friend struct VarArgAMD64Helper;
420 friend struct VarArgMIPS64Helper;
421 friend struct VarArgAArch64Helper;
422 friend struct VarArgPowerPC64Helper;
423
424 void initializeCallbacks(Module &M);
425 void createUserspaceApi(Module &M);
426
427 /// Track origins (allocation points) of uninitialized values.
428 int TrackOrigins;
429 bool Recover;
430
431 LLVMContext *C;
432 Type *IntptrTy;
433 Type *OriginTy;
434
435 /// Thread-local shadow storage for function parameters.
436 GlobalVariable *ParamTLS;
437
438 /// Thread-local origin storage for function parameters.
439 GlobalVariable *ParamOriginTLS;
440
441 /// Thread-local shadow storage for function return value.
442 GlobalVariable *RetvalTLS;
443
444 /// Thread-local origin storage for function return value.
445 GlobalVariable *RetvalOriginTLS;
446
447 /// Thread-local shadow storage for in-register va_arg function
448 /// parameters (x86_64-specific).
449 GlobalVariable *VAArgTLS;
450
451 /// Thread-local shadow storage for va_arg overflow area
452 /// (x86_64-specific).
453 GlobalVariable *VAArgOverflowSizeTLS;
454
455 /// Thread-local space used to pass origin value to the UMR reporting
456 /// function.
457 GlobalVariable *OriginTLS;
458
459 /// Are the instrumentation callbacks set up?
460 bool CallbacksInitialized = false;
461
462 /// The run-time callback to print a warning.
463 Value *WarningFn;
464
465 // These arrays are indexed by log2(AccessSize).
466 Value *MaybeWarningFn[kNumberOfAccessSizes];
467 Value *MaybeStoreOriginFn[kNumberOfAccessSizes];
468
469 /// Run-time helper that generates a new origin value for a stack
470 /// allocation.
471 Value *MsanSetAllocaOrigin4Fn;
472
473 /// Run-time helper that poisons stack on function entry.
474 Value *MsanPoisonStackFn;
475
476 /// Run-time helper that records a store (or any event) of an
477 /// uninitialized value and returns an updated origin id encoding this info.
478 Value *MsanChainOriginFn;
479
480 /// MSan runtime replacements for memmove, memcpy and memset.
481 Value *MemmoveFn, *MemcpyFn, *MemsetFn;
482
483 /// Memory map parameters used in application-to-shadow calculation.
484 const MemoryMapParams *MapParams;
485
486 /// Custom memory map parameters used when -msan-shadow-base or
487 // -msan-origin-base is provided.
488 MemoryMapParams CustomMapParams;
489
490 MDNode *ColdCallWeights;
491
492 /// Branch weights for origin store.
493 MDNode *OriginStoreWeights;
494
495 /// An empty volatile inline asm that prevents callback merge.
496 InlineAsm *EmptyAsm;
497
498 Function *MsanCtorFunction;
499 };
500
501 } // end anonymous namespace
502
503 char MemorySanitizer::ID = 0;
504
505 INITIALIZE_PASS_BEGIN(
506 MemorySanitizer, "msan",
507 "MemorySanitizer: detects uninitialized reads.", false, false)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)508 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
509 INITIALIZE_PASS_END(
510 MemorySanitizer, "msan",
511 "MemorySanitizer: detects uninitialized reads.", false, false)
512
513 FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover) {
514 return new MemorySanitizer(TrackOrigins, Recover);
515 }
516
517 /// Create a non-const global initialized with the given string.
518 ///
519 /// Creates a writable global for Str so that we can pass it to the
520 /// run-time lib. Runtime uses first 4 bytes of the string to store the
521 /// frame ID, so the string needs to be mutable.
createPrivateNonConstGlobalForString(Module & M,StringRef Str)522 static GlobalVariable *createPrivateNonConstGlobalForString(Module &M,
523 StringRef Str) {
524 Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
525 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false,
526 GlobalValue::PrivateLinkage, StrConst, "");
527 }
528
529 /// Insert declarations for userspace-specific functions and globals.
createUserspaceApi(Module & M)530 void MemorySanitizer::createUserspaceApi(Module &M) {
531 IRBuilder<> IRB(*C);
532 // Create the callback.
533 // FIXME: this function should have "Cold" calling conv,
534 // which is not yet implemented.
535 StringRef WarningFnName = Recover ? "__msan_warning"
536 : "__msan_warning_noreturn";
537 WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
538
539 // Create the global TLS variables.
540 RetvalTLS = new GlobalVariable(
541 M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false,
542 GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr,
543 GlobalVariable::InitialExecTLSModel);
544
545 RetvalOriginTLS = new GlobalVariable(
546 M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr,
547 "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
548
549 ParamTLS = new GlobalVariable(
550 M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
551 GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr,
552 GlobalVariable::InitialExecTLSModel);
553
554 ParamOriginTLS = new GlobalVariable(
555 M, ArrayType::get(OriginTy, kParamTLSSize / 4), false,
556 GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls",
557 nullptr, GlobalVariable::InitialExecTLSModel);
558
559 VAArgTLS = new GlobalVariable(
560 M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
561 GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr,
562 GlobalVariable::InitialExecTLSModel);
563 VAArgOverflowSizeTLS = new GlobalVariable(
564 M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
565 "__msan_va_arg_overflow_size_tls", nullptr,
566 GlobalVariable::InitialExecTLSModel);
567 OriginTLS = new GlobalVariable(
568 M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
569 "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
570
571 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
572 AccessSizeIndex++) {
573 unsigned AccessSize = 1 << AccessSizeIndex;
574 std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
575 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
576 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
577 IRB.getInt32Ty());
578
579 FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
580 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
581 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
582 IRB.getInt8PtrTy(), IRB.getInt32Ty());
583 }
584
585 MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
586 "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
587 IRB.getInt8PtrTy(), IntptrTy);
588 MsanPoisonStackFn =
589 M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
590 IRB.getInt8PtrTy(), IntptrTy);
591 }
592
593 /// Insert extern declaration of runtime-provided functions and globals.
initializeCallbacks(Module & M)594 void MemorySanitizer::initializeCallbacks(Module &M) {
595 // Only do this once.
596 if (CallbacksInitialized)
597 return;
598
599 IRBuilder<> IRB(*C);
600 // Initialize callbacks that are common for kernel and userspace
601 // instrumentation.
602 MsanChainOriginFn = M.getOrInsertFunction(
603 "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
604 MemmoveFn = M.getOrInsertFunction(
605 "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
606 IRB.getInt8PtrTy(), IntptrTy);
607 MemcpyFn = M.getOrInsertFunction(
608 "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
609 IntptrTy);
610 MemsetFn = M.getOrInsertFunction(
611 "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
612 IntptrTy);
613 // We insert an empty inline asm after __msan_report* to avoid callback merge.
614 EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
615 StringRef(""), StringRef(""),
616 /*hasSideEffects=*/true);
617
618 createUserspaceApi(M);
619 CallbacksInitialized = true;
620 }
621
622 /// Module-level initialization.
623 ///
624 /// inserts a call to __msan_init to the module's constructor list.
doInitialization(Module & M)625 bool MemorySanitizer::doInitialization(Module &M) {
626 auto &DL = M.getDataLayout();
627
628 bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
629 bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
630 // Check the overrides first
631 if (ShadowPassed || OriginPassed) {
632 CustomMapParams.AndMask = ClAndMask;
633 CustomMapParams.XorMask = ClXorMask;
634 CustomMapParams.ShadowBase = ClShadowBase;
635 CustomMapParams.OriginBase = ClOriginBase;
636 MapParams = &CustomMapParams;
637 } else {
638 Triple TargetTriple(M.getTargetTriple());
639 switch (TargetTriple.getOS()) {
640 case Triple::FreeBSD:
641 switch (TargetTriple.getArch()) {
642 case Triple::x86_64:
643 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
644 break;
645 case Triple::x86:
646 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
647 break;
648 default:
649 report_fatal_error("unsupported architecture");
650 }
651 break;
652 case Triple::NetBSD:
653 switch (TargetTriple.getArch()) {
654 case Triple::x86_64:
655 MapParams = NetBSD_X86_MemoryMapParams.bits64;
656 break;
657 default:
658 report_fatal_error("unsupported architecture");
659 }
660 break;
661 case Triple::Linux:
662 switch (TargetTriple.getArch()) {
663 case Triple::x86_64:
664 MapParams = Linux_X86_MemoryMapParams.bits64;
665 break;
666 case Triple::x86:
667 MapParams = Linux_X86_MemoryMapParams.bits32;
668 break;
669 case Triple::mips64:
670 case Triple::mips64el:
671 MapParams = Linux_MIPS_MemoryMapParams.bits64;
672 break;
673 case Triple::ppc64:
674 case Triple::ppc64le:
675 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
676 break;
677 case Triple::aarch64:
678 case Triple::aarch64_be:
679 MapParams = Linux_ARM_MemoryMapParams.bits64;
680 break;
681 default:
682 report_fatal_error("unsupported architecture");
683 }
684 break;
685 default:
686 report_fatal_error("unsupported operating system");
687 }
688 }
689
690 C = &(M.getContext());
691 IRBuilder<> IRB(*C);
692 IntptrTy = IRB.getIntPtrTy(DL);
693 OriginTy = IRB.getInt32Ty();
694
695 ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
696 OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
697
698 std::tie(MsanCtorFunction, std::ignore) =
699 createSanitizerCtorAndInitFunctions(M, kMsanModuleCtorName, kMsanInitName,
700 /*InitArgTypes=*/{},
701 /*InitArgs=*/{});
702 if (ClWithComdat) {
703 Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
704 MsanCtorFunction->setComdat(MsanCtorComdat);
705 appendToGlobalCtors(M, MsanCtorFunction, 0, MsanCtorFunction);
706 } else {
707 appendToGlobalCtors(M, MsanCtorFunction, 0);
708 }
709
710
711 if (TrackOrigins)
712 new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
713 IRB.getInt32(TrackOrigins), "__msan_track_origins");
714
715 if (Recover)
716 new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
717 IRB.getInt32(Recover), "__msan_keep_going");
718
719 return true;
720 }
721
722 namespace {
723
724 /// A helper class that handles instrumentation of VarArg
725 /// functions on a particular platform.
726 ///
727 /// Implementations are expected to insert the instrumentation
728 /// necessary to propagate argument shadow through VarArg function
729 /// calls. Visit* methods are called during an InstVisitor pass over
730 /// the function, and should avoid creating new basic blocks. A new
731 /// instance of this class is created for each instrumented function.
732 struct VarArgHelper {
733 virtual ~VarArgHelper() = default;
734
735 /// Visit a CallSite.
736 virtual void visitCallSite(CallSite &CS, IRBuilder<> &IRB) = 0;
737
738 /// Visit a va_start call.
739 virtual void visitVAStartInst(VAStartInst &I) = 0;
740
741 /// Visit a va_copy call.
742 virtual void visitVACopyInst(VACopyInst &I) = 0;
743
744 /// Finalize function instrumentation.
745 ///
746 /// This method is called after visiting all interesting (see above)
747 /// instructions in a function.
748 virtual void finalizeInstrumentation() = 0;
749 };
750
751 struct MemorySanitizerVisitor;
752
753 } // end anonymous namespace
754
755 static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
756 MemorySanitizerVisitor &Visitor);
757
TypeSizeToSizeIndex(unsigned TypeSize)758 static unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
759 if (TypeSize <= 8) return 0;
760 return Log2_32_Ceil((TypeSize + 7) / 8);
761 }
762
763 namespace {
764
765 /// This class does all the work for a given function. Store and Load
766 /// instructions store and load corresponding shadow and origin
767 /// values. Most instructions propagate shadow from arguments to their
768 /// return values. Certain instructions (most importantly, BranchInst)
769 /// test their argument shadow and print reports (with a runtime call) if it's
770 /// non-zero.
771 struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
772 Function &F;
773 MemorySanitizer &MS;
774 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
775 ValueMap<Value*, Value*> ShadowMap, OriginMap;
776 std::unique_ptr<VarArgHelper> VAHelper;
777 const TargetLibraryInfo *TLI;
778 BasicBlock *ActualFnStart;
779
780 // The following flags disable parts of MSan instrumentation based on
781 // blacklist contents and command-line options.
782 bool InsertChecks;
783 bool PropagateShadow;
784 bool PoisonStack;
785 bool PoisonUndef;
786 bool CheckReturnValue;
787
788 struct ShadowOriginAndInsertPoint {
789 Value *Shadow;
790 Value *Origin;
791 Instruction *OrigIns;
792
ShadowOriginAndInsertPoint__anonde7c31650411::MemorySanitizerVisitor::ShadowOriginAndInsertPoint793 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
794 : Shadow(S), Origin(O), OrigIns(I) {}
795 };
796 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
797 SmallVector<StoreInst *, 16> StoreList;
798
MemorySanitizerVisitor__anonde7c31650411::MemorySanitizerVisitor799 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS)
800 : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)) {
801 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
802 InsertChecks = SanitizeFunction;
803 PropagateShadow = SanitizeFunction;
804 PoisonStack = SanitizeFunction && ClPoisonStack;
805 PoisonUndef = SanitizeFunction && ClPoisonUndef;
806 // FIXME: Consider using SpecialCaseList to specify a list of functions that
807 // must always return fully initialized values. For now, we hardcode "main".
808 CheckReturnValue = SanitizeFunction && (F.getName() == "main");
809 TLI = &MS.getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
810
811 MS.initializeCallbacks(*F.getParent());
812 ActualFnStart = &F.getEntryBlock();
813
814 LLVM_DEBUG(if (!InsertChecks) dbgs()
815 << "MemorySanitizer is not inserting checks into '"
816 << F.getName() << "'\n");
817 }
818
updateOrigin__anonde7c31650411::MemorySanitizerVisitor819 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
820 if (MS.TrackOrigins <= 1) return V;
821 return IRB.CreateCall(MS.MsanChainOriginFn, V);
822 }
823
originToIntptr__anonde7c31650411::MemorySanitizerVisitor824 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
825 const DataLayout &DL = F.getParent()->getDataLayout();
826 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
827 if (IntptrSize == kOriginSize) return Origin;
828 assert(IntptrSize == kOriginSize * 2);
829 Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
830 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
831 }
832
833 /// Fill memory range with the given origin value.
paintOrigin__anonde7c31650411::MemorySanitizerVisitor834 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
835 unsigned Size, unsigned Alignment) {
836 const DataLayout &DL = F.getParent()->getDataLayout();
837 unsigned IntptrAlignment = DL.getABITypeAlignment(MS.IntptrTy);
838 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
839 assert(IntptrAlignment >= kMinOriginAlignment);
840 assert(IntptrSize >= kOriginSize);
841
842 unsigned Ofs = 0;
843 unsigned CurrentAlignment = Alignment;
844 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
845 Value *IntptrOrigin = originToIntptr(IRB, Origin);
846 Value *IntptrOriginPtr =
847 IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
848 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
849 Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
850 : IntptrOriginPtr;
851 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
852 Ofs += IntptrSize / kOriginSize;
853 CurrentAlignment = IntptrAlignment;
854 }
855 }
856
857 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
858 Value *GEP =
859 i ? IRB.CreateConstGEP1_32(nullptr, OriginPtr, i) : OriginPtr;
860 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
861 CurrentAlignment = kMinOriginAlignment;
862 }
863 }
864
storeOrigin__anonde7c31650411::MemorySanitizerVisitor865 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
866 Value *OriginPtr, unsigned Alignment, bool AsCall) {
867 const DataLayout &DL = F.getParent()->getDataLayout();
868 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
869 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
870 if (Shadow->getType()->isAggregateType()) {
871 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
872 OriginAlignment);
873 } else {
874 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
875 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
876 if (ConstantShadow) {
877 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
878 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
879 OriginAlignment);
880 return;
881 }
882
883 unsigned TypeSizeInBits =
884 DL.getTypeSizeInBits(ConvertedShadow->getType());
885 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
886 if (AsCall && SizeIndex < kNumberOfAccessSizes) {
887 Value *Fn = MS.MaybeStoreOriginFn[SizeIndex];
888 Value *ConvertedShadow2 = IRB.CreateZExt(
889 ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
890 IRB.CreateCall(Fn, {ConvertedShadow2,
891 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
892 Origin});
893 } else {
894 Value *Cmp = IRB.CreateICmpNE(
895 ConvertedShadow, getCleanShadow(ConvertedShadow), "_mscmp");
896 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
897 Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
898 IRBuilder<> IRBNew(CheckTerm);
899 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
900 OriginAlignment);
901 }
902 }
903 }
904
materializeStores__anonde7c31650411::MemorySanitizerVisitor905 void materializeStores(bool InstrumentWithCalls) {
906 for (StoreInst *SI : StoreList) {
907 IRBuilder<> IRB(SI);
908 Value *Val = SI->getValueOperand();
909 Value *Addr = SI->getPointerOperand();
910 Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
911 Value *ShadowPtr, *OriginPtr;
912 Type *ShadowTy = Shadow->getType();
913 unsigned Alignment = SI->getAlignment();
914 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
915 std::tie(ShadowPtr, OriginPtr) =
916 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
917
918 StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
919 LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
920 (void)NewSI;
921
922 if (SI->isAtomic())
923 SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
924
925 if (MS.TrackOrigins && !SI->isAtomic())
926 storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
927 OriginAlignment, InstrumentWithCalls);
928 }
929 }
930
931 /// Helper function to insert a warning at IRB's current insert point.
insertWarningFn__anonde7c31650411::MemorySanitizerVisitor932 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
933 if (!Origin)
934 Origin = (Value *)IRB.getInt32(0);
935 if (MS.TrackOrigins) {
936 IRB.CreateStore(Origin, MS.OriginTLS);
937 }
938 IRB.CreateCall(MS.WarningFn, {});
939 IRB.CreateCall(MS.EmptyAsm, {});
940 // FIXME: Insert UnreachableInst if !MS.Recover?
941 // This may invalidate some of the following checks and needs to be done
942 // at the very end.
943 }
944
materializeOneCheck__anonde7c31650411::MemorySanitizerVisitor945 void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
946 bool AsCall) {
947 IRBuilder<> IRB(OrigIns);
948 LLVM_DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n");
949 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
950 LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n");
951
952 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
953 if (ConstantShadow) {
954 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) {
955 insertWarningFn(IRB, Origin);
956 }
957 return;
958 }
959
960 const DataLayout &DL = OrigIns->getModule()->getDataLayout();
961
962 unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
963 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
964 if (AsCall && SizeIndex < kNumberOfAccessSizes) {
965 Value *Fn = MS.MaybeWarningFn[SizeIndex];
966 Value *ConvertedShadow2 =
967 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
968 IRB.CreateCall(Fn, {ConvertedShadow2, MS.TrackOrigins && Origin
969 ? Origin
970 : (Value *)IRB.getInt32(0)});
971 } else {
972 Value *Cmp = IRB.CreateICmpNE(ConvertedShadow,
973 getCleanShadow(ConvertedShadow), "_mscmp");
974 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
975 Cmp, OrigIns,
976 /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
977
978 IRB.SetInsertPoint(CheckTerm);
979 insertWarningFn(IRB, Origin);
980 LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
981 }
982 }
983
materializeChecks__anonde7c31650411::MemorySanitizerVisitor984 void materializeChecks(bool InstrumentWithCalls) {
985 for (const auto &ShadowData : InstrumentationList) {
986 Instruction *OrigIns = ShadowData.OrigIns;
987 Value *Shadow = ShadowData.Shadow;
988 Value *Origin = ShadowData.Origin;
989 materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
990 }
991 LLVM_DEBUG(dbgs() << "DONE:\n" << F);
992 }
993
994 /// Add MemorySanitizer instrumentation to a function.
runOnFunction__anonde7c31650411::MemorySanitizerVisitor995 bool runOnFunction() {
996 // In the presence of unreachable blocks, we may see Phi nodes with
997 // incoming nodes from such blocks. Since InstVisitor skips unreachable
998 // blocks, such nodes will not have any shadow value associated with them.
999 // It's easier to remove unreachable blocks than deal with missing shadow.
1000 removeUnreachableBlocks(F);
1001
1002 // Iterate all BBs in depth-first order and create shadow instructions
1003 // for all instructions (where applicable).
1004 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1005 for (BasicBlock *BB : depth_first(ActualFnStart))
1006 visit(*BB);
1007
1008 // Finalize PHI nodes.
1009 for (PHINode *PN : ShadowPHINodes) {
1010 PHINode *PNS = cast<PHINode>(getShadow(PN));
1011 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1012 size_t NumValues = PN->getNumIncomingValues();
1013 for (size_t v = 0; v < NumValues; v++) {
1014 PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1015 if (PNO) PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1016 }
1017 }
1018
1019 VAHelper->finalizeInstrumentation();
1020
1021 bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 &&
1022 InstrumentationList.size() + StoreList.size() >
1023 (unsigned)ClInstrumentationWithCallThreshold;
1024
1025 // Insert shadow value checks.
1026 materializeChecks(InstrumentWithCalls);
1027
1028 // Delayed instrumentation of StoreInst.
1029 // This may not add new address checks.
1030 materializeStores(InstrumentWithCalls);
1031
1032 return true;
1033 }
1034
1035 /// Compute the shadow type that corresponds to a given Value.
getShadowTy__anonde7c31650411::MemorySanitizerVisitor1036 Type *getShadowTy(Value *V) {
1037 return getShadowTy(V->getType());
1038 }
1039
1040 /// Compute the shadow type that corresponds to a given Type.
getShadowTy__anonde7c31650411::MemorySanitizerVisitor1041 Type *getShadowTy(Type *OrigTy) {
1042 if (!OrigTy->isSized()) {
1043 return nullptr;
1044 }
1045 // For integer type, shadow is the same as the original type.
1046 // This may return weird-sized types like i1.
1047 if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
1048 return IT;
1049 const DataLayout &DL = F.getParent()->getDataLayout();
1050 if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1051 uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
1052 return VectorType::get(IntegerType::get(*MS.C, EltSize),
1053 VT->getNumElements());
1054 }
1055 if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1056 return ArrayType::get(getShadowTy(AT->getElementType()),
1057 AT->getNumElements());
1058 }
1059 if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1060 SmallVector<Type*, 4> Elements;
1061 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1062 Elements.push_back(getShadowTy(ST->getElementType(i)));
1063 StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
1064 LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
1065 return Res;
1066 }
1067 uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
1068 return IntegerType::get(*MS.C, TypeSize);
1069 }
1070
1071 /// Flatten a vector type.
getShadowTyNoVec__anonde7c31650411::MemorySanitizerVisitor1072 Type *getShadowTyNoVec(Type *ty) {
1073 if (VectorType *vt = dyn_cast<VectorType>(ty))
1074 return IntegerType::get(*MS.C, vt->getBitWidth());
1075 return ty;
1076 }
1077
1078 /// Convert a shadow value to it's flattened variant.
convertToShadowTyNoVec__anonde7c31650411::MemorySanitizerVisitor1079 Value *convertToShadowTyNoVec(Value *V, IRBuilder<> &IRB) {
1080 Type *Ty = V->getType();
1081 Type *NoVecTy = getShadowTyNoVec(Ty);
1082 if (Ty == NoVecTy) return V;
1083 return IRB.CreateBitCast(V, NoVecTy);
1084 }
1085
1086 /// Compute the integer shadow offset that corresponds to a given
1087 /// application address.
1088 ///
1089 /// Offset = (Addr & ~AndMask) ^ XorMask
getShadowPtrOffset__anonde7c31650411::MemorySanitizerVisitor1090 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1091 Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy);
1092
1093 uint64_t AndMask = MS.MapParams->AndMask;
1094 if (AndMask)
1095 OffsetLong =
1096 IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask));
1097
1098 uint64_t XorMask = MS.MapParams->XorMask;
1099 if (XorMask)
1100 OffsetLong =
1101 IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask));
1102 return OffsetLong;
1103 }
1104
1105 /// Compute the shadow and origin addresses corresponding to a given
1106 /// application address.
1107 ///
1108 /// Shadow = ShadowBase + Offset
1109 /// Origin = (OriginBase + Offset) & ~3ULL
getShadowOriginPtrUserspace__anonde7c31650411::MemorySanitizerVisitor1110 std::pair<Value *, Value *> getShadowOriginPtrUserspace(Value *Addr,
1111 IRBuilder<> &IRB,
1112 Type *ShadowTy,
1113 unsigned Alignment) {
1114 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1115 Value *ShadowLong = ShadowOffset;
1116 uint64_t ShadowBase = MS.MapParams->ShadowBase;
1117 if (ShadowBase != 0) {
1118 ShadowLong =
1119 IRB.CreateAdd(ShadowLong,
1120 ConstantInt::get(MS.IntptrTy, ShadowBase));
1121 }
1122 Value *ShadowPtr =
1123 IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1124 Value *OriginPtr = nullptr;
1125 if (MS.TrackOrigins) {
1126 Value *OriginLong = ShadowOffset;
1127 uint64_t OriginBase = MS.MapParams->OriginBase;
1128 if (OriginBase != 0)
1129 OriginLong = IRB.CreateAdd(OriginLong,
1130 ConstantInt::get(MS.IntptrTy, OriginBase));
1131 if (Alignment < kMinOriginAlignment) {
1132 uint64_t Mask = kMinOriginAlignment - 1;
1133 OriginLong =
1134 IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask));
1135 }
1136 OriginPtr =
1137 IRB.CreateIntToPtr(OriginLong, PointerType::get(IRB.getInt32Ty(), 0));
1138 }
1139 return std::make_pair(ShadowPtr, OriginPtr);
1140 }
1141
getShadowOriginPtr__anonde7c31650411::MemorySanitizerVisitor1142 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1143 Type *ShadowTy,
1144 unsigned Alignment,
1145 bool isStore) {
1146 std::pair<Value *, Value *> ret =
1147 getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1148 return ret;
1149 }
1150
1151 /// Compute the shadow address for a given function argument.
1152 ///
1153 /// Shadow = ParamTLS+ArgOffset.
getShadowPtrForArgument__anonde7c31650411::MemorySanitizerVisitor1154 Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
1155 int ArgOffset) {
1156 Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
1157 if (ArgOffset)
1158 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1159 return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1160 "_msarg");
1161 }
1162
1163 /// Compute the origin address for a given function argument.
getOriginPtrForArgument__anonde7c31650411::MemorySanitizerVisitor1164 Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB,
1165 int ArgOffset) {
1166 if (!MS.TrackOrigins) return nullptr;
1167 Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1168 if (ArgOffset)
1169 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1170 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
1171 "_msarg_o");
1172 }
1173
1174 /// Compute the shadow address for a retval.
getShadowPtrForRetval__anonde7c31650411::MemorySanitizerVisitor1175 Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
1176 return IRB.CreatePointerCast(MS.RetvalTLS,
1177 PointerType::get(getShadowTy(A), 0),
1178 "_msret");
1179 }
1180
1181 /// Compute the origin address for a retval.
getOriginPtrForRetval__anonde7c31650411::MemorySanitizerVisitor1182 Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
1183 // We keep a single origin for the entire retval. Might be too optimistic.
1184 return MS.RetvalOriginTLS;
1185 }
1186
1187 /// Set SV to be the shadow value for V.
setShadow__anonde7c31650411::MemorySanitizerVisitor1188 void setShadow(Value *V, Value *SV) {
1189 assert(!ShadowMap.count(V) && "Values may only have one shadow");
1190 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1191 }
1192
1193 /// Set Origin to be the origin value for V.
setOrigin__anonde7c31650411::MemorySanitizerVisitor1194 void setOrigin(Value *V, Value *Origin) {
1195 if (!MS.TrackOrigins) return;
1196 assert(!OriginMap.count(V) && "Values may only have one origin");
1197 LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
1198 OriginMap[V] = Origin;
1199 }
1200
getCleanShadow__anonde7c31650411::MemorySanitizerVisitor1201 Constant *getCleanShadow(Type *OrigTy) {
1202 Type *ShadowTy = getShadowTy(OrigTy);
1203 if (!ShadowTy)
1204 return nullptr;
1205 return Constant::getNullValue(ShadowTy);
1206 }
1207
1208 /// Create a clean shadow value for a given value.
1209 ///
1210 /// Clean shadow (all zeroes) means all bits of the value are defined
1211 /// (initialized).
getCleanShadow__anonde7c31650411::MemorySanitizerVisitor1212 Constant *getCleanShadow(Value *V) {
1213 return getCleanShadow(V->getType());
1214 }
1215
1216 /// Create a dirty shadow of a given shadow type.
getPoisonedShadow__anonde7c31650411::MemorySanitizerVisitor1217 Constant *getPoisonedShadow(Type *ShadowTy) {
1218 assert(ShadowTy);
1219 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1220 return Constant::getAllOnesValue(ShadowTy);
1221 if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1222 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1223 getPoisonedShadow(AT->getElementType()));
1224 return ConstantArray::get(AT, Vals);
1225 }
1226 if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1227 SmallVector<Constant *, 4> Vals;
1228 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1229 Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
1230 return ConstantStruct::get(ST, Vals);
1231 }
1232 llvm_unreachable("Unexpected shadow type");
1233 }
1234
1235 /// Create a dirty shadow for a given value.
getPoisonedShadow__anonde7c31650411::MemorySanitizerVisitor1236 Constant *getPoisonedShadow(Value *V) {
1237 Type *ShadowTy = getShadowTy(V);
1238 if (!ShadowTy)
1239 return nullptr;
1240 return getPoisonedShadow(ShadowTy);
1241 }
1242
1243 /// Create a clean (zero) origin.
getCleanOrigin__anonde7c31650411::MemorySanitizerVisitor1244 Value *getCleanOrigin() {
1245 return Constant::getNullValue(MS.OriginTy);
1246 }
1247
1248 /// Get the shadow value for a given Value.
1249 ///
1250 /// This function either returns the value set earlier with setShadow,
1251 /// or extracts if from ParamTLS (for function arguments).
getShadow__anonde7c31650411::MemorySanitizerVisitor1252 Value *getShadow(Value *V) {
1253 if (!PropagateShadow) return getCleanShadow(V);
1254 if (Instruction *I = dyn_cast<Instruction>(V)) {
1255 if (I->getMetadata("nosanitize"))
1256 return getCleanShadow(V);
1257 // For instructions the shadow is already stored in the map.
1258 Value *Shadow = ShadowMap[V];
1259 if (!Shadow) {
1260 LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
1261 (void)I;
1262 assert(Shadow && "No shadow for a value");
1263 }
1264 return Shadow;
1265 }
1266 if (UndefValue *U = dyn_cast<UndefValue>(V)) {
1267 Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V);
1268 LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
1269 (void)U;
1270 return AllOnes;
1271 }
1272 if (Argument *A = dyn_cast<Argument>(V)) {
1273 // For arguments we compute the shadow on demand and store it in the map.
1274 Value **ShadowPtr = &ShadowMap[V];
1275 if (*ShadowPtr)
1276 return *ShadowPtr;
1277 Function *F = A->getParent();
1278 IRBuilder<> EntryIRB(ActualFnStart->getFirstNonPHI());
1279 unsigned ArgOffset = 0;
1280 const DataLayout &DL = F->getParent()->getDataLayout();
1281 for (auto &FArg : F->args()) {
1282 if (!FArg.getType()->isSized()) {
1283 LLVM_DEBUG(dbgs() << "Arg is not sized\n");
1284 continue;
1285 }
1286 unsigned Size =
1287 FArg.hasByValAttr()
1288 ? DL.getTypeAllocSize(FArg.getType()->getPointerElementType())
1289 : DL.getTypeAllocSize(FArg.getType());
1290 if (A == &FArg) {
1291 bool Overflow = ArgOffset + Size > kParamTLSSize;
1292 Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1293 if (FArg.hasByValAttr()) {
1294 // ByVal pointer itself has clean shadow. We copy the actual
1295 // argument shadow to the underlying memory.
1296 // Figure out maximal valid memcpy alignment.
1297 unsigned ArgAlign = FArg.getParamAlignment();
1298 if (ArgAlign == 0) {
1299 Type *EltType = A->getType()->getPointerElementType();
1300 ArgAlign = DL.getABITypeAlignment(EltType);
1301 }
1302 Value *CpShadowPtr =
1303 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1304 /*isStore*/ true)
1305 .first;
1306 if (Overflow) {
1307 // ParamTLS overflow.
1308 EntryIRB.CreateMemSet(
1309 CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()),
1310 Size, ArgAlign);
1311 } else {
1312 unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
1313 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
1314 CopyAlign, Size);
1315 LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
1316 (void)Cpy;
1317 }
1318 *ShadowPtr = getCleanShadow(V);
1319 } else {
1320 if (Overflow) {
1321 // ParamTLS overflow.
1322 *ShadowPtr = getCleanShadow(V);
1323 } else {
1324 *ShadowPtr =
1325 EntryIRB.CreateAlignedLoad(Base, kShadowTLSAlignment);
1326 }
1327 }
1328 LLVM_DEBUG(dbgs()
1329 << " ARG: " << FArg << " ==> " << **ShadowPtr << "\n");
1330 if (MS.TrackOrigins && !Overflow) {
1331 Value *OriginPtr =
1332 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1333 setOrigin(A, EntryIRB.CreateLoad(OriginPtr));
1334 } else {
1335 setOrigin(A, getCleanOrigin());
1336 }
1337 }
1338 ArgOffset += alignTo(Size, kShadowTLSAlignment);
1339 }
1340 assert(*ShadowPtr && "Could not find shadow for an argument");
1341 return *ShadowPtr;
1342 }
1343 // For everything else the shadow is zero.
1344 return getCleanShadow(V);
1345 }
1346
1347 /// Get the shadow for i-th argument of the instruction I.
getShadow__anonde7c31650411::MemorySanitizerVisitor1348 Value *getShadow(Instruction *I, int i) {
1349 return getShadow(I->getOperand(i));
1350 }
1351
1352 /// Get the origin for a value.
getOrigin__anonde7c31650411::MemorySanitizerVisitor1353 Value *getOrigin(Value *V) {
1354 if (!MS.TrackOrigins) return nullptr;
1355 if (!PropagateShadow) return getCleanOrigin();
1356 if (isa<Constant>(V)) return getCleanOrigin();
1357 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
1358 "Unexpected value type in getOrigin()");
1359 if (Instruction *I = dyn_cast<Instruction>(V)) {
1360 if (I->getMetadata("nosanitize"))
1361 return getCleanOrigin();
1362 }
1363 Value *Origin = OriginMap[V];
1364 assert(Origin && "Missing origin");
1365 return Origin;
1366 }
1367
1368 /// Get the origin for i-th argument of the instruction I.
getOrigin__anonde7c31650411::MemorySanitizerVisitor1369 Value *getOrigin(Instruction *I, int i) {
1370 return getOrigin(I->getOperand(i));
1371 }
1372
1373 /// Remember the place where a shadow check should be inserted.
1374 ///
1375 /// This location will be later instrumented with a check that will print a
1376 /// UMR warning in runtime if the shadow value is not 0.
insertShadowCheck__anonde7c31650411::MemorySanitizerVisitor1377 void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
1378 assert(Shadow);
1379 if (!InsertChecks) return;
1380 #ifndef NDEBUG
1381 Type *ShadowTy = Shadow->getType();
1382 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) &&
1383 "Can only insert checks for integer and vector shadow types");
1384 #endif
1385 InstrumentationList.push_back(
1386 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
1387 }
1388
1389 /// Remember the place where a shadow check should be inserted.
1390 ///
1391 /// This location will be later instrumented with a check that will print a
1392 /// UMR warning in runtime if the value is not fully defined.
insertShadowCheck__anonde7c31650411::MemorySanitizerVisitor1393 void insertShadowCheck(Value *Val, Instruction *OrigIns) {
1394 assert(Val);
1395 Value *Shadow, *Origin;
1396 if (ClCheckConstantShadow) {
1397 Shadow = getShadow(Val);
1398 if (!Shadow) return;
1399 Origin = getOrigin(Val);
1400 } else {
1401 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
1402 if (!Shadow) return;
1403 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
1404 }
1405 insertShadowCheck(Shadow, Origin, OrigIns);
1406 }
1407
addReleaseOrdering__anonde7c31650411::MemorySanitizerVisitor1408 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
1409 switch (a) {
1410 case AtomicOrdering::NotAtomic:
1411 return AtomicOrdering::NotAtomic;
1412 case AtomicOrdering::Unordered:
1413 case AtomicOrdering::Monotonic:
1414 case AtomicOrdering::Release:
1415 return AtomicOrdering::Release;
1416 case AtomicOrdering::Acquire:
1417 case AtomicOrdering::AcquireRelease:
1418 return AtomicOrdering::AcquireRelease;
1419 case AtomicOrdering::SequentiallyConsistent:
1420 return AtomicOrdering::SequentiallyConsistent;
1421 }
1422 llvm_unreachable("Unknown ordering");
1423 }
1424
addAcquireOrdering__anonde7c31650411::MemorySanitizerVisitor1425 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
1426 switch (a) {
1427 case AtomicOrdering::NotAtomic:
1428 return AtomicOrdering::NotAtomic;
1429 case AtomicOrdering::Unordered:
1430 case AtomicOrdering::Monotonic:
1431 case AtomicOrdering::Acquire:
1432 return AtomicOrdering::Acquire;
1433 case AtomicOrdering::Release:
1434 case AtomicOrdering::AcquireRelease:
1435 return AtomicOrdering::AcquireRelease;
1436 case AtomicOrdering::SequentiallyConsistent:
1437 return AtomicOrdering::SequentiallyConsistent;
1438 }
1439 llvm_unreachable("Unknown ordering");
1440 }
1441
1442 // ------------------- Visitors.
1443 using InstVisitor<MemorySanitizerVisitor>::visit;
visit__anonde7c31650411::MemorySanitizerVisitor1444 void visit(Instruction &I) {
1445 if (!I.getMetadata("nosanitize"))
1446 InstVisitor<MemorySanitizerVisitor>::visit(I);
1447 }
1448
1449 /// Instrument LoadInst
1450 ///
1451 /// Loads the corresponding shadow and (optionally) origin.
1452 /// Optionally, checks that the load address is fully defined.
visitLoadInst__anonde7c31650411::MemorySanitizerVisitor1453 void visitLoadInst(LoadInst &I) {
1454 assert(I.getType()->isSized() && "Load type must have size");
1455 assert(!I.getMetadata("nosanitize"));
1456 IRBuilder<> IRB(I.getNextNode());
1457 Type *ShadowTy = getShadowTy(&I);
1458 Value *Addr = I.getPointerOperand();
1459 Value *ShadowPtr, *OriginPtr;
1460 unsigned Alignment = I.getAlignment();
1461 if (PropagateShadow) {
1462 std::tie(ShadowPtr, OriginPtr) =
1463 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
1464 setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_msld"));
1465 } else {
1466 setShadow(&I, getCleanShadow(&I));
1467 }
1468
1469 if (ClCheckAccessAddress)
1470 insertShadowCheck(I.getPointerOperand(), &I);
1471
1472 if (I.isAtomic())
1473 I.setOrdering(addAcquireOrdering(I.getOrdering()));
1474
1475 if (MS.TrackOrigins) {
1476 if (PropagateShadow) {
1477 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1478 setOrigin(&I, IRB.CreateAlignedLoad(OriginPtr, OriginAlignment));
1479 } else {
1480 setOrigin(&I, getCleanOrigin());
1481 }
1482 }
1483 }
1484
1485 /// Instrument StoreInst
1486 ///
1487 /// Stores the corresponding shadow and (optionally) origin.
1488 /// Optionally, checks that the store address is fully defined.
visitStoreInst__anonde7c31650411::MemorySanitizerVisitor1489 void visitStoreInst(StoreInst &I) {
1490 StoreList.push_back(&I);
1491 if (ClCheckAccessAddress)
1492 insertShadowCheck(I.getPointerOperand(), &I);
1493 }
1494
handleCASOrRMW__anonde7c31650411::MemorySanitizerVisitor1495 void handleCASOrRMW(Instruction &I) {
1496 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
1497
1498 IRBuilder<> IRB(&I);
1499 Value *Addr = I.getOperand(0);
1500 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, I.getType(),
1501 /*Alignment*/ 1, /*isStore*/ true)
1502 .first;
1503
1504 if (ClCheckAccessAddress)
1505 insertShadowCheck(Addr, &I);
1506
1507 // Only test the conditional argument of cmpxchg instruction.
1508 // The other argument can potentially be uninitialized, but we can not
1509 // detect this situation reliably without possible false positives.
1510 if (isa<AtomicCmpXchgInst>(I))
1511 insertShadowCheck(I.getOperand(1), &I);
1512
1513 IRB.CreateStore(getCleanShadow(&I), ShadowPtr);
1514
1515 setShadow(&I, getCleanShadow(&I));
1516 setOrigin(&I, getCleanOrigin());
1517 }
1518
visitAtomicRMWInst__anonde7c31650411::MemorySanitizerVisitor1519 void visitAtomicRMWInst(AtomicRMWInst &I) {
1520 handleCASOrRMW(I);
1521 I.setOrdering(addReleaseOrdering(I.getOrdering()));
1522 }
1523
visitAtomicCmpXchgInst__anonde7c31650411::MemorySanitizerVisitor1524 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1525 handleCASOrRMW(I);
1526 I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
1527 }
1528
1529 // Vector manipulation.
visitExtractElementInst__anonde7c31650411::MemorySanitizerVisitor1530 void visitExtractElementInst(ExtractElementInst &I) {
1531 insertShadowCheck(I.getOperand(1), &I);
1532 IRBuilder<> IRB(&I);
1533 setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
1534 "_msprop"));
1535 setOrigin(&I, getOrigin(&I, 0));
1536 }
1537
visitInsertElementInst__anonde7c31650411::MemorySanitizerVisitor1538 void visitInsertElementInst(InsertElementInst &I) {
1539 insertShadowCheck(I.getOperand(2), &I);
1540 IRBuilder<> IRB(&I);
1541 setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1),
1542 I.getOperand(2), "_msprop"));
1543 setOriginForNaryOp(I);
1544 }
1545
visitShuffleVectorInst__anonde7c31650411::MemorySanitizerVisitor1546 void visitShuffleVectorInst(ShuffleVectorInst &I) {
1547 insertShadowCheck(I.getOperand(2), &I);
1548 IRBuilder<> IRB(&I);
1549 setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
1550 I.getOperand(2), "_msprop"));
1551 setOriginForNaryOp(I);
1552 }
1553
1554 // Casts.
visitSExtInst__anonde7c31650411::MemorySanitizerVisitor1555 void visitSExtInst(SExtInst &I) {
1556 IRBuilder<> IRB(&I);
1557 setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
1558 setOrigin(&I, getOrigin(&I, 0));
1559 }
1560
visitZExtInst__anonde7c31650411::MemorySanitizerVisitor1561 void visitZExtInst(ZExtInst &I) {
1562 IRBuilder<> IRB(&I);
1563 setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
1564 setOrigin(&I, getOrigin(&I, 0));
1565 }
1566
visitTruncInst__anonde7c31650411::MemorySanitizerVisitor1567 void visitTruncInst(TruncInst &I) {
1568 IRBuilder<> IRB(&I);
1569 setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
1570 setOrigin(&I, getOrigin(&I, 0));
1571 }
1572
visitBitCastInst__anonde7c31650411::MemorySanitizerVisitor1573 void visitBitCastInst(BitCastInst &I) {
1574 // Special case: if this is the bitcast (there is exactly 1 allowed) between
1575 // a musttail call and a ret, don't instrument. New instructions are not
1576 // allowed after a musttail call.
1577 if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
1578 if (CI->isMustTailCall())
1579 return;
1580 IRBuilder<> IRB(&I);
1581 setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
1582 setOrigin(&I, getOrigin(&I, 0));
1583 }
1584
visitPtrToIntInst__anonde7c31650411::MemorySanitizerVisitor1585 void visitPtrToIntInst(PtrToIntInst &I) {
1586 IRBuilder<> IRB(&I);
1587 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1588 "_msprop_ptrtoint"));
1589 setOrigin(&I, getOrigin(&I, 0));
1590 }
1591
visitIntToPtrInst__anonde7c31650411::MemorySanitizerVisitor1592 void visitIntToPtrInst(IntToPtrInst &I) {
1593 IRBuilder<> IRB(&I);
1594 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1595 "_msprop_inttoptr"));
1596 setOrigin(&I, getOrigin(&I, 0));
1597 }
1598
visitFPToSIInst__anonde7c31650411::MemorySanitizerVisitor1599 void visitFPToSIInst(CastInst& I) { handleShadowOr(I); }
visitFPToUIInst__anonde7c31650411::MemorySanitizerVisitor1600 void visitFPToUIInst(CastInst& I) { handleShadowOr(I); }
visitSIToFPInst__anonde7c31650411::MemorySanitizerVisitor1601 void visitSIToFPInst(CastInst& I) { handleShadowOr(I); }
visitUIToFPInst__anonde7c31650411::MemorySanitizerVisitor1602 void visitUIToFPInst(CastInst& I) { handleShadowOr(I); }
visitFPExtInst__anonde7c31650411::MemorySanitizerVisitor1603 void visitFPExtInst(CastInst& I) { handleShadowOr(I); }
visitFPTruncInst__anonde7c31650411::MemorySanitizerVisitor1604 void visitFPTruncInst(CastInst& I) { handleShadowOr(I); }
1605
1606 /// Propagate shadow for bitwise AND.
1607 ///
1608 /// This code is exact, i.e. if, for example, a bit in the left argument
1609 /// is defined and 0, then neither the value not definedness of the
1610 /// corresponding bit in B don't affect the resulting shadow.
visitAnd__anonde7c31650411::MemorySanitizerVisitor1611 void visitAnd(BinaryOperator &I) {
1612 IRBuilder<> IRB(&I);
1613 // "And" of 0 and a poisoned value results in unpoisoned value.
1614 // 1&1 => 1; 0&1 => 0; p&1 => p;
1615 // 1&0 => 0; 0&0 => 0; p&0 => 0;
1616 // 1&p => p; 0&p => 0; p&p => p;
1617 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
1618 Value *S1 = getShadow(&I, 0);
1619 Value *S2 = getShadow(&I, 1);
1620 Value *V1 = I.getOperand(0);
1621 Value *V2 = I.getOperand(1);
1622 if (V1->getType() != S1->getType()) {
1623 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1624 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1625 }
1626 Value *S1S2 = IRB.CreateAnd(S1, S2);
1627 Value *V1S2 = IRB.CreateAnd(V1, S2);
1628 Value *S1V2 = IRB.CreateAnd(S1, V2);
1629 setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2)));
1630 setOriginForNaryOp(I);
1631 }
1632
visitOr__anonde7c31650411::MemorySanitizerVisitor1633 void visitOr(BinaryOperator &I) {
1634 IRBuilder<> IRB(&I);
1635 // "Or" of 1 and a poisoned value results in unpoisoned value.
1636 // 1|1 => 1; 0|1 => 1; p|1 => 1;
1637 // 1|0 => 1; 0|0 => 0; p|0 => p;
1638 // 1|p => 1; 0|p => p; p|p => p;
1639 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
1640 Value *S1 = getShadow(&I, 0);
1641 Value *S2 = getShadow(&I, 1);
1642 Value *V1 = IRB.CreateNot(I.getOperand(0));
1643 Value *V2 = IRB.CreateNot(I.getOperand(1));
1644 if (V1->getType() != S1->getType()) {
1645 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1646 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1647 }
1648 Value *S1S2 = IRB.CreateAnd(S1, S2);
1649 Value *V1S2 = IRB.CreateAnd(V1, S2);
1650 Value *S1V2 = IRB.CreateAnd(S1, V2);
1651 setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2)));
1652 setOriginForNaryOp(I);
1653 }
1654
1655 /// Default propagation of shadow and/or origin.
1656 ///
1657 /// This class implements the general case of shadow propagation, used in all
1658 /// cases where we don't know and/or don't care about what the operation
1659 /// actually does. It converts all input shadow values to a common type
1660 /// (extending or truncating as necessary), and bitwise OR's them.
1661 ///
1662 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
1663 /// fully initialized), and less prone to false positives.
1664 ///
1665 /// This class also implements the general case of origin propagation. For a
1666 /// Nary operation, result origin is set to the origin of an argument that is
1667 /// not entirely initialized. If there is more than one such arguments, the
1668 /// rightmost of them is picked. It does not matter which one is picked if all
1669 /// arguments are initialized.
1670 template <bool CombineShadow>
1671 class Combiner {
1672 Value *Shadow = nullptr;
1673 Value *Origin = nullptr;
1674 IRBuilder<> &IRB;
1675 MemorySanitizerVisitor *MSV;
1676
1677 public:
Combiner(MemorySanitizerVisitor * MSV,IRBuilder<> & IRB)1678 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
1679 : IRB(IRB), MSV(MSV) {}
1680
1681 /// Add a pair of shadow and origin values to the mix.
Add(Value * OpShadow,Value * OpOrigin)1682 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
1683 if (CombineShadow) {
1684 assert(OpShadow);
1685 if (!Shadow)
1686 Shadow = OpShadow;
1687 else {
1688 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
1689 Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
1690 }
1691 }
1692
1693 if (MSV->MS.TrackOrigins) {
1694 assert(OpOrigin);
1695 if (!Origin) {
1696 Origin = OpOrigin;
1697 } else {
1698 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
1699 // No point in adding something that might result in 0 origin value.
1700 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
1701 Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB);
1702 Value *Cond =
1703 IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
1704 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
1705 }
1706 }
1707 }
1708 return *this;
1709 }
1710
1711 /// Add an application value to the mix.
Add(Value * V)1712 Combiner &Add(Value *V) {
1713 Value *OpShadow = MSV->getShadow(V);
1714 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
1715 return Add(OpShadow, OpOrigin);
1716 }
1717
1718 /// Set the current combined values as the given instruction's shadow
1719 /// and origin.
Done(Instruction * I)1720 void Done(Instruction *I) {
1721 if (CombineShadow) {
1722 assert(Shadow);
1723 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
1724 MSV->setShadow(I, Shadow);
1725 }
1726 if (MSV->MS.TrackOrigins) {
1727 assert(Origin);
1728 MSV->setOrigin(I, Origin);
1729 }
1730 }
1731 };
1732
1733 using ShadowAndOriginCombiner = Combiner<true>;
1734 using OriginCombiner = Combiner<false>;
1735
1736 /// Propagate origin for arbitrary operation.
setOriginForNaryOp__anonde7c31650411::MemorySanitizerVisitor1737 void setOriginForNaryOp(Instruction &I) {
1738 if (!MS.TrackOrigins) return;
1739 IRBuilder<> IRB(&I);
1740 OriginCombiner OC(this, IRB);
1741 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
1742 OC.Add(OI->get());
1743 OC.Done(&I);
1744 }
1745
VectorOrPrimitiveTypeSizeInBits__anonde7c31650411::MemorySanitizerVisitor1746 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
1747 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
1748 "Vector of pointers is not a valid shadow type");
1749 return Ty->isVectorTy() ?
1750 Ty->getVectorNumElements() * Ty->getScalarSizeInBits() :
1751 Ty->getPrimitiveSizeInBits();
1752 }
1753
1754 /// Cast between two shadow types, extending or truncating as
1755 /// necessary.
CreateShadowCast__anonde7c31650411::MemorySanitizerVisitor1756 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
1757 bool Signed = false) {
1758 Type *srcTy = V->getType();
1759 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
1760 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
1761 if (srcSizeInBits > 1 && dstSizeInBits == 1)
1762 return IRB.CreateICmpNE(V, getCleanShadow(V));
1763
1764 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
1765 return IRB.CreateIntCast(V, dstTy, Signed);
1766 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
1767 dstTy->getVectorNumElements() == srcTy->getVectorNumElements())
1768 return IRB.CreateIntCast(V, dstTy, Signed);
1769 Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
1770 Value *V2 =
1771 IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
1772 return IRB.CreateBitCast(V2, dstTy);
1773 // TODO: handle struct types.
1774 }
1775
1776 /// Cast an application value to the type of its own shadow.
CreateAppToShadowCast__anonde7c31650411::MemorySanitizerVisitor1777 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
1778 Type *ShadowTy = getShadowTy(V);
1779 if (V->getType() == ShadowTy)
1780 return V;
1781 if (V->getType()->isPtrOrPtrVectorTy())
1782 return IRB.CreatePtrToInt(V, ShadowTy);
1783 else
1784 return IRB.CreateBitCast(V, ShadowTy);
1785 }
1786
1787 /// Propagate shadow for arbitrary operation.
handleShadowOr__anonde7c31650411::MemorySanitizerVisitor1788 void handleShadowOr(Instruction &I) {
1789 IRBuilder<> IRB(&I);
1790 ShadowAndOriginCombiner SC(this, IRB);
1791 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
1792 SC.Add(OI->get());
1793 SC.Done(&I);
1794 }
1795
1796 // Handle multiplication by constant.
1797 //
1798 // Handle a special case of multiplication by constant that may have one or
1799 // more zeros in the lower bits. This makes corresponding number of lower bits
1800 // of the result zero as well. We model it by shifting the other operand
1801 // shadow left by the required number of bits. Effectively, we transform
1802 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
1803 // We use multiplication by 2**N instead of shift to cover the case of
1804 // multiplication by 0, which may occur in some elements of a vector operand.
handleMulByConstant__anonde7c31650411::MemorySanitizerVisitor1805 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
1806 Value *OtherArg) {
1807 Constant *ShadowMul;
1808 Type *Ty = ConstArg->getType();
1809 if (Ty->isVectorTy()) {
1810 unsigned NumElements = Ty->getVectorNumElements();
1811 Type *EltTy = Ty->getSequentialElementType();
1812 SmallVector<Constant *, 16> Elements;
1813 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
1814 if (ConstantInt *Elt =
1815 dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
1816 const APInt &V = Elt->getValue();
1817 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
1818 Elements.push_back(ConstantInt::get(EltTy, V2));
1819 } else {
1820 Elements.push_back(ConstantInt::get(EltTy, 1));
1821 }
1822 }
1823 ShadowMul = ConstantVector::get(Elements);
1824 } else {
1825 if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
1826 const APInt &V = Elt->getValue();
1827 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
1828 ShadowMul = ConstantInt::get(Ty, V2);
1829 } else {
1830 ShadowMul = ConstantInt::get(Ty, 1);
1831 }
1832 }
1833
1834 IRBuilder<> IRB(&I);
1835 setShadow(&I,
1836 IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
1837 setOrigin(&I, getOrigin(OtherArg));
1838 }
1839
visitMul__anonde7c31650411::MemorySanitizerVisitor1840 void visitMul(BinaryOperator &I) {
1841 Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
1842 Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
1843 if (constOp0 && !constOp1)
1844 handleMulByConstant(I, constOp0, I.getOperand(1));
1845 else if (constOp1 && !constOp0)
1846 handleMulByConstant(I, constOp1, I.getOperand(0));
1847 else
1848 handleShadowOr(I);
1849 }
1850
visitFAdd__anonde7c31650411::MemorySanitizerVisitor1851 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
visitFSub__anonde7c31650411::MemorySanitizerVisitor1852 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
visitFMul__anonde7c31650411::MemorySanitizerVisitor1853 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
visitAdd__anonde7c31650411::MemorySanitizerVisitor1854 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
visitSub__anonde7c31650411::MemorySanitizerVisitor1855 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
visitXor__anonde7c31650411::MemorySanitizerVisitor1856 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
1857
handleIntegerDiv__anonde7c31650411::MemorySanitizerVisitor1858 void handleIntegerDiv(Instruction &I) {
1859 IRBuilder<> IRB(&I);
1860 // Strict on the second argument.
1861 insertShadowCheck(I.getOperand(1), &I);
1862 setShadow(&I, getShadow(&I, 0));
1863 setOrigin(&I, getOrigin(&I, 0));
1864 }
1865
visitUDiv__anonde7c31650411::MemorySanitizerVisitor1866 void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
visitSDiv__anonde7c31650411::MemorySanitizerVisitor1867 void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
visitURem__anonde7c31650411::MemorySanitizerVisitor1868 void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
visitSRem__anonde7c31650411::MemorySanitizerVisitor1869 void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
1870
1871 // Floating point division is side-effect free. We can not require that the
1872 // divisor is fully initialized and must propagate shadow. See PR37523.
visitFDiv__anonde7c31650411::MemorySanitizerVisitor1873 void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
visitFRem__anonde7c31650411::MemorySanitizerVisitor1874 void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
1875
1876 /// Instrument == and != comparisons.
1877 ///
1878 /// Sometimes the comparison result is known even if some of the bits of the
1879 /// arguments are not.
handleEqualityComparison__anonde7c31650411::MemorySanitizerVisitor1880 void handleEqualityComparison(ICmpInst &I) {
1881 IRBuilder<> IRB(&I);
1882 Value *A = I.getOperand(0);
1883 Value *B = I.getOperand(1);
1884 Value *Sa = getShadow(A);
1885 Value *Sb = getShadow(B);
1886
1887 // Get rid of pointers and vectors of pointers.
1888 // For ints (and vectors of ints), types of A and Sa match,
1889 // and this is a no-op.
1890 A = IRB.CreatePointerCast(A, Sa->getType());
1891 B = IRB.CreatePointerCast(B, Sb->getType());
1892
1893 // A == B <==> (C = A^B) == 0
1894 // A != B <==> (C = A^B) != 0
1895 // Sc = Sa | Sb
1896 Value *C = IRB.CreateXor(A, B);
1897 Value *Sc = IRB.CreateOr(Sa, Sb);
1898 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
1899 // Result is defined if one of the following is true
1900 // * there is a defined 1 bit in C
1901 // * C is fully defined
1902 // Si = !(C & ~Sc) && Sc
1903 Value *Zero = Constant::getNullValue(Sc->getType());
1904 Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
1905 Value *Si =
1906 IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero),
1907 IRB.CreateICmpEQ(
1908 IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero));
1909 Si->setName("_msprop_icmp");
1910 setShadow(&I, Si);
1911 setOriginForNaryOp(I);
1912 }
1913
1914 /// Build the lowest possible value of V, taking into account V's
1915 /// uninitialized bits.
getLowestPossibleValue__anonde7c31650411::MemorySanitizerVisitor1916 Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
1917 bool isSigned) {
1918 if (isSigned) {
1919 // Split shadow into sign bit and other bits.
1920 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
1921 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
1922 // Maximise the undefined shadow bit, minimize other undefined bits.
1923 return
1924 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit);
1925 } else {
1926 // Minimize undefined bits.
1927 return IRB.CreateAnd(A, IRB.CreateNot(Sa));
1928 }
1929 }
1930
1931 /// Build the highest possible value of V, taking into account V's
1932 /// uninitialized bits.
getHighestPossibleValue__anonde7c31650411::MemorySanitizerVisitor1933 Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
1934 bool isSigned) {
1935 if (isSigned) {
1936 // Split shadow into sign bit and other bits.
1937 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
1938 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
1939 // Minimise the undefined shadow bit, maximise other undefined bits.
1940 return
1941 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits);
1942 } else {
1943 // Maximize undefined bits.
1944 return IRB.CreateOr(A, Sa);
1945 }
1946 }
1947
1948 /// Instrument relational comparisons.
1949 ///
1950 /// This function does exact shadow propagation for all relational
1951 /// comparisons of integers, pointers and vectors of those.
1952 /// FIXME: output seems suboptimal when one of the operands is a constant
handleRelationalComparisonExact__anonde7c31650411::MemorySanitizerVisitor1953 void handleRelationalComparisonExact(ICmpInst &I) {
1954 IRBuilder<> IRB(&I);
1955 Value *A = I.getOperand(0);
1956 Value *B = I.getOperand(1);
1957 Value *Sa = getShadow(A);
1958 Value *Sb = getShadow(B);
1959
1960 // Get rid of pointers and vectors of pointers.
1961 // For ints (and vectors of ints), types of A and Sa match,
1962 // and this is a no-op.
1963 A = IRB.CreatePointerCast(A, Sa->getType());
1964 B = IRB.CreatePointerCast(B, Sb->getType());
1965
1966 // Let [a0, a1] be the interval of possible values of A, taking into account
1967 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
1968 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
1969 bool IsSigned = I.isSigned();
1970 Value *S1 = IRB.CreateICmp(I.getPredicate(),
1971 getLowestPossibleValue(IRB, A, Sa, IsSigned),
1972 getHighestPossibleValue(IRB, B, Sb, IsSigned));
1973 Value *S2 = IRB.CreateICmp(I.getPredicate(),
1974 getHighestPossibleValue(IRB, A, Sa, IsSigned),
1975 getLowestPossibleValue(IRB, B, Sb, IsSigned));
1976 Value *Si = IRB.CreateXor(S1, S2);
1977 setShadow(&I, Si);
1978 setOriginForNaryOp(I);
1979 }
1980
1981 /// Instrument signed relational comparisons.
1982 ///
1983 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
1984 /// bit of the shadow. Everything else is delegated to handleShadowOr().
handleSignedRelationalComparison__anonde7c31650411::MemorySanitizerVisitor1985 void handleSignedRelationalComparison(ICmpInst &I) {
1986 Constant *constOp;
1987 Value *op = nullptr;
1988 CmpInst::Predicate pre;
1989 if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) {
1990 op = I.getOperand(0);
1991 pre = I.getPredicate();
1992 } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) {
1993 op = I.getOperand(1);
1994 pre = I.getSwappedPredicate();
1995 } else {
1996 handleShadowOr(I);
1997 return;
1998 }
1999
2000 if ((constOp->isNullValue() &&
2001 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
2002 (constOp->isAllOnesValue() &&
2003 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
2004 IRBuilder<> IRB(&I);
2005 Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
2006 "_msprop_icmp_s");
2007 setShadow(&I, Shadow);
2008 setOrigin(&I, getOrigin(op));
2009 } else {
2010 handleShadowOr(I);
2011 }
2012 }
2013
visitICmpInst__anonde7c31650411::MemorySanitizerVisitor2014 void visitICmpInst(ICmpInst &I) {
2015 if (!ClHandleICmp) {
2016 handleShadowOr(I);
2017 return;
2018 }
2019 if (I.isEquality()) {
2020 handleEqualityComparison(I);
2021 return;
2022 }
2023
2024 assert(I.isRelational());
2025 if (ClHandleICmpExact) {
2026 handleRelationalComparisonExact(I);
2027 return;
2028 }
2029 if (I.isSigned()) {
2030 handleSignedRelationalComparison(I);
2031 return;
2032 }
2033
2034 assert(I.isUnsigned());
2035 if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) {
2036 handleRelationalComparisonExact(I);
2037 return;
2038 }
2039
2040 handleShadowOr(I);
2041 }
2042
visitFCmpInst__anonde7c31650411::MemorySanitizerVisitor2043 void visitFCmpInst(FCmpInst &I) {
2044 handleShadowOr(I);
2045 }
2046
handleShift__anonde7c31650411::MemorySanitizerVisitor2047 void handleShift(BinaryOperator &I) {
2048 IRBuilder<> IRB(&I);
2049 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2050 // Otherwise perform the same shift on S1.
2051 Value *S1 = getShadow(&I, 0);
2052 Value *S2 = getShadow(&I, 1);
2053 Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)),
2054 S2->getType());
2055 Value *V2 = I.getOperand(1);
2056 Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
2057 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2058 setOriginForNaryOp(I);
2059 }
2060
visitShl__anonde7c31650411::MemorySanitizerVisitor2061 void visitShl(BinaryOperator &I) { handleShift(I); }
visitAShr__anonde7c31650411::MemorySanitizerVisitor2062 void visitAShr(BinaryOperator &I) { handleShift(I); }
visitLShr__anonde7c31650411::MemorySanitizerVisitor2063 void visitLShr(BinaryOperator &I) { handleShift(I); }
2064
2065 /// Instrument llvm.memmove
2066 ///
2067 /// At this point we don't know if llvm.memmove will be inlined or not.
2068 /// If we don't instrument it and it gets inlined,
2069 /// our interceptor will not kick in and we will lose the memmove.
2070 /// If we instrument the call here, but it does not get inlined,
2071 /// we will memove the shadow twice: which is bad in case
2072 /// of overlapping regions. So, we simply lower the intrinsic to a call.
2073 ///
2074 /// Similar situation exists for memcpy and memset.
visitMemMoveInst__anonde7c31650411::MemorySanitizerVisitor2075 void visitMemMoveInst(MemMoveInst &I) {
2076 IRBuilder<> IRB(&I);
2077 IRB.CreateCall(
2078 MS.MemmoveFn,
2079 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2080 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2081 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2082 I.eraseFromParent();
2083 }
2084
2085 // Similar to memmove: avoid copying shadow twice.
2086 // This is somewhat unfortunate as it may slowdown small constant memcpys.
2087 // FIXME: consider doing manual inline for small constant sizes and proper
2088 // alignment.
visitMemCpyInst__anonde7c31650411::MemorySanitizerVisitor2089 void visitMemCpyInst(MemCpyInst &I) {
2090 IRBuilder<> IRB(&I);
2091 IRB.CreateCall(
2092 MS.MemcpyFn,
2093 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2094 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2095 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2096 I.eraseFromParent();
2097 }
2098
2099 // Same as memcpy.
visitMemSetInst__anonde7c31650411::MemorySanitizerVisitor2100 void visitMemSetInst(MemSetInst &I) {
2101 IRBuilder<> IRB(&I);
2102 IRB.CreateCall(
2103 MS.MemsetFn,
2104 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2105 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2106 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2107 I.eraseFromParent();
2108 }
2109
visitVAStartInst__anonde7c31650411::MemorySanitizerVisitor2110 void visitVAStartInst(VAStartInst &I) {
2111 VAHelper->visitVAStartInst(I);
2112 }
2113
visitVACopyInst__anonde7c31650411::MemorySanitizerVisitor2114 void visitVACopyInst(VACopyInst &I) {
2115 VAHelper->visitVACopyInst(I);
2116 }
2117
2118 /// Handle vector store-like intrinsics.
2119 ///
2120 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
2121 /// has 1 pointer argument and 1 vector argument, returns void.
handleVectorStoreIntrinsic__anonde7c31650411::MemorySanitizerVisitor2122 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
2123 IRBuilder<> IRB(&I);
2124 Value* Addr = I.getArgOperand(0);
2125 Value *Shadow = getShadow(&I, 1);
2126 Value *ShadowPtr, *OriginPtr;
2127
2128 // We don't know the pointer alignment (could be unaligned SSE store!).
2129 // Have to assume to worst case.
2130 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2131 Addr, IRB, Shadow->getType(), /*Alignment*/ 1, /*isStore*/ true);
2132 IRB.CreateAlignedStore(Shadow, ShadowPtr, 1);
2133
2134 if (ClCheckAccessAddress)
2135 insertShadowCheck(Addr, &I);
2136
2137 // FIXME: factor out common code from materializeStores
2138 if (MS.TrackOrigins) IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
2139 return true;
2140 }
2141
2142 /// Handle vector load-like intrinsics.
2143 ///
2144 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
2145 /// has 1 pointer argument, returns a vector.
handleVectorLoadIntrinsic__anonde7c31650411::MemorySanitizerVisitor2146 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
2147 IRBuilder<> IRB(&I);
2148 Value *Addr = I.getArgOperand(0);
2149
2150 Type *ShadowTy = getShadowTy(&I);
2151 Value *ShadowPtr, *OriginPtr;
2152 if (PropagateShadow) {
2153 // We don't know the pointer alignment (could be unaligned SSE load!).
2154 // Have to assume to worst case.
2155 unsigned Alignment = 1;
2156 std::tie(ShadowPtr, OriginPtr) =
2157 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2158 setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_msld"));
2159 } else {
2160 setShadow(&I, getCleanShadow(&I));
2161 }
2162
2163 if (ClCheckAccessAddress)
2164 insertShadowCheck(Addr, &I);
2165
2166 if (MS.TrackOrigins) {
2167 if (PropagateShadow)
2168 setOrigin(&I, IRB.CreateLoad(OriginPtr));
2169 else
2170 setOrigin(&I, getCleanOrigin());
2171 }
2172 return true;
2173 }
2174
2175 /// Handle (SIMD arithmetic)-like intrinsics.
2176 ///
2177 /// Instrument intrinsics with any number of arguments of the same type,
2178 /// equal to the return type. The type should be simple (no aggregates or
2179 /// pointers; vectors are fine).
2180 /// Caller guarantees that this intrinsic does not access memory.
maybeHandleSimpleNomemIntrinsic__anonde7c31650411::MemorySanitizerVisitor2181 bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
2182 Type *RetTy = I.getType();
2183 if (!(RetTy->isIntOrIntVectorTy() ||
2184 RetTy->isFPOrFPVectorTy() ||
2185 RetTy->isX86_MMXTy()))
2186 return false;
2187
2188 unsigned NumArgOperands = I.getNumArgOperands();
2189
2190 for (unsigned i = 0; i < NumArgOperands; ++i) {
2191 Type *Ty = I.getArgOperand(i)->getType();
2192 if (Ty != RetTy)
2193 return false;
2194 }
2195
2196 IRBuilder<> IRB(&I);
2197 ShadowAndOriginCombiner SC(this, IRB);
2198 for (unsigned i = 0; i < NumArgOperands; ++i)
2199 SC.Add(I.getArgOperand(i));
2200 SC.Done(&I);
2201
2202 return true;
2203 }
2204
2205 /// Heuristically instrument unknown intrinsics.
2206 ///
2207 /// The main purpose of this code is to do something reasonable with all
2208 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2209 /// We recognize several classes of intrinsics by their argument types and
2210 /// ModRefBehaviour and apply special intrumentation when we are reasonably
2211 /// sure that we know what the intrinsic does.
2212 ///
2213 /// We special-case intrinsics where this approach fails. See llvm.bswap
2214 /// handling as an example of that.
handleUnknownIntrinsic__anonde7c31650411::MemorySanitizerVisitor2215 bool handleUnknownIntrinsic(IntrinsicInst &I) {
2216 unsigned NumArgOperands = I.getNumArgOperands();
2217 if (NumArgOperands == 0)
2218 return false;
2219
2220 if (NumArgOperands == 2 &&
2221 I.getArgOperand(0)->getType()->isPointerTy() &&
2222 I.getArgOperand(1)->getType()->isVectorTy() &&
2223 I.getType()->isVoidTy() &&
2224 !I.onlyReadsMemory()) {
2225 // This looks like a vector store.
2226 return handleVectorStoreIntrinsic(I);
2227 }
2228
2229 if (NumArgOperands == 1 &&
2230 I.getArgOperand(0)->getType()->isPointerTy() &&
2231 I.getType()->isVectorTy() &&
2232 I.onlyReadsMemory()) {
2233 // This looks like a vector load.
2234 return handleVectorLoadIntrinsic(I);
2235 }
2236
2237 if (I.doesNotAccessMemory())
2238 if (maybeHandleSimpleNomemIntrinsic(I))
2239 return true;
2240
2241 // FIXME: detect and handle SSE maskstore/maskload
2242 return false;
2243 }
2244
handleBswap__anonde7c31650411::MemorySanitizerVisitor2245 void handleBswap(IntrinsicInst &I) {
2246 IRBuilder<> IRB(&I);
2247 Value *Op = I.getArgOperand(0);
2248 Type *OpType = Op->getType();
2249 Function *BswapFunc = Intrinsic::getDeclaration(
2250 F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
2251 setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
2252 setOrigin(&I, getOrigin(Op));
2253 }
2254
2255 // Instrument vector convert instrinsic.
2256 //
2257 // This function instruments intrinsics like cvtsi2ss:
2258 // %Out = int_xxx_cvtyyy(%ConvertOp)
2259 // or
2260 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
2261 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
2262 // number \p Out elements, and (if has 2 arguments) copies the rest of the
2263 // elements from \p CopyOp.
2264 // In most cases conversion involves floating-point value which may trigger a
2265 // hardware exception when not fully initialized. For this reason we require
2266 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
2267 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
2268 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
2269 // return a fully initialized value.
handleVectorConvertIntrinsic__anonde7c31650411::MemorySanitizerVisitor2270 void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements) {
2271 IRBuilder<> IRB(&I);
2272 Value *CopyOp, *ConvertOp;
2273
2274 switch (I.getNumArgOperands()) {
2275 case 3:
2276 assert(isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode");
2277 LLVM_FALLTHROUGH;
2278 case 2:
2279 CopyOp = I.getArgOperand(0);
2280 ConvertOp = I.getArgOperand(1);
2281 break;
2282 case 1:
2283 ConvertOp = I.getArgOperand(0);
2284 CopyOp = nullptr;
2285 break;
2286 default:
2287 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
2288 }
2289
2290 // The first *NumUsedElements* elements of ConvertOp are converted to the
2291 // same number of output elements. The rest of the output is copied from
2292 // CopyOp, or (if not available) filled with zeroes.
2293 // Combine shadow for elements of ConvertOp that are used in this operation,
2294 // and insert a check.
2295 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
2296 // int->any conversion.
2297 Value *ConvertShadow = getShadow(ConvertOp);
2298 Value *AggShadow = nullptr;
2299 if (ConvertOp->getType()->isVectorTy()) {
2300 AggShadow = IRB.CreateExtractElement(
2301 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2302 for (int i = 1; i < NumUsedElements; ++i) {
2303 Value *MoreShadow = IRB.CreateExtractElement(
2304 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2305 AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
2306 }
2307 } else {
2308 AggShadow = ConvertShadow;
2309 }
2310 assert(AggShadow->getType()->isIntegerTy());
2311 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
2312
2313 // Build result shadow by zero-filling parts of CopyOp shadow that come from
2314 // ConvertOp.
2315 if (CopyOp) {
2316 assert(CopyOp->getType() == I.getType());
2317 assert(CopyOp->getType()->isVectorTy());
2318 Value *ResultShadow = getShadow(CopyOp);
2319 Type *EltTy = ResultShadow->getType()->getVectorElementType();
2320 for (int i = 0; i < NumUsedElements; ++i) {
2321 ResultShadow = IRB.CreateInsertElement(
2322 ResultShadow, ConstantInt::getNullValue(EltTy),
2323 ConstantInt::get(IRB.getInt32Ty(), i));
2324 }
2325 setShadow(&I, ResultShadow);
2326 setOrigin(&I, getOrigin(CopyOp));
2327 } else {
2328 setShadow(&I, getCleanShadow(&I));
2329 setOrigin(&I, getCleanOrigin());
2330 }
2331 }
2332
2333 // Given a scalar or vector, extract lower 64 bits (or less), and return all
2334 // zeroes if it is zero, and all ones otherwise.
Lower64ShadowExtend__anonde7c31650411::MemorySanitizerVisitor2335 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2336 if (S->getType()->isVectorTy())
2337 S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
2338 assert(S->getType()->getPrimitiveSizeInBits() <= 64);
2339 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2340 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2341 }
2342
2343 // Given a vector, extract its first element, and return all
2344 // zeroes if it is zero, and all ones otherwise.
LowerElementShadowExtend__anonde7c31650411::MemorySanitizerVisitor2345 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2346 Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
2347 Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
2348 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2349 }
2350
VariableShadowExtend__anonde7c31650411::MemorySanitizerVisitor2351 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
2352 Type *T = S->getType();
2353 assert(T->isVectorTy());
2354 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2355 return IRB.CreateSExt(S2, T);
2356 }
2357
2358 // Instrument vector shift instrinsic.
2359 //
2360 // This function instruments intrinsics like int_x86_avx2_psll_w.
2361 // Intrinsic shifts %In by %ShiftSize bits.
2362 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
2363 // size, and the rest is ignored. Behavior is defined even if shift size is
2364 // greater than register (or field) width.
handleVectorShiftIntrinsic__anonde7c31650411::MemorySanitizerVisitor2365 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
2366 assert(I.getNumArgOperands() == 2);
2367 IRBuilder<> IRB(&I);
2368 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2369 // Otherwise perform the same shift on S1.
2370 Value *S1 = getShadow(&I, 0);
2371 Value *S2 = getShadow(&I, 1);
2372 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
2373 : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
2374 Value *V1 = I.getOperand(0);
2375 Value *V2 = I.getOperand(1);
2376 Value *Shift = IRB.CreateCall(I.getCalledValue(),
2377 {IRB.CreateBitCast(S1, V1->getType()), V2});
2378 Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
2379 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2380 setOriginForNaryOp(I);
2381 }
2382
2383 // Get an X86_MMX-sized vector type.
getMMXVectorTy__anonde7c31650411::MemorySanitizerVisitor2384 Type *getMMXVectorTy(unsigned EltSizeInBits) {
2385 const unsigned X86_MMXSizeInBits = 64;
2386 return VectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
2387 X86_MMXSizeInBits / EltSizeInBits);
2388 }
2389
2390 // Returns a signed counterpart for an (un)signed-saturate-and-pack
2391 // intrinsic.
getSignedPackIntrinsic__anonde7c31650411::MemorySanitizerVisitor2392 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
2393 switch (id) {
2394 case Intrinsic::x86_sse2_packsswb_128:
2395 case Intrinsic::x86_sse2_packuswb_128:
2396 return Intrinsic::x86_sse2_packsswb_128;
2397
2398 case Intrinsic::x86_sse2_packssdw_128:
2399 case Intrinsic::x86_sse41_packusdw:
2400 return Intrinsic::x86_sse2_packssdw_128;
2401
2402 case Intrinsic::x86_avx2_packsswb:
2403 case Intrinsic::x86_avx2_packuswb:
2404 return Intrinsic::x86_avx2_packsswb;
2405
2406 case Intrinsic::x86_avx2_packssdw:
2407 case Intrinsic::x86_avx2_packusdw:
2408 return Intrinsic::x86_avx2_packssdw;
2409
2410 case Intrinsic::x86_mmx_packsswb:
2411 case Intrinsic::x86_mmx_packuswb:
2412 return Intrinsic::x86_mmx_packsswb;
2413
2414 case Intrinsic::x86_mmx_packssdw:
2415 return Intrinsic::x86_mmx_packssdw;
2416 default:
2417 llvm_unreachable("unexpected intrinsic id");
2418 }
2419 }
2420
2421 // Instrument vector pack instrinsic.
2422 //
2423 // This function instruments intrinsics like x86_mmx_packsswb, that
2424 // packs elements of 2 input vectors into half as many bits with saturation.
2425 // Shadow is propagated with the signed variant of the same intrinsic applied
2426 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
2427 // EltSizeInBits is used only for x86mmx arguments.
handleVectorPackIntrinsic__anonde7c31650411::MemorySanitizerVisitor2428 void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
2429 assert(I.getNumArgOperands() == 2);
2430 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2431 IRBuilder<> IRB(&I);
2432 Value *S1 = getShadow(&I, 0);
2433 Value *S2 = getShadow(&I, 1);
2434 assert(isX86_MMX || S1->getType()->isVectorTy());
2435
2436 // SExt and ICmpNE below must apply to individual elements of input vectors.
2437 // In case of x86mmx arguments, cast them to appropriate vector types and
2438 // back.
2439 Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
2440 if (isX86_MMX) {
2441 S1 = IRB.CreateBitCast(S1, T);
2442 S2 = IRB.CreateBitCast(S2, T);
2443 }
2444 Value *S1_ext = IRB.CreateSExt(
2445 IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
2446 Value *S2_ext = IRB.CreateSExt(
2447 IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
2448 if (isX86_MMX) {
2449 Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
2450 S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
2451 S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
2452 }
2453
2454 Function *ShadowFn = Intrinsic::getDeclaration(
2455 F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
2456
2457 Value *S =
2458 IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
2459 if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I));
2460 setShadow(&I, S);
2461 setOriginForNaryOp(I);
2462 }
2463
2464 // Instrument sum-of-absolute-differencies intrinsic.
handleVectorSadIntrinsic__anonde7c31650411::MemorySanitizerVisitor2465 void handleVectorSadIntrinsic(IntrinsicInst &I) {
2466 const unsigned SignificantBitsPerResultElement = 16;
2467 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2468 Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType();
2469 unsigned ZeroBitsPerResultElement =
2470 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
2471
2472 IRBuilder<> IRB(&I);
2473 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2474 S = IRB.CreateBitCast(S, ResTy);
2475 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2476 ResTy);
2477 S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
2478 S = IRB.CreateBitCast(S, getShadowTy(&I));
2479 setShadow(&I, S);
2480 setOriginForNaryOp(I);
2481 }
2482
2483 // Instrument multiply-add intrinsic.
handleVectorPmaddIntrinsic__anonde7c31650411::MemorySanitizerVisitor2484 void handleVectorPmaddIntrinsic(IntrinsicInst &I,
2485 unsigned EltSizeInBits = 0) {
2486 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2487 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType();
2488 IRBuilder<> IRB(&I);
2489 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2490 S = IRB.CreateBitCast(S, ResTy);
2491 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2492 ResTy);
2493 S = IRB.CreateBitCast(S, getShadowTy(&I));
2494 setShadow(&I, S);
2495 setOriginForNaryOp(I);
2496 }
2497
2498 // Instrument compare-packed intrinsic.
2499 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
2500 // all-ones shadow.
handleVectorComparePackedIntrinsic__anonde7c31650411::MemorySanitizerVisitor2501 void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
2502 IRBuilder<> IRB(&I);
2503 Type *ResTy = getShadowTy(&I);
2504 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2505 Value *S = IRB.CreateSExt(
2506 IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
2507 setShadow(&I, S);
2508 setOriginForNaryOp(I);
2509 }
2510
2511 // Instrument compare-scalar intrinsic.
2512 // This handles both cmp* intrinsics which return the result in the first
2513 // element of a vector, and comi* which return the result as i32.
handleVectorCompareScalarIntrinsic__anonde7c31650411::MemorySanitizerVisitor2514 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
2515 IRBuilder<> IRB(&I);
2516 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2517 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
2518 setShadow(&I, S);
2519 setOriginForNaryOp(I);
2520 }
2521
handleStmxcsr__anonde7c31650411::MemorySanitizerVisitor2522 void handleStmxcsr(IntrinsicInst &I) {
2523 IRBuilder<> IRB(&I);
2524 Value* Addr = I.getArgOperand(0);
2525 Type *Ty = IRB.getInt32Ty();
2526 Value *ShadowPtr =
2527 getShadowOriginPtr(Addr, IRB, Ty, /*Alignment*/ 1, /*isStore*/ true)
2528 .first;
2529
2530 IRB.CreateStore(getCleanShadow(Ty),
2531 IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
2532
2533 if (ClCheckAccessAddress)
2534 insertShadowCheck(Addr, &I);
2535 }
2536
handleLdmxcsr__anonde7c31650411::MemorySanitizerVisitor2537 void handleLdmxcsr(IntrinsicInst &I) {
2538 if (!InsertChecks) return;
2539
2540 IRBuilder<> IRB(&I);
2541 Value *Addr = I.getArgOperand(0);
2542 Type *Ty = IRB.getInt32Ty();
2543 unsigned Alignment = 1;
2544 Value *ShadowPtr, *OriginPtr;
2545 std::tie(ShadowPtr, OriginPtr) =
2546 getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
2547
2548 if (ClCheckAccessAddress)
2549 insertShadowCheck(Addr, &I);
2550
2551 Value *Shadow = IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_ldmxcsr");
2552 Value *Origin =
2553 MS.TrackOrigins ? IRB.CreateLoad(OriginPtr) : getCleanOrigin();
2554 insertShadowCheck(Shadow, Origin, &I);
2555 }
2556
handleMaskedStore__anonde7c31650411::MemorySanitizerVisitor2557 void handleMaskedStore(IntrinsicInst &I) {
2558 IRBuilder<> IRB(&I);
2559 Value *V = I.getArgOperand(0);
2560 Value *Addr = I.getArgOperand(1);
2561 unsigned Align = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
2562 Value *Mask = I.getArgOperand(3);
2563 Value *Shadow = getShadow(V);
2564
2565 Value *ShadowPtr;
2566 Value *OriginPtr;
2567 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2568 Addr, IRB, Shadow->getType(), Align, /*isStore*/ true);
2569
2570 if (ClCheckAccessAddress) {
2571 insertShadowCheck(Addr, &I);
2572 // Uninitialized mask is kind of like uninitialized address, but not as
2573 // scary.
2574 insertShadowCheck(Mask, &I);
2575 }
2576
2577 IRB.CreateMaskedStore(Shadow, ShadowPtr, Align, Mask);
2578
2579 if (MS.TrackOrigins) {
2580 auto &DL = F.getParent()->getDataLayout();
2581 paintOrigin(IRB, getOrigin(V), OriginPtr,
2582 DL.getTypeStoreSize(Shadow->getType()),
2583 std::max(Align, kMinOriginAlignment));
2584 }
2585 }
2586
handleMaskedLoad__anonde7c31650411::MemorySanitizerVisitor2587 bool handleMaskedLoad(IntrinsicInst &I) {
2588 IRBuilder<> IRB(&I);
2589 Value *Addr = I.getArgOperand(0);
2590 unsigned Align = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
2591 Value *Mask = I.getArgOperand(2);
2592 Value *PassThru = I.getArgOperand(3);
2593
2594 Type *ShadowTy = getShadowTy(&I);
2595 Value *ShadowPtr, *OriginPtr;
2596 if (PropagateShadow) {
2597 std::tie(ShadowPtr, OriginPtr) =
2598 getShadowOriginPtr(Addr, IRB, ShadowTy, Align, /*isStore*/ false);
2599 setShadow(&I, IRB.CreateMaskedLoad(ShadowPtr, Align, Mask,
2600 getShadow(PassThru), "_msmaskedld"));
2601 } else {
2602 setShadow(&I, getCleanShadow(&I));
2603 }
2604
2605 if (ClCheckAccessAddress) {
2606 insertShadowCheck(Addr, &I);
2607 insertShadowCheck(Mask, &I);
2608 }
2609
2610 if (MS.TrackOrigins) {
2611 if (PropagateShadow) {
2612 // Choose between PassThru's and the loaded value's origins.
2613 Value *MaskedPassThruShadow = IRB.CreateAnd(
2614 getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
2615
2616 Value *Acc = IRB.CreateExtractElement(
2617 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2618 for (int i = 1, N = PassThru->getType()->getVectorNumElements(); i < N;
2619 ++i) {
2620 Value *More = IRB.CreateExtractElement(
2621 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2622 Acc = IRB.CreateOr(Acc, More);
2623 }
2624
2625 Value *Origin = IRB.CreateSelect(
2626 IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())),
2627 getOrigin(PassThru), IRB.CreateLoad(OriginPtr));
2628
2629 setOrigin(&I, Origin);
2630 } else {
2631 setOrigin(&I, getCleanOrigin());
2632 }
2633 }
2634 return true;
2635 }
2636
2637
visitIntrinsicInst__anonde7c31650411::MemorySanitizerVisitor2638 void visitIntrinsicInst(IntrinsicInst &I) {
2639 switch (I.getIntrinsicID()) {
2640 case Intrinsic::bswap:
2641 handleBswap(I);
2642 break;
2643 case Intrinsic::masked_store:
2644 handleMaskedStore(I);
2645 break;
2646 case Intrinsic::masked_load:
2647 handleMaskedLoad(I);
2648 break;
2649 case Intrinsic::x86_sse_stmxcsr:
2650 handleStmxcsr(I);
2651 break;
2652 case Intrinsic::x86_sse_ldmxcsr:
2653 handleLdmxcsr(I);
2654 break;
2655 case Intrinsic::x86_avx512_vcvtsd2usi64:
2656 case Intrinsic::x86_avx512_vcvtsd2usi32:
2657 case Intrinsic::x86_avx512_vcvtss2usi64:
2658 case Intrinsic::x86_avx512_vcvtss2usi32:
2659 case Intrinsic::x86_avx512_cvttss2usi64:
2660 case Intrinsic::x86_avx512_cvttss2usi:
2661 case Intrinsic::x86_avx512_cvttsd2usi64:
2662 case Intrinsic::x86_avx512_cvttsd2usi:
2663 case Intrinsic::x86_avx512_cvtusi2ss:
2664 case Intrinsic::x86_avx512_cvtusi642sd:
2665 case Intrinsic::x86_avx512_cvtusi642ss:
2666 case Intrinsic::x86_sse2_cvtsd2si64:
2667 case Intrinsic::x86_sse2_cvtsd2si:
2668 case Intrinsic::x86_sse2_cvtsd2ss:
2669 case Intrinsic::x86_sse2_cvttsd2si64:
2670 case Intrinsic::x86_sse2_cvttsd2si:
2671 case Intrinsic::x86_sse_cvtss2si64:
2672 case Intrinsic::x86_sse_cvtss2si:
2673 case Intrinsic::x86_sse_cvttss2si64:
2674 case Intrinsic::x86_sse_cvttss2si:
2675 handleVectorConvertIntrinsic(I, 1);
2676 break;
2677 case Intrinsic::x86_sse_cvtps2pi:
2678 case Intrinsic::x86_sse_cvttps2pi:
2679 handleVectorConvertIntrinsic(I, 2);
2680 break;
2681
2682 case Intrinsic::x86_avx512_psll_w_512:
2683 case Intrinsic::x86_avx512_psll_d_512:
2684 case Intrinsic::x86_avx512_psll_q_512:
2685 case Intrinsic::x86_avx512_pslli_w_512:
2686 case Intrinsic::x86_avx512_pslli_d_512:
2687 case Intrinsic::x86_avx512_pslli_q_512:
2688 case Intrinsic::x86_avx512_psrl_w_512:
2689 case Intrinsic::x86_avx512_psrl_d_512:
2690 case Intrinsic::x86_avx512_psrl_q_512:
2691 case Intrinsic::x86_avx512_psra_w_512:
2692 case Intrinsic::x86_avx512_psra_d_512:
2693 case Intrinsic::x86_avx512_psra_q_512:
2694 case Intrinsic::x86_avx512_psrli_w_512:
2695 case Intrinsic::x86_avx512_psrli_d_512:
2696 case Intrinsic::x86_avx512_psrli_q_512:
2697 case Intrinsic::x86_avx512_psrai_w_512:
2698 case Intrinsic::x86_avx512_psrai_d_512:
2699 case Intrinsic::x86_avx512_psrai_q_512:
2700 case Intrinsic::x86_avx512_psra_q_256:
2701 case Intrinsic::x86_avx512_psra_q_128:
2702 case Intrinsic::x86_avx512_psrai_q_256:
2703 case Intrinsic::x86_avx512_psrai_q_128:
2704 case Intrinsic::x86_avx2_psll_w:
2705 case Intrinsic::x86_avx2_psll_d:
2706 case Intrinsic::x86_avx2_psll_q:
2707 case Intrinsic::x86_avx2_pslli_w:
2708 case Intrinsic::x86_avx2_pslli_d:
2709 case Intrinsic::x86_avx2_pslli_q:
2710 case Intrinsic::x86_avx2_psrl_w:
2711 case Intrinsic::x86_avx2_psrl_d:
2712 case Intrinsic::x86_avx2_psrl_q:
2713 case Intrinsic::x86_avx2_psra_w:
2714 case Intrinsic::x86_avx2_psra_d:
2715 case Intrinsic::x86_avx2_psrli_w:
2716 case Intrinsic::x86_avx2_psrli_d:
2717 case Intrinsic::x86_avx2_psrli_q:
2718 case Intrinsic::x86_avx2_psrai_w:
2719 case Intrinsic::x86_avx2_psrai_d:
2720 case Intrinsic::x86_sse2_psll_w:
2721 case Intrinsic::x86_sse2_psll_d:
2722 case Intrinsic::x86_sse2_psll_q:
2723 case Intrinsic::x86_sse2_pslli_w:
2724 case Intrinsic::x86_sse2_pslli_d:
2725 case Intrinsic::x86_sse2_pslli_q:
2726 case Intrinsic::x86_sse2_psrl_w:
2727 case Intrinsic::x86_sse2_psrl_d:
2728 case Intrinsic::x86_sse2_psrl_q:
2729 case Intrinsic::x86_sse2_psra_w:
2730 case Intrinsic::x86_sse2_psra_d:
2731 case Intrinsic::x86_sse2_psrli_w:
2732 case Intrinsic::x86_sse2_psrli_d:
2733 case Intrinsic::x86_sse2_psrli_q:
2734 case Intrinsic::x86_sse2_psrai_w:
2735 case Intrinsic::x86_sse2_psrai_d:
2736 case Intrinsic::x86_mmx_psll_w:
2737 case Intrinsic::x86_mmx_psll_d:
2738 case Intrinsic::x86_mmx_psll_q:
2739 case Intrinsic::x86_mmx_pslli_w:
2740 case Intrinsic::x86_mmx_pslli_d:
2741 case Intrinsic::x86_mmx_pslli_q:
2742 case Intrinsic::x86_mmx_psrl_w:
2743 case Intrinsic::x86_mmx_psrl_d:
2744 case Intrinsic::x86_mmx_psrl_q:
2745 case Intrinsic::x86_mmx_psra_w:
2746 case Intrinsic::x86_mmx_psra_d:
2747 case Intrinsic::x86_mmx_psrli_w:
2748 case Intrinsic::x86_mmx_psrli_d:
2749 case Intrinsic::x86_mmx_psrli_q:
2750 case Intrinsic::x86_mmx_psrai_w:
2751 case Intrinsic::x86_mmx_psrai_d:
2752 handleVectorShiftIntrinsic(I, /* Variable */ false);
2753 break;
2754 case Intrinsic::x86_avx2_psllv_d:
2755 case Intrinsic::x86_avx2_psllv_d_256:
2756 case Intrinsic::x86_avx512_psllv_d_512:
2757 case Intrinsic::x86_avx2_psllv_q:
2758 case Intrinsic::x86_avx2_psllv_q_256:
2759 case Intrinsic::x86_avx512_psllv_q_512:
2760 case Intrinsic::x86_avx2_psrlv_d:
2761 case Intrinsic::x86_avx2_psrlv_d_256:
2762 case Intrinsic::x86_avx512_psrlv_d_512:
2763 case Intrinsic::x86_avx2_psrlv_q:
2764 case Intrinsic::x86_avx2_psrlv_q_256:
2765 case Intrinsic::x86_avx512_psrlv_q_512:
2766 case Intrinsic::x86_avx2_psrav_d:
2767 case Intrinsic::x86_avx2_psrav_d_256:
2768 case Intrinsic::x86_avx512_psrav_d_512:
2769 case Intrinsic::x86_avx512_psrav_q_128:
2770 case Intrinsic::x86_avx512_psrav_q_256:
2771 case Intrinsic::x86_avx512_psrav_q_512:
2772 handleVectorShiftIntrinsic(I, /* Variable */ true);
2773 break;
2774
2775 case Intrinsic::x86_sse2_packsswb_128:
2776 case Intrinsic::x86_sse2_packssdw_128:
2777 case Intrinsic::x86_sse2_packuswb_128:
2778 case Intrinsic::x86_sse41_packusdw:
2779 case Intrinsic::x86_avx2_packsswb:
2780 case Intrinsic::x86_avx2_packssdw:
2781 case Intrinsic::x86_avx2_packuswb:
2782 case Intrinsic::x86_avx2_packusdw:
2783 handleVectorPackIntrinsic(I);
2784 break;
2785
2786 case Intrinsic::x86_mmx_packsswb:
2787 case Intrinsic::x86_mmx_packuswb:
2788 handleVectorPackIntrinsic(I, 16);
2789 break;
2790
2791 case Intrinsic::x86_mmx_packssdw:
2792 handleVectorPackIntrinsic(I, 32);
2793 break;
2794
2795 case Intrinsic::x86_mmx_psad_bw:
2796 case Intrinsic::x86_sse2_psad_bw:
2797 case Intrinsic::x86_avx2_psad_bw:
2798 handleVectorSadIntrinsic(I);
2799 break;
2800
2801 case Intrinsic::x86_sse2_pmadd_wd:
2802 case Intrinsic::x86_avx2_pmadd_wd:
2803 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
2804 case Intrinsic::x86_avx2_pmadd_ub_sw:
2805 handleVectorPmaddIntrinsic(I);
2806 break;
2807
2808 case Intrinsic::x86_ssse3_pmadd_ub_sw:
2809 handleVectorPmaddIntrinsic(I, 8);
2810 break;
2811
2812 case Intrinsic::x86_mmx_pmadd_wd:
2813 handleVectorPmaddIntrinsic(I, 16);
2814 break;
2815
2816 case Intrinsic::x86_sse_cmp_ss:
2817 case Intrinsic::x86_sse2_cmp_sd:
2818 case Intrinsic::x86_sse_comieq_ss:
2819 case Intrinsic::x86_sse_comilt_ss:
2820 case Intrinsic::x86_sse_comile_ss:
2821 case Intrinsic::x86_sse_comigt_ss:
2822 case Intrinsic::x86_sse_comige_ss:
2823 case Intrinsic::x86_sse_comineq_ss:
2824 case Intrinsic::x86_sse_ucomieq_ss:
2825 case Intrinsic::x86_sse_ucomilt_ss:
2826 case Intrinsic::x86_sse_ucomile_ss:
2827 case Intrinsic::x86_sse_ucomigt_ss:
2828 case Intrinsic::x86_sse_ucomige_ss:
2829 case Intrinsic::x86_sse_ucomineq_ss:
2830 case Intrinsic::x86_sse2_comieq_sd:
2831 case Intrinsic::x86_sse2_comilt_sd:
2832 case Intrinsic::x86_sse2_comile_sd:
2833 case Intrinsic::x86_sse2_comigt_sd:
2834 case Intrinsic::x86_sse2_comige_sd:
2835 case Intrinsic::x86_sse2_comineq_sd:
2836 case Intrinsic::x86_sse2_ucomieq_sd:
2837 case Intrinsic::x86_sse2_ucomilt_sd:
2838 case Intrinsic::x86_sse2_ucomile_sd:
2839 case Intrinsic::x86_sse2_ucomigt_sd:
2840 case Intrinsic::x86_sse2_ucomige_sd:
2841 case Intrinsic::x86_sse2_ucomineq_sd:
2842 handleVectorCompareScalarIntrinsic(I);
2843 break;
2844
2845 case Intrinsic::x86_sse_cmp_ps:
2846 case Intrinsic::x86_sse2_cmp_pd:
2847 // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function
2848 // generates reasonably looking IR that fails in the backend with "Do not
2849 // know how to split the result of this operator!".
2850 handleVectorComparePackedIntrinsic(I);
2851 break;
2852
2853 default:
2854 if (!handleUnknownIntrinsic(I))
2855 visitInstruction(I);
2856 break;
2857 }
2858 }
2859
visitCallSite__anonde7c31650411::MemorySanitizerVisitor2860 void visitCallSite(CallSite CS) {
2861 Instruction &I = *CS.getInstruction();
2862 assert(!I.getMetadata("nosanitize"));
2863 assert((CS.isCall() || CS.isInvoke()) && "Unknown type of CallSite");
2864 if (CS.isCall()) {
2865 CallInst *Call = cast<CallInst>(&I);
2866
2867 // For inline asm, do the usual thing: check argument shadow and mark all
2868 // outputs as clean. Note that any side effects of the inline asm that are
2869 // not immediately visible in its constraints are not handled.
2870 if (Call->isInlineAsm()) {
2871 if (ClHandleAsmConservative)
2872 visitAsmInstruction(I);
2873 else
2874 visitInstruction(I);
2875 return;
2876 }
2877
2878 assert(!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere");
2879
2880 // We are going to insert code that relies on the fact that the callee
2881 // will become a non-readonly function after it is instrumented by us. To
2882 // prevent this code from being optimized out, mark that function
2883 // non-readonly in advance.
2884 if (Function *Func = Call->getCalledFunction()) {
2885 // Clear out readonly/readnone attributes.
2886 AttrBuilder B;
2887 B.addAttribute(Attribute::ReadOnly)
2888 .addAttribute(Attribute::ReadNone);
2889 Func->removeAttributes(AttributeList::FunctionIndex, B);
2890 }
2891
2892 maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
2893 }
2894 IRBuilder<> IRB(&I);
2895
2896 unsigned ArgOffset = 0;
2897 LLVM_DEBUG(dbgs() << " CallSite: " << I << "\n");
2898 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
2899 ArgIt != End; ++ArgIt) {
2900 Value *A = *ArgIt;
2901 unsigned i = ArgIt - CS.arg_begin();
2902 if (!A->getType()->isSized()) {
2903 LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << I << "\n");
2904 continue;
2905 }
2906 unsigned Size = 0;
2907 Value *Store = nullptr;
2908 // Compute the Shadow for arg even if it is ByVal, because
2909 // in that case getShadow() will copy the actual arg shadow to
2910 // __msan_param_tls.
2911 Value *ArgShadow = getShadow(A);
2912 Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
2913 LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *A
2914 << " Shadow: " << *ArgShadow << "\n");
2915 bool ArgIsInitialized = false;
2916 const DataLayout &DL = F.getParent()->getDataLayout();
2917 if (CS.paramHasAttr(i, Attribute::ByVal)) {
2918 assert(A->getType()->isPointerTy() &&
2919 "ByVal argument is not a pointer!");
2920 Size = DL.getTypeAllocSize(A->getType()->getPointerElementType());
2921 if (ArgOffset + Size > kParamTLSSize) break;
2922 unsigned ParamAlignment = CS.getParamAlignment(i);
2923 unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment);
2924 Value *AShadowPtr = getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
2925 Alignment, /*isStore*/ false)
2926 .first;
2927
2928 Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
2929 Alignment, Size);
2930 } else {
2931 Size = DL.getTypeAllocSize(A->getType());
2932 if (ArgOffset + Size > kParamTLSSize) break;
2933 Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
2934 kShadowTLSAlignment);
2935 Constant *Cst = dyn_cast<Constant>(ArgShadow);
2936 if (Cst && Cst->isNullValue()) ArgIsInitialized = true;
2937 }
2938 if (MS.TrackOrigins && !ArgIsInitialized)
2939 IRB.CreateStore(getOrigin(A),
2940 getOriginPtrForArgument(A, IRB, ArgOffset));
2941 (void)Store;
2942 assert(Size != 0 && Store != nullptr);
2943 LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
2944 ArgOffset += alignTo(Size, 8);
2945 }
2946 LLVM_DEBUG(dbgs() << " done with call args\n");
2947
2948 FunctionType *FT =
2949 cast<FunctionType>(CS.getCalledValue()->getType()->getContainedType(0));
2950 if (FT->isVarArg()) {
2951 VAHelper->visitCallSite(CS, IRB);
2952 }
2953
2954 // Now, get the shadow for the RetVal.
2955 if (!I.getType()->isSized()) return;
2956 // Don't emit the epilogue for musttail call returns.
2957 if (CS.isCall() && cast<CallInst>(&I)->isMustTailCall()) return;
2958 IRBuilder<> IRBBefore(&I);
2959 // Until we have full dynamic coverage, make sure the retval shadow is 0.
2960 Value *Base = getShadowPtrForRetval(&I, IRBBefore);
2961 IRBBefore.CreateAlignedStore(getCleanShadow(&I), Base, kShadowTLSAlignment);
2962 BasicBlock::iterator NextInsn;
2963 if (CS.isCall()) {
2964 NextInsn = ++I.getIterator();
2965 assert(NextInsn != I.getParent()->end());
2966 } else {
2967 BasicBlock *NormalDest = cast<InvokeInst>(&I)->getNormalDest();
2968 if (!NormalDest->getSinglePredecessor()) {
2969 // FIXME: this case is tricky, so we are just conservative here.
2970 // Perhaps we need to split the edge between this BB and NormalDest,
2971 // but a naive attempt to use SplitEdge leads to a crash.
2972 setShadow(&I, getCleanShadow(&I));
2973 setOrigin(&I, getCleanOrigin());
2974 return;
2975 }
2976 // FIXME: NextInsn is likely in a basic block that has not been visited yet.
2977 // Anything inserted there will be instrumented by MSan later!
2978 NextInsn = NormalDest->getFirstInsertionPt();
2979 assert(NextInsn != NormalDest->end() &&
2980 "Could not find insertion point for retval shadow load");
2981 }
2982 IRBuilder<> IRBAfter(&*NextInsn);
2983 Value *RetvalShadow =
2984 IRBAfter.CreateAlignedLoad(getShadowPtrForRetval(&I, IRBAfter),
2985 kShadowTLSAlignment, "_msret");
2986 setShadow(&I, RetvalShadow);
2987 if (MS.TrackOrigins)
2988 setOrigin(&I, IRBAfter.CreateLoad(getOriginPtrForRetval(IRBAfter)));
2989 }
2990
isAMustTailRetVal__anonde7c31650411::MemorySanitizerVisitor2991 bool isAMustTailRetVal(Value *RetVal) {
2992 if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
2993 RetVal = I->getOperand(0);
2994 }
2995 if (auto *I = dyn_cast<CallInst>(RetVal)) {
2996 return I->isMustTailCall();
2997 }
2998 return false;
2999 }
3000
visitReturnInst__anonde7c31650411::MemorySanitizerVisitor3001 void visitReturnInst(ReturnInst &I) {
3002 IRBuilder<> IRB(&I);
3003 Value *RetVal = I.getReturnValue();
3004 if (!RetVal) return;
3005 // Don't emit the epilogue for musttail call returns.
3006 if (isAMustTailRetVal(RetVal)) return;
3007 Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
3008 if (CheckReturnValue) {
3009 insertShadowCheck(RetVal, &I);
3010 Value *Shadow = getCleanShadow(RetVal);
3011 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3012 } else {
3013 Value *Shadow = getShadow(RetVal);
3014 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3015 if (MS.TrackOrigins)
3016 IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
3017 }
3018 }
3019
visitPHINode__anonde7c31650411::MemorySanitizerVisitor3020 void visitPHINode(PHINode &I) {
3021 IRBuilder<> IRB(&I);
3022 if (!PropagateShadow) {
3023 setShadow(&I, getCleanShadow(&I));
3024 setOrigin(&I, getCleanOrigin());
3025 return;
3026 }
3027
3028 ShadowPHINodes.push_back(&I);
3029 setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
3030 "_msphi_s"));
3031 if (MS.TrackOrigins)
3032 setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(),
3033 "_msphi_o"));
3034 }
3035
visitAllocaInst__anonde7c31650411::MemorySanitizerVisitor3036 void visitAllocaInst(AllocaInst &I) {
3037 setShadow(&I, getCleanShadow(&I));
3038 setOrigin(&I, getCleanOrigin());
3039 IRBuilder<> IRB(I.getNextNode());
3040 const DataLayout &DL = F.getParent()->getDataLayout();
3041 uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
3042 Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
3043 if (I.isArrayAllocation())
3044 Len = IRB.CreateMul(Len, I.getArraySize());
3045 if (PoisonStack && ClPoisonStackWithCall) {
3046 IRB.CreateCall(MS.MsanPoisonStackFn,
3047 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3048 } else {
3049 Value *ShadowBase = getShadowOriginPtr(&I, IRB, IRB.getInt8Ty(),
3050 I.getAlignment(), /*isStore*/ true)
3051 .first;
3052
3053 Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
3054 IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlignment());
3055 }
3056
3057 if (PoisonStack && MS.TrackOrigins) {
3058 SmallString<2048> StackDescriptionStorage;
3059 raw_svector_ostream StackDescription(StackDescriptionStorage);
3060 // We create a string with a description of the stack allocation and
3061 // pass it into __msan_set_alloca_origin.
3062 // It will be printed by the run-time if stack-originated UMR is found.
3063 // The first 4 bytes of the string are set to '----' and will be replaced
3064 // by __msan_va_arg_overflow_size_tls at the first call.
3065 StackDescription << "----" << I.getName() << "@" << F.getName();
3066 Value *Descr =
3067 createPrivateNonConstGlobalForString(*F.getParent(),
3068 StackDescription.str());
3069
3070 IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
3071 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3072 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
3073 IRB.CreatePointerCast(&F, MS.IntptrTy)});
3074 }
3075 }
3076
visitSelectInst__anonde7c31650411::MemorySanitizerVisitor3077 void visitSelectInst(SelectInst& I) {
3078 IRBuilder<> IRB(&I);
3079 // a = select b, c, d
3080 Value *B = I.getCondition();
3081 Value *C = I.getTrueValue();
3082 Value *D = I.getFalseValue();
3083 Value *Sb = getShadow(B);
3084 Value *Sc = getShadow(C);
3085 Value *Sd = getShadow(D);
3086
3087 // Result shadow if condition shadow is 0.
3088 Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
3089 Value *Sa1;
3090 if (I.getType()->isAggregateType()) {
3091 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
3092 // an extra "select". This results in much more compact IR.
3093 // Sa = select Sb, poisoned, (select b, Sc, Sd)
3094 Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
3095 } else {
3096 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
3097 // If Sb (condition is poisoned), look for bits in c and d that are equal
3098 // and both unpoisoned.
3099 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
3100
3101 // Cast arguments to shadow-compatible type.
3102 C = CreateAppToShadowCast(IRB, C);
3103 D = CreateAppToShadowCast(IRB, D);
3104
3105 // Result shadow if condition shadow is 1.
3106 Sa1 = IRB.CreateOr(IRB.CreateXor(C, D), IRB.CreateOr(Sc, Sd));
3107 }
3108 Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
3109 setShadow(&I, Sa);
3110 if (MS.TrackOrigins) {
3111 // Origins are always i32, so any vector conditions must be flattened.
3112 // FIXME: consider tracking vector origins for app vectors?
3113 if (B->getType()->isVectorTy()) {
3114 Type *FlatTy = getShadowTyNoVec(B->getType());
3115 B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
3116 ConstantInt::getNullValue(FlatTy));
3117 Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
3118 ConstantInt::getNullValue(FlatTy));
3119 }
3120 // a = select b, c, d
3121 // Oa = Sb ? Ob : (b ? Oc : Od)
3122 setOrigin(
3123 &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
3124 IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
3125 getOrigin(I.getFalseValue()))));
3126 }
3127 }
3128
visitLandingPadInst__anonde7c31650411::MemorySanitizerVisitor3129 void visitLandingPadInst(LandingPadInst &I) {
3130 // Do nothing.
3131 // See https://github.com/google/sanitizers/issues/504
3132 setShadow(&I, getCleanShadow(&I));
3133 setOrigin(&I, getCleanOrigin());
3134 }
3135
visitCatchSwitchInst__anonde7c31650411::MemorySanitizerVisitor3136 void visitCatchSwitchInst(CatchSwitchInst &I) {
3137 setShadow(&I, getCleanShadow(&I));
3138 setOrigin(&I, getCleanOrigin());
3139 }
3140
visitFuncletPadInst__anonde7c31650411::MemorySanitizerVisitor3141 void visitFuncletPadInst(FuncletPadInst &I) {
3142 setShadow(&I, getCleanShadow(&I));
3143 setOrigin(&I, getCleanOrigin());
3144 }
3145
visitGetElementPtrInst__anonde7c31650411::MemorySanitizerVisitor3146 void visitGetElementPtrInst(GetElementPtrInst &I) {
3147 handleShadowOr(I);
3148 }
3149
visitExtractValueInst__anonde7c31650411::MemorySanitizerVisitor3150 void visitExtractValueInst(ExtractValueInst &I) {
3151 IRBuilder<> IRB(&I);
3152 Value *Agg = I.getAggregateOperand();
3153 LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n");
3154 Value *AggShadow = getShadow(Agg);
3155 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
3156 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
3157 LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
3158 setShadow(&I, ResShadow);
3159 setOriginForNaryOp(I);
3160 }
3161
visitInsertValueInst__anonde7c31650411::MemorySanitizerVisitor3162 void visitInsertValueInst(InsertValueInst &I) {
3163 IRBuilder<> IRB(&I);
3164 LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n");
3165 Value *AggShadow = getShadow(I.getAggregateOperand());
3166 Value *InsShadow = getShadow(I.getInsertedValueOperand());
3167 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
3168 LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
3169 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
3170 LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n");
3171 setShadow(&I, Res);
3172 setOriginForNaryOp(I);
3173 }
3174
dumpInst__anonde7c31650411::MemorySanitizerVisitor3175 void dumpInst(Instruction &I) {
3176 if (CallInst *CI = dyn_cast<CallInst>(&I)) {
3177 errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
3178 } else {
3179 errs() << "ZZZ " << I.getOpcodeName() << "\n";
3180 }
3181 errs() << "QQQ " << I << "\n";
3182 }
3183
visitResumeInst__anonde7c31650411::MemorySanitizerVisitor3184 void visitResumeInst(ResumeInst &I) {
3185 LLVM_DEBUG(dbgs() << "Resume: " << I << "\n");
3186 // Nothing to do here.
3187 }
3188
visitCleanupReturnInst__anonde7c31650411::MemorySanitizerVisitor3189 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
3190 LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
3191 // Nothing to do here.
3192 }
3193
visitCatchReturnInst__anonde7c31650411::MemorySanitizerVisitor3194 void visitCatchReturnInst(CatchReturnInst &CRI) {
3195 LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
3196 // Nothing to do here.
3197 }
3198
visitAsmInstruction__anonde7c31650411::MemorySanitizerVisitor3199 void visitAsmInstruction(Instruction &I) {
3200 // Conservative inline assembly handling: check for poisoned shadow of
3201 // asm() arguments, then unpoison the result and all the memory locations
3202 // pointed to by those arguments.
3203 CallInst *CI = dyn_cast<CallInst>(&I);
3204
3205 for (size_t i = 0, n = CI->getNumOperands(); i < n; i++) {
3206 Value *Operand = CI->getOperand(i);
3207 if (Operand->getType()->isSized())
3208 insertShadowCheck(Operand, &I);
3209 }
3210 setShadow(&I, getCleanShadow(&I));
3211 setOrigin(&I, getCleanOrigin());
3212 IRBuilder<> IRB(&I);
3213 IRB.SetInsertPoint(I.getNextNode());
3214 for (size_t i = 0, n = CI->getNumOperands(); i < n; i++) {
3215 Value *Operand = CI->getOperand(i);
3216 Type *OpType = Operand->getType();
3217 if (!OpType->isPointerTy())
3218 continue;
3219 Type *ElType = OpType->getPointerElementType();
3220 if (!ElType->isSized())
3221 continue;
3222 Value *ShadowPtr, *OriginPtr;
3223 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3224 Operand, IRB, ElType, /*Alignment*/ 1, /*isStore*/ true);
3225 Value *CShadow = getCleanShadow(ElType);
3226 IRB.CreateStore(
3227 CShadow,
3228 IRB.CreatePointerCast(ShadowPtr, CShadow->getType()->getPointerTo()));
3229 }
3230 }
3231
visitInstruction__anonde7c31650411::MemorySanitizerVisitor3232 void visitInstruction(Instruction &I) {
3233 // Everything else: stop propagating and check for poisoned shadow.
3234 if (ClDumpStrictInstructions)
3235 dumpInst(I);
3236 LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n");
3237 for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
3238 Value *Operand = I.getOperand(i);
3239 if (Operand->getType()->isSized())
3240 insertShadowCheck(Operand, &I);
3241 }
3242 setShadow(&I, getCleanShadow(&I));
3243 setOrigin(&I, getCleanOrigin());
3244 }
3245 };
3246
3247 /// AMD64-specific implementation of VarArgHelper.
3248 struct VarArgAMD64Helper : public VarArgHelper {
3249 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
3250 // See a comment in visitCallSite for more details.
3251 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
3252 static const unsigned AMD64FpEndOffset = 176;
3253
3254 Function &F;
3255 MemorySanitizer &MS;
3256 MemorySanitizerVisitor &MSV;
3257 Value *VAArgTLSCopy = nullptr;
3258 Value *VAArgOverflowSize = nullptr;
3259
3260 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3261
3262 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
3263
VarArgAMD64Helper__anonde7c31650411::VarArgAMD64Helper3264 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
3265 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
3266
classifyArgument__anonde7c31650411::VarArgAMD64Helper3267 ArgKind classifyArgument(Value* arg) {
3268 // A very rough approximation of X86_64 argument classification rules.
3269 Type *T = arg->getType();
3270 if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
3271 return AK_FloatingPoint;
3272 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
3273 return AK_GeneralPurpose;
3274 if (T->isPointerTy())
3275 return AK_GeneralPurpose;
3276 return AK_Memory;
3277 }
3278
3279 // For VarArg functions, store the argument shadow in an ABI-specific format
3280 // that corresponds to va_list layout.
3281 // We do this because Clang lowers va_arg in the frontend, and this pass
3282 // only sees the low level code that deals with va_list internals.
3283 // A much easier alternative (provided that Clang emits va_arg instructions)
3284 // would have been to associate each live instance of va_list with a copy of
3285 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
3286 // order.
visitCallSite__anonde7c31650411::VarArgAMD64Helper3287 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3288 unsigned GpOffset = 0;
3289 unsigned FpOffset = AMD64GpEndOffset;
3290 unsigned OverflowOffset = AMD64FpEndOffset;
3291 const DataLayout &DL = F.getParent()->getDataLayout();
3292 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3293 ArgIt != End; ++ArgIt) {
3294 Value *A = *ArgIt;
3295 unsigned ArgNo = CS.getArgumentNo(ArgIt);
3296 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3297 bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
3298 if (IsByVal) {
3299 // ByVal arguments always go to the overflow area.
3300 // Fixed arguments passed through the overflow area will be stepped
3301 // over by va_start, so don't count them towards the offset.
3302 if (IsFixed)
3303 continue;
3304 assert(A->getType()->isPointerTy());
3305 Type *RealTy = A->getType()->getPointerElementType();
3306 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
3307 Value *ShadowBase =
3308 getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
3309 OverflowOffset += alignTo(ArgSize, 8);
3310 Value *ShadowPtr, *OriginPtr;
3311 std::tie(ShadowPtr, OriginPtr) =
3312 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
3313 /*isStore*/ false);
3314
3315 IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
3316 kShadowTLSAlignment, ArgSize);
3317 } else {
3318 ArgKind AK = classifyArgument(A);
3319 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
3320 AK = AK_Memory;
3321 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
3322 AK = AK_Memory;
3323 Value *ShadowBase;
3324 switch (AK) {
3325 case AK_GeneralPurpose:
3326 ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, GpOffset);
3327 GpOffset += 8;
3328 break;
3329 case AK_FloatingPoint:
3330 ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, FpOffset);
3331 FpOffset += 16;
3332 break;
3333 case AK_Memory:
3334 if (IsFixed)
3335 continue;
3336 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3337 ShadowBase =
3338 getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset);
3339 OverflowOffset += alignTo(ArgSize, 8);
3340 }
3341 // Take fixed arguments into account for GpOffset and FpOffset,
3342 // but don't actually store shadows for them.
3343 if (IsFixed)
3344 continue;
3345 IRB.CreateAlignedStore(MSV.getShadow(A), ShadowBase,
3346 kShadowTLSAlignment);
3347 }
3348 }
3349 Constant *OverflowSize =
3350 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
3351 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
3352 }
3353
3354 /// Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anonde7c31650411::VarArgAMD64Helper3355 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3356 int ArgOffset) {
3357 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3358 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3359 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3360 "_msarg");
3361 }
3362
unpoisonVAListTagForInst__anonde7c31650411::VarArgAMD64Helper3363 void unpoisonVAListTagForInst(IntrinsicInst &I) {
3364 IRBuilder<> IRB(&I);
3365 Value *VAListTag = I.getArgOperand(0);
3366 Value *ShadowPtr, *OriginPtr;
3367 unsigned Alignment = 8;
3368 std::tie(ShadowPtr, OriginPtr) =
3369 MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
3370 /*isStore*/ true);
3371
3372 // Unpoison the whole __va_list_tag.
3373 // FIXME: magic ABI constants.
3374 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3375 /* size */ 24, Alignment, false);
3376 // We shouldn't need to zero out the origins, as they're only checked for
3377 // nonzero shadow.
3378 }
3379
visitVAStartInst__anonde7c31650411::VarArgAMD64Helper3380 void visitVAStartInst(VAStartInst &I) override {
3381 if (F.getCallingConv() == CallingConv::Win64)
3382 return;
3383 VAStartInstrumentationList.push_back(&I);
3384 unpoisonVAListTagForInst(I);
3385 }
3386
visitVACopyInst__anonde7c31650411::VarArgAMD64Helper3387 void visitVACopyInst(VACopyInst &I) override {
3388 if (F.getCallingConv() == CallingConv::Win64) return;
3389 unpoisonVAListTagForInst(I);
3390 }
3391
finalizeInstrumentation__anonde7c31650411::VarArgAMD64Helper3392 void finalizeInstrumentation() override {
3393 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
3394 "finalizeInstrumentation called twice");
3395 if (!VAStartInstrumentationList.empty()) {
3396 // If there is a va_start in this function, make a backup copy of
3397 // va_arg_tls somewhere in the function entry block.
3398 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
3399 VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3400 Value *CopySize =
3401 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
3402 VAArgOverflowSize);
3403 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3404 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
3405 }
3406
3407 // Instrument va_start.
3408 // Copy va_list shadow from the backup copy of the TLS contents.
3409 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3410 CallInst *OrigInst = VAStartInstrumentationList[i];
3411 IRBuilder<> IRB(OrigInst->getNextNode());
3412 Value *VAListTag = OrigInst->getArgOperand(0);
3413
3414 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
3415 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3416 ConstantInt::get(MS.IntptrTy, 16)),
3417 PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
3418 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3419 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
3420 unsigned Alignment = 16;
3421 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
3422 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3423 Alignment, /*isStore*/ true);
3424 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
3425 AMD64FpEndOffset);
3426 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
3427 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3428 ConstantInt::get(MS.IntptrTy, 8)),
3429 PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
3430 Value *OverflowArgAreaPtr = IRB.CreateLoad(OverflowArgAreaPtrPtr);
3431 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
3432 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
3433 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
3434 Alignment, /*isStore*/ true);
3435 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
3436 AMD64FpEndOffset);
3437 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
3438 VAArgOverflowSize);
3439 }
3440 }
3441 };
3442
3443 /// MIPS64-specific implementation of VarArgHelper.
3444 struct VarArgMIPS64Helper : public VarArgHelper {
3445 Function &F;
3446 MemorySanitizer &MS;
3447 MemorySanitizerVisitor &MSV;
3448 Value *VAArgTLSCopy = nullptr;
3449 Value *VAArgSize = nullptr;
3450
3451 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3452
VarArgMIPS64Helper__anonde7c31650411::VarArgMIPS64Helper3453 VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
3454 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
3455
visitCallSite__anonde7c31650411::VarArgMIPS64Helper3456 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3457 unsigned VAArgOffset = 0;
3458 const DataLayout &DL = F.getParent()->getDataLayout();
3459 for (CallSite::arg_iterator ArgIt = CS.arg_begin() +
3460 CS.getFunctionType()->getNumParams(), End = CS.arg_end();
3461 ArgIt != End; ++ArgIt) {
3462 Triple TargetTriple(F.getParent()->getTargetTriple());
3463 Value *A = *ArgIt;
3464 Value *Base;
3465 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3466 if (TargetTriple.getArch() == Triple::mips64) {
3467 // Adjusting the shadow for argument with size < 8 to match the placement
3468 // of bits in big endian system
3469 if (ArgSize < 8)
3470 VAArgOffset += (8 - ArgSize);
3471 }
3472 Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset);
3473 VAArgOffset += ArgSize;
3474 VAArgOffset = alignTo(VAArgOffset, 8);
3475 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3476 }
3477
3478 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
3479 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
3480 // a new class member i.e. it is the total size of all VarArgs.
3481 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
3482 }
3483
3484 /// Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anonde7c31650411::VarArgMIPS64Helper3485 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3486 int ArgOffset) {
3487 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3488 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3489 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3490 "_msarg");
3491 }
3492
visitVAStartInst__anonde7c31650411::VarArgMIPS64Helper3493 void visitVAStartInst(VAStartInst &I) override {
3494 IRBuilder<> IRB(&I);
3495 VAStartInstrumentationList.push_back(&I);
3496 Value *VAListTag = I.getArgOperand(0);
3497 Value *ShadowPtr, *OriginPtr;
3498 unsigned Alignment = 8;
3499 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3500 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3501 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3502 /* size */ 8, Alignment, false);
3503 }
3504
visitVACopyInst__anonde7c31650411::VarArgMIPS64Helper3505 void visitVACopyInst(VACopyInst &I) override {
3506 IRBuilder<> IRB(&I);
3507 VAStartInstrumentationList.push_back(&I);
3508 Value *VAListTag = I.getArgOperand(0);
3509 Value *ShadowPtr, *OriginPtr;
3510 unsigned Alignment = 8;
3511 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3512 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3513 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3514 /* size */ 8, Alignment, false);
3515 }
3516
finalizeInstrumentation__anonde7c31650411::VarArgMIPS64Helper3517 void finalizeInstrumentation() override {
3518 assert(!VAArgSize && !VAArgTLSCopy &&
3519 "finalizeInstrumentation called twice");
3520 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
3521 VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3522 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
3523 VAArgSize);
3524
3525 if (!VAStartInstrumentationList.empty()) {
3526 // If there is a va_start in this function, make a backup copy of
3527 // va_arg_tls somewhere in the function entry block.
3528 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3529 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
3530 }
3531
3532 // Instrument va_start.
3533 // Copy va_list shadow from the backup copy of the TLS contents.
3534 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3535 CallInst *OrigInst = VAStartInstrumentationList[i];
3536 IRBuilder<> IRB(OrigInst->getNextNode());
3537 Value *VAListTag = OrigInst->getArgOperand(0);
3538 Value *RegSaveAreaPtrPtr =
3539 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3540 PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
3541 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3542 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
3543 unsigned Alignment = 8;
3544 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
3545 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3546 Alignment, /*isStore*/ true);
3547 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
3548 CopySize);
3549 }
3550 }
3551 };
3552
3553 /// AArch64-specific implementation of VarArgHelper.
3554 struct VarArgAArch64Helper : public VarArgHelper {
3555 static const unsigned kAArch64GrArgSize = 64;
3556 static const unsigned kAArch64VrArgSize = 128;
3557
3558 static const unsigned AArch64GrBegOffset = 0;
3559 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
3560 // Make VR space aligned to 16 bytes.
3561 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
3562 static const unsigned AArch64VrEndOffset = AArch64VrBegOffset
3563 + kAArch64VrArgSize;
3564 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
3565
3566 Function &F;
3567 MemorySanitizer &MS;
3568 MemorySanitizerVisitor &MSV;
3569 Value *VAArgTLSCopy = nullptr;
3570 Value *VAArgOverflowSize = nullptr;
3571
3572 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3573
3574 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
3575
VarArgAArch64Helper__anonde7c31650411::VarArgAArch64Helper3576 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
3577 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
3578
classifyArgument__anonde7c31650411::VarArgAArch64Helper3579 ArgKind classifyArgument(Value* arg) {
3580 Type *T = arg->getType();
3581 if (T->isFPOrFPVectorTy())
3582 return AK_FloatingPoint;
3583 if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
3584 || (T->isPointerTy()))
3585 return AK_GeneralPurpose;
3586 return AK_Memory;
3587 }
3588
3589 // The instrumentation stores the argument shadow in a non ABI-specific
3590 // format because it does not know which argument is named (since Clang,
3591 // like x86_64 case, lowers the va_args in the frontend and this pass only
3592 // sees the low level code that deals with va_list internals).
3593 // The first seven GR registers are saved in the first 56 bytes of the
3594 // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
3595 // the remaining arguments.
3596 // Using constant offset within the va_arg TLS array allows fast copy
3597 // in the finalize instrumentation.
visitCallSite__anonde7c31650411::VarArgAArch64Helper3598 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3599 unsigned GrOffset = AArch64GrBegOffset;
3600 unsigned VrOffset = AArch64VrBegOffset;
3601 unsigned OverflowOffset = AArch64VAEndOffset;
3602
3603 const DataLayout &DL = F.getParent()->getDataLayout();
3604 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3605 ArgIt != End; ++ArgIt) {
3606 Value *A = *ArgIt;
3607 unsigned ArgNo = CS.getArgumentNo(ArgIt);
3608 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3609 ArgKind AK = classifyArgument(A);
3610 if (AK == AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset)
3611 AK = AK_Memory;
3612 if (AK == AK_FloatingPoint && VrOffset >= AArch64VrEndOffset)
3613 AK = AK_Memory;
3614 Value *Base;
3615 switch (AK) {
3616 case AK_GeneralPurpose:
3617 Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset);
3618 GrOffset += 8;
3619 break;
3620 case AK_FloatingPoint:
3621 Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset);
3622 VrOffset += 16;
3623 break;
3624 case AK_Memory:
3625 // Don't count fixed arguments in the overflow area - va_start will
3626 // skip right over them.
3627 if (IsFixed)
3628 continue;
3629 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3630 Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset);
3631 OverflowOffset += alignTo(ArgSize, 8);
3632 break;
3633 }
3634 // Count Gp/Vr fixed arguments to their respective offsets, but don't
3635 // bother to actually store a shadow.
3636 if (IsFixed)
3637 continue;
3638 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3639 }
3640 Constant *OverflowSize =
3641 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
3642 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
3643 }
3644
3645 /// Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anonde7c31650411::VarArgAArch64Helper3646 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3647 int ArgOffset) {
3648 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3649 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3650 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3651 "_msarg");
3652 }
3653
visitVAStartInst__anonde7c31650411::VarArgAArch64Helper3654 void visitVAStartInst(VAStartInst &I) override {
3655 IRBuilder<> IRB(&I);
3656 VAStartInstrumentationList.push_back(&I);
3657 Value *VAListTag = I.getArgOperand(0);
3658 Value *ShadowPtr, *OriginPtr;
3659 unsigned Alignment = 8;
3660 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3661 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3662 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3663 /* size */ 32, Alignment, false);
3664 }
3665
visitVACopyInst__anonde7c31650411::VarArgAArch64Helper3666 void visitVACopyInst(VACopyInst &I) override {
3667 IRBuilder<> IRB(&I);
3668 VAStartInstrumentationList.push_back(&I);
3669 Value *VAListTag = I.getArgOperand(0);
3670 Value *ShadowPtr, *OriginPtr;
3671 unsigned Alignment = 8;
3672 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3673 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3674 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3675 /* size */ 32, Alignment, false);
3676 }
3677
3678 // Retrieve a va_list field of 'void*' size.
getVAField64__anonde7c31650411::VarArgAArch64Helper3679 Value* getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
3680 Value *SaveAreaPtrPtr =
3681 IRB.CreateIntToPtr(
3682 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3683 ConstantInt::get(MS.IntptrTy, offset)),
3684 Type::getInt64PtrTy(*MS.C));
3685 return IRB.CreateLoad(SaveAreaPtrPtr);
3686 }
3687
3688 // Retrieve a va_list field of 'int' size.
getVAField32__anonde7c31650411::VarArgAArch64Helper3689 Value* getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
3690 Value *SaveAreaPtr =
3691 IRB.CreateIntToPtr(
3692 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3693 ConstantInt::get(MS.IntptrTy, offset)),
3694 Type::getInt32PtrTy(*MS.C));
3695 Value *SaveArea32 = IRB.CreateLoad(SaveAreaPtr);
3696 return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
3697 }
3698
finalizeInstrumentation__anonde7c31650411::VarArgAArch64Helper3699 void finalizeInstrumentation() override {
3700 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
3701 "finalizeInstrumentation called twice");
3702 if (!VAStartInstrumentationList.empty()) {
3703 // If there is a va_start in this function, make a backup copy of
3704 // va_arg_tls somewhere in the function entry block.
3705 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
3706 VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3707 Value *CopySize =
3708 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
3709 VAArgOverflowSize);
3710 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3711 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
3712 }
3713
3714 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
3715 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
3716
3717 // Instrument va_start, copy va_list shadow from the backup copy of
3718 // the TLS contents.
3719 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3720 CallInst *OrigInst = VAStartInstrumentationList[i];
3721 IRBuilder<> IRB(OrigInst->getNextNode());
3722
3723 Value *VAListTag = OrigInst->getArgOperand(0);
3724
3725 // The variadic ABI for AArch64 creates two areas to save the incoming
3726 // argument registers (one for 64-bit general register xn-x7 and another
3727 // for 128-bit FP/SIMD vn-v7).
3728 // We need then to propagate the shadow arguments on both regions
3729 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
3730 // The remaning arguments are saved on shadow for 'va::stack'.
3731 // One caveat is it requires only to propagate the non-named arguments,
3732 // however on the call site instrumentation 'all' the arguments are
3733 // saved. So to copy the shadow values from the va_arg TLS array
3734 // we need to adjust the offset for both GR and VR fields based on
3735 // the __{gr,vr}_offs value (since they are stores based on incoming
3736 // named arguments).
3737
3738 // Read the stack pointer from the va_list.
3739 Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
3740
3741 // Read both the __gr_top and __gr_off and add them up.
3742 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
3743 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
3744
3745 Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
3746
3747 // Read both the __vr_top and __vr_off and add them up.
3748 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
3749 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
3750
3751 Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
3752
3753 // It does not know how many named arguments is being used and, on the
3754 // callsite all the arguments were saved. Since __gr_off is defined as
3755 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
3756 // argument by ignoring the bytes of shadow from named arguments.
3757 Value *GrRegSaveAreaShadowPtrOff =
3758 IRB.CreateAdd(GrArgSize, GrOffSaveArea);
3759
3760 Value *GrRegSaveAreaShadowPtr =
3761 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3762 /*Alignment*/ 8, /*isStore*/ true)
3763 .first;
3764
3765 Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3766 GrRegSaveAreaShadowPtrOff);
3767 Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
3768
3769 IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, 8, GrSrcPtr, 8, GrCopySize);
3770
3771 // Again, but for FP/SIMD values.
3772 Value *VrRegSaveAreaShadowPtrOff =
3773 IRB.CreateAdd(VrArgSize, VrOffSaveArea);
3774
3775 Value *VrRegSaveAreaShadowPtr =
3776 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3777 /*Alignment*/ 8, /*isStore*/ true)
3778 .first;
3779
3780 Value *VrSrcPtr = IRB.CreateInBoundsGEP(
3781 IRB.getInt8Ty(),
3782 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3783 IRB.getInt32(AArch64VrBegOffset)),
3784 VrRegSaveAreaShadowPtrOff);
3785 Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
3786
3787 IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, 8, VrSrcPtr, 8, VrCopySize);
3788
3789 // And finally for remaining arguments.
3790 Value *StackSaveAreaShadowPtr =
3791 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
3792 /*Alignment*/ 16, /*isStore*/ true)
3793 .first;
3794
3795 Value *StackSrcPtr =
3796 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3797 IRB.getInt32(AArch64VAEndOffset));
3798
3799 IRB.CreateMemCpy(StackSaveAreaShadowPtr, 16, StackSrcPtr, 16,
3800 VAArgOverflowSize);
3801 }
3802 }
3803 };
3804
3805 /// PowerPC64-specific implementation of VarArgHelper.
3806 struct VarArgPowerPC64Helper : public VarArgHelper {
3807 Function &F;
3808 MemorySanitizer &MS;
3809 MemorySanitizerVisitor &MSV;
3810 Value *VAArgTLSCopy = nullptr;
3811 Value *VAArgSize = nullptr;
3812
3813 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3814
VarArgPowerPC64Helper__anonde7c31650411::VarArgPowerPC64Helper3815 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
3816 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
3817
visitCallSite__anonde7c31650411::VarArgPowerPC64Helper3818 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3819 // For PowerPC, we need to deal with alignment of stack arguments -
3820 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
3821 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
3822 // and QPX vectors are aligned to 32 bytes. For that reason, we
3823 // compute current offset from stack pointer (which is always properly
3824 // aligned), and offset for the first vararg, then subtract them.
3825 unsigned VAArgBase;
3826 Triple TargetTriple(F.getParent()->getTargetTriple());
3827 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
3828 // and 32 bytes for ABIv2. This is usually determined by target
3829 // endianness, but in theory could be overriden by function attribute.
3830 // For simplicity, we ignore it here (it'd only matter for QPX vectors).
3831 if (TargetTriple.getArch() == Triple::ppc64)
3832 VAArgBase = 48;
3833 else
3834 VAArgBase = 32;
3835 unsigned VAArgOffset = VAArgBase;
3836 const DataLayout &DL = F.getParent()->getDataLayout();
3837 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3838 ArgIt != End; ++ArgIt) {
3839 Value *A = *ArgIt;
3840 unsigned ArgNo = CS.getArgumentNo(ArgIt);
3841 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3842 bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
3843 if (IsByVal) {
3844 assert(A->getType()->isPointerTy());
3845 Type *RealTy = A->getType()->getPointerElementType();
3846 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
3847 uint64_t ArgAlign = CS.getParamAlignment(ArgNo);
3848 if (ArgAlign < 8)
3849 ArgAlign = 8;
3850 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
3851 if (!IsFixed) {
3852 Value *Base = getShadowPtrForVAArgument(RealTy, IRB,
3853 VAArgOffset - VAArgBase);
3854 Value *AShadowPtr, *AOriginPtr;
3855 std::tie(AShadowPtr, AOriginPtr) = MSV.getShadowOriginPtr(
3856 A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment, /*isStore*/ false);
3857
3858 IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
3859 kShadowTLSAlignment, ArgSize);
3860 }
3861 VAArgOffset += alignTo(ArgSize, 8);
3862 } else {
3863 Value *Base;
3864 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3865 uint64_t ArgAlign = 8;
3866 if (A->getType()->isArrayTy()) {
3867 // Arrays are aligned to element size, except for long double
3868 // arrays, which are aligned to 8 bytes.
3869 Type *ElementTy = A->getType()->getArrayElementType();
3870 if (!ElementTy->isPPC_FP128Ty())
3871 ArgAlign = DL.getTypeAllocSize(ElementTy);
3872 } else if (A->getType()->isVectorTy()) {
3873 // Vectors are naturally aligned.
3874 ArgAlign = DL.getTypeAllocSize(A->getType());
3875 }
3876 if (ArgAlign < 8)
3877 ArgAlign = 8;
3878 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
3879 if (DL.isBigEndian()) {
3880 // Adjusting the shadow for argument with size < 8 to match the placement
3881 // of bits in big endian system
3882 if (ArgSize < 8)
3883 VAArgOffset += (8 - ArgSize);
3884 }
3885 if (!IsFixed) {
3886 Base = getShadowPtrForVAArgument(A->getType(), IRB,
3887 VAArgOffset - VAArgBase);
3888 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3889 }
3890 VAArgOffset += ArgSize;
3891 VAArgOffset = alignTo(VAArgOffset, 8);
3892 }
3893 if (IsFixed)
3894 VAArgBase = VAArgOffset;
3895 }
3896
3897 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(),
3898 VAArgOffset - VAArgBase);
3899 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
3900 // a new class member i.e. it is the total size of all VarArgs.
3901 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
3902 }
3903
3904 /// Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anonde7c31650411::VarArgPowerPC64Helper3905 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3906 int ArgOffset) {
3907 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3908 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3909 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3910 "_msarg");
3911 }
3912
visitVAStartInst__anonde7c31650411::VarArgPowerPC64Helper3913 void visitVAStartInst(VAStartInst &I) override {
3914 IRBuilder<> IRB(&I);
3915 VAStartInstrumentationList.push_back(&I);
3916 Value *VAListTag = I.getArgOperand(0);
3917 Value *ShadowPtr, *OriginPtr;
3918 unsigned Alignment = 8;
3919 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3920 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3921 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3922 /* size */ 8, Alignment, false);
3923 }
3924
visitVACopyInst__anonde7c31650411::VarArgPowerPC64Helper3925 void visitVACopyInst(VACopyInst &I) override {
3926 IRBuilder<> IRB(&I);
3927 Value *VAListTag = I.getArgOperand(0);
3928 Value *ShadowPtr, *OriginPtr;
3929 unsigned Alignment = 8;
3930 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3931 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3932 // Unpoison the whole __va_list_tag.
3933 // FIXME: magic ABI constants.
3934 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3935 /* size */ 8, Alignment, false);
3936 }
3937
finalizeInstrumentation__anonde7c31650411::VarArgPowerPC64Helper3938 void finalizeInstrumentation() override {
3939 assert(!VAArgSize && !VAArgTLSCopy &&
3940 "finalizeInstrumentation called twice");
3941 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
3942 VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3943 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
3944 VAArgSize);
3945
3946 if (!VAStartInstrumentationList.empty()) {
3947 // If there is a va_start in this function, make a backup copy of
3948 // va_arg_tls somewhere in the function entry block.
3949 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3950 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
3951 }
3952
3953 // Instrument va_start.
3954 // Copy va_list shadow from the backup copy of the TLS contents.
3955 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3956 CallInst *OrigInst = VAStartInstrumentationList[i];
3957 IRBuilder<> IRB(OrigInst->getNextNode());
3958 Value *VAListTag = OrigInst->getArgOperand(0);
3959 Value *RegSaveAreaPtrPtr =
3960 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3961 PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
3962 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3963 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
3964 unsigned Alignment = 8;
3965 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
3966 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3967 Alignment, /*isStore*/ true);
3968 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
3969 CopySize);
3970 }
3971 }
3972 };
3973
3974 /// A no-op implementation of VarArgHelper.
3975 struct VarArgNoOpHelper : public VarArgHelper {
VarArgNoOpHelper__anonde7c31650411::VarArgNoOpHelper3976 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
3977 MemorySanitizerVisitor &MSV) {}
3978
visitCallSite__anonde7c31650411::VarArgNoOpHelper3979 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {}
3980
visitVAStartInst__anonde7c31650411::VarArgNoOpHelper3981 void visitVAStartInst(VAStartInst &I) override {}
3982
visitVACopyInst__anonde7c31650411::VarArgNoOpHelper3983 void visitVACopyInst(VACopyInst &I) override {}
3984
finalizeInstrumentation__anonde7c31650411::VarArgNoOpHelper3985 void finalizeInstrumentation() override {}
3986 };
3987
3988 } // end anonymous namespace
3989
CreateVarArgHelper(Function & Func,MemorySanitizer & Msan,MemorySanitizerVisitor & Visitor)3990 static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
3991 MemorySanitizerVisitor &Visitor) {
3992 // VarArg handling is only implemented on AMD64. False positives are possible
3993 // on other platforms.
3994 Triple TargetTriple(Func.getParent()->getTargetTriple());
3995 if (TargetTriple.getArch() == Triple::x86_64)
3996 return new VarArgAMD64Helper(Func, Msan, Visitor);
3997 else if (TargetTriple.isMIPS64())
3998 return new VarArgMIPS64Helper(Func, Msan, Visitor);
3999 else if (TargetTriple.getArch() == Triple::aarch64)
4000 return new VarArgAArch64Helper(Func, Msan, Visitor);
4001 else if (TargetTriple.getArch() == Triple::ppc64 ||
4002 TargetTriple.getArch() == Triple::ppc64le)
4003 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
4004 else
4005 return new VarArgNoOpHelper(Func, Msan, Visitor);
4006 }
4007
runOnFunction(Function & F)4008 bool MemorySanitizer::runOnFunction(Function &F) {
4009 if (&F == MsanCtorFunction)
4010 return false;
4011 MemorySanitizerVisitor Visitor(F, *this);
4012
4013 // Clear out readonly/readnone attributes.
4014 AttrBuilder B;
4015 B.addAttribute(Attribute::ReadOnly)
4016 .addAttribute(Attribute::ReadNone);
4017 F.removeAttributes(AttributeList::FunctionIndex, B);
4018
4019 return Visitor.runOnFunction();
4020 }
4021