109467b48Spatrick //===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
209467b48Spatrick //
309467b48Spatrick // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
409467b48Spatrick // See https://llvm.org/LICENSE.txt for license information.
509467b48Spatrick // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
609467b48Spatrick //
709467b48Spatrick //===----------------------------------------------------------------------===//
809467b48Spatrick //
909467b48Spatrick /// \file
1009467b48Spatrick /// This file is a part of MemorySanitizer, a detector of uninitialized
1109467b48Spatrick /// reads.
1209467b48Spatrick ///
1309467b48Spatrick /// The algorithm of the tool is similar to Memcheck
1409467b48Spatrick /// (http://goo.gl/QKbem). We associate a few shadow bits with every
1509467b48Spatrick /// byte of the application memory, poison the shadow of the malloc-ed
1609467b48Spatrick /// or alloca-ed memory, load the shadow bits on every memory read,
1709467b48Spatrick /// propagate the shadow bits through some of the arithmetic
1809467b48Spatrick /// instruction (including MOV), store the shadow bits on every memory
1909467b48Spatrick /// write, report a bug on some other instructions (e.g. JMP) if the
2009467b48Spatrick /// associated shadow is poisoned.
2109467b48Spatrick ///
2209467b48Spatrick /// But there are differences too. The first and the major one:
2309467b48Spatrick /// compiler instrumentation instead of binary instrumentation. This
2409467b48Spatrick /// gives us much better register allocation, possible compiler
2509467b48Spatrick /// optimizations and a fast start-up. But this brings the major issue
2609467b48Spatrick /// as well: msan needs to see all program events, including system
2709467b48Spatrick /// calls and reads/writes in system libraries, so we either need to
2809467b48Spatrick /// compile *everything* with msan or use a binary translation
2909467b48Spatrick /// component (e.g. DynamoRIO) to instrument pre-built libraries.
3009467b48Spatrick /// Another difference from Memcheck is that we use 8 shadow bits per
3109467b48Spatrick /// byte of application memory and use a direct shadow mapping. This
3209467b48Spatrick /// greatly simplifies the instrumentation code and avoids races on
3309467b48Spatrick /// shadow updates (Memcheck is single-threaded so races are not a
3409467b48Spatrick /// concern there. Memcheck uses 2 shadow bits per byte with a slow
3509467b48Spatrick /// path storage that uses 8 bits per byte).
3609467b48Spatrick ///
3709467b48Spatrick /// The default value of shadow is 0, which means "clean" (not poisoned).
3809467b48Spatrick ///
3909467b48Spatrick /// Every module initializer should call __msan_init to ensure that the
4009467b48Spatrick /// shadow memory is ready. On error, __msan_warning is called. Since
4109467b48Spatrick /// parameters and return values may be passed via registers, we have a
4209467b48Spatrick /// specialized thread-local shadow for return values
4309467b48Spatrick /// (__msan_retval_tls) and parameters (__msan_param_tls).
4409467b48Spatrick ///
4509467b48Spatrick /// Origin tracking.
4609467b48Spatrick ///
4709467b48Spatrick /// MemorySanitizer can track origins (allocation points) of all uninitialized
4809467b48Spatrick /// values. This behavior is controlled with a flag (msan-track-origins) and is
4909467b48Spatrick /// disabled by default.
5009467b48Spatrick ///
5109467b48Spatrick /// Origins are 4-byte values created and interpreted by the runtime library.
5209467b48Spatrick /// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
5309467b48Spatrick /// of application memory. Propagation of origins is basically a bunch of
5409467b48Spatrick /// "select" instructions that pick the origin of a dirty argument, if an
5509467b48Spatrick /// instruction has one.
5609467b48Spatrick ///
5709467b48Spatrick /// Every 4 aligned, consecutive bytes of application memory have one origin
5809467b48Spatrick /// value associated with them. If these bytes contain uninitialized data
5909467b48Spatrick /// coming from 2 different allocations, the last store wins. Because of this,
6009467b48Spatrick /// MemorySanitizer reports can show unrelated origins, but this is unlikely in
6109467b48Spatrick /// practice.
6209467b48Spatrick ///
6309467b48Spatrick /// Origins are meaningless for fully initialized values, so MemorySanitizer
6409467b48Spatrick /// avoids storing origin to memory when a fully initialized value is stored.
65097a140dSpatrick /// This way it avoids needless overwriting origin of the 4-byte region on
6609467b48Spatrick /// a short (i.e. 1 byte) clean store, and it is also good for performance.
6709467b48Spatrick ///
6809467b48Spatrick /// Atomic handling.
6909467b48Spatrick ///
7009467b48Spatrick /// Ideally, every atomic store of application value should update the
7109467b48Spatrick /// corresponding shadow location in an atomic way. Unfortunately, atomic store
7209467b48Spatrick /// of two disjoint locations can not be done without severe slowdown.
7309467b48Spatrick ///
7409467b48Spatrick /// Therefore, we implement an approximation that may err on the safe side.
7509467b48Spatrick /// In this implementation, every atomically accessed location in the program
7609467b48Spatrick /// may only change from (partially) uninitialized to fully initialized, but
7709467b48Spatrick /// not the other way around. We load the shadow _after_ the application load,
7809467b48Spatrick /// and we store the shadow _before_ the app store. Also, we always store clean
7909467b48Spatrick /// shadow (if the application store is atomic). This way, if the store-load
8009467b48Spatrick /// pair constitutes a happens-before arc, shadow store and load are correctly
8109467b48Spatrick /// ordered such that the load will get either the value that was stored, or
8209467b48Spatrick /// some later value (which is always clean).
8309467b48Spatrick ///
8409467b48Spatrick /// This does not work very well with Compare-And-Swap (CAS) and
8509467b48Spatrick /// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
8609467b48Spatrick /// must store the new shadow before the app operation, and load the shadow
8709467b48Spatrick /// after the app operation. Computers don't work this way. Current
8809467b48Spatrick /// implementation ignores the load aspect of CAS/RMW, always returning a clean
8909467b48Spatrick /// value. It implements the store part as a simple atomic store by storing a
9009467b48Spatrick /// clean shadow.
9109467b48Spatrick ///
9209467b48Spatrick /// Instrumenting inline assembly.
9309467b48Spatrick ///
9409467b48Spatrick /// For inline assembly code LLVM has little idea about which memory locations
9509467b48Spatrick /// become initialized depending on the arguments. It can be possible to figure
9609467b48Spatrick /// out which arguments are meant to point to inputs and outputs, but the
9709467b48Spatrick /// actual semantics can be only visible at runtime. In the Linux kernel it's
9809467b48Spatrick /// also possible that the arguments only indicate the offset for a base taken
9909467b48Spatrick /// from a segment register, so it's dangerous to treat any asm() arguments as
10009467b48Spatrick /// pointers. We take a conservative approach generating calls to
10109467b48Spatrick /// __msan_instrument_asm_store(ptr, size)
10209467b48Spatrick /// , which defer the memory unpoisoning to the runtime library.
10309467b48Spatrick /// The latter can perform more complex address checks to figure out whether
10409467b48Spatrick /// it's safe to touch the shadow memory.
10509467b48Spatrick /// Like with atomic operations, we call __msan_instrument_asm_store() before
10609467b48Spatrick /// the assembly call, so that changes to the shadow memory will be seen by
10709467b48Spatrick /// other threads together with main memory initialization.
10809467b48Spatrick ///
10909467b48Spatrick /// KernelMemorySanitizer (KMSAN) implementation.
11009467b48Spatrick ///
11109467b48Spatrick /// The major differences between KMSAN and MSan instrumentation are:
11209467b48Spatrick /// - KMSAN always tracks the origins and implies msan-keep-going=true;
11309467b48Spatrick /// - KMSAN allocates shadow and origin memory for each page separately, so
11409467b48Spatrick /// there are no explicit accesses to shadow and origin in the
11509467b48Spatrick /// instrumentation.
11609467b48Spatrick /// Shadow and origin values for a particular X-byte memory location
11709467b48Spatrick /// (X=1,2,4,8) are accessed through pointers obtained via the
11809467b48Spatrick /// __msan_metadata_ptr_for_load_X(ptr)
11909467b48Spatrick /// __msan_metadata_ptr_for_store_X(ptr)
12009467b48Spatrick /// functions. The corresponding functions check that the X-byte accesses
12109467b48Spatrick /// are possible and returns the pointers to shadow and origin memory.
12209467b48Spatrick /// Arbitrary sized accesses are handled with:
12309467b48Spatrick /// __msan_metadata_ptr_for_load_n(ptr, size)
12409467b48Spatrick /// __msan_metadata_ptr_for_store_n(ptr, size);
12509467b48Spatrick /// - TLS variables are stored in a single per-task struct. A call to a
12609467b48Spatrick /// function __msan_get_context_state() returning a pointer to that struct
12709467b48Spatrick /// is inserted into every instrumented function before the entry block;
12809467b48Spatrick /// - __msan_warning() takes a 32-bit origin parameter;
12909467b48Spatrick /// - local variables are poisoned with __msan_poison_alloca() upon function
13009467b48Spatrick /// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
13109467b48Spatrick /// function;
13209467b48Spatrick /// - the pass doesn't declare any global variables or add global constructors
13309467b48Spatrick /// to the translation unit.
13409467b48Spatrick ///
13509467b48Spatrick /// Also, KMSAN currently ignores uninitialized memory passed into inline asm
13609467b48Spatrick /// calls, making sure we're on the safe side wrt. possible false positives.
13709467b48Spatrick ///
13809467b48Spatrick /// KernelMemorySanitizer only supports X86_64 at the moment.
13909467b48Spatrick ///
140097a140dSpatrick //
141097a140dSpatrick // FIXME: This sanitizer does not yet handle scalable vectors
142097a140dSpatrick //
14309467b48Spatrick //===----------------------------------------------------------------------===//
14409467b48Spatrick
14509467b48Spatrick #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
14609467b48Spatrick #include "llvm/ADT/APInt.h"
14709467b48Spatrick #include "llvm/ADT/ArrayRef.h"
148*d415bd75Srobert #include "llvm/ADT/DenseMap.h"
14909467b48Spatrick #include "llvm/ADT/DepthFirstIterator.h"
150*d415bd75Srobert #include "llvm/ADT/SetVector.h"
15109467b48Spatrick #include "llvm/ADT/SmallString.h"
15209467b48Spatrick #include "llvm/ADT/SmallVector.h"
15309467b48Spatrick #include "llvm/ADT/StringExtras.h"
15409467b48Spatrick #include "llvm/ADT/StringRef.h"
15509467b48Spatrick #include "llvm/ADT/Triple.h"
156*d415bd75Srobert #include "llvm/Analysis/GlobalsModRef.h"
15709467b48Spatrick #include "llvm/Analysis/TargetLibraryInfo.h"
15873471bf0Spatrick #include "llvm/Analysis/ValueTracking.h"
15909467b48Spatrick #include "llvm/IR/Argument.h"
16009467b48Spatrick #include "llvm/IR/Attributes.h"
16109467b48Spatrick #include "llvm/IR/BasicBlock.h"
16209467b48Spatrick #include "llvm/IR/CallingConv.h"
16309467b48Spatrick #include "llvm/IR/Constant.h"
16409467b48Spatrick #include "llvm/IR/Constants.h"
16509467b48Spatrick #include "llvm/IR/DataLayout.h"
16609467b48Spatrick #include "llvm/IR/DerivedTypes.h"
16709467b48Spatrick #include "llvm/IR/Function.h"
16809467b48Spatrick #include "llvm/IR/GlobalValue.h"
16909467b48Spatrick #include "llvm/IR/GlobalVariable.h"
17009467b48Spatrick #include "llvm/IR/IRBuilder.h"
17109467b48Spatrick #include "llvm/IR/InlineAsm.h"
17209467b48Spatrick #include "llvm/IR/InstVisitor.h"
17309467b48Spatrick #include "llvm/IR/InstrTypes.h"
17409467b48Spatrick #include "llvm/IR/Instruction.h"
17509467b48Spatrick #include "llvm/IR/Instructions.h"
17609467b48Spatrick #include "llvm/IR/IntrinsicInst.h"
17709467b48Spatrick #include "llvm/IR/Intrinsics.h"
17809467b48Spatrick #include "llvm/IR/IntrinsicsX86.h"
17909467b48Spatrick #include "llvm/IR/MDBuilder.h"
18009467b48Spatrick #include "llvm/IR/Module.h"
18109467b48Spatrick #include "llvm/IR/Type.h"
18209467b48Spatrick #include "llvm/IR/Value.h"
18309467b48Spatrick #include "llvm/IR/ValueMap.h"
184*d415bd75Srobert #include "llvm/Support/Alignment.h"
18509467b48Spatrick #include "llvm/Support/AtomicOrdering.h"
18609467b48Spatrick #include "llvm/Support/Casting.h"
18709467b48Spatrick #include "llvm/Support/CommandLine.h"
18809467b48Spatrick #include "llvm/Support/Debug.h"
189*d415bd75Srobert #include "llvm/Support/DebugCounter.h"
19009467b48Spatrick #include "llvm/Support/ErrorHandling.h"
19109467b48Spatrick #include "llvm/Support/MathExtras.h"
19209467b48Spatrick #include "llvm/Support/raw_ostream.h"
19309467b48Spatrick #include "llvm/Transforms/Utils/BasicBlockUtils.h"
19409467b48Spatrick #include "llvm/Transforms/Utils/Local.h"
19509467b48Spatrick #include "llvm/Transforms/Utils/ModuleUtils.h"
19609467b48Spatrick #include <algorithm>
19709467b48Spatrick #include <cassert>
19809467b48Spatrick #include <cstddef>
19909467b48Spatrick #include <cstdint>
20009467b48Spatrick #include <memory>
20109467b48Spatrick #include <string>
20209467b48Spatrick #include <tuple>
20309467b48Spatrick
20409467b48Spatrick using namespace llvm;
20509467b48Spatrick
20609467b48Spatrick #define DEBUG_TYPE "msan"
20709467b48Spatrick
208*d415bd75Srobert DEBUG_COUNTER(DebugInsertCheck, "msan-insert-check",
209*d415bd75Srobert "Controls which checks to insert");
210*d415bd75Srobert
21109467b48Spatrick static const unsigned kOriginSize = 4;
21209467b48Spatrick static const Align kMinOriginAlignment = Align(4);
21309467b48Spatrick static const Align kShadowTLSAlignment = Align(8);
21409467b48Spatrick
21509467b48Spatrick // These constants must be kept in sync with the ones in msan.h.
21609467b48Spatrick static const unsigned kParamTLSSize = 800;
21709467b48Spatrick static const unsigned kRetvalTLSSize = 800;
21809467b48Spatrick
21909467b48Spatrick // Accesses sizes are powers of two: 1, 2, 4, 8.
22009467b48Spatrick static const size_t kNumberOfAccessSizes = 4;
22109467b48Spatrick
22209467b48Spatrick /// Track origins of uninitialized values.
22309467b48Spatrick ///
22409467b48Spatrick /// Adds a section to MemorySanitizer report that points to the allocation
22509467b48Spatrick /// (stack or heap) the uninitialized bits came from originally.
226*d415bd75Srobert static cl::opt<int> ClTrackOrigins(
227*d415bd75Srobert "msan-track-origins",
228*d415bd75Srobert cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden,
229*d415bd75Srobert cl::init(0));
23009467b48Spatrick
23109467b48Spatrick static cl::opt<bool> ClKeepGoing("msan-keep-going",
23209467b48Spatrick cl::desc("keep going after reporting a UMR"),
23309467b48Spatrick cl::Hidden, cl::init(false));
23409467b48Spatrick
235*d415bd75Srobert static cl::opt<bool>
236*d415bd75Srobert ClPoisonStack("msan-poison-stack",
237*d415bd75Srobert cl::desc("poison uninitialized stack variables"), cl::Hidden,
238*d415bd75Srobert cl::init(true));
23909467b48Spatrick
240*d415bd75Srobert static cl::opt<bool> ClPoisonStackWithCall(
241*d415bd75Srobert "msan-poison-stack-with-call",
242*d415bd75Srobert cl::desc("poison uninitialized stack variables with a call"), cl::Hidden,
243*d415bd75Srobert cl::init(false));
24409467b48Spatrick
245*d415bd75Srobert static cl::opt<int> ClPoisonStackPattern(
246*d415bd75Srobert "msan-poison-stack-pattern",
24709467b48Spatrick cl::desc("poison uninitialized stack variables with the given pattern"),
24809467b48Spatrick cl::Hidden, cl::init(0xff));
24909467b48Spatrick
250*d415bd75Srobert static cl::opt<bool>
251*d415bd75Srobert ClPrintStackNames("msan-print-stack-names",
252*d415bd75Srobert cl::desc("Print name of local stack variable"),
25309467b48Spatrick cl::Hidden, cl::init(true));
25409467b48Spatrick
255*d415bd75Srobert static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
256*d415bd75Srobert cl::desc("poison undef temps"), cl::Hidden,
257*d415bd75Srobert cl::init(true));
258*d415bd75Srobert
259*d415bd75Srobert static cl::opt<bool>
260*d415bd75Srobert ClHandleICmp("msan-handle-icmp",
26109467b48Spatrick cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
26209467b48Spatrick cl::Hidden, cl::init(true));
26309467b48Spatrick
264*d415bd75Srobert static cl::opt<bool>
265*d415bd75Srobert ClHandleICmpExact("msan-handle-icmp-exact",
26609467b48Spatrick cl::desc("exact handling of relational integer ICmp"),
26709467b48Spatrick cl::Hidden, cl::init(false));
26809467b48Spatrick
26909467b48Spatrick static cl::opt<bool> ClHandleLifetimeIntrinsics(
27009467b48Spatrick "msan-handle-lifetime-intrinsics",
27109467b48Spatrick cl::desc(
27209467b48Spatrick "when possible, poison scoped variables at the beginning of the scope "
27309467b48Spatrick "(slower, but more precise)"),
27409467b48Spatrick cl::Hidden, cl::init(true));
27509467b48Spatrick
27609467b48Spatrick // When compiling the Linux kernel, we sometimes see false positives related to
27709467b48Spatrick // MSan being unable to understand that inline assembly calls may initialize
27809467b48Spatrick // local variables.
27909467b48Spatrick // This flag makes the compiler conservatively unpoison every memory location
28009467b48Spatrick // passed into an assembly call. Note that this may cause false positives.
28109467b48Spatrick // Because it's impossible to figure out the array sizes, we can only unpoison
28209467b48Spatrick // the first sizeof(type) bytes for each type* pointer.
28309467b48Spatrick // The instrumentation is only enabled in KMSAN builds, and only if
28409467b48Spatrick // -msan-handle-asm-conservative is on. This is done because we may want to
28509467b48Spatrick // quickly disable assembly instrumentation when it breaks.
28609467b48Spatrick static cl::opt<bool> ClHandleAsmConservative(
28709467b48Spatrick "msan-handle-asm-conservative",
28809467b48Spatrick cl::desc("conservative handling of inline assembly"), cl::Hidden,
28909467b48Spatrick cl::init(true));
29009467b48Spatrick
29109467b48Spatrick // This flag controls whether we check the shadow of the address
29209467b48Spatrick // operand of load or store. Such bugs are very rare, since load from
29309467b48Spatrick // a garbage address typically results in SEGV, but still happen
29409467b48Spatrick // (e.g. only lower bits of address are garbage, or the access happens
29509467b48Spatrick // early at program startup where malloc-ed memory is more likely to
29609467b48Spatrick // be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
297*d415bd75Srobert static cl::opt<bool> ClCheckAccessAddress(
298*d415bd75Srobert "msan-check-access-address",
29909467b48Spatrick cl::desc("report accesses through a pointer which has poisoned shadow"),
30009467b48Spatrick cl::Hidden, cl::init(true));
30109467b48Spatrick
302097a140dSpatrick static cl::opt<bool> ClEagerChecks(
303097a140dSpatrick "msan-eager-checks",
304097a140dSpatrick cl::desc("check arguments and return values at function call boundaries"),
305097a140dSpatrick cl::Hidden, cl::init(false));
306097a140dSpatrick
307*d415bd75Srobert static cl::opt<bool> ClDumpStrictInstructions(
308*d415bd75Srobert "msan-dump-strict-instructions",
30909467b48Spatrick cl::desc("print out instructions with default strict semantics"),
31009467b48Spatrick cl::Hidden, cl::init(false));
31109467b48Spatrick
31209467b48Spatrick static cl::opt<int> ClInstrumentationWithCallThreshold(
31309467b48Spatrick "msan-instrumentation-with-call-threshold",
31409467b48Spatrick cl::desc(
31509467b48Spatrick "If the function being instrumented requires more than "
31609467b48Spatrick "this number of checks and origin stores, use callbacks instead of "
31709467b48Spatrick "inline checks (-1 means never use callbacks)."),
31809467b48Spatrick cl::Hidden, cl::init(3500));
31909467b48Spatrick
32009467b48Spatrick static cl::opt<bool>
32109467b48Spatrick ClEnableKmsan("msan-kernel",
32209467b48Spatrick cl::desc("Enable KernelMemorySanitizer instrumentation"),
32309467b48Spatrick cl::Hidden, cl::init(false));
32409467b48Spatrick
325*d415bd75Srobert static cl::opt<bool>
326*d415bd75Srobert ClDisableChecks("msan-disable-checks",
327*d415bd75Srobert cl::desc("Apply no_sanitize to the whole file"), cl::Hidden,
328*d415bd75Srobert cl::init(false));
329*d415bd75Srobert
330*d415bd75Srobert static cl::opt<bool>
331*d415bd75Srobert ClCheckConstantShadow("msan-check-constant-shadow",
33209467b48Spatrick cl::desc("Insert checks for constant shadow values"),
333*d415bd75Srobert cl::Hidden, cl::init(true));
33409467b48Spatrick
33509467b48Spatrick // This is off by default because of a bug in gold:
33609467b48Spatrick // https://sourceware.org/bugzilla/show_bug.cgi?id=19002
337*d415bd75Srobert static cl::opt<bool>
338*d415bd75Srobert ClWithComdat("msan-with-comdat",
33909467b48Spatrick cl::desc("Place MSan constructors in comdat sections"),
34009467b48Spatrick cl::Hidden, cl::init(false));
34109467b48Spatrick
34209467b48Spatrick // These options allow to specify custom memory map parameters
34309467b48Spatrick // See MemoryMapParams for details.
34409467b48Spatrick static cl::opt<uint64_t> ClAndMask("msan-and-mask",
34509467b48Spatrick cl::desc("Define custom MSan AndMask"),
34609467b48Spatrick cl::Hidden, cl::init(0));
34709467b48Spatrick
34809467b48Spatrick static cl::opt<uint64_t> ClXorMask("msan-xor-mask",
34909467b48Spatrick cl::desc("Define custom MSan XorMask"),
35009467b48Spatrick cl::Hidden, cl::init(0));
35109467b48Spatrick
35209467b48Spatrick static cl::opt<uint64_t> ClShadowBase("msan-shadow-base",
35309467b48Spatrick cl::desc("Define custom MSan ShadowBase"),
35409467b48Spatrick cl::Hidden, cl::init(0));
35509467b48Spatrick
35609467b48Spatrick static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
35709467b48Spatrick cl::desc("Define custom MSan OriginBase"),
35809467b48Spatrick cl::Hidden, cl::init(0));
35909467b48Spatrick
360*d415bd75Srobert static cl::opt<int>
361*d415bd75Srobert ClDisambiguateWarning("msan-disambiguate-warning-threshold",
362*d415bd75Srobert cl::desc("Define threshold for number of checks per "
363*d415bd75Srobert "debug location to force origin update."),
364*d415bd75Srobert cl::Hidden, cl::init(3));
365*d415bd75Srobert
36673471bf0Spatrick const char kMsanModuleCtorName[] = "msan.module_ctor";
36773471bf0Spatrick const char kMsanInitName[] = "__msan_init";
36809467b48Spatrick
36909467b48Spatrick namespace {
37009467b48Spatrick
37109467b48Spatrick // Memory map parameters used in application-to-shadow address calculation.
37209467b48Spatrick // Offset = (Addr & ~AndMask) ^ XorMask
37309467b48Spatrick // Shadow = ShadowBase + Offset
37409467b48Spatrick // Origin = OriginBase + Offset
37509467b48Spatrick struct MemoryMapParams {
37609467b48Spatrick uint64_t AndMask;
37709467b48Spatrick uint64_t XorMask;
37809467b48Spatrick uint64_t ShadowBase;
37909467b48Spatrick uint64_t OriginBase;
38009467b48Spatrick };
38109467b48Spatrick
38209467b48Spatrick struct PlatformMemoryMapParams {
38309467b48Spatrick const MemoryMapParams *bits32;
38409467b48Spatrick const MemoryMapParams *bits64;
38509467b48Spatrick };
38609467b48Spatrick
38709467b48Spatrick } // end anonymous namespace
38809467b48Spatrick
38909467b48Spatrick // i386 Linux
39009467b48Spatrick static const MemoryMapParams Linux_I386_MemoryMapParams = {
39109467b48Spatrick 0x000080000000, // AndMask
39209467b48Spatrick 0, // XorMask (not used)
39309467b48Spatrick 0, // ShadowBase (not used)
39409467b48Spatrick 0x000040000000, // OriginBase
39509467b48Spatrick };
39609467b48Spatrick
39709467b48Spatrick // x86_64 Linux
39809467b48Spatrick static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
39909467b48Spatrick 0, // AndMask (not used)
40009467b48Spatrick 0x500000000000, // XorMask
40109467b48Spatrick 0, // ShadowBase (not used)
40209467b48Spatrick 0x100000000000, // OriginBase
40309467b48Spatrick };
40409467b48Spatrick
40509467b48Spatrick // mips64 Linux
40609467b48Spatrick static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
40709467b48Spatrick 0, // AndMask (not used)
40809467b48Spatrick 0x008000000000, // XorMask
40909467b48Spatrick 0, // ShadowBase (not used)
41009467b48Spatrick 0x002000000000, // OriginBase
41109467b48Spatrick };
41209467b48Spatrick
41309467b48Spatrick // ppc64 Linux
41409467b48Spatrick static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
41509467b48Spatrick 0xE00000000000, // AndMask
41609467b48Spatrick 0x100000000000, // XorMask
41709467b48Spatrick 0x080000000000, // ShadowBase
41809467b48Spatrick 0x1C0000000000, // OriginBase
41909467b48Spatrick };
42009467b48Spatrick
421097a140dSpatrick // s390x Linux
422097a140dSpatrick static const MemoryMapParams Linux_S390X_MemoryMapParams = {
423097a140dSpatrick 0xC00000000000, // AndMask
424097a140dSpatrick 0, // XorMask (not used)
425097a140dSpatrick 0x080000000000, // ShadowBase
426097a140dSpatrick 0x1C0000000000, // OriginBase
427097a140dSpatrick };
428097a140dSpatrick
42909467b48Spatrick // aarch64 Linux
43009467b48Spatrick static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
43109467b48Spatrick 0, // AndMask (not used)
432*d415bd75Srobert 0x0B00000000000, // XorMask
43309467b48Spatrick 0, // ShadowBase (not used)
434*d415bd75Srobert 0x0200000000000, // OriginBase
435*d415bd75Srobert };
436*d415bd75Srobert
437*d415bd75Srobert // aarch64 FreeBSD
438*d415bd75Srobert static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams = {
439*d415bd75Srobert 0x1800000000000, // AndMask
440*d415bd75Srobert 0x0400000000000, // XorMask
441*d415bd75Srobert 0x0200000000000, // ShadowBase
442*d415bd75Srobert 0x0700000000000, // OriginBase
44309467b48Spatrick };
44409467b48Spatrick
44509467b48Spatrick // i386 FreeBSD
44609467b48Spatrick static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
44709467b48Spatrick 0x000180000000, // AndMask
44809467b48Spatrick 0x000040000000, // XorMask
44909467b48Spatrick 0x000020000000, // ShadowBase
45009467b48Spatrick 0x000700000000, // OriginBase
45109467b48Spatrick };
45209467b48Spatrick
45309467b48Spatrick // x86_64 FreeBSD
45409467b48Spatrick static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
45509467b48Spatrick 0xc00000000000, // AndMask
45609467b48Spatrick 0x200000000000, // XorMask
45709467b48Spatrick 0x100000000000, // ShadowBase
45809467b48Spatrick 0x380000000000, // OriginBase
45909467b48Spatrick };
46009467b48Spatrick
46109467b48Spatrick // x86_64 NetBSD
46209467b48Spatrick static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
46309467b48Spatrick 0, // AndMask
46409467b48Spatrick 0x500000000000, // XorMask
46509467b48Spatrick 0, // ShadowBase
46609467b48Spatrick 0x100000000000, // OriginBase
46709467b48Spatrick };
46809467b48Spatrick
46909467b48Spatrick static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
47009467b48Spatrick &Linux_I386_MemoryMapParams,
47109467b48Spatrick &Linux_X86_64_MemoryMapParams,
47209467b48Spatrick };
47309467b48Spatrick
47409467b48Spatrick static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
47509467b48Spatrick nullptr,
47609467b48Spatrick &Linux_MIPS64_MemoryMapParams,
47709467b48Spatrick };
47809467b48Spatrick
47909467b48Spatrick static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
48009467b48Spatrick nullptr,
48109467b48Spatrick &Linux_PowerPC64_MemoryMapParams,
48209467b48Spatrick };
48309467b48Spatrick
484097a140dSpatrick static const PlatformMemoryMapParams Linux_S390_MemoryMapParams = {
485097a140dSpatrick nullptr,
486097a140dSpatrick &Linux_S390X_MemoryMapParams,
487097a140dSpatrick };
488097a140dSpatrick
48909467b48Spatrick static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
49009467b48Spatrick nullptr,
49109467b48Spatrick &Linux_AArch64_MemoryMapParams,
49209467b48Spatrick };
49309467b48Spatrick
494*d415bd75Srobert static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams = {
495*d415bd75Srobert nullptr,
496*d415bd75Srobert &FreeBSD_AArch64_MemoryMapParams,
497*d415bd75Srobert };
498*d415bd75Srobert
49909467b48Spatrick static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
50009467b48Spatrick &FreeBSD_I386_MemoryMapParams,
50109467b48Spatrick &FreeBSD_X86_64_MemoryMapParams,
50209467b48Spatrick };
50309467b48Spatrick
50409467b48Spatrick static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
50509467b48Spatrick nullptr,
50609467b48Spatrick &NetBSD_X86_64_MemoryMapParams,
50709467b48Spatrick };
50809467b48Spatrick
50909467b48Spatrick namespace {
51009467b48Spatrick
51109467b48Spatrick /// Instrument functions of a module to detect uninitialized reads.
51209467b48Spatrick ///
51309467b48Spatrick /// Instantiating MemorySanitizer inserts the msan runtime library API function
51409467b48Spatrick /// declarations into the module if they don't exist already. Instantiating
51509467b48Spatrick /// ensures the __msan_init function is in the list of global constructors for
51609467b48Spatrick /// the module.
51709467b48Spatrick class MemorySanitizer {
51809467b48Spatrick public:
MemorySanitizer(Module & M,MemorySanitizerOptions Options)51909467b48Spatrick MemorySanitizer(Module &M, MemorySanitizerOptions Options)
52009467b48Spatrick : CompileKernel(Options.Kernel), TrackOrigins(Options.TrackOrigins),
521*d415bd75Srobert Recover(Options.Recover), EagerChecks(Options.EagerChecks) {
52209467b48Spatrick initializeModule(M);
52309467b48Spatrick }
52409467b48Spatrick
52509467b48Spatrick // MSan cannot be moved or copied because of MapParams.
52609467b48Spatrick MemorySanitizer(MemorySanitizer &&) = delete;
52709467b48Spatrick MemorySanitizer &operator=(MemorySanitizer &&) = delete;
52809467b48Spatrick MemorySanitizer(const MemorySanitizer &) = delete;
52909467b48Spatrick MemorySanitizer &operator=(const MemorySanitizer &) = delete;
53009467b48Spatrick
53109467b48Spatrick bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
53209467b48Spatrick
53309467b48Spatrick private:
53409467b48Spatrick friend struct MemorySanitizerVisitor;
53509467b48Spatrick friend struct VarArgAMD64Helper;
53609467b48Spatrick friend struct VarArgMIPS64Helper;
53709467b48Spatrick friend struct VarArgAArch64Helper;
53809467b48Spatrick friend struct VarArgPowerPC64Helper;
539097a140dSpatrick friend struct VarArgSystemZHelper;
54009467b48Spatrick
54109467b48Spatrick void initializeModule(Module &M);
542*d415bd75Srobert void initializeCallbacks(Module &M, const TargetLibraryInfo &TLI);
543*d415bd75Srobert void createKernelApi(Module &M, const TargetLibraryInfo &TLI);
544*d415bd75Srobert void createUserspaceApi(Module &M, const TargetLibraryInfo &TLI);
54509467b48Spatrick
54609467b48Spatrick /// True if we're compiling the Linux kernel.
54709467b48Spatrick bool CompileKernel;
54809467b48Spatrick /// Track origins (allocation points) of uninitialized values.
54909467b48Spatrick int TrackOrigins;
55009467b48Spatrick bool Recover;
551*d415bd75Srobert bool EagerChecks;
55209467b48Spatrick
55309467b48Spatrick LLVMContext *C;
55409467b48Spatrick Type *IntptrTy;
55509467b48Spatrick Type *OriginTy;
55609467b48Spatrick
55709467b48Spatrick // XxxTLS variables represent the per-thread state in MSan and per-task state
55809467b48Spatrick // in KMSAN.
55909467b48Spatrick // For the userspace these point to thread-local globals. In the kernel land
56009467b48Spatrick // they point to the members of a per-task struct obtained via a call to
56109467b48Spatrick // __msan_get_context_state().
56209467b48Spatrick
56309467b48Spatrick /// Thread-local shadow storage for function parameters.
56409467b48Spatrick Value *ParamTLS;
56509467b48Spatrick
56609467b48Spatrick /// Thread-local origin storage for function parameters.
56709467b48Spatrick Value *ParamOriginTLS;
56809467b48Spatrick
56909467b48Spatrick /// Thread-local shadow storage for function return value.
57009467b48Spatrick Value *RetvalTLS;
57109467b48Spatrick
57209467b48Spatrick /// Thread-local origin storage for function return value.
57309467b48Spatrick Value *RetvalOriginTLS;
57409467b48Spatrick
57509467b48Spatrick /// Thread-local shadow storage for in-register va_arg function
57609467b48Spatrick /// parameters (x86_64-specific).
57709467b48Spatrick Value *VAArgTLS;
57809467b48Spatrick
57909467b48Spatrick /// Thread-local shadow storage for in-register va_arg function
58009467b48Spatrick /// parameters (x86_64-specific).
58109467b48Spatrick Value *VAArgOriginTLS;
58209467b48Spatrick
58309467b48Spatrick /// Thread-local shadow storage for va_arg overflow area
58409467b48Spatrick /// (x86_64-specific).
58509467b48Spatrick Value *VAArgOverflowSizeTLS;
58609467b48Spatrick
58709467b48Spatrick /// Are the instrumentation callbacks set up?
58809467b48Spatrick bool CallbacksInitialized = false;
58909467b48Spatrick
59009467b48Spatrick /// The run-time callback to print a warning.
59109467b48Spatrick FunctionCallee WarningFn;
59209467b48Spatrick
59309467b48Spatrick // These arrays are indexed by log2(AccessSize).
59409467b48Spatrick FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
59509467b48Spatrick FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
59609467b48Spatrick
59709467b48Spatrick /// Run-time helper that generates a new origin value for a stack
59809467b48Spatrick /// allocation.
599*d415bd75Srobert FunctionCallee MsanSetAllocaOriginWithDescriptionFn;
600*d415bd75Srobert // No description version
601*d415bd75Srobert FunctionCallee MsanSetAllocaOriginNoDescriptionFn;
60209467b48Spatrick
60309467b48Spatrick /// Run-time helper that poisons stack on function entry.
60409467b48Spatrick FunctionCallee MsanPoisonStackFn;
60509467b48Spatrick
60609467b48Spatrick /// Run-time helper that records a store (or any event) of an
60709467b48Spatrick /// uninitialized value and returns an updated origin id encoding this info.
60809467b48Spatrick FunctionCallee MsanChainOriginFn;
60909467b48Spatrick
61073471bf0Spatrick /// Run-time helper that paints an origin over a region.
61173471bf0Spatrick FunctionCallee MsanSetOriginFn;
61273471bf0Spatrick
61309467b48Spatrick /// MSan runtime replacements for memmove, memcpy and memset.
61409467b48Spatrick FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
61509467b48Spatrick
61609467b48Spatrick /// KMSAN callback for task-local function argument shadow.
61709467b48Spatrick StructType *MsanContextStateTy;
61809467b48Spatrick FunctionCallee MsanGetContextStateFn;
61909467b48Spatrick
62009467b48Spatrick /// Functions for poisoning/unpoisoning local variables
62109467b48Spatrick FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
62209467b48Spatrick
62309467b48Spatrick /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin
62409467b48Spatrick /// pointers.
62509467b48Spatrick FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
62609467b48Spatrick FunctionCallee MsanMetadataPtrForLoad_1_8[4];
62709467b48Spatrick FunctionCallee MsanMetadataPtrForStore_1_8[4];
62809467b48Spatrick FunctionCallee MsanInstrumentAsmStoreFn;
62909467b48Spatrick
63009467b48Spatrick /// Helper to choose between different MsanMetadataPtrXxx().
63109467b48Spatrick FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
63209467b48Spatrick
63309467b48Spatrick /// Memory map parameters used in application-to-shadow calculation.
63409467b48Spatrick const MemoryMapParams *MapParams;
63509467b48Spatrick
63609467b48Spatrick /// Custom memory map parameters used when -msan-shadow-base or
63709467b48Spatrick // -msan-origin-base is provided.
63809467b48Spatrick MemoryMapParams CustomMapParams;
63909467b48Spatrick
64009467b48Spatrick MDNode *ColdCallWeights;
64109467b48Spatrick
64209467b48Spatrick /// Branch weights for origin store.
64309467b48Spatrick MDNode *OriginStoreWeights;
64409467b48Spatrick };
64509467b48Spatrick
insertModuleCtor(Module & M)64609467b48Spatrick void insertModuleCtor(Module &M) {
64709467b48Spatrick getOrCreateSanitizerCtorAndInitFunctions(
64809467b48Spatrick M, kMsanModuleCtorName, kMsanInitName,
64909467b48Spatrick /*InitArgTypes=*/{},
65009467b48Spatrick /*InitArgs=*/{},
65109467b48Spatrick // This callback is invoked when the functions are created the first
65209467b48Spatrick // time. Hook them into the global ctors list in that case:
65309467b48Spatrick [&](Function *Ctor, FunctionCallee) {
65409467b48Spatrick if (!ClWithComdat) {
65509467b48Spatrick appendToGlobalCtors(M, Ctor, 0);
65609467b48Spatrick return;
65709467b48Spatrick }
65809467b48Spatrick Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
65909467b48Spatrick Ctor->setComdat(MsanCtorComdat);
66009467b48Spatrick appendToGlobalCtors(M, Ctor, 0, Ctor);
66109467b48Spatrick });
66209467b48Spatrick }
66309467b48Spatrick
getOptOrDefault(const cl::opt<T> & Opt,T Default)66409467b48Spatrick template <class T> T getOptOrDefault(const cl::opt<T> &Opt, T Default) {
66509467b48Spatrick return (Opt.getNumOccurrences() > 0) ? Opt : Default;
66609467b48Spatrick }
66709467b48Spatrick
66809467b48Spatrick } // end anonymous namespace
66909467b48Spatrick
MemorySanitizerOptions(int TO,bool R,bool K,bool EagerChecks)670*d415bd75Srobert MemorySanitizerOptions::MemorySanitizerOptions(int TO, bool R, bool K,
671*d415bd75Srobert bool EagerChecks)
67209467b48Spatrick : Kernel(getOptOrDefault(ClEnableKmsan, K)),
67309467b48Spatrick TrackOrigins(getOptOrDefault(ClTrackOrigins, Kernel ? 2 : TO)),
674*d415bd75Srobert Recover(getOptOrDefault(ClKeepGoing, Kernel || R)),
675*d415bd75Srobert EagerChecks(getOptOrDefault(ClEagerChecks, EagerChecks)) {}
67609467b48Spatrick
run(Module & M,ModuleAnalysisManager & AM)67709467b48Spatrick PreservedAnalyses MemorySanitizerPass::run(Module &M,
67809467b48Spatrick ModuleAnalysisManager &AM) {
679*d415bd75Srobert bool Modified = false;
680*d415bd75Srobert if (!Options.Kernel) {
68109467b48Spatrick insertModuleCtor(M);
682*d415bd75Srobert Modified = true;
68309467b48Spatrick }
68409467b48Spatrick
685*d415bd75Srobert auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
686*d415bd75Srobert for (Function &F : M) {
687*d415bd75Srobert if (F.empty())
688*d415bd75Srobert continue;
689*d415bd75Srobert MemorySanitizer Msan(*F.getParent(), Options);
690*d415bd75Srobert Modified |=
691*d415bd75Srobert Msan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F));
692*d415bd75Srobert }
69309467b48Spatrick
694*d415bd75Srobert if (!Modified)
695*d415bd75Srobert return PreservedAnalyses::all();
69609467b48Spatrick
697*d415bd75Srobert PreservedAnalyses PA = PreservedAnalyses::none();
698*d415bd75Srobert // GlobalsAA is considered stateless and does not get invalidated unless
699*d415bd75Srobert // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
700*d415bd75Srobert // make changes that require GlobalsAA to be invalidated.
701*d415bd75Srobert PA.abandon<GlobalsAA>();
702*d415bd75Srobert return PA;
703*d415bd75Srobert }
704*d415bd75Srobert
printPipeline(raw_ostream & OS,function_ref<StringRef (StringRef)> MapClassName2PassName)705*d415bd75Srobert void MemorySanitizerPass::printPipeline(
706*d415bd75Srobert raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
707*d415bd75Srobert static_cast<PassInfoMixin<MemorySanitizerPass> *>(this)->printPipeline(
708*d415bd75Srobert OS, MapClassName2PassName);
709*d415bd75Srobert OS << "<";
710*d415bd75Srobert if (Options.Recover)
711*d415bd75Srobert OS << "recover;";
712*d415bd75Srobert if (Options.Kernel)
713*d415bd75Srobert OS << "kernel;";
714*d415bd75Srobert if (Options.EagerChecks)
715*d415bd75Srobert OS << "eager-checks;";
716*d415bd75Srobert OS << "track-origins=" << Options.TrackOrigins;
717*d415bd75Srobert OS << ">";
71809467b48Spatrick }
71909467b48Spatrick
72009467b48Spatrick /// Create a non-const global initialized with the given string.
72109467b48Spatrick ///
72209467b48Spatrick /// Creates a writable global for Str so that we can pass it to the
72309467b48Spatrick /// run-time lib. Runtime uses first 4 bytes of the string to store the
72409467b48Spatrick /// frame ID, so the string needs to be mutable.
createPrivateConstGlobalForString(Module & M,StringRef Str)725*d415bd75Srobert static GlobalVariable *createPrivateConstGlobalForString(Module &M,
72609467b48Spatrick StringRef Str) {
72709467b48Spatrick Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
728*d415bd75Srobert return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/true,
72909467b48Spatrick GlobalValue::PrivateLinkage, StrConst, "");
73009467b48Spatrick }
73109467b48Spatrick
73209467b48Spatrick /// Create KMSAN API callbacks.
createKernelApi(Module & M,const TargetLibraryInfo & TLI)733*d415bd75Srobert void MemorySanitizer::createKernelApi(Module &M, const TargetLibraryInfo &TLI) {
73409467b48Spatrick IRBuilder<> IRB(*C);
73509467b48Spatrick
73609467b48Spatrick // These will be initialized in insertKmsanPrologue().
73709467b48Spatrick RetvalTLS = nullptr;
73809467b48Spatrick RetvalOriginTLS = nullptr;
73909467b48Spatrick ParamTLS = nullptr;
74009467b48Spatrick ParamOriginTLS = nullptr;
74109467b48Spatrick VAArgTLS = nullptr;
74209467b48Spatrick VAArgOriginTLS = nullptr;
74309467b48Spatrick VAArgOverflowSizeTLS = nullptr;
74409467b48Spatrick
745*d415bd75Srobert WarningFn = M.getOrInsertFunction("__msan_warning",
746*d415bd75Srobert TLI.getAttrList(C, {0}, /*Signed=*/false),
747*d415bd75Srobert IRB.getVoidTy(), IRB.getInt32Ty());
748*d415bd75Srobert
74909467b48Spatrick // Requests the per-task context state (kmsan_context_state*) from the
75009467b48Spatrick // runtime library.
75109467b48Spatrick MsanContextStateTy = StructType::get(
75209467b48Spatrick ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
75309467b48Spatrick ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
75409467b48Spatrick ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
75509467b48Spatrick ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */
75609467b48Spatrick IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
75709467b48Spatrick OriginTy);
75809467b48Spatrick MsanGetContextStateFn = M.getOrInsertFunction(
75909467b48Spatrick "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0));
76009467b48Spatrick
76109467b48Spatrick Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
76209467b48Spatrick PointerType::get(IRB.getInt32Ty(), 0));
76309467b48Spatrick
76409467b48Spatrick for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
76509467b48Spatrick std::string name_load =
76609467b48Spatrick "__msan_metadata_ptr_for_load_" + std::to_string(size);
76709467b48Spatrick std::string name_store =
76809467b48Spatrick "__msan_metadata_ptr_for_store_" + std::to_string(size);
76909467b48Spatrick MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction(
77009467b48Spatrick name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
77109467b48Spatrick MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction(
77209467b48Spatrick name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
77309467b48Spatrick }
77409467b48Spatrick
77509467b48Spatrick MsanMetadataPtrForLoadN = M.getOrInsertFunction(
77609467b48Spatrick "__msan_metadata_ptr_for_load_n", RetTy,
77709467b48Spatrick PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
77809467b48Spatrick MsanMetadataPtrForStoreN = M.getOrInsertFunction(
77909467b48Spatrick "__msan_metadata_ptr_for_store_n", RetTy,
78009467b48Spatrick PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
78109467b48Spatrick
78209467b48Spatrick // Functions for poisoning and unpoisoning memory.
78309467b48Spatrick MsanPoisonAllocaFn =
78409467b48Spatrick M.getOrInsertFunction("__msan_poison_alloca", IRB.getVoidTy(),
78509467b48Spatrick IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy());
78609467b48Spatrick MsanUnpoisonAllocaFn = M.getOrInsertFunction(
78709467b48Spatrick "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
78809467b48Spatrick }
78909467b48Spatrick
getOrInsertGlobal(Module & M,StringRef Name,Type * Ty)79009467b48Spatrick static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
79109467b48Spatrick return M.getOrInsertGlobal(Name, Ty, [&] {
79209467b48Spatrick return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
79309467b48Spatrick nullptr, Name, nullptr,
79409467b48Spatrick GlobalVariable::InitialExecTLSModel);
79509467b48Spatrick });
79609467b48Spatrick }
79709467b48Spatrick
79809467b48Spatrick /// Insert declarations for userspace-specific functions and globals.
createUserspaceApi(Module & M,const TargetLibraryInfo & TLI)799*d415bd75Srobert void MemorySanitizer::createUserspaceApi(Module &M, const TargetLibraryInfo &TLI) {
80009467b48Spatrick IRBuilder<> IRB(*C);
801097a140dSpatrick
80209467b48Spatrick // Create the callback.
80309467b48Spatrick // FIXME: this function should have "Cold" calling conv,
80409467b48Spatrick // which is not yet implemented.
805*d415bd75Srobert if (TrackOrigins) {
806097a140dSpatrick StringRef WarningFnName = Recover ? "__msan_warning_with_origin"
807097a140dSpatrick : "__msan_warning_with_origin_noreturn";
808*d415bd75Srobert WarningFn = M.getOrInsertFunction(WarningFnName,
809*d415bd75Srobert TLI.getAttrList(C, {0}, /*Signed=*/false),
810*d415bd75Srobert IRB.getVoidTy(), IRB.getInt32Ty());
811*d415bd75Srobert } else {
812*d415bd75Srobert StringRef WarningFnName =
813*d415bd75Srobert Recover ? "__msan_warning" : "__msan_warning_noreturn";
814*d415bd75Srobert WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
815*d415bd75Srobert }
81609467b48Spatrick
81709467b48Spatrick // Create the global TLS variables.
81809467b48Spatrick RetvalTLS =
81909467b48Spatrick getOrInsertGlobal(M, "__msan_retval_tls",
82009467b48Spatrick ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8));
82109467b48Spatrick
82209467b48Spatrick RetvalOriginTLS = getOrInsertGlobal(M, "__msan_retval_origin_tls", OriginTy);
82309467b48Spatrick
82409467b48Spatrick ParamTLS =
82509467b48Spatrick getOrInsertGlobal(M, "__msan_param_tls",
82609467b48Spatrick ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
82709467b48Spatrick
82809467b48Spatrick ParamOriginTLS =
82909467b48Spatrick getOrInsertGlobal(M, "__msan_param_origin_tls",
83009467b48Spatrick ArrayType::get(OriginTy, kParamTLSSize / 4));
83109467b48Spatrick
83209467b48Spatrick VAArgTLS =
83309467b48Spatrick getOrInsertGlobal(M, "__msan_va_arg_tls",
83409467b48Spatrick ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
83509467b48Spatrick
83609467b48Spatrick VAArgOriginTLS =
83709467b48Spatrick getOrInsertGlobal(M, "__msan_va_arg_origin_tls",
83809467b48Spatrick ArrayType::get(OriginTy, kParamTLSSize / 4));
83909467b48Spatrick
84009467b48Spatrick VAArgOverflowSizeTLS =
84109467b48Spatrick getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty());
84209467b48Spatrick
84309467b48Spatrick for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
84409467b48Spatrick AccessSizeIndex++) {
84509467b48Spatrick unsigned AccessSize = 1 << AccessSizeIndex;
84609467b48Spatrick std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
84709467b48Spatrick MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
848*d415bd75Srobert FunctionName, TLI.getAttrList(C, {0, 1}, /*Signed=*/false),
849097a140dSpatrick IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
85009467b48Spatrick
85109467b48Spatrick FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
85209467b48Spatrick MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
853*d415bd75Srobert FunctionName, TLI.getAttrList(C, {0, 2}, /*Signed=*/false),
854097a140dSpatrick IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt8PtrTy(),
855097a140dSpatrick IRB.getInt32Ty());
85609467b48Spatrick }
85709467b48Spatrick
858*d415bd75Srobert MsanSetAllocaOriginWithDescriptionFn = M.getOrInsertFunction(
859*d415bd75Srobert "__msan_set_alloca_origin_with_descr", IRB.getVoidTy(),
860*d415bd75Srobert IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy(), IRB.getInt8PtrTy());
861*d415bd75Srobert MsanSetAllocaOriginNoDescriptionFn = M.getOrInsertFunction(
862*d415bd75Srobert "__msan_set_alloca_origin_no_descr", IRB.getVoidTy(), IRB.getInt8PtrTy(),
863*d415bd75Srobert IntptrTy, IRB.getInt8PtrTy());
864*d415bd75Srobert MsanPoisonStackFn = M.getOrInsertFunction(
865*d415bd75Srobert "__msan_poison_stack", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
86609467b48Spatrick }
86709467b48Spatrick
86809467b48Spatrick /// Insert extern declaration of runtime-provided functions and globals.
initializeCallbacks(Module & M,const TargetLibraryInfo & TLI)869*d415bd75Srobert void MemorySanitizer::initializeCallbacks(Module &M, const TargetLibraryInfo &TLI) {
87009467b48Spatrick // Only do this once.
87109467b48Spatrick if (CallbacksInitialized)
87209467b48Spatrick return;
87309467b48Spatrick
87409467b48Spatrick IRBuilder<> IRB(*C);
87509467b48Spatrick // Initialize callbacks that are common for kernel and userspace
87609467b48Spatrick // instrumentation.
87709467b48Spatrick MsanChainOriginFn = M.getOrInsertFunction(
878*d415bd75Srobert "__msan_chain_origin",
879*d415bd75Srobert TLI.getAttrList(C, {0}, /*Signed=*/false, /*Ret=*/true), IRB.getInt32Ty(),
880*d415bd75Srobert IRB.getInt32Ty());
881*d415bd75Srobert MsanSetOriginFn = M.getOrInsertFunction(
882*d415bd75Srobert "__msan_set_origin", TLI.getAttrList(C, {2}, /*Signed=*/false),
883*d415bd75Srobert IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy, IRB.getInt32Ty());
884*d415bd75Srobert MemmoveFn =
885*d415bd75Srobert M.getOrInsertFunction("__msan_memmove", IRB.getInt8PtrTy(),
886*d415bd75Srobert IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
887*d415bd75Srobert MemcpyFn =
888*d415bd75Srobert M.getOrInsertFunction("__msan_memcpy", IRB.getInt8PtrTy(),
889*d415bd75Srobert IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
89009467b48Spatrick MemsetFn = M.getOrInsertFunction(
891*d415bd75Srobert "__msan_memset", TLI.getAttrList(C, {1}, /*Signed=*/true),
892*d415bd75Srobert IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy);
89309467b48Spatrick
89409467b48Spatrick MsanInstrumentAsmStoreFn =
89509467b48Spatrick M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(),
89609467b48Spatrick PointerType::get(IRB.getInt8Ty(), 0), IntptrTy);
89709467b48Spatrick
89809467b48Spatrick if (CompileKernel) {
899*d415bd75Srobert createKernelApi(M, TLI);
90009467b48Spatrick } else {
901*d415bd75Srobert createUserspaceApi(M, TLI);
90209467b48Spatrick }
90309467b48Spatrick CallbacksInitialized = true;
90409467b48Spatrick }
90509467b48Spatrick
getKmsanShadowOriginAccessFn(bool isStore,int size)90609467b48Spatrick FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
90709467b48Spatrick int size) {
90809467b48Spatrick FunctionCallee *Fns =
90909467b48Spatrick isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
91009467b48Spatrick switch (size) {
91109467b48Spatrick case 1:
91209467b48Spatrick return Fns[0];
91309467b48Spatrick case 2:
91409467b48Spatrick return Fns[1];
91509467b48Spatrick case 4:
91609467b48Spatrick return Fns[2];
91709467b48Spatrick case 8:
91809467b48Spatrick return Fns[3];
91909467b48Spatrick default:
92009467b48Spatrick return nullptr;
92109467b48Spatrick }
92209467b48Spatrick }
92309467b48Spatrick
92409467b48Spatrick /// Module-level initialization.
92509467b48Spatrick ///
92609467b48Spatrick /// inserts a call to __msan_init to the module's constructor list.
initializeModule(Module & M)92709467b48Spatrick void MemorySanitizer::initializeModule(Module &M) {
92809467b48Spatrick auto &DL = M.getDataLayout();
92909467b48Spatrick
93009467b48Spatrick bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
93109467b48Spatrick bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
93209467b48Spatrick // Check the overrides first
93309467b48Spatrick if (ShadowPassed || OriginPassed) {
93409467b48Spatrick CustomMapParams.AndMask = ClAndMask;
93509467b48Spatrick CustomMapParams.XorMask = ClXorMask;
93609467b48Spatrick CustomMapParams.ShadowBase = ClShadowBase;
93709467b48Spatrick CustomMapParams.OriginBase = ClOriginBase;
93809467b48Spatrick MapParams = &CustomMapParams;
93909467b48Spatrick } else {
94009467b48Spatrick Triple TargetTriple(M.getTargetTriple());
94109467b48Spatrick switch (TargetTriple.getOS()) {
94209467b48Spatrick case Triple::FreeBSD:
94309467b48Spatrick switch (TargetTriple.getArch()) {
944*d415bd75Srobert case Triple::aarch64:
945*d415bd75Srobert MapParams = FreeBSD_ARM_MemoryMapParams.bits64;
946*d415bd75Srobert break;
94709467b48Spatrick case Triple::x86_64:
94809467b48Spatrick MapParams = FreeBSD_X86_MemoryMapParams.bits64;
94909467b48Spatrick break;
95009467b48Spatrick case Triple::x86:
95109467b48Spatrick MapParams = FreeBSD_X86_MemoryMapParams.bits32;
95209467b48Spatrick break;
95309467b48Spatrick default:
95409467b48Spatrick report_fatal_error("unsupported architecture");
95509467b48Spatrick }
95609467b48Spatrick break;
95709467b48Spatrick case Triple::NetBSD:
95809467b48Spatrick switch (TargetTriple.getArch()) {
95909467b48Spatrick case Triple::x86_64:
96009467b48Spatrick MapParams = NetBSD_X86_MemoryMapParams.bits64;
96109467b48Spatrick break;
96209467b48Spatrick default:
96309467b48Spatrick report_fatal_error("unsupported architecture");
96409467b48Spatrick }
96509467b48Spatrick break;
96609467b48Spatrick case Triple::Linux:
96709467b48Spatrick switch (TargetTriple.getArch()) {
96809467b48Spatrick case Triple::x86_64:
96909467b48Spatrick MapParams = Linux_X86_MemoryMapParams.bits64;
97009467b48Spatrick break;
97109467b48Spatrick case Triple::x86:
97209467b48Spatrick MapParams = Linux_X86_MemoryMapParams.bits32;
97309467b48Spatrick break;
97409467b48Spatrick case Triple::mips64:
97509467b48Spatrick case Triple::mips64el:
97609467b48Spatrick MapParams = Linux_MIPS_MemoryMapParams.bits64;
97709467b48Spatrick break;
97809467b48Spatrick case Triple::ppc64:
97909467b48Spatrick case Triple::ppc64le:
98009467b48Spatrick MapParams = Linux_PowerPC_MemoryMapParams.bits64;
98109467b48Spatrick break;
982097a140dSpatrick case Triple::systemz:
983097a140dSpatrick MapParams = Linux_S390_MemoryMapParams.bits64;
984097a140dSpatrick break;
98509467b48Spatrick case Triple::aarch64:
98609467b48Spatrick case Triple::aarch64_be:
98709467b48Spatrick MapParams = Linux_ARM_MemoryMapParams.bits64;
98809467b48Spatrick break;
98909467b48Spatrick default:
99009467b48Spatrick report_fatal_error("unsupported architecture");
99109467b48Spatrick }
99209467b48Spatrick break;
99309467b48Spatrick default:
99409467b48Spatrick report_fatal_error("unsupported operating system");
99509467b48Spatrick }
99609467b48Spatrick }
99709467b48Spatrick
99809467b48Spatrick C = &(M.getContext());
99909467b48Spatrick IRBuilder<> IRB(*C);
100009467b48Spatrick IntptrTy = IRB.getIntPtrTy(DL);
100109467b48Spatrick OriginTy = IRB.getInt32Ty();
100209467b48Spatrick
100309467b48Spatrick ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
100409467b48Spatrick OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
100509467b48Spatrick
100609467b48Spatrick if (!CompileKernel) {
100709467b48Spatrick if (TrackOrigins)
100809467b48Spatrick M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] {
100909467b48Spatrick return new GlobalVariable(
101009467b48Spatrick M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
101109467b48Spatrick IRB.getInt32(TrackOrigins), "__msan_track_origins");
101209467b48Spatrick });
101309467b48Spatrick
101409467b48Spatrick if (Recover)
101509467b48Spatrick M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] {
101609467b48Spatrick return new GlobalVariable(M, IRB.getInt32Ty(), true,
101709467b48Spatrick GlobalValue::WeakODRLinkage,
101809467b48Spatrick IRB.getInt32(Recover), "__msan_keep_going");
101909467b48Spatrick });
102009467b48Spatrick }
102109467b48Spatrick }
102209467b48Spatrick
102309467b48Spatrick namespace {
102409467b48Spatrick
102509467b48Spatrick /// A helper class that handles instrumentation of VarArg
102609467b48Spatrick /// functions on a particular platform.
102709467b48Spatrick ///
102809467b48Spatrick /// Implementations are expected to insert the instrumentation
102909467b48Spatrick /// necessary to propagate argument shadow through VarArg function
103009467b48Spatrick /// calls. Visit* methods are called during an InstVisitor pass over
103109467b48Spatrick /// the function, and should avoid creating new basic blocks. A new
103209467b48Spatrick /// instance of this class is created for each instrumented function.
103309467b48Spatrick struct VarArgHelper {
103409467b48Spatrick virtual ~VarArgHelper() = default;
103509467b48Spatrick
1036097a140dSpatrick /// Visit a CallBase.
1037097a140dSpatrick virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
103809467b48Spatrick
103909467b48Spatrick /// Visit a va_start call.
104009467b48Spatrick virtual void visitVAStartInst(VAStartInst &I) = 0;
104109467b48Spatrick
104209467b48Spatrick /// Visit a va_copy call.
104309467b48Spatrick virtual void visitVACopyInst(VACopyInst &I) = 0;
104409467b48Spatrick
104509467b48Spatrick /// Finalize function instrumentation.
104609467b48Spatrick ///
104709467b48Spatrick /// This method is called after visiting all interesting (see above)
104809467b48Spatrick /// instructions in a function.
104909467b48Spatrick virtual void finalizeInstrumentation() = 0;
105009467b48Spatrick };
105109467b48Spatrick
105209467b48Spatrick struct MemorySanitizerVisitor;
105309467b48Spatrick
105409467b48Spatrick } // end anonymous namespace
105509467b48Spatrick
105609467b48Spatrick static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
105709467b48Spatrick MemorySanitizerVisitor &Visitor);
105809467b48Spatrick
TypeSizeToSizeIndex(unsigned TypeSize)105909467b48Spatrick static unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
1060*d415bd75Srobert if (TypeSize <= 8)
1061*d415bd75Srobert return 0;
106209467b48Spatrick return Log2_32_Ceil((TypeSize + 7) / 8);
106309467b48Spatrick }
106409467b48Spatrick
106509467b48Spatrick namespace {
106609467b48Spatrick
1067*d415bd75Srobert /// Helper class to attach debug information of the given instruction onto new
1068*d415bd75Srobert /// instructions inserted after.
1069*d415bd75Srobert class NextNodeIRBuilder : public IRBuilder<> {
1070*d415bd75Srobert public:
NextNodeIRBuilder(Instruction * IP)1071*d415bd75Srobert explicit NextNodeIRBuilder(Instruction *IP) : IRBuilder<>(IP->getNextNode()) {
1072*d415bd75Srobert SetCurrentDebugLocation(IP->getDebugLoc());
1073*d415bd75Srobert }
1074*d415bd75Srobert };
1075*d415bd75Srobert
107609467b48Spatrick /// This class does all the work for a given function. Store and Load
107709467b48Spatrick /// instructions store and load corresponding shadow and origin
107809467b48Spatrick /// values. Most instructions propagate shadow from arguments to their
107909467b48Spatrick /// return values. Certain instructions (most importantly, BranchInst)
108009467b48Spatrick /// test their argument shadow and print reports (with a runtime call) if it's
108109467b48Spatrick /// non-zero.
108209467b48Spatrick struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
108309467b48Spatrick Function &F;
108409467b48Spatrick MemorySanitizer &MS;
108509467b48Spatrick SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
108609467b48Spatrick ValueMap<Value *, Value *> ShadowMap, OriginMap;
108709467b48Spatrick std::unique_ptr<VarArgHelper> VAHelper;
108809467b48Spatrick const TargetLibraryInfo *TLI;
108973471bf0Spatrick Instruction *FnPrologueEnd;
109009467b48Spatrick
109109467b48Spatrick // The following flags disable parts of MSan instrumentation based on
1092097a140dSpatrick // exclusion list contents and command-line options.
109309467b48Spatrick bool InsertChecks;
109409467b48Spatrick bool PropagateShadow;
109509467b48Spatrick bool PoisonStack;
109609467b48Spatrick bool PoisonUndef;
109709467b48Spatrick
109809467b48Spatrick struct ShadowOriginAndInsertPoint {
109909467b48Spatrick Value *Shadow;
110009467b48Spatrick Value *Origin;
110109467b48Spatrick Instruction *OrigIns;
110209467b48Spatrick
ShadowOriginAndInsertPoint__anon9f4f4c900811::MemorySanitizerVisitor::ShadowOriginAndInsertPoint110309467b48Spatrick ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
110409467b48Spatrick : Shadow(S), Origin(O), OrigIns(I) {}
110509467b48Spatrick };
110609467b48Spatrick SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1107*d415bd75Srobert DenseMap<const DILocation *, int> LazyWarningDebugLocationCount;
110809467b48Spatrick bool InstrumentLifetimeStart = ClHandleLifetimeIntrinsics;
1109*d415bd75Srobert SmallSetVector<AllocaInst *, 16> AllocaSet;
111009467b48Spatrick SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList;
111109467b48Spatrick SmallVector<StoreInst *, 16> StoreList;
1112*d415bd75Srobert int64_t SplittableBlocksCount = 0;
111309467b48Spatrick
MemorySanitizerVisitor__anon9f4f4c900811::MemorySanitizerVisitor111409467b48Spatrick MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
111509467b48Spatrick const TargetLibraryInfo &TLI)
111609467b48Spatrick : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) {
1117*d415bd75Srobert bool SanitizeFunction =
1118*d415bd75Srobert F.hasFnAttribute(Attribute::SanitizeMemory) && !ClDisableChecks;
111909467b48Spatrick InsertChecks = SanitizeFunction;
112009467b48Spatrick PropagateShadow = SanitizeFunction;
112109467b48Spatrick PoisonStack = SanitizeFunction && ClPoisonStack;
112209467b48Spatrick PoisonUndef = SanitizeFunction && ClPoisonUndef;
112309467b48Spatrick
112473471bf0Spatrick // In the presence of unreachable blocks, we may see Phi nodes with
112573471bf0Spatrick // incoming nodes from such blocks. Since InstVisitor skips unreachable
112673471bf0Spatrick // blocks, such nodes will not have any shadow value associated with them.
112773471bf0Spatrick // It's easier to remove unreachable blocks than deal with missing shadow.
112873471bf0Spatrick removeUnreachableBlocks(F);
112973471bf0Spatrick
1130*d415bd75Srobert MS.initializeCallbacks(*F.getParent(), TLI);
113173471bf0Spatrick FnPrologueEnd = IRBuilder<>(F.getEntryBlock().getFirstNonPHI())
113273471bf0Spatrick .CreateIntrinsic(Intrinsic::donothing, {}, {});
113373471bf0Spatrick
113473471bf0Spatrick if (MS.CompileKernel) {
113573471bf0Spatrick IRBuilder<> IRB(FnPrologueEnd);
113673471bf0Spatrick insertKmsanPrologue(IRB);
113773471bf0Spatrick }
113809467b48Spatrick
113909467b48Spatrick LLVM_DEBUG(if (!InsertChecks) dbgs()
114009467b48Spatrick << "MemorySanitizer is not inserting checks into '"
114109467b48Spatrick << F.getName() << "'\n");
114209467b48Spatrick }
114309467b48Spatrick
instrumentWithCalls__anon9f4f4c900811::MemorySanitizerVisitor1144*d415bd75Srobert bool instrumentWithCalls(Value *V) {
1145*d415bd75Srobert // Constants likely will be eliminated by follow-up passes.
1146*d415bd75Srobert if (isa<Constant>(V))
1147*d415bd75Srobert return false;
1148*d415bd75Srobert
1149*d415bd75Srobert ++SplittableBlocksCount;
1150*d415bd75Srobert return ClInstrumentationWithCallThreshold >= 0 &&
1151*d415bd75Srobert SplittableBlocksCount > ClInstrumentationWithCallThreshold;
1152*d415bd75Srobert }
1153*d415bd75Srobert
isInPrologue__anon9f4f4c900811::MemorySanitizerVisitor115473471bf0Spatrick bool isInPrologue(Instruction &I) {
115573471bf0Spatrick return I.getParent() == FnPrologueEnd->getParent() &&
115673471bf0Spatrick (&I == FnPrologueEnd || I.comesBefore(FnPrologueEnd));
115773471bf0Spatrick }
115873471bf0Spatrick
1159*d415bd75Srobert // Creates a new origin and records the stack trace. In general we can call
1160*d415bd75Srobert // this function for any origin manipulation we like. However it will cost
1161*d415bd75Srobert // runtime resources. So use this wisely only if it can provide additional
1162*d415bd75Srobert // information helpful to a user.
updateOrigin__anon9f4f4c900811::MemorySanitizerVisitor116309467b48Spatrick Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1164*d415bd75Srobert if (MS.TrackOrigins <= 1)
1165*d415bd75Srobert return V;
116609467b48Spatrick return IRB.CreateCall(MS.MsanChainOriginFn, V);
116709467b48Spatrick }
116809467b48Spatrick
originToIntptr__anon9f4f4c900811::MemorySanitizerVisitor116909467b48Spatrick Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
117009467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
117109467b48Spatrick unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1172*d415bd75Srobert if (IntptrSize == kOriginSize)
1173*d415bd75Srobert return Origin;
117409467b48Spatrick assert(IntptrSize == kOriginSize * 2);
117509467b48Spatrick Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
117609467b48Spatrick return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
117709467b48Spatrick }
117809467b48Spatrick
117909467b48Spatrick /// Fill memory range with the given origin value.
paintOrigin__anon9f4f4c900811::MemorySanitizerVisitor118009467b48Spatrick void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
118109467b48Spatrick unsigned Size, Align Alignment) {
118209467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
1183097a140dSpatrick const Align IntptrAlignment = DL.getABITypeAlign(MS.IntptrTy);
118409467b48Spatrick unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
118509467b48Spatrick assert(IntptrAlignment >= kMinOriginAlignment);
118609467b48Spatrick assert(IntptrSize >= kOriginSize);
118709467b48Spatrick
118809467b48Spatrick unsigned Ofs = 0;
118909467b48Spatrick Align CurrentAlignment = Alignment;
119009467b48Spatrick if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
119109467b48Spatrick Value *IntptrOrigin = originToIntptr(IRB, Origin);
119209467b48Spatrick Value *IntptrOriginPtr =
119309467b48Spatrick IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
119409467b48Spatrick for (unsigned i = 0; i < Size / IntptrSize; ++i) {
119509467b48Spatrick Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
119609467b48Spatrick : IntptrOriginPtr;
1197097a140dSpatrick IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
119809467b48Spatrick Ofs += IntptrSize / kOriginSize;
119909467b48Spatrick CurrentAlignment = IntptrAlignment;
120009467b48Spatrick }
120109467b48Spatrick }
120209467b48Spatrick
120309467b48Spatrick for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
120409467b48Spatrick Value *GEP =
120509467b48Spatrick i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr;
1206097a140dSpatrick IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
120709467b48Spatrick CurrentAlignment = kMinOriginAlignment;
120809467b48Spatrick }
120909467b48Spatrick }
121009467b48Spatrick
storeOrigin__anon9f4f4c900811::MemorySanitizerVisitor121109467b48Spatrick void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1212*d415bd75Srobert Value *OriginPtr, Align Alignment) {
121309467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
121409467b48Spatrick const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
121509467b48Spatrick unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
121673471bf0Spatrick Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
121709467b48Spatrick if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1218*d415bd75Srobert if (!ClCheckConstantShadow || ConstantShadow->isZeroValue()) {
1219*d415bd75Srobert // Origin is not needed: value is initialized or const shadow is
1220*d415bd75Srobert // ignored.
1221*d415bd75Srobert return;
1222*d415bd75Srobert }
1223*d415bd75Srobert if (llvm::isKnownNonZero(ConvertedShadow, DL)) {
1224*d415bd75Srobert // Copy origin as the value is definitely uninitialized.
122509467b48Spatrick paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
122609467b48Spatrick OriginAlignment);
122709467b48Spatrick return;
122809467b48Spatrick }
1229*d415bd75Srobert // Fallback to runtime check, which still can be optimized out later.
1230*d415bd75Srobert }
123109467b48Spatrick
123273471bf0Spatrick unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
123309467b48Spatrick unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1234*d415bd75Srobert if (instrumentWithCalls(ConvertedShadow) &&
1235*d415bd75Srobert SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
123609467b48Spatrick FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
123773471bf0Spatrick Value *ConvertedShadow2 =
123873471bf0Spatrick IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
123973471bf0Spatrick CallBase *CB = IRB.CreateCall(
124073471bf0Spatrick Fn, {ConvertedShadow2,
124173471bf0Spatrick IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), Origin});
124273471bf0Spatrick CB->addParamAttr(0, Attribute::ZExt);
124373471bf0Spatrick CB->addParamAttr(2, Attribute::ZExt);
124409467b48Spatrick } else {
124573471bf0Spatrick Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
124609467b48Spatrick Instruction *CheckTerm = SplitBlockAndInsertIfThen(
124709467b48Spatrick Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
124809467b48Spatrick IRBuilder<> IRBNew(CheckTerm);
124909467b48Spatrick paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
125009467b48Spatrick OriginAlignment);
125109467b48Spatrick }
125209467b48Spatrick }
125309467b48Spatrick
materializeStores__anon9f4f4c900811::MemorySanitizerVisitor1254*d415bd75Srobert void materializeStores() {
125509467b48Spatrick for (StoreInst *SI : StoreList) {
125609467b48Spatrick IRBuilder<> IRB(SI);
125709467b48Spatrick Value *Val = SI->getValueOperand();
125809467b48Spatrick Value *Addr = SI->getPointerOperand();
125909467b48Spatrick Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
126009467b48Spatrick Value *ShadowPtr, *OriginPtr;
126109467b48Spatrick Type *ShadowTy = Shadow->getType();
1262*d415bd75Srobert const Align Alignment = SI->getAlign();
126309467b48Spatrick const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
126409467b48Spatrick std::tie(ShadowPtr, OriginPtr) =
126509467b48Spatrick getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
126609467b48Spatrick
1267097a140dSpatrick StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
126809467b48Spatrick LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
126909467b48Spatrick (void)NewSI;
127009467b48Spatrick
127109467b48Spatrick if (SI->isAtomic())
127209467b48Spatrick SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
127309467b48Spatrick
127409467b48Spatrick if (MS.TrackOrigins && !SI->isAtomic())
127509467b48Spatrick storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
1276*d415bd75Srobert OriginAlignment);
127709467b48Spatrick }
127809467b48Spatrick }
127909467b48Spatrick
1280*d415bd75Srobert // Returns true if Debug Location curresponds to multiple warnings.
shouldDisambiguateWarningLocation__anon9f4f4c900811::MemorySanitizerVisitor1281*d415bd75Srobert bool shouldDisambiguateWarningLocation(const DebugLoc &DebugLoc) {
1282*d415bd75Srobert if (MS.TrackOrigins < 2)
1283*d415bd75Srobert return false;
1284*d415bd75Srobert
1285*d415bd75Srobert if (LazyWarningDebugLocationCount.empty())
1286*d415bd75Srobert for (const auto &I : InstrumentationList)
1287*d415bd75Srobert ++LazyWarningDebugLocationCount[I.OrigIns->getDebugLoc()];
1288*d415bd75Srobert
1289*d415bd75Srobert return LazyWarningDebugLocationCount[DebugLoc] >= ClDisambiguateWarning;
1290*d415bd75Srobert }
1291*d415bd75Srobert
129209467b48Spatrick /// Helper function to insert a warning at IRB's current insert point.
insertWarningFn__anon9f4f4c900811::MemorySanitizerVisitor129309467b48Spatrick void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
129409467b48Spatrick if (!Origin)
129509467b48Spatrick Origin = (Value *)IRB.getInt32(0);
1296097a140dSpatrick assert(Origin->getType()->isIntegerTy());
1297*d415bd75Srobert
1298*d415bd75Srobert if (shouldDisambiguateWarningLocation(IRB.getCurrentDebugLocation())) {
1299*d415bd75Srobert // Try to create additional origin with debug info of the last origin
1300*d415bd75Srobert // instruction. It may provide additional information to the user.
1301*d415bd75Srobert if (Instruction *OI = dyn_cast_or_null<Instruction>(Origin)) {
1302*d415bd75Srobert assert(MS.TrackOrigins);
1303*d415bd75Srobert auto NewDebugLoc = OI->getDebugLoc();
1304*d415bd75Srobert // Origin update with missing or the same debug location provides no
1305*d415bd75Srobert // additional value.
1306*d415bd75Srobert if (NewDebugLoc && NewDebugLoc != IRB.getCurrentDebugLocation()) {
1307*d415bd75Srobert // Insert update just before the check, so we call runtime only just
1308*d415bd75Srobert // before the report.
1309*d415bd75Srobert IRBuilder<> IRBOrigin(&*IRB.GetInsertPoint());
1310*d415bd75Srobert IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1311*d415bd75Srobert Origin = updateOrigin(Origin, IRBOrigin);
1312*d415bd75Srobert }
1313*d415bd75Srobert }
1314*d415bd75Srobert }
1315*d415bd75Srobert
1316*d415bd75Srobert if (MS.CompileKernel || MS.TrackOrigins)
1317097a140dSpatrick IRB.CreateCall(MS.WarningFn, Origin)->setCannotMerge();
1318*d415bd75Srobert else
1319*d415bd75Srobert IRB.CreateCall(MS.WarningFn)->setCannotMerge();
132009467b48Spatrick // FIXME: Insert UnreachableInst if !MS.Recover?
132109467b48Spatrick // This may invalidate some of the following checks and needs to be done
132209467b48Spatrick // at the very end.
132309467b48Spatrick }
132409467b48Spatrick
materializeOneCheck__anon9f4f4c900811::MemorySanitizerVisitor1325*d415bd75Srobert void materializeOneCheck(IRBuilder<> &IRB, Value *ConvertedShadow,
1326*d415bd75Srobert Value *Origin) {
1327*d415bd75Srobert const DataLayout &DL = F.getParent()->getDataLayout();
132809467b48Spatrick unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
132909467b48Spatrick unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1330*d415bd75Srobert if (instrumentWithCalls(ConvertedShadow) &&
1331*d415bd75Srobert SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
133209467b48Spatrick FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
133309467b48Spatrick Value *ConvertedShadow2 =
133409467b48Spatrick IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
133573471bf0Spatrick CallBase *CB = IRB.CreateCall(
133673471bf0Spatrick Fn, {ConvertedShadow2,
133773471bf0Spatrick MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(0)});
133873471bf0Spatrick CB->addParamAttr(0, Attribute::ZExt);
133973471bf0Spatrick CB->addParamAttr(1, Attribute::ZExt);
134009467b48Spatrick } else {
134173471bf0Spatrick Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
134209467b48Spatrick Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1343*d415bd75Srobert Cmp, &*IRB.GetInsertPoint(),
134409467b48Spatrick /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
134509467b48Spatrick
134609467b48Spatrick IRB.SetInsertPoint(CheckTerm);
134709467b48Spatrick insertWarningFn(IRB, Origin);
134809467b48Spatrick LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
134909467b48Spatrick }
135009467b48Spatrick }
135109467b48Spatrick
materializeInstructionChecks__anon9f4f4c900811::MemorySanitizerVisitor1352*d415bd75Srobert void materializeInstructionChecks(
1353*d415bd75Srobert ArrayRef<ShadowOriginAndInsertPoint> InstructionChecks) {
1354*d415bd75Srobert const DataLayout &DL = F.getParent()->getDataLayout();
1355*d415bd75Srobert // Disable combining in some cases. TrackOrigins checks each shadow to pick
1356*d415bd75Srobert // correct origin.
1357*d415bd75Srobert bool Combine = !MS.TrackOrigins;
1358*d415bd75Srobert Instruction *Instruction = InstructionChecks.front().OrigIns;
1359*d415bd75Srobert Value *Shadow = nullptr;
1360*d415bd75Srobert for (const auto &ShadowData : InstructionChecks) {
1361*d415bd75Srobert assert(ShadowData.OrigIns == Instruction);
1362*d415bd75Srobert IRBuilder<> IRB(Instruction);
1363*d415bd75Srobert
1364*d415bd75Srobert Value *ConvertedShadow = ShadowData.Shadow;
1365*d415bd75Srobert
1366*d415bd75Srobert if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1367*d415bd75Srobert if (!ClCheckConstantShadow || ConstantShadow->isZeroValue()) {
1368*d415bd75Srobert // Skip, value is initialized or const shadow is ignored.
1369*d415bd75Srobert continue;
137009467b48Spatrick }
1371*d415bd75Srobert if (llvm::isKnownNonZero(ConvertedShadow, DL)) {
1372*d415bd75Srobert // Report as the value is definitely uninitialized.
1373*d415bd75Srobert insertWarningFn(IRB, ShadowData.Origin);
1374*d415bd75Srobert if (!MS.Recover)
1375*d415bd75Srobert return; // Always fail and stop here, not need to check the rest.
1376*d415bd75Srobert // Skip entire instruction,
1377*d415bd75Srobert continue;
1378*d415bd75Srobert }
1379*d415bd75Srobert // Fallback to runtime check, which still can be optimized out later.
1380*d415bd75Srobert }
1381*d415bd75Srobert
1382*d415bd75Srobert if (!Combine) {
1383*d415bd75Srobert materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1384*d415bd75Srobert continue;
1385*d415bd75Srobert }
1386*d415bd75Srobert
1387*d415bd75Srobert if (!Shadow) {
1388*d415bd75Srobert Shadow = ConvertedShadow;
1389*d415bd75Srobert continue;
1390*d415bd75Srobert }
1391*d415bd75Srobert
1392*d415bd75Srobert Shadow = convertToBool(Shadow, IRB, "_mscmp");
1393*d415bd75Srobert ConvertedShadow = convertToBool(ConvertedShadow, IRB, "_mscmp");
1394*d415bd75Srobert Shadow = IRB.CreateOr(Shadow, ConvertedShadow, "_msor");
1395*d415bd75Srobert }
1396*d415bd75Srobert
1397*d415bd75Srobert if (Shadow) {
1398*d415bd75Srobert assert(Combine);
1399*d415bd75Srobert IRBuilder<> IRB(Instruction);
1400*d415bd75Srobert materializeOneCheck(IRB, Shadow, nullptr);
1401*d415bd75Srobert }
1402*d415bd75Srobert }
1403*d415bd75Srobert
materializeChecks__anon9f4f4c900811::MemorySanitizerVisitor1404*d415bd75Srobert void materializeChecks() {
1405*d415bd75Srobert llvm::stable_sort(InstrumentationList,
1406*d415bd75Srobert [](const ShadowOriginAndInsertPoint &L,
1407*d415bd75Srobert const ShadowOriginAndInsertPoint &R) {
1408*d415bd75Srobert return L.OrigIns < R.OrigIns;
1409*d415bd75Srobert });
1410*d415bd75Srobert
1411*d415bd75Srobert for (auto I = InstrumentationList.begin();
1412*d415bd75Srobert I != InstrumentationList.end();) {
1413*d415bd75Srobert auto J =
1414*d415bd75Srobert std::find_if(I + 1, InstrumentationList.end(),
1415*d415bd75Srobert [L = I->OrigIns](const ShadowOriginAndInsertPoint &R) {
1416*d415bd75Srobert return L != R.OrigIns;
1417*d415bd75Srobert });
1418*d415bd75Srobert // Process all checks of instruction at once.
1419*d415bd75Srobert materializeInstructionChecks(ArrayRef<ShadowOriginAndInsertPoint>(I, J));
1420*d415bd75Srobert I = J;
1421*d415bd75Srobert }
1422*d415bd75Srobert
142309467b48Spatrick LLVM_DEBUG(dbgs() << "DONE:\n" << F);
142409467b48Spatrick }
142509467b48Spatrick
142673471bf0Spatrick // Returns the last instruction in the new prologue
insertKmsanPrologue__anon9f4f4c900811::MemorySanitizerVisitor142773471bf0Spatrick void insertKmsanPrologue(IRBuilder<> &IRB) {
142809467b48Spatrick Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
142909467b48Spatrick Constant *Zero = IRB.getInt32(0);
143009467b48Spatrick MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
143109467b48Spatrick {Zero, IRB.getInt32(0)}, "param_shadow");
143209467b48Spatrick MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
143309467b48Spatrick {Zero, IRB.getInt32(1)}, "retval_shadow");
143409467b48Spatrick MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
143509467b48Spatrick {Zero, IRB.getInt32(2)}, "va_arg_shadow");
143609467b48Spatrick MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
143709467b48Spatrick {Zero, IRB.getInt32(3)}, "va_arg_origin");
143809467b48Spatrick MS.VAArgOverflowSizeTLS =
143909467b48Spatrick IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
144009467b48Spatrick {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
144109467b48Spatrick MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
144209467b48Spatrick {Zero, IRB.getInt32(5)}, "param_origin");
144309467b48Spatrick MS.RetvalOriginTLS =
144409467b48Spatrick IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
144509467b48Spatrick {Zero, IRB.getInt32(6)}, "retval_origin");
144609467b48Spatrick }
144709467b48Spatrick
144809467b48Spatrick /// Add MemorySanitizer instrumentation to a function.
runOnFunction__anon9f4f4c900811::MemorySanitizerVisitor144909467b48Spatrick bool runOnFunction() {
145009467b48Spatrick // Iterate all BBs in depth-first order and create shadow instructions
145109467b48Spatrick // for all instructions (where applicable).
145209467b48Spatrick // For PHI nodes we create dummy shadow PHIs which will be finalized later.
145373471bf0Spatrick for (BasicBlock *BB : depth_first(FnPrologueEnd->getParent()))
145409467b48Spatrick visit(*BB);
145509467b48Spatrick
145609467b48Spatrick // Finalize PHI nodes.
145709467b48Spatrick for (PHINode *PN : ShadowPHINodes) {
145809467b48Spatrick PHINode *PNS = cast<PHINode>(getShadow(PN));
145909467b48Spatrick PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
146009467b48Spatrick size_t NumValues = PN->getNumIncomingValues();
146109467b48Spatrick for (size_t v = 0; v < NumValues; v++) {
146209467b48Spatrick PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1463*d415bd75Srobert if (PNO)
1464*d415bd75Srobert PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
146509467b48Spatrick }
146609467b48Spatrick }
146709467b48Spatrick
146809467b48Spatrick VAHelper->finalizeInstrumentation();
146909467b48Spatrick
147009467b48Spatrick // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
147109467b48Spatrick // instrumenting only allocas.
147209467b48Spatrick if (InstrumentLifetimeStart) {
147309467b48Spatrick for (auto Item : LifetimeStartList) {
147409467b48Spatrick instrumentAlloca(*Item.second, Item.first);
1475*d415bd75Srobert AllocaSet.remove(Item.second);
147609467b48Spatrick }
147709467b48Spatrick }
147809467b48Spatrick // Poison the allocas for which we didn't instrument the corresponding
147909467b48Spatrick // lifetime intrinsics.
148009467b48Spatrick for (AllocaInst *AI : AllocaSet)
148109467b48Spatrick instrumentAlloca(*AI);
148209467b48Spatrick
148309467b48Spatrick // Insert shadow value checks.
1484*d415bd75Srobert materializeChecks();
148509467b48Spatrick
148609467b48Spatrick // Delayed instrumentation of StoreInst.
148709467b48Spatrick // This may not add new address checks.
1488*d415bd75Srobert materializeStores();
148909467b48Spatrick
149009467b48Spatrick return true;
149109467b48Spatrick }
149209467b48Spatrick
149309467b48Spatrick /// Compute the shadow type that corresponds to a given Value.
getShadowTy__anon9f4f4c900811::MemorySanitizerVisitor1494*d415bd75Srobert Type *getShadowTy(Value *V) { return getShadowTy(V->getType()); }
149509467b48Spatrick
149609467b48Spatrick /// Compute the shadow type that corresponds to a given Type.
getShadowTy__anon9f4f4c900811::MemorySanitizerVisitor149709467b48Spatrick Type *getShadowTy(Type *OrigTy) {
149809467b48Spatrick if (!OrigTy->isSized()) {
149909467b48Spatrick return nullptr;
150009467b48Spatrick }
150109467b48Spatrick // For integer type, shadow is the same as the original type.
150209467b48Spatrick // This may return weird-sized types like i1.
150309467b48Spatrick if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
150409467b48Spatrick return IT;
150509467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
150609467b48Spatrick if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
150709467b48Spatrick uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
1508097a140dSpatrick return FixedVectorType::get(IntegerType::get(*MS.C, EltSize),
1509097a140dSpatrick cast<FixedVectorType>(VT)->getNumElements());
151009467b48Spatrick }
151109467b48Spatrick if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
151209467b48Spatrick return ArrayType::get(getShadowTy(AT->getElementType()),
151309467b48Spatrick AT->getNumElements());
151409467b48Spatrick }
151509467b48Spatrick if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
151609467b48Spatrick SmallVector<Type *, 4> Elements;
151709467b48Spatrick for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
151809467b48Spatrick Elements.push_back(getShadowTy(ST->getElementType(i)));
151909467b48Spatrick StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
152009467b48Spatrick LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
152109467b48Spatrick return Res;
152209467b48Spatrick }
152309467b48Spatrick uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
152409467b48Spatrick return IntegerType::get(*MS.C, TypeSize);
152509467b48Spatrick }
152609467b48Spatrick
152709467b48Spatrick /// Flatten a vector type.
getShadowTyNoVec__anon9f4f4c900811::MemorySanitizerVisitor152809467b48Spatrick Type *getShadowTyNoVec(Type *ty) {
152909467b48Spatrick if (VectorType *vt = dyn_cast<VectorType>(ty))
1530097a140dSpatrick return IntegerType::get(*MS.C,
1531*d415bd75Srobert vt->getPrimitiveSizeInBits().getFixedValue());
153209467b48Spatrick return ty;
153309467b48Spatrick }
153409467b48Spatrick
153573471bf0Spatrick /// Extract combined shadow of struct elements as a bool
collapseStructShadow__anon9f4f4c900811::MemorySanitizerVisitor153673471bf0Spatrick Value *collapseStructShadow(StructType *Struct, Value *Shadow,
153773471bf0Spatrick IRBuilder<> &IRB) {
153873471bf0Spatrick Value *FalseVal = IRB.getIntN(/* width */ 1, /* value */ 0);
153973471bf0Spatrick Value *Aggregator = FalseVal;
154073471bf0Spatrick
154173471bf0Spatrick for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
154273471bf0Spatrick // Combine by ORing together each element's bool shadow
154373471bf0Spatrick Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
154473471bf0Spatrick Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
154573471bf0Spatrick Value *ShadowBool = convertToBool(ShadowInner, IRB);
154673471bf0Spatrick
154773471bf0Spatrick if (Aggregator != FalseVal)
154873471bf0Spatrick Aggregator = IRB.CreateOr(Aggregator, ShadowBool);
154973471bf0Spatrick else
155073471bf0Spatrick Aggregator = ShadowBool;
155173471bf0Spatrick }
155273471bf0Spatrick
155373471bf0Spatrick return Aggregator;
155473471bf0Spatrick }
155573471bf0Spatrick
155673471bf0Spatrick // Extract combined shadow of array elements
collapseArrayShadow__anon9f4f4c900811::MemorySanitizerVisitor155773471bf0Spatrick Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
155873471bf0Spatrick IRBuilder<> &IRB) {
155973471bf0Spatrick if (!Array->getNumElements())
156073471bf0Spatrick return IRB.getIntN(/* width */ 1, /* value */ 0);
156173471bf0Spatrick
156273471bf0Spatrick Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
156373471bf0Spatrick Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
156473471bf0Spatrick
156573471bf0Spatrick for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
156673471bf0Spatrick Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
156773471bf0Spatrick Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
156873471bf0Spatrick Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
156973471bf0Spatrick }
157073471bf0Spatrick return Aggregator;
157173471bf0Spatrick }
157273471bf0Spatrick
157373471bf0Spatrick /// Convert a shadow value to it's flattened variant. The resulting
157473471bf0Spatrick /// shadow may not necessarily have the same bit width as the input
157573471bf0Spatrick /// value, but it will always be comparable to zero.
convertShadowToScalar__anon9f4f4c900811::MemorySanitizerVisitor157673471bf0Spatrick Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
157773471bf0Spatrick if (StructType *Struct = dyn_cast<StructType>(V->getType()))
157873471bf0Spatrick return collapseStructShadow(Struct, V, IRB);
157973471bf0Spatrick if (ArrayType *Array = dyn_cast<ArrayType>(V->getType()))
158073471bf0Spatrick return collapseArrayShadow(Array, V, IRB);
158109467b48Spatrick Type *Ty = V->getType();
158209467b48Spatrick Type *NoVecTy = getShadowTyNoVec(Ty);
1583*d415bd75Srobert if (Ty == NoVecTy)
1584*d415bd75Srobert return V;
158509467b48Spatrick return IRB.CreateBitCast(V, NoVecTy);
158609467b48Spatrick }
158709467b48Spatrick
158873471bf0Spatrick // Convert a scalar value to an i1 by comparing with 0
convertToBool__anon9f4f4c900811::MemorySanitizerVisitor158973471bf0Spatrick Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
159073471bf0Spatrick Type *VTy = V->getType();
1591*d415bd75Srobert if (!VTy->isIntegerTy())
1592*d415bd75Srobert return convertToBool(convertShadowToScalar(V, IRB), IRB, name);
159373471bf0Spatrick if (VTy->getIntegerBitWidth() == 1)
159473471bf0Spatrick // Just converting a bool to a bool, so do nothing.
159573471bf0Spatrick return V;
159673471bf0Spatrick return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), name);
159773471bf0Spatrick }
159873471bf0Spatrick
ptrToIntPtrType__anon9f4f4c900811::MemorySanitizerVisitor1599*d415bd75Srobert Type *ptrToIntPtrType(Type *PtrTy) const {
1600*d415bd75Srobert if (FixedVectorType *VectTy = dyn_cast<FixedVectorType>(PtrTy)) {
1601*d415bd75Srobert return FixedVectorType::get(ptrToIntPtrType(VectTy->getElementType()),
1602*d415bd75Srobert VectTy->getNumElements());
1603*d415bd75Srobert }
1604*d415bd75Srobert assert(PtrTy->isIntOrPtrTy());
1605*d415bd75Srobert return MS.IntptrTy;
1606*d415bd75Srobert }
1607*d415bd75Srobert
getPtrToShadowPtrType__anon9f4f4c900811::MemorySanitizerVisitor1608*d415bd75Srobert Type *getPtrToShadowPtrType(Type *IntPtrTy, Type *ShadowTy) const {
1609*d415bd75Srobert if (FixedVectorType *VectTy = dyn_cast<FixedVectorType>(IntPtrTy)) {
1610*d415bd75Srobert return FixedVectorType::get(
1611*d415bd75Srobert getPtrToShadowPtrType(VectTy->getElementType(), ShadowTy),
1612*d415bd75Srobert VectTy->getNumElements());
1613*d415bd75Srobert }
1614*d415bd75Srobert assert(IntPtrTy == MS.IntptrTy);
1615*d415bd75Srobert return ShadowTy->getPointerTo();
1616*d415bd75Srobert }
1617*d415bd75Srobert
constToIntPtr__anon9f4f4c900811::MemorySanitizerVisitor1618*d415bd75Srobert Constant *constToIntPtr(Type *IntPtrTy, uint64_t C) const {
1619*d415bd75Srobert if (FixedVectorType *VectTy = dyn_cast<FixedVectorType>(IntPtrTy)) {
1620*d415bd75Srobert return ConstantDataVector::getSplat(
1621*d415bd75Srobert VectTy->getNumElements(), constToIntPtr(VectTy->getElementType(), C));
1622*d415bd75Srobert }
1623*d415bd75Srobert assert(IntPtrTy == MS.IntptrTy);
1624*d415bd75Srobert return ConstantInt::get(MS.IntptrTy, C);
1625*d415bd75Srobert }
1626*d415bd75Srobert
162709467b48Spatrick /// Compute the integer shadow offset that corresponds to a given
162809467b48Spatrick /// application address.
162909467b48Spatrick ///
163009467b48Spatrick /// Offset = (Addr & ~AndMask) ^ XorMask
1631*d415bd75Srobert /// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1632*d415bd75Srobert /// a single pointee.
1633*d415bd75Srobert /// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
getShadowPtrOffset__anon9f4f4c900811::MemorySanitizerVisitor163409467b48Spatrick Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1635*d415bd75Srobert Type *IntptrTy = ptrToIntPtrType(Addr->getType());
1636*d415bd75Srobert Value *OffsetLong = IRB.CreatePointerCast(Addr, IntptrTy);
163709467b48Spatrick
1638*d415bd75Srobert if (uint64_t AndMask = MS.MapParams->AndMask)
1639*d415bd75Srobert OffsetLong = IRB.CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
164009467b48Spatrick
1641*d415bd75Srobert if (uint64_t XorMask = MS.MapParams->XorMask)
1642*d415bd75Srobert OffsetLong = IRB.CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
164309467b48Spatrick return OffsetLong;
164409467b48Spatrick }
164509467b48Spatrick
164609467b48Spatrick /// Compute the shadow and origin addresses corresponding to a given
164709467b48Spatrick /// application address.
164809467b48Spatrick ///
164909467b48Spatrick /// Shadow = ShadowBase + Offset
165009467b48Spatrick /// Origin = (OriginBase + Offset) & ~3ULL
1651*d415bd75Srobert /// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1652*d415bd75Srobert /// a single pointee.
1653*d415bd75Srobert /// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
165409467b48Spatrick std::pair<Value *, Value *>
getShadowOriginPtrUserspace__anon9f4f4c900811::MemorySanitizerVisitor165509467b48Spatrick getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
165609467b48Spatrick MaybeAlign Alignment) {
1657*d415bd75Srobert Type *IntptrTy = ptrToIntPtrType(Addr->getType());
165809467b48Spatrick Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
165909467b48Spatrick Value *ShadowLong = ShadowOffset;
1660*d415bd75Srobert if (uint64_t ShadowBase = MS.MapParams->ShadowBase) {
166109467b48Spatrick ShadowLong =
1662*d415bd75Srobert IRB.CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
166309467b48Spatrick }
1664*d415bd75Srobert Value *ShadowPtr = IRB.CreateIntToPtr(
1665*d415bd75Srobert ShadowLong, getPtrToShadowPtrType(IntptrTy, ShadowTy));
1666*d415bd75Srobert
166709467b48Spatrick Value *OriginPtr = nullptr;
166809467b48Spatrick if (MS.TrackOrigins) {
166909467b48Spatrick Value *OriginLong = ShadowOffset;
167009467b48Spatrick uint64_t OriginBase = MS.MapParams->OriginBase;
167109467b48Spatrick if (OriginBase != 0)
1672*d415bd75Srobert OriginLong =
1673*d415bd75Srobert IRB.CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
167409467b48Spatrick if (!Alignment || *Alignment < kMinOriginAlignment) {
167509467b48Spatrick uint64_t Mask = kMinOriginAlignment.value() - 1;
1676*d415bd75Srobert OriginLong = IRB.CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
167709467b48Spatrick }
1678*d415bd75Srobert OriginPtr = IRB.CreateIntToPtr(
1679*d415bd75Srobert OriginLong, getPtrToShadowPtrType(IntptrTy, MS.OriginTy));
168009467b48Spatrick }
168109467b48Spatrick return std::make_pair(ShadowPtr, OriginPtr);
168209467b48Spatrick }
168309467b48Spatrick
getShadowOriginPtrKernelNoVec__anon9f4f4c900811::MemorySanitizerVisitor1684*d415bd75Srobert std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(Value *Addr,
168509467b48Spatrick IRBuilder<> &IRB,
168609467b48Spatrick Type *ShadowTy,
168709467b48Spatrick bool isStore) {
168809467b48Spatrick Value *ShadowOriginPtrs;
168909467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
169009467b48Spatrick int Size = DL.getTypeStoreSize(ShadowTy);
169109467b48Spatrick
169209467b48Spatrick FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size);
169309467b48Spatrick Value *AddrCast =
169409467b48Spatrick IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0));
169509467b48Spatrick if (Getter) {
169609467b48Spatrick ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast);
169709467b48Spatrick } else {
169809467b48Spatrick Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
169909467b48Spatrick ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN
170009467b48Spatrick : MS.MsanMetadataPtrForLoadN,
170109467b48Spatrick {AddrCast, SizeVal});
170209467b48Spatrick }
170309467b48Spatrick Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
170409467b48Spatrick ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
170509467b48Spatrick Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
170609467b48Spatrick
170709467b48Spatrick return std::make_pair(ShadowPtr, OriginPtr);
170809467b48Spatrick }
170909467b48Spatrick
1710*d415bd75Srobert /// Addr can be a ptr or <N x ptr>. In both cases ShadowTy the shadow type of
1711*d415bd75Srobert /// a single pointee.
1712*d415bd75Srobert /// Returns <shadow_ptr, origin_ptr> or <<N x shadow_ptr>, <N x origin_ptr>>.
getShadowOriginPtrKernel__anon9f4f4c900811::MemorySanitizerVisitor1713*d415bd75Srobert std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr,
1714*d415bd75Srobert IRBuilder<> &IRB,
1715*d415bd75Srobert Type *ShadowTy,
1716*d415bd75Srobert bool isStore) {
1717*d415bd75Srobert FixedVectorType *VectTy = dyn_cast<FixedVectorType>(Addr->getType());
1718*d415bd75Srobert if (!VectTy) {
1719*d415bd75Srobert assert(Addr->getType()->isPointerTy());
1720*d415bd75Srobert return getShadowOriginPtrKernelNoVec(Addr, IRB, ShadowTy, isStore);
1721*d415bd75Srobert }
1722*d415bd75Srobert
1723*d415bd75Srobert // TODO: Support callbacs with vectors of addresses.
1724*d415bd75Srobert unsigned NumElements = VectTy->getNumElements();
1725*d415bd75Srobert Value *ShadowPtrs = ConstantInt::getNullValue(
1726*d415bd75Srobert FixedVectorType::get(ShadowTy->getPointerTo(), NumElements));
1727*d415bd75Srobert Value *OriginPtrs = nullptr;
1728*d415bd75Srobert if (MS.TrackOrigins)
1729*d415bd75Srobert OriginPtrs = ConstantInt::getNullValue(
1730*d415bd75Srobert FixedVectorType::get(MS.OriginTy->getPointerTo(), NumElements));
1731*d415bd75Srobert for (unsigned i = 0; i < NumElements; ++i) {
1732*d415bd75Srobert Value *OneAddr =
1733*d415bd75Srobert IRB.CreateExtractElement(Addr, ConstantInt::get(IRB.getInt32Ty(), i));
1734*d415bd75Srobert auto [ShadowPtr, OriginPtr] =
1735*d415bd75Srobert getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy, isStore);
1736*d415bd75Srobert
1737*d415bd75Srobert ShadowPtrs = IRB.CreateInsertElement(
1738*d415bd75Srobert ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.getInt32Ty(), i));
1739*d415bd75Srobert if (MS.TrackOrigins)
1740*d415bd75Srobert OriginPtrs = IRB.CreateInsertElement(
1741*d415bd75Srobert OriginPtrs, OriginPtr, ConstantInt::get(IRB.getInt32Ty(), i));
1742*d415bd75Srobert }
1743*d415bd75Srobert return {ShadowPtrs, OriginPtrs};
1744*d415bd75Srobert }
1745*d415bd75Srobert
getShadowOriginPtr__anon9f4f4c900811::MemorySanitizerVisitor174609467b48Spatrick std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
174709467b48Spatrick Type *ShadowTy,
174809467b48Spatrick MaybeAlign Alignment,
174909467b48Spatrick bool isStore) {
175009467b48Spatrick if (MS.CompileKernel)
175109467b48Spatrick return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
175209467b48Spatrick return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
175309467b48Spatrick }
175409467b48Spatrick
175509467b48Spatrick /// Compute the shadow address for a given function argument.
175609467b48Spatrick ///
175709467b48Spatrick /// Shadow = ParamTLS+ArgOffset.
getShadowPtrForArgument__anon9f4f4c900811::MemorySanitizerVisitor1758*d415bd75Srobert Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB, int ArgOffset) {
175909467b48Spatrick Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
176009467b48Spatrick if (ArgOffset)
176109467b48Spatrick Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
176209467b48Spatrick return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
176309467b48Spatrick "_msarg");
176409467b48Spatrick }
176509467b48Spatrick
176609467b48Spatrick /// Compute the origin address for a given function argument.
getOriginPtrForArgument__anon9f4f4c900811::MemorySanitizerVisitor1767*d415bd75Srobert Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB, int ArgOffset) {
176809467b48Spatrick if (!MS.TrackOrigins)
176909467b48Spatrick return nullptr;
177009467b48Spatrick Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
177109467b48Spatrick if (ArgOffset)
177209467b48Spatrick Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
177309467b48Spatrick return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
177409467b48Spatrick "_msarg_o");
177509467b48Spatrick }
177609467b48Spatrick
177709467b48Spatrick /// Compute the shadow address for a retval.
getShadowPtrForRetval__anon9f4f4c900811::MemorySanitizerVisitor177809467b48Spatrick Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
177909467b48Spatrick return IRB.CreatePointerCast(MS.RetvalTLS,
1780*d415bd75Srobert PointerType::get(getShadowTy(A), 0), "_msret");
178109467b48Spatrick }
178209467b48Spatrick
178309467b48Spatrick /// Compute the origin address for a retval.
getOriginPtrForRetval__anon9f4f4c900811::MemorySanitizerVisitor178409467b48Spatrick Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
178509467b48Spatrick // We keep a single origin for the entire retval. Might be too optimistic.
178609467b48Spatrick return MS.RetvalOriginTLS;
178709467b48Spatrick }
178809467b48Spatrick
178909467b48Spatrick /// Set SV to be the shadow value for V.
setShadow__anon9f4f4c900811::MemorySanitizerVisitor179009467b48Spatrick void setShadow(Value *V, Value *SV) {
179109467b48Spatrick assert(!ShadowMap.count(V) && "Values may only have one shadow");
179209467b48Spatrick ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
179309467b48Spatrick }
179409467b48Spatrick
179509467b48Spatrick /// Set Origin to be the origin value for V.
setOrigin__anon9f4f4c900811::MemorySanitizerVisitor179609467b48Spatrick void setOrigin(Value *V, Value *Origin) {
1797*d415bd75Srobert if (!MS.TrackOrigins)
1798*d415bd75Srobert return;
179909467b48Spatrick assert(!OriginMap.count(V) && "Values may only have one origin");
180009467b48Spatrick LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
180109467b48Spatrick OriginMap[V] = Origin;
180209467b48Spatrick }
180309467b48Spatrick
getCleanShadow__anon9f4f4c900811::MemorySanitizerVisitor180409467b48Spatrick Constant *getCleanShadow(Type *OrigTy) {
180509467b48Spatrick Type *ShadowTy = getShadowTy(OrigTy);
180609467b48Spatrick if (!ShadowTy)
180709467b48Spatrick return nullptr;
180809467b48Spatrick return Constant::getNullValue(ShadowTy);
180909467b48Spatrick }
181009467b48Spatrick
181109467b48Spatrick /// Create a clean shadow value for a given value.
181209467b48Spatrick ///
181309467b48Spatrick /// Clean shadow (all zeroes) means all bits of the value are defined
181409467b48Spatrick /// (initialized).
getCleanShadow__anon9f4f4c900811::MemorySanitizerVisitor1815*d415bd75Srobert Constant *getCleanShadow(Value *V) { return getCleanShadow(V->getType()); }
181609467b48Spatrick
181709467b48Spatrick /// Create a dirty shadow of a given shadow type.
getPoisonedShadow__anon9f4f4c900811::MemorySanitizerVisitor181809467b48Spatrick Constant *getPoisonedShadow(Type *ShadowTy) {
181909467b48Spatrick assert(ShadowTy);
182009467b48Spatrick if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
182109467b48Spatrick return Constant::getAllOnesValue(ShadowTy);
182209467b48Spatrick if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
182309467b48Spatrick SmallVector<Constant *, 4> Vals(AT->getNumElements(),
182409467b48Spatrick getPoisonedShadow(AT->getElementType()));
182509467b48Spatrick return ConstantArray::get(AT, Vals);
182609467b48Spatrick }
182709467b48Spatrick if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
182809467b48Spatrick SmallVector<Constant *, 4> Vals;
182909467b48Spatrick for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
183009467b48Spatrick Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
183109467b48Spatrick return ConstantStruct::get(ST, Vals);
183209467b48Spatrick }
183309467b48Spatrick llvm_unreachable("Unexpected shadow type");
183409467b48Spatrick }
183509467b48Spatrick
183609467b48Spatrick /// Create a dirty shadow for a given value.
getPoisonedShadow__anon9f4f4c900811::MemorySanitizerVisitor183709467b48Spatrick Constant *getPoisonedShadow(Value *V) {
183809467b48Spatrick Type *ShadowTy = getShadowTy(V);
183909467b48Spatrick if (!ShadowTy)
184009467b48Spatrick return nullptr;
184109467b48Spatrick return getPoisonedShadow(ShadowTy);
184209467b48Spatrick }
184309467b48Spatrick
184409467b48Spatrick /// Create a clean (zero) origin.
getCleanOrigin__anon9f4f4c900811::MemorySanitizerVisitor1845*d415bd75Srobert Value *getCleanOrigin() { return Constant::getNullValue(MS.OriginTy); }
184609467b48Spatrick
184709467b48Spatrick /// Get the shadow value for a given Value.
184809467b48Spatrick ///
184909467b48Spatrick /// This function either returns the value set earlier with setShadow,
185009467b48Spatrick /// or extracts if from ParamTLS (for function arguments).
getShadow__anon9f4f4c900811::MemorySanitizerVisitor185109467b48Spatrick Value *getShadow(Value *V) {
185209467b48Spatrick if (Instruction *I = dyn_cast<Instruction>(V)) {
1853*d415bd75Srobert if (!PropagateShadow || I->getMetadata(LLVMContext::MD_nosanitize))
185409467b48Spatrick return getCleanShadow(V);
185509467b48Spatrick // For instructions the shadow is already stored in the map.
185609467b48Spatrick Value *Shadow = ShadowMap[V];
185709467b48Spatrick if (!Shadow) {
185809467b48Spatrick LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
185909467b48Spatrick (void)I;
186009467b48Spatrick assert(Shadow && "No shadow for a value");
186109467b48Spatrick }
186209467b48Spatrick return Shadow;
186309467b48Spatrick }
186409467b48Spatrick if (UndefValue *U = dyn_cast<UndefValue>(V)) {
1865*d415bd75Srobert Value *AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1866*d415bd75Srobert : getCleanShadow(V);
186709467b48Spatrick LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
186809467b48Spatrick (void)U;
186909467b48Spatrick return AllOnes;
187009467b48Spatrick }
187109467b48Spatrick if (Argument *A = dyn_cast<Argument>(V)) {
187209467b48Spatrick // For arguments we compute the shadow on demand and store it in the map.
1873*d415bd75Srobert Value *&ShadowPtr = ShadowMap[V];
1874*d415bd75Srobert if (ShadowPtr)
1875*d415bd75Srobert return ShadowPtr;
187609467b48Spatrick Function *F = A->getParent();
187773471bf0Spatrick IRBuilder<> EntryIRB(FnPrologueEnd);
187809467b48Spatrick unsigned ArgOffset = 0;
187909467b48Spatrick const DataLayout &DL = F->getParent()->getDataLayout();
188009467b48Spatrick for (auto &FArg : F->args()) {
188109467b48Spatrick if (!FArg.getType()->isSized()) {
188209467b48Spatrick LLVM_DEBUG(dbgs() << "Arg is not sized\n");
188309467b48Spatrick continue;
188409467b48Spatrick }
1885097a140dSpatrick
1886*d415bd75Srobert unsigned Size = FArg.hasByValAttr()
1887097a140dSpatrick ? DL.getTypeAllocSize(FArg.getParamByValType())
188809467b48Spatrick : DL.getTypeAllocSize(FArg.getType());
1889097a140dSpatrick
189009467b48Spatrick if (A == &FArg) {
189109467b48Spatrick bool Overflow = ArgOffset + Size > kParamTLSSize;
1892*d415bd75Srobert if (FArg.hasByValAttr()) {
189309467b48Spatrick // ByVal pointer itself has clean shadow. We copy the actual
189409467b48Spatrick // argument shadow to the underlying memory.
189509467b48Spatrick // Figure out maximal valid memcpy alignment.
189609467b48Spatrick const Align ArgAlign = DL.getValueOrABITypeAlignment(
1897*d415bd75Srobert FArg.getParamAlign(), FArg.getParamByValType());
1898*d415bd75Srobert Value *CpShadowPtr, *CpOriginPtr;
1899*d415bd75Srobert std::tie(CpShadowPtr, CpOriginPtr) =
190009467b48Spatrick getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1901*d415bd75Srobert /*isStore*/ true);
1902*d415bd75Srobert if (!PropagateShadow || Overflow) {
190309467b48Spatrick // ParamTLS overflow.
190409467b48Spatrick EntryIRB.CreateMemSet(
190509467b48Spatrick CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()),
190609467b48Spatrick Size, ArgAlign);
190709467b48Spatrick } else {
1908*d415bd75Srobert Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
190909467b48Spatrick const Align CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
191009467b48Spatrick Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
191109467b48Spatrick CopyAlign, Size);
191209467b48Spatrick LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
191309467b48Spatrick (void)Cpy;
1914*d415bd75Srobert
1915*d415bd75Srobert if (MS.TrackOrigins) {
1916*d415bd75Srobert Value *OriginPtr =
1917*d415bd75Srobert getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1918*d415bd75Srobert // FIXME: OriginSize should be:
1919*d415bd75Srobert // alignTo(V % kMinOriginAlignment + Size, kMinOriginAlignment)
1920*d415bd75Srobert unsigned OriginSize = alignTo(Size, kMinOriginAlignment);
1921*d415bd75Srobert EntryIRB.CreateMemCpy(
1922*d415bd75Srobert CpOriginPtr,
1923*d415bd75Srobert /* by getShadowOriginPtr */ kMinOriginAlignment, OriginPtr,
1924*d415bd75Srobert /* by origin_tls[ArgOffset] */ kMinOriginAlignment,
1925*d415bd75Srobert OriginSize);
192609467b48Spatrick }
1927*d415bd75Srobert }
1928*d415bd75Srobert }
1929*d415bd75Srobert
1930*d415bd75Srobert if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
1931*d415bd75Srobert (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
1932*d415bd75Srobert ShadowPtr = getCleanShadow(V);
1933*d415bd75Srobert setOrigin(A, getCleanOrigin());
193409467b48Spatrick } else {
1935097a140dSpatrick // Shadow over TLS
1936097a140dSpatrick Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1937*d415bd75Srobert ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg), Base,
1938097a140dSpatrick kShadowTLSAlignment);
1939*d415bd75Srobert if (MS.TrackOrigins) {
194009467b48Spatrick Value *OriginPtr =
194109467b48Spatrick getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
194209467b48Spatrick setOrigin(A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
194309467b48Spatrick }
1944*d415bd75Srobert }
1945*d415bd75Srobert LLVM_DEBUG(dbgs()
1946*d415bd75Srobert << " ARG: " << FArg << " ==> " << *ShadowPtr << "\n");
194773471bf0Spatrick break;
194809467b48Spatrick }
1949097a140dSpatrick
195009467b48Spatrick ArgOffset += alignTo(Size, kShadowTLSAlignment);
195109467b48Spatrick }
1952*d415bd75Srobert assert(ShadowPtr && "Could not find shadow for an argument");
1953*d415bd75Srobert return ShadowPtr;
195409467b48Spatrick }
195509467b48Spatrick // For everything else the shadow is zero.
195609467b48Spatrick return getCleanShadow(V);
195709467b48Spatrick }
195809467b48Spatrick
195909467b48Spatrick /// Get the shadow for i-th argument of the instruction I.
getShadow__anon9f4f4c900811::MemorySanitizerVisitor196009467b48Spatrick Value *getShadow(Instruction *I, int i) {
196109467b48Spatrick return getShadow(I->getOperand(i));
196209467b48Spatrick }
196309467b48Spatrick
196409467b48Spatrick /// Get the origin for a value.
getOrigin__anon9f4f4c900811::MemorySanitizerVisitor196509467b48Spatrick Value *getOrigin(Value *V) {
1966*d415bd75Srobert if (!MS.TrackOrigins)
1967*d415bd75Srobert return nullptr;
1968*d415bd75Srobert if (!PropagateShadow || isa<Constant>(V) || isa<InlineAsm>(V))
1969*d415bd75Srobert return getCleanOrigin();
197009467b48Spatrick assert((isa<Instruction>(V) || isa<Argument>(V)) &&
197109467b48Spatrick "Unexpected value type in getOrigin()");
197209467b48Spatrick if (Instruction *I = dyn_cast<Instruction>(V)) {
1973*d415bd75Srobert if (I->getMetadata(LLVMContext::MD_nosanitize))
197409467b48Spatrick return getCleanOrigin();
197509467b48Spatrick }
197609467b48Spatrick Value *Origin = OriginMap[V];
197709467b48Spatrick assert(Origin && "Missing origin");
197809467b48Spatrick return Origin;
197909467b48Spatrick }
198009467b48Spatrick
198109467b48Spatrick /// Get the origin for i-th argument of the instruction I.
getOrigin__anon9f4f4c900811::MemorySanitizerVisitor198209467b48Spatrick Value *getOrigin(Instruction *I, int i) {
198309467b48Spatrick return getOrigin(I->getOperand(i));
198409467b48Spatrick }
198509467b48Spatrick
198609467b48Spatrick /// Remember the place where a shadow check should be inserted.
198709467b48Spatrick ///
198809467b48Spatrick /// This location will be later instrumented with a check that will print a
198909467b48Spatrick /// UMR warning in runtime if the shadow value is not 0.
insertShadowCheck__anon9f4f4c900811::MemorySanitizerVisitor199009467b48Spatrick void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
199109467b48Spatrick assert(Shadow);
1992*d415bd75Srobert if (!InsertChecks)
1993*d415bd75Srobert return;
1994*d415bd75Srobert
1995*d415bd75Srobert if (!DebugCounter::shouldExecute(DebugInsertCheck)) {
1996*d415bd75Srobert LLVM_DEBUG(dbgs() << "Skipping check of " << *Shadow << " before "
1997*d415bd75Srobert << *OrigIns << "\n");
1998*d415bd75Srobert return;
1999*d415bd75Srobert }
200009467b48Spatrick #ifndef NDEBUG
200109467b48Spatrick Type *ShadowTy = Shadow->getType();
200273471bf0Spatrick assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
200373471bf0Spatrick isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
200473471bf0Spatrick "Can only insert checks for integer, vector, and aggregate shadow "
200573471bf0Spatrick "types");
200609467b48Spatrick #endif
200709467b48Spatrick InstrumentationList.push_back(
200809467b48Spatrick ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
200909467b48Spatrick }
201009467b48Spatrick
201109467b48Spatrick /// Remember the place where a shadow check should be inserted.
201209467b48Spatrick ///
201309467b48Spatrick /// This location will be later instrumented with a check that will print a
201409467b48Spatrick /// UMR warning in runtime if the value is not fully defined.
insertShadowCheck__anon9f4f4c900811::MemorySanitizerVisitor201509467b48Spatrick void insertShadowCheck(Value *Val, Instruction *OrigIns) {
201609467b48Spatrick assert(Val);
201709467b48Spatrick Value *Shadow, *Origin;
201809467b48Spatrick if (ClCheckConstantShadow) {
201909467b48Spatrick Shadow = getShadow(Val);
2020*d415bd75Srobert if (!Shadow)
2021*d415bd75Srobert return;
202209467b48Spatrick Origin = getOrigin(Val);
202309467b48Spatrick } else {
202409467b48Spatrick Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
2025*d415bd75Srobert if (!Shadow)
2026*d415bd75Srobert return;
202709467b48Spatrick Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
202809467b48Spatrick }
202909467b48Spatrick insertShadowCheck(Shadow, Origin, OrigIns);
203009467b48Spatrick }
203109467b48Spatrick
addReleaseOrdering__anon9f4f4c900811::MemorySanitizerVisitor203209467b48Spatrick AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
203309467b48Spatrick switch (a) {
203409467b48Spatrick case AtomicOrdering::NotAtomic:
203509467b48Spatrick return AtomicOrdering::NotAtomic;
203609467b48Spatrick case AtomicOrdering::Unordered:
203709467b48Spatrick case AtomicOrdering::Monotonic:
203809467b48Spatrick case AtomicOrdering::Release:
203909467b48Spatrick return AtomicOrdering::Release;
204009467b48Spatrick case AtomicOrdering::Acquire:
204109467b48Spatrick case AtomicOrdering::AcquireRelease:
204209467b48Spatrick return AtomicOrdering::AcquireRelease;
204309467b48Spatrick case AtomicOrdering::SequentiallyConsistent:
204409467b48Spatrick return AtomicOrdering::SequentiallyConsistent;
204509467b48Spatrick }
204609467b48Spatrick llvm_unreachable("Unknown ordering");
204709467b48Spatrick }
204809467b48Spatrick
makeAddReleaseOrderingTable__anon9f4f4c900811::MemorySanitizerVisitor204973471bf0Spatrick Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
205073471bf0Spatrick constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
205173471bf0Spatrick uint32_t OrderingTable[NumOrderings] = {};
205273471bf0Spatrick
205373471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::relaxed] =
205473471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::release] =
205573471bf0Spatrick (int)AtomicOrderingCABI::release;
205673471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::consume] =
205773471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::acquire] =
205873471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
205973471bf0Spatrick (int)AtomicOrderingCABI::acq_rel;
206073471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
206173471bf0Spatrick (int)AtomicOrderingCABI::seq_cst;
206273471bf0Spatrick
206373471bf0Spatrick return ConstantDataVector::get(IRB.getContext(),
2064*d415bd75Srobert ArrayRef(OrderingTable, NumOrderings));
206573471bf0Spatrick }
206673471bf0Spatrick
addAcquireOrdering__anon9f4f4c900811::MemorySanitizerVisitor206709467b48Spatrick AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
206809467b48Spatrick switch (a) {
206909467b48Spatrick case AtomicOrdering::NotAtomic:
207009467b48Spatrick return AtomicOrdering::NotAtomic;
207109467b48Spatrick case AtomicOrdering::Unordered:
207209467b48Spatrick case AtomicOrdering::Monotonic:
207309467b48Spatrick case AtomicOrdering::Acquire:
207409467b48Spatrick return AtomicOrdering::Acquire;
207509467b48Spatrick case AtomicOrdering::Release:
207609467b48Spatrick case AtomicOrdering::AcquireRelease:
207709467b48Spatrick return AtomicOrdering::AcquireRelease;
207809467b48Spatrick case AtomicOrdering::SequentiallyConsistent:
207909467b48Spatrick return AtomicOrdering::SequentiallyConsistent;
208009467b48Spatrick }
208109467b48Spatrick llvm_unreachable("Unknown ordering");
208209467b48Spatrick }
208309467b48Spatrick
makeAddAcquireOrderingTable__anon9f4f4c900811::MemorySanitizerVisitor208473471bf0Spatrick Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
208573471bf0Spatrick constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
208673471bf0Spatrick uint32_t OrderingTable[NumOrderings] = {};
208773471bf0Spatrick
208873471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::relaxed] =
208973471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::acquire] =
209073471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::consume] =
209173471bf0Spatrick (int)AtomicOrderingCABI::acquire;
209273471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::release] =
209373471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
209473471bf0Spatrick (int)AtomicOrderingCABI::acq_rel;
209573471bf0Spatrick OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
209673471bf0Spatrick (int)AtomicOrderingCABI::seq_cst;
209773471bf0Spatrick
209873471bf0Spatrick return ConstantDataVector::get(IRB.getContext(),
2099*d415bd75Srobert ArrayRef(OrderingTable, NumOrderings));
210073471bf0Spatrick }
210173471bf0Spatrick
210209467b48Spatrick // ------------------- Visitors.
210309467b48Spatrick using InstVisitor<MemorySanitizerVisitor>::visit;
visit__anon9f4f4c900811::MemorySanitizerVisitor210409467b48Spatrick void visit(Instruction &I) {
2105*d415bd75Srobert if (I.getMetadata(LLVMContext::MD_nosanitize))
210673471bf0Spatrick return;
210773471bf0Spatrick // Don't want to visit if we're in the prologue
210873471bf0Spatrick if (isInPrologue(I))
210973471bf0Spatrick return;
211009467b48Spatrick InstVisitor<MemorySanitizerVisitor>::visit(I);
211109467b48Spatrick }
211209467b48Spatrick
211309467b48Spatrick /// Instrument LoadInst
211409467b48Spatrick ///
211509467b48Spatrick /// Loads the corresponding shadow and (optionally) origin.
211609467b48Spatrick /// Optionally, checks that the load address is fully defined.
visitLoadInst__anon9f4f4c900811::MemorySanitizerVisitor211709467b48Spatrick void visitLoadInst(LoadInst &I) {
211809467b48Spatrick assert(I.getType()->isSized() && "Load type must have size");
2119*d415bd75Srobert assert(!I.getMetadata(LLVMContext::MD_nosanitize));
2120*d415bd75Srobert NextNodeIRBuilder IRB(&I);
212109467b48Spatrick Type *ShadowTy = getShadowTy(&I);
212209467b48Spatrick Value *Addr = I.getPointerOperand();
212309467b48Spatrick Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2124*d415bd75Srobert const Align Alignment = I.getAlign();
212509467b48Spatrick if (PropagateShadow) {
212609467b48Spatrick std::tie(ShadowPtr, OriginPtr) =
212709467b48Spatrick getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2128097a140dSpatrick setShadow(&I,
2129097a140dSpatrick IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
213009467b48Spatrick } else {
213109467b48Spatrick setShadow(&I, getCleanShadow(&I));
213209467b48Spatrick }
213309467b48Spatrick
213409467b48Spatrick if (ClCheckAccessAddress)
213509467b48Spatrick insertShadowCheck(I.getPointerOperand(), &I);
213609467b48Spatrick
213709467b48Spatrick if (I.isAtomic())
213809467b48Spatrick I.setOrdering(addAcquireOrdering(I.getOrdering()));
213909467b48Spatrick
214009467b48Spatrick if (MS.TrackOrigins) {
214109467b48Spatrick if (PropagateShadow) {
214209467b48Spatrick const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
2143097a140dSpatrick setOrigin(
2144097a140dSpatrick &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment));
214509467b48Spatrick } else {
214609467b48Spatrick setOrigin(&I, getCleanOrigin());
214709467b48Spatrick }
214809467b48Spatrick }
214909467b48Spatrick }
215009467b48Spatrick
215109467b48Spatrick /// Instrument StoreInst
215209467b48Spatrick ///
215309467b48Spatrick /// Stores the corresponding shadow and (optionally) origin.
215409467b48Spatrick /// Optionally, checks that the store address is fully defined.
visitStoreInst__anon9f4f4c900811::MemorySanitizerVisitor215509467b48Spatrick void visitStoreInst(StoreInst &I) {
215609467b48Spatrick StoreList.push_back(&I);
215709467b48Spatrick if (ClCheckAccessAddress)
215809467b48Spatrick insertShadowCheck(I.getPointerOperand(), &I);
215909467b48Spatrick }
216009467b48Spatrick
handleCASOrRMW__anon9f4f4c900811::MemorySanitizerVisitor216109467b48Spatrick void handleCASOrRMW(Instruction &I) {
216209467b48Spatrick assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
216309467b48Spatrick
216409467b48Spatrick IRBuilder<> IRB(&I);
216509467b48Spatrick Value *Addr = I.getOperand(0);
216673471bf0Spatrick Value *Val = I.getOperand(1);
2167*d415bd75Srobert Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, getShadowTy(Val), Align(1),
216809467b48Spatrick /*isStore*/ true)
216909467b48Spatrick .first;
217009467b48Spatrick
217109467b48Spatrick if (ClCheckAccessAddress)
217209467b48Spatrick insertShadowCheck(Addr, &I);
217309467b48Spatrick
217409467b48Spatrick // Only test the conditional argument of cmpxchg instruction.
217509467b48Spatrick // The other argument can potentially be uninitialized, but we can not
217609467b48Spatrick // detect this situation reliably without possible false positives.
217709467b48Spatrick if (isa<AtomicCmpXchgInst>(I))
217873471bf0Spatrick insertShadowCheck(Val, &I);
217909467b48Spatrick
218073471bf0Spatrick IRB.CreateStore(getCleanShadow(Val), ShadowPtr);
218109467b48Spatrick
218209467b48Spatrick setShadow(&I, getCleanShadow(&I));
218309467b48Spatrick setOrigin(&I, getCleanOrigin());
218409467b48Spatrick }
218509467b48Spatrick
visitAtomicRMWInst__anon9f4f4c900811::MemorySanitizerVisitor218609467b48Spatrick void visitAtomicRMWInst(AtomicRMWInst &I) {
218709467b48Spatrick handleCASOrRMW(I);
218809467b48Spatrick I.setOrdering(addReleaseOrdering(I.getOrdering()));
218909467b48Spatrick }
219009467b48Spatrick
visitAtomicCmpXchgInst__anon9f4f4c900811::MemorySanitizerVisitor219109467b48Spatrick void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
219209467b48Spatrick handleCASOrRMW(I);
219309467b48Spatrick I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
219409467b48Spatrick }
219509467b48Spatrick
219609467b48Spatrick // Vector manipulation.
visitExtractElementInst__anon9f4f4c900811::MemorySanitizerVisitor219709467b48Spatrick void visitExtractElementInst(ExtractElementInst &I) {
219809467b48Spatrick insertShadowCheck(I.getOperand(1), &I);
219909467b48Spatrick IRBuilder<> IRB(&I);
220009467b48Spatrick setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
220109467b48Spatrick "_msprop"));
220209467b48Spatrick setOrigin(&I, getOrigin(&I, 0));
220309467b48Spatrick }
220409467b48Spatrick
visitInsertElementInst__anon9f4f4c900811::MemorySanitizerVisitor220509467b48Spatrick void visitInsertElementInst(InsertElementInst &I) {
220609467b48Spatrick insertShadowCheck(I.getOperand(2), &I);
220709467b48Spatrick IRBuilder<> IRB(&I);
2208*d415bd75Srobert auto *Shadow0 = getShadow(&I, 0);
2209*d415bd75Srobert auto *Shadow1 = getShadow(&I, 1);
2210*d415bd75Srobert setShadow(&I, IRB.CreateInsertElement(Shadow0, Shadow1, I.getOperand(2),
2211*d415bd75Srobert "_msprop"));
221209467b48Spatrick setOriginForNaryOp(I);
221309467b48Spatrick }
221409467b48Spatrick
visitShuffleVectorInst__anon9f4f4c900811::MemorySanitizerVisitor221509467b48Spatrick void visitShuffleVectorInst(ShuffleVectorInst &I) {
221609467b48Spatrick IRBuilder<> IRB(&I);
2217*d415bd75Srobert auto *Shadow0 = getShadow(&I, 0);
2218*d415bd75Srobert auto *Shadow1 = getShadow(&I, 1);
2219*d415bd75Srobert setShadow(&I, IRB.CreateShuffleVector(Shadow0, Shadow1, I.getShuffleMask(),
2220*d415bd75Srobert "_msprop"));
222109467b48Spatrick setOriginForNaryOp(I);
222209467b48Spatrick }
222309467b48Spatrick
222409467b48Spatrick // Casts.
visitSExtInst__anon9f4f4c900811::MemorySanitizerVisitor222509467b48Spatrick void visitSExtInst(SExtInst &I) {
222609467b48Spatrick IRBuilder<> IRB(&I);
222709467b48Spatrick setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
222809467b48Spatrick setOrigin(&I, getOrigin(&I, 0));
222909467b48Spatrick }
223009467b48Spatrick
visitZExtInst__anon9f4f4c900811::MemorySanitizerVisitor223109467b48Spatrick void visitZExtInst(ZExtInst &I) {
223209467b48Spatrick IRBuilder<> IRB(&I);
223309467b48Spatrick setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
223409467b48Spatrick setOrigin(&I, getOrigin(&I, 0));
223509467b48Spatrick }
223609467b48Spatrick
visitTruncInst__anon9f4f4c900811::MemorySanitizerVisitor223709467b48Spatrick void visitTruncInst(TruncInst &I) {
223809467b48Spatrick IRBuilder<> IRB(&I);
223909467b48Spatrick setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
224009467b48Spatrick setOrigin(&I, getOrigin(&I, 0));
224109467b48Spatrick }
224209467b48Spatrick
visitBitCastInst__anon9f4f4c900811::MemorySanitizerVisitor224309467b48Spatrick void visitBitCastInst(BitCastInst &I) {
224409467b48Spatrick // Special case: if this is the bitcast (there is exactly 1 allowed) between
224509467b48Spatrick // a musttail call and a ret, don't instrument. New instructions are not
224609467b48Spatrick // allowed after a musttail call.
224709467b48Spatrick if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
224809467b48Spatrick if (CI->isMustTailCall())
224909467b48Spatrick return;
225009467b48Spatrick IRBuilder<> IRB(&I);
225109467b48Spatrick setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
225209467b48Spatrick setOrigin(&I, getOrigin(&I, 0));
225309467b48Spatrick }
225409467b48Spatrick
visitPtrToIntInst__anon9f4f4c900811::MemorySanitizerVisitor225509467b48Spatrick void visitPtrToIntInst(PtrToIntInst &I) {
225609467b48Spatrick IRBuilder<> IRB(&I);
225709467b48Spatrick setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
225809467b48Spatrick "_msprop_ptrtoint"));
225909467b48Spatrick setOrigin(&I, getOrigin(&I, 0));
226009467b48Spatrick }
226109467b48Spatrick
visitIntToPtrInst__anon9f4f4c900811::MemorySanitizerVisitor226209467b48Spatrick void visitIntToPtrInst(IntToPtrInst &I) {
226309467b48Spatrick IRBuilder<> IRB(&I);
226409467b48Spatrick setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
226509467b48Spatrick "_msprop_inttoptr"));
226609467b48Spatrick setOrigin(&I, getOrigin(&I, 0));
226709467b48Spatrick }
226809467b48Spatrick
visitFPToSIInst__anon9f4f4c900811::MemorySanitizerVisitor226909467b48Spatrick void visitFPToSIInst(CastInst &I) { handleShadowOr(I); }
visitFPToUIInst__anon9f4f4c900811::MemorySanitizerVisitor227009467b48Spatrick void visitFPToUIInst(CastInst &I) { handleShadowOr(I); }
visitSIToFPInst__anon9f4f4c900811::MemorySanitizerVisitor227109467b48Spatrick void visitSIToFPInst(CastInst &I) { handleShadowOr(I); }
visitUIToFPInst__anon9f4f4c900811::MemorySanitizerVisitor227209467b48Spatrick void visitUIToFPInst(CastInst &I) { handleShadowOr(I); }
visitFPExtInst__anon9f4f4c900811::MemorySanitizerVisitor227309467b48Spatrick void visitFPExtInst(CastInst &I) { handleShadowOr(I); }
visitFPTruncInst__anon9f4f4c900811::MemorySanitizerVisitor227409467b48Spatrick void visitFPTruncInst(CastInst &I) { handleShadowOr(I); }
227509467b48Spatrick
227609467b48Spatrick /// Propagate shadow for bitwise AND.
227709467b48Spatrick ///
227809467b48Spatrick /// This code is exact, i.e. if, for example, a bit in the left argument
227909467b48Spatrick /// is defined and 0, then neither the value not definedness of the
228009467b48Spatrick /// corresponding bit in B don't affect the resulting shadow.
visitAnd__anon9f4f4c900811::MemorySanitizerVisitor228109467b48Spatrick void visitAnd(BinaryOperator &I) {
228209467b48Spatrick IRBuilder<> IRB(&I);
228309467b48Spatrick // "And" of 0 and a poisoned value results in unpoisoned value.
228409467b48Spatrick // 1&1 => 1; 0&1 => 0; p&1 => p;
228509467b48Spatrick // 1&0 => 0; 0&0 => 0; p&0 => 0;
228609467b48Spatrick // 1&p => p; 0&p => 0; p&p => p;
228709467b48Spatrick // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
228809467b48Spatrick Value *S1 = getShadow(&I, 0);
228909467b48Spatrick Value *S2 = getShadow(&I, 1);
229009467b48Spatrick Value *V1 = I.getOperand(0);
229109467b48Spatrick Value *V2 = I.getOperand(1);
229209467b48Spatrick if (V1->getType() != S1->getType()) {
229309467b48Spatrick V1 = IRB.CreateIntCast(V1, S1->getType(), false);
229409467b48Spatrick V2 = IRB.CreateIntCast(V2, S2->getType(), false);
229509467b48Spatrick }
229609467b48Spatrick Value *S1S2 = IRB.CreateAnd(S1, S2);
229709467b48Spatrick Value *V1S2 = IRB.CreateAnd(V1, S2);
229809467b48Spatrick Value *S1V2 = IRB.CreateAnd(S1, V2);
229909467b48Spatrick setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
230009467b48Spatrick setOriginForNaryOp(I);
230109467b48Spatrick }
230209467b48Spatrick
visitOr__anon9f4f4c900811::MemorySanitizerVisitor230309467b48Spatrick void visitOr(BinaryOperator &I) {
230409467b48Spatrick IRBuilder<> IRB(&I);
230509467b48Spatrick // "Or" of 1 and a poisoned value results in unpoisoned value.
230609467b48Spatrick // 1|1 => 1; 0|1 => 1; p|1 => 1;
230709467b48Spatrick // 1|0 => 1; 0|0 => 0; p|0 => p;
230809467b48Spatrick // 1|p => 1; 0|p => p; p|p => p;
230909467b48Spatrick // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
231009467b48Spatrick Value *S1 = getShadow(&I, 0);
231109467b48Spatrick Value *S2 = getShadow(&I, 1);
231209467b48Spatrick Value *V1 = IRB.CreateNot(I.getOperand(0));
231309467b48Spatrick Value *V2 = IRB.CreateNot(I.getOperand(1));
231409467b48Spatrick if (V1->getType() != S1->getType()) {
231509467b48Spatrick V1 = IRB.CreateIntCast(V1, S1->getType(), false);
231609467b48Spatrick V2 = IRB.CreateIntCast(V2, S2->getType(), false);
231709467b48Spatrick }
231809467b48Spatrick Value *S1S2 = IRB.CreateAnd(S1, S2);
231909467b48Spatrick Value *V1S2 = IRB.CreateAnd(V1, S2);
232009467b48Spatrick Value *S1V2 = IRB.CreateAnd(S1, V2);
232109467b48Spatrick setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
232209467b48Spatrick setOriginForNaryOp(I);
232309467b48Spatrick }
232409467b48Spatrick
232509467b48Spatrick /// Default propagation of shadow and/or origin.
232609467b48Spatrick ///
232709467b48Spatrick /// This class implements the general case of shadow propagation, used in all
232809467b48Spatrick /// cases where we don't know and/or don't care about what the operation
232909467b48Spatrick /// actually does. It converts all input shadow values to a common type
233009467b48Spatrick /// (extending or truncating as necessary), and bitwise OR's them.
233109467b48Spatrick ///
233209467b48Spatrick /// This is much cheaper than inserting checks (i.e. requiring inputs to be
233309467b48Spatrick /// fully initialized), and less prone to false positives.
233409467b48Spatrick ///
233509467b48Spatrick /// This class also implements the general case of origin propagation. For a
233609467b48Spatrick /// Nary operation, result origin is set to the origin of an argument that is
233709467b48Spatrick /// not entirely initialized. If there is more than one such arguments, the
233809467b48Spatrick /// rightmost of them is picked. It does not matter which one is picked if all
233909467b48Spatrick /// arguments are initialized.
2340*d415bd75Srobert template <bool CombineShadow> class Combiner {
234109467b48Spatrick Value *Shadow = nullptr;
234209467b48Spatrick Value *Origin = nullptr;
234309467b48Spatrick IRBuilder<> &IRB;
234409467b48Spatrick MemorySanitizerVisitor *MSV;
234509467b48Spatrick
234609467b48Spatrick public:
Combiner(MemorySanitizerVisitor * MSV,IRBuilder<> & IRB)234709467b48Spatrick Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
234809467b48Spatrick : IRB(IRB), MSV(MSV) {}
234909467b48Spatrick
235009467b48Spatrick /// Add a pair of shadow and origin values to the mix.
Add(Value * OpShadow,Value * OpOrigin)235109467b48Spatrick Combiner &Add(Value *OpShadow, Value *OpOrigin) {
235209467b48Spatrick if (CombineShadow) {
235309467b48Spatrick assert(OpShadow);
235409467b48Spatrick if (!Shadow)
235509467b48Spatrick Shadow = OpShadow;
235609467b48Spatrick else {
235709467b48Spatrick OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
235809467b48Spatrick Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
235909467b48Spatrick }
236009467b48Spatrick }
236109467b48Spatrick
236209467b48Spatrick if (MSV->MS.TrackOrigins) {
236309467b48Spatrick assert(OpOrigin);
236409467b48Spatrick if (!Origin) {
236509467b48Spatrick Origin = OpOrigin;
236609467b48Spatrick } else {
236709467b48Spatrick Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
236809467b48Spatrick // No point in adding something that might result in 0 origin value.
236909467b48Spatrick if (!ConstOrigin || !ConstOrigin->isNullValue()) {
237073471bf0Spatrick Value *FlatShadow = MSV->convertShadowToScalar(OpShadow, IRB);
237109467b48Spatrick Value *Cond =
237209467b48Spatrick IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
237309467b48Spatrick Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
237409467b48Spatrick }
237509467b48Spatrick }
237609467b48Spatrick }
237709467b48Spatrick return *this;
237809467b48Spatrick }
237909467b48Spatrick
238009467b48Spatrick /// Add an application value to the mix.
Add(Value * V)238109467b48Spatrick Combiner &Add(Value *V) {
238209467b48Spatrick Value *OpShadow = MSV->getShadow(V);
238309467b48Spatrick Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
238409467b48Spatrick return Add(OpShadow, OpOrigin);
238509467b48Spatrick }
238609467b48Spatrick
238709467b48Spatrick /// Set the current combined values as the given instruction's shadow
238809467b48Spatrick /// and origin.
Done(Instruction * I)238909467b48Spatrick void Done(Instruction *I) {
239009467b48Spatrick if (CombineShadow) {
239109467b48Spatrick assert(Shadow);
239209467b48Spatrick Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
239309467b48Spatrick MSV->setShadow(I, Shadow);
239409467b48Spatrick }
239509467b48Spatrick if (MSV->MS.TrackOrigins) {
239609467b48Spatrick assert(Origin);
239709467b48Spatrick MSV->setOrigin(I, Origin);
239809467b48Spatrick }
239909467b48Spatrick }
240009467b48Spatrick };
240109467b48Spatrick
240209467b48Spatrick using ShadowAndOriginCombiner = Combiner<true>;
240309467b48Spatrick using OriginCombiner = Combiner<false>;
240409467b48Spatrick
240509467b48Spatrick /// Propagate origin for arbitrary operation.
setOriginForNaryOp__anon9f4f4c900811::MemorySanitizerVisitor240609467b48Spatrick void setOriginForNaryOp(Instruction &I) {
2407*d415bd75Srobert if (!MS.TrackOrigins)
2408*d415bd75Srobert return;
240909467b48Spatrick IRBuilder<> IRB(&I);
241009467b48Spatrick OriginCombiner OC(this, IRB);
241173471bf0Spatrick for (Use &Op : I.operands())
241273471bf0Spatrick OC.Add(Op.get());
241309467b48Spatrick OC.Done(&I);
241409467b48Spatrick }
241509467b48Spatrick
VectorOrPrimitiveTypeSizeInBits__anon9f4f4c900811::MemorySanitizerVisitor241609467b48Spatrick size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
241709467b48Spatrick assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
241809467b48Spatrick "Vector of pointers is not a valid shadow type");
2419097a140dSpatrick return Ty->isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2420097a140dSpatrick Ty->getScalarSizeInBits()
2421097a140dSpatrick : Ty->getPrimitiveSizeInBits();
242209467b48Spatrick }
242309467b48Spatrick
242409467b48Spatrick /// Cast between two shadow types, extending or truncating as
242509467b48Spatrick /// necessary.
CreateShadowCast__anon9f4f4c900811::MemorySanitizerVisitor242609467b48Spatrick Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
242709467b48Spatrick bool Signed = false) {
242809467b48Spatrick Type *srcTy = V->getType();
242909467b48Spatrick size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
243009467b48Spatrick size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
243109467b48Spatrick if (srcSizeInBits > 1 && dstSizeInBits == 1)
243209467b48Spatrick return IRB.CreateICmpNE(V, getCleanShadow(V));
243309467b48Spatrick
243409467b48Spatrick if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
243509467b48Spatrick return IRB.CreateIntCast(V, dstTy, Signed);
243609467b48Spatrick if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2437097a140dSpatrick cast<FixedVectorType>(dstTy)->getNumElements() ==
2438097a140dSpatrick cast<FixedVectorType>(srcTy)->getNumElements())
243909467b48Spatrick return IRB.CreateIntCast(V, dstTy, Signed);
244009467b48Spatrick Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
244109467b48Spatrick Value *V2 =
244209467b48Spatrick IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
244309467b48Spatrick return IRB.CreateBitCast(V2, dstTy);
244409467b48Spatrick // TODO: handle struct types.
244509467b48Spatrick }
244609467b48Spatrick
244709467b48Spatrick /// Cast an application value to the type of its own shadow.
CreateAppToShadowCast__anon9f4f4c900811::MemorySanitizerVisitor244809467b48Spatrick Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
244909467b48Spatrick Type *ShadowTy = getShadowTy(V);
245009467b48Spatrick if (V->getType() == ShadowTy)
245109467b48Spatrick return V;
245209467b48Spatrick if (V->getType()->isPtrOrPtrVectorTy())
245309467b48Spatrick return IRB.CreatePtrToInt(V, ShadowTy);
245409467b48Spatrick else
245509467b48Spatrick return IRB.CreateBitCast(V, ShadowTy);
245609467b48Spatrick }
245709467b48Spatrick
245809467b48Spatrick /// Propagate shadow for arbitrary operation.
handleShadowOr__anon9f4f4c900811::MemorySanitizerVisitor245909467b48Spatrick void handleShadowOr(Instruction &I) {
246009467b48Spatrick IRBuilder<> IRB(&I);
246109467b48Spatrick ShadowAndOriginCombiner SC(this, IRB);
246273471bf0Spatrick for (Use &Op : I.operands())
246373471bf0Spatrick SC.Add(Op.get());
246409467b48Spatrick SC.Done(&I);
246509467b48Spatrick }
246609467b48Spatrick
visitFNeg__anon9f4f4c900811::MemorySanitizerVisitor246709467b48Spatrick void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
246809467b48Spatrick
246909467b48Spatrick // Handle multiplication by constant.
247009467b48Spatrick //
247109467b48Spatrick // Handle a special case of multiplication by constant that may have one or
247209467b48Spatrick // more zeros in the lower bits. This makes corresponding number of lower bits
247309467b48Spatrick // of the result zero as well. We model it by shifting the other operand
247409467b48Spatrick // shadow left by the required number of bits. Effectively, we transform
247509467b48Spatrick // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
247609467b48Spatrick // We use multiplication by 2**N instead of shift to cover the case of
247709467b48Spatrick // multiplication by 0, which may occur in some elements of a vector operand.
handleMulByConstant__anon9f4f4c900811::MemorySanitizerVisitor247809467b48Spatrick void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
247909467b48Spatrick Value *OtherArg) {
248009467b48Spatrick Constant *ShadowMul;
248109467b48Spatrick Type *Ty = ConstArg->getType();
2482097a140dSpatrick if (auto *VTy = dyn_cast<VectorType>(Ty)) {
2483097a140dSpatrick unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2484097a140dSpatrick Type *EltTy = VTy->getElementType();
248509467b48Spatrick SmallVector<Constant *, 16> Elements;
248609467b48Spatrick for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
248709467b48Spatrick if (ConstantInt *Elt =
248809467b48Spatrick dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
248909467b48Spatrick const APInt &V = Elt->getValue();
249009467b48Spatrick APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
249109467b48Spatrick Elements.push_back(ConstantInt::get(EltTy, V2));
249209467b48Spatrick } else {
249309467b48Spatrick Elements.push_back(ConstantInt::get(EltTy, 1));
249409467b48Spatrick }
249509467b48Spatrick }
249609467b48Spatrick ShadowMul = ConstantVector::get(Elements);
249709467b48Spatrick } else {
249809467b48Spatrick if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
249909467b48Spatrick const APInt &V = Elt->getValue();
250009467b48Spatrick APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
250109467b48Spatrick ShadowMul = ConstantInt::get(Ty, V2);
250209467b48Spatrick } else {
250309467b48Spatrick ShadowMul = ConstantInt::get(Ty, 1);
250409467b48Spatrick }
250509467b48Spatrick }
250609467b48Spatrick
250709467b48Spatrick IRBuilder<> IRB(&I);
250809467b48Spatrick setShadow(&I,
250909467b48Spatrick IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
251009467b48Spatrick setOrigin(&I, getOrigin(OtherArg));
251109467b48Spatrick }
251209467b48Spatrick
visitMul__anon9f4f4c900811::MemorySanitizerVisitor251309467b48Spatrick void visitMul(BinaryOperator &I) {
251409467b48Spatrick Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
251509467b48Spatrick Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
251609467b48Spatrick if (constOp0 && !constOp1)
251709467b48Spatrick handleMulByConstant(I, constOp0, I.getOperand(1));
251809467b48Spatrick else if (constOp1 && !constOp0)
251909467b48Spatrick handleMulByConstant(I, constOp1, I.getOperand(0));
252009467b48Spatrick else
252109467b48Spatrick handleShadowOr(I);
252209467b48Spatrick }
252309467b48Spatrick
visitFAdd__anon9f4f4c900811::MemorySanitizerVisitor252409467b48Spatrick void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
visitFSub__anon9f4f4c900811::MemorySanitizerVisitor252509467b48Spatrick void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
visitFMul__anon9f4f4c900811::MemorySanitizerVisitor252609467b48Spatrick void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
visitAdd__anon9f4f4c900811::MemorySanitizerVisitor252709467b48Spatrick void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
visitSub__anon9f4f4c900811::MemorySanitizerVisitor252809467b48Spatrick void visitSub(BinaryOperator &I) { handleShadowOr(I); }
visitXor__anon9f4f4c900811::MemorySanitizerVisitor252909467b48Spatrick void visitXor(BinaryOperator &I) { handleShadowOr(I); }
253009467b48Spatrick
handleIntegerDiv__anon9f4f4c900811::MemorySanitizerVisitor253109467b48Spatrick void handleIntegerDiv(Instruction &I) {
253209467b48Spatrick IRBuilder<> IRB(&I);
253309467b48Spatrick // Strict on the second argument.
253409467b48Spatrick insertShadowCheck(I.getOperand(1), &I);
253509467b48Spatrick setShadow(&I, getShadow(&I, 0));
253609467b48Spatrick setOrigin(&I, getOrigin(&I, 0));
253709467b48Spatrick }
253809467b48Spatrick
visitUDiv__anon9f4f4c900811::MemorySanitizerVisitor253909467b48Spatrick void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
visitSDiv__anon9f4f4c900811::MemorySanitizerVisitor254009467b48Spatrick void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
visitURem__anon9f4f4c900811::MemorySanitizerVisitor254109467b48Spatrick void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
visitSRem__anon9f4f4c900811::MemorySanitizerVisitor254209467b48Spatrick void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
254309467b48Spatrick
254409467b48Spatrick // Floating point division is side-effect free. We can not require that the
254509467b48Spatrick // divisor is fully initialized and must propagate shadow. See PR37523.
visitFDiv__anon9f4f4c900811::MemorySanitizerVisitor254609467b48Spatrick void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
visitFRem__anon9f4f4c900811::MemorySanitizerVisitor254709467b48Spatrick void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
254809467b48Spatrick
254909467b48Spatrick /// Instrument == and != comparisons.
255009467b48Spatrick ///
255109467b48Spatrick /// Sometimes the comparison result is known even if some of the bits of the
255209467b48Spatrick /// arguments are not.
handleEqualityComparison__anon9f4f4c900811::MemorySanitizerVisitor255309467b48Spatrick void handleEqualityComparison(ICmpInst &I) {
255409467b48Spatrick IRBuilder<> IRB(&I);
255509467b48Spatrick Value *A = I.getOperand(0);
255609467b48Spatrick Value *B = I.getOperand(1);
255709467b48Spatrick Value *Sa = getShadow(A);
255809467b48Spatrick Value *Sb = getShadow(B);
255909467b48Spatrick
256009467b48Spatrick // Get rid of pointers and vectors of pointers.
256109467b48Spatrick // For ints (and vectors of ints), types of A and Sa match,
256209467b48Spatrick // and this is a no-op.
256309467b48Spatrick A = IRB.CreatePointerCast(A, Sa->getType());
256409467b48Spatrick B = IRB.CreatePointerCast(B, Sb->getType());
256509467b48Spatrick
256609467b48Spatrick // A == B <==> (C = A^B) == 0
256709467b48Spatrick // A != B <==> (C = A^B) != 0
256809467b48Spatrick // Sc = Sa | Sb
256909467b48Spatrick Value *C = IRB.CreateXor(A, B);
257009467b48Spatrick Value *Sc = IRB.CreateOr(Sa, Sb);
257109467b48Spatrick // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
257209467b48Spatrick // Result is defined if one of the following is true
257309467b48Spatrick // * there is a defined 1 bit in C
257409467b48Spatrick // * C is fully defined
257509467b48Spatrick // Si = !(C & ~Sc) && Sc
257609467b48Spatrick Value *Zero = Constant::getNullValue(Sc->getType());
257709467b48Spatrick Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
2578*d415bd75Srobert Value *LHS = IRB.CreateICmpNE(Sc, Zero);
2579*d415bd75Srobert Value *RHS =
2580*d415bd75Srobert IRB.CreateICmpEQ(IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero);
2581*d415bd75Srobert Value *Si = IRB.CreateAnd(LHS, RHS);
258209467b48Spatrick Si->setName("_msprop_icmp");
258309467b48Spatrick setShadow(&I, Si);
258409467b48Spatrick setOriginForNaryOp(I);
258509467b48Spatrick }
258609467b48Spatrick
258709467b48Spatrick /// Build the lowest possible value of V, taking into account V's
258809467b48Spatrick /// uninitialized bits.
getLowestPossibleValue__anon9f4f4c900811::MemorySanitizerVisitor258909467b48Spatrick Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
259009467b48Spatrick bool isSigned) {
259109467b48Spatrick if (isSigned) {
259209467b48Spatrick // Split shadow into sign bit and other bits.
259309467b48Spatrick Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
259409467b48Spatrick Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
259509467b48Spatrick // Maximise the undefined shadow bit, minimize other undefined bits.
2596*d415bd75Srobert return IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)),
2597*d415bd75Srobert SaSignBit);
259809467b48Spatrick } else {
259909467b48Spatrick // Minimize undefined bits.
260009467b48Spatrick return IRB.CreateAnd(A, IRB.CreateNot(Sa));
260109467b48Spatrick }
260209467b48Spatrick }
260309467b48Spatrick
260409467b48Spatrick /// Build the highest possible value of V, taking into account V's
260509467b48Spatrick /// uninitialized bits.
getHighestPossibleValue__anon9f4f4c900811::MemorySanitizerVisitor260609467b48Spatrick Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
260709467b48Spatrick bool isSigned) {
260809467b48Spatrick if (isSigned) {
260909467b48Spatrick // Split shadow into sign bit and other bits.
261009467b48Spatrick Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
261109467b48Spatrick Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
261209467b48Spatrick // Minimise the undefined shadow bit, maximise other undefined bits.
2613*d415bd75Srobert return IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)),
2614*d415bd75Srobert SaOtherBits);
261509467b48Spatrick } else {
261609467b48Spatrick // Maximize undefined bits.
261709467b48Spatrick return IRB.CreateOr(A, Sa);
261809467b48Spatrick }
261909467b48Spatrick }
262009467b48Spatrick
262109467b48Spatrick /// Instrument relational comparisons.
262209467b48Spatrick ///
262309467b48Spatrick /// This function does exact shadow propagation for all relational
262409467b48Spatrick /// comparisons of integers, pointers and vectors of those.
262509467b48Spatrick /// FIXME: output seems suboptimal when one of the operands is a constant
handleRelationalComparisonExact__anon9f4f4c900811::MemorySanitizerVisitor262609467b48Spatrick void handleRelationalComparisonExact(ICmpInst &I) {
262709467b48Spatrick IRBuilder<> IRB(&I);
262809467b48Spatrick Value *A = I.getOperand(0);
262909467b48Spatrick Value *B = I.getOperand(1);
263009467b48Spatrick Value *Sa = getShadow(A);
263109467b48Spatrick Value *Sb = getShadow(B);
263209467b48Spatrick
263309467b48Spatrick // Get rid of pointers and vectors of pointers.
263409467b48Spatrick // For ints (and vectors of ints), types of A and Sa match,
263509467b48Spatrick // and this is a no-op.
263609467b48Spatrick A = IRB.CreatePointerCast(A, Sa->getType());
263709467b48Spatrick B = IRB.CreatePointerCast(B, Sb->getType());
263809467b48Spatrick
263909467b48Spatrick // Let [a0, a1] be the interval of possible values of A, taking into account
264009467b48Spatrick // its undefined bits. Let [b0, b1] be the interval of possible values of B.
264109467b48Spatrick // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
264209467b48Spatrick bool IsSigned = I.isSigned();
264309467b48Spatrick Value *S1 = IRB.CreateICmp(I.getPredicate(),
264409467b48Spatrick getLowestPossibleValue(IRB, A, Sa, IsSigned),
264509467b48Spatrick getHighestPossibleValue(IRB, B, Sb, IsSigned));
264609467b48Spatrick Value *S2 = IRB.CreateICmp(I.getPredicate(),
264709467b48Spatrick getHighestPossibleValue(IRB, A, Sa, IsSigned),
264809467b48Spatrick getLowestPossibleValue(IRB, B, Sb, IsSigned));
264909467b48Spatrick Value *Si = IRB.CreateXor(S1, S2);
265009467b48Spatrick setShadow(&I, Si);
265109467b48Spatrick setOriginForNaryOp(I);
265209467b48Spatrick }
265309467b48Spatrick
265409467b48Spatrick /// Instrument signed relational comparisons.
265509467b48Spatrick ///
265609467b48Spatrick /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
265709467b48Spatrick /// bit of the shadow. Everything else is delegated to handleShadowOr().
handleSignedRelationalComparison__anon9f4f4c900811::MemorySanitizerVisitor265809467b48Spatrick void handleSignedRelationalComparison(ICmpInst &I) {
265909467b48Spatrick Constant *constOp;
266009467b48Spatrick Value *op = nullptr;
266109467b48Spatrick CmpInst::Predicate pre;
266209467b48Spatrick if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) {
266309467b48Spatrick op = I.getOperand(0);
266409467b48Spatrick pre = I.getPredicate();
266509467b48Spatrick } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) {
266609467b48Spatrick op = I.getOperand(1);
266709467b48Spatrick pre = I.getSwappedPredicate();
266809467b48Spatrick } else {
266909467b48Spatrick handleShadowOr(I);
267009467b48Spatrick return;
267109467b48Spatrick }
267209467b48Spatrick
267309467b48Spatrick if ((constOp->isNullValue() &&
267409467b48Spatrick (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
267509467b48Spatrick (constOp->isAllOnesValue() &&
267609467b48Spatrick (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
267709467b48Spatrick IRBuilder<> IRB(&I);
267809467b48Spatrick Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
267909467b48Spatrick "_msprop_icmp_s");
268009467b48Spatrick setShadow(&I, Shadow);
268109467b48Spatrick setOrigin(&I, getOrigin(op));
268209467b48Spatrick } else {
268309467b48Spatrick handleShadowOr(I);
268409467b48Spatrick }
268509467b48Spatrick }
268609467b48Spatrick
visitICmpInst__anon9f4f4c900811::MemorySanitizerVisitor268709467b48Spatrick void visitICmpInst(ICmpInst &I) {
268809467b48Spatrick if (!ClHandleICmp) {
268909467b48Spatrick handleShadowOr(I);
269009467b48Spatrick return;
269109467b48Spatrick }
269209467b48Spatrick if (I.isEquality()) {
269309467b48Spatrick handleEqualityComparison(I);
269409467b48Spatrick return;
269509467b48Spatrick }
269609467b48Spatrick
269709467b48Spatrick assert(I.isRelational());
269809467b48Spatrick if (ClHandleICmpExact) {
269909467b48Spatrick handleRelationalComparisonExact(I);
270009467b48Spatrick return;
270109467b48Spatrick }
270209467b48Spatrick if (I.isSigned()) {
270309467b48Spatrick handleSignedRelationalComparison(I);
270409467b48Spatrick return;
270509467b48Spatrick }
270609467b48Spatrick
270709467b48Spatrick assert(I.isUnsigned());
270809467b48Spatrick if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) {
270909467b48Spatrick handleRelationalComparisonExact(I);
271009467b48Spatrick return;
271109467b48Spatrick }
271209467b48Spatrick
271309467b48Spatrick handleShadowOr(I);
271409467b48Spatrick }
271509467b48Spatrick
visitFCmpInst__anon9f4f4c900811::MemorySanitizerVisitor2716*d415bd75Srobert void visitFCmpInst(FCmpInst &I) { handleShadowOr(I); }
271709467b48Spatrick
handleShift__anon9f4f4c900811::MemorySanitizerVisitor271809467b48Spatrick void handleShift(BinaryOperator &I) {
271909467b48Spatrick IRBuilder<> IRB(&I);
272009467b48Spatrick // If any of the S2 bits are poisoned, the whole thing is poisoned.
272109467b48Spatrick // Otherwise perform the same shift on S1.
272209467b48Spatrick Value *S1 = getShadow(&I, 0);
272309467b48Spatrick Value *S2 = getShadow(&I, 1);
2724*d415bd75Srobert Value *S2Conv =
2725*d415bd75Srobert IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
272609467b48Spatrick Value *V2 = I.getOperand(1);
272709467b48Spatrick Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
272809467b48Spatrick setShadow(&I, IRB.CreateOr(Shift, S2Conv));
272909467b48Spatrick setOriginForNaryOp(I);
273009467b48Spatrick }
273109467b48Spatrick
visitShl__anon9f4f4c900811::MemorySanitizerVisitor273209467b48Spatrick void visitShl(BinaryOperator &I) { handleShift(I); }
visitAShr__anon9f4f4c900811::MemorySanitizerVisitor273309467b48Spatrick void visitAShr(BinaryOperator &I) { handleShift(I); }
visitLShr__anon9f4f4c900811::MemorySanitizerVisitor273409467b48Spatrick void visitLShr(BinaryOperator &I) { handleShift(I); }
273509467b48Spatrick
handleFunnelShift__anon9f4f4c900811::MemorySanitizerVisitor273673471bf0Spatrick void handleFunnelShift(IntrinsicInst &I) {
273773471bf0Spatrick IRBuilder<> IRB(&I);
273873471bf0Spatrick // If any of the S2 bits are poisoned, the whole thing is poisoned.
273973471bf0Spatrick // Otherwise perform the same shift on S0 and S1.
274073471bf0Spatrick Value *S0 = getShadow(&I, 0);
274173471bf0Spatrick Value *S1 = getShadow(&I, 1);
274273471bf0Spatrick Value *S2 = getShadow(&I, 2);
274373471bf0Spatrick Value *S2Conv =
274473471bf0Spatrick IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
274573471bf0Spatrick Value *V2 = I.getOperand(2);
274673471bf0Spatrick Function *Intrin = Intrinsic::getDeclaration(
274773471bf0Spatrick I.getModule(), I.getIntrinsicID(), S2Conv->getType());
274873471bf0Spatrick Value *Shift = IRB.CreateCall(Intrin, {S0, S1, V2});
274973471bf0Spatrick setShadow(&I, IRB.CreateOr(Shift, S2Conv));
275073471bf0Spatrick setOriginForNaryOp(I);
275173471bf0Spatrick }
275273471bf0Spatrick
275309467b48Spatrick /// Instrument llvm.memmove
275409467b48Spatrick ///
275509467b48Spatrick /// At this point we don't know if llvm.memmove will be inlined or not.
275609467b48Spatrick /// If we don't instrument it and it gets inlined,
275709467b48Spatrick /// our interceptor will not kick in and we will lose the memmove.
275809467b48Spatrick /// If we instrument the call here, but it does not get inlined,
275909467b48Spatrick /// we will memove the shadow twice: which is bad in case
276009467b48Spatrick /// of overlapping regions. So, we simply lower the intrinsic to a call.
276109467b48Spatrick ///
276209467b48Spatrick /// Similar situation exists for memcpy and memset.
visitMemMoveInst__anon9f4f4c900811::MemorySanitizerVisitor276309467b48Spatrick void visitMemMoveInst(MemMoveInst &I) {
2764*d415bd75Srobert getShadow(I.getArgOperand(1)); // Ensure shadow initialized
276509467b48Spatrick IRBuilder<> IRB(&I);
276609467b48Spatrick IRB.CreateCall(
276709467b48Spatrick MS.MemmoveFn,
276809467b48Spatrick {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
276909467b48Spatrick IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
277009467b48Spatrick IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
277109467b48Spatrick I.eraseFromParent();
277209467b48Spatrick }
277309467b48Spatrick
2774*d415bd75Srobert /// Instrument memcpy
2775*d415bd75Srobert ///
2776*d415bd75Srobert /// Similar to memmove: avoid copying shadow twice. This is somewhat
2777*d415bd75Srobert /// unfortunate as it may slowdown small constant memcpys.
2778*d415bd75Srobert /// FIXME: consider doing manual inline for small constant sizes and proper
2779*d415bd75Srobert /// alignment.
2780*d415bd75Srobert ///
2781*d415bd75Srobert /// Note: This also handles memcpy.inline, which promises no calls to external
2782*d415bd75Srobert /// functions as an optimization. However, with instrumentation enabled this
2783*d415bd75Srobert /// is difficult to promise; additionally, we know that the MSan runtime
2784*d415bd75Srobert /// exists and provides __msan_memcpy(). Therefore, we assume that with
2785*d415bd75Srobert /// instrumentation it's safe to turn memcpy.inline into a call to
2786*d415bd75Srobert /// __msan_memcpy(). Should this be wrong, such as when implementing memcpy()
2787*d415bd75Srobert /// itself, instrumentation should be disabled with the no_sanitize attribute.
visitMemCpyInst__anon9f4f4c900811::MemorySanitizerVisitor278809467b48Spatrick void visitMemCpyInst(MemCpyInst &I) {
2789*d415bd75Srobert getShadow(I.getArgOperand(1)); // Ensure shadow initialized
279009467b48Spatrick IRBuilder<> IRB(&I);
279109467b48Spatrick IRB.CreateCall(
279209467b48Spatrick MS.MemcpyFn,
279309467b48Spatrick {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
279409467b48Spatrick IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
279509467b48Spatrick IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
279609467b48Spatrick I.eraseFromParent();
279709467b48Spatrick }
279809467b48Spatrick
279909467b48Spatrick // Same as memcpy.
visitMemSetInst__anon9f4f4c900811::MemorySanitizerVisitor280009467b48Spatrick void visitMemSetInst(MemSetInst &I) {
280109467b48Spatrick IRBuilder<> IRB(&I);
280209467b48Spatrick IRB.CreateCall(
280309467b48Spatrick MS.MemsetFn,
280409467b48Spatrick {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
280509467b48Spatrick IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
280609467b48Spatrick IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
280709467b48Spatrick I.eraseFromParent();
280809467b48Spatrick }
280909467b48Spatrick
visitVAStartInst__anon9f4f4c900811::MemorySanitizerVisitor2810*d415bd75Srobert void visitVAStartInst(VAStartInst &I) { VAHelper->visitVAStartInst(I); }
281109467b48Spatrick
visitVACopyInst__anon9f4f4c900811::MemorySanitizerVisitor2812*d415bd75Srobert void visitVACopyInst(VACopyInst &I) { VAHelper->visitVACopyInst(I); }
281309467b48Spatrick
281409467b48Spatrick /// Handle vector store-like intrinsics.
281509467b48Spatrick ///
281609467b48Spatrick /// Instrument intrinsics that look like a simple SIMD store: writes memory,
281709467b48Spatrick /// has 1 pointer argument and 1 vector argument, returns void.
handleVectorStoreIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor281809467b48Spatrick bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
281909467b48Spatrick IRBuilder<> IRB(&I);
282009467b48Spatrick Value *Addr = I.getArgOperand(0);
282109467b48Spatrick Value *Shadow = getShadow(&I, 1);
282209467b48Spatrick Value *ShadowPtr, *OriginPtr;
282309467b48Spatrick
282409467b48Spatrick // We don't know the pointer alignment (could be unaligned SSE store!).
282509467b48Spatrick // Have to assume to worst case.
282609467b48Spatrick std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2827097a140dSpatrick Addr, IRB, Shadow->getType(), Align(1), /*isStore*/ true);
2828097a140dSpatrick IRB.CreateAlignedStore(Shadow, ShadowPtr, Align(1));
282909467b48Spatrick
283009467b48Spatrick if (ClCheckAccessAddress)
283109467b48Spatrick insertShadowCheck(Addr, &I);
283209467b48Spatrick
283309467b48Spatrick // FIXME: factor out common code from materializeStores
2834*d415bd75Srobert if (MS.TrackOrigins)
2835*d415bd75Srobert IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
283609467b48Spatrick return true;
283709467b48Spatrick }
283809467b48Spatrick
283909467b48Spatrick /// Handle vector load-like intrinsics.
284009467b48Spatrick ///
284109467b48Spatrick /// Instrument intrinsics that look like a simple SIMD load: reads memory,
284209467b48Spatrick /// has 1 pointer argument, returns a vector.
handleVectorLoadIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor284309467b48Spatrick bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
284409467b48Spatrick IRBuilder<> IRB(&I);
284509467b48Spatrick Value *Addr = I.getArgOperand(0);
284609467b48Spatrick
284709467b48Spatrick Type *ShadowTy = getShadowTy(&I);
284809467b48Spatrick Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
284909467b48Spatrick if (PropagateShadow) {
285009467b48Spatrick // We don't know the pointer alignment (could be unaligned SSE load!).
285109467b48Spatrick // Have to assume to worst case.
2852097a140dSpatrick const Align Alignment = Align(1);
285309467b48Spatrick std::tie(ShadowPtr, OriginPtr) =
285409467b48Spatrick getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2855097a140dSpatrick setShadow(&I,
2856097a140dSpatrick IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
285709467b48Spatrick } else {
285809467b48Spatrick setShadow(&I, getCleanShadow(&I));
285909467b48Spatrick }
286009467b48Spatrick
286109467b48Spatrick if (ClCheckAccessAddress)
286209467b48Spatrick insertShadowCheck(Addr, &I);
286309467b48Spatrick
286409467b48Spatrick if (MS.TrackOrigins) {
286509467b48Spatrick if (PropagateShadow)
286609467b48Spatrick setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr));
286709467b48Spatrick else
286809467b48Spatrick setOrigin(&I, getCleanOrigin());
286909467b48Spatrick }
287009467b48Spatrick return true;
287109467b48Spatrick }
287209467b48Spatrick
287309467b48Spatrick /// Handle (SIMD arithmetic)-like intrinsics.
287409467b48Spatrick ///
287509467b48Spatrick /// Instrument intrinsics with any number of arguments of the same type,
287609467b48Spatrick /// equal to the return type. The type should be simple (no aggregates or
287709467b48Spatrick /// pointers; vectors are fine).
287809467b48Spatrick /// Caller guarantees that this intrinsic does not access memory.
maybeHandleSimpleNomemIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor287909467b48Spatrick bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
288009467b48Spatrick Type *RetTy = I.getType();
2881*d415bd75Srobert if (!(RetTy->isIntOrIntVectorTy() || RetTy->isFPOrFPVectorTy() ||
288209467b48Spatrick RetTy->isX86_MMXTy()))
288309467b48Spatrick return false;
288409467b48Spatrick
2885*d415bd75Srobert unsigned NumArgOperands = I.arg_size();
288609467b48Spatrick for (unsigned i = 0; i < NumArgOperands; ++i) {
288709467b48Spatrick Type *Ty = I.getArgOperand(i)->getType();
288809467b48Spatrick if (Ty != RetTy)
288909467b48Spatrick return false;
289009467b48Spatrick }
289109467b48Spatrick
289209467b48Spatrick IRBuilder<> IRB(&I);
289309467b48Spatrick ShadowAndOriginCombiner SC(this, IRB);
289409467b48Spatrick for (unsigned i = 0; i < NumArgOperands; ++i)
289509467b48Spatrick SC.Add(I.getArgOperand(i));
289609467b48Spatrick SC.Done(&I);
289709467b48Spatrick
289809467b48Spatrick return true;
289909467b48Spatrick }
290009467b48Spatrick
290109467b48Spatrick /// Heuristically instrument unknown intrinsics.
290209467b48Spatrick ///
290309467b48Spatrick /// The main purpose of this code is to do something reasonable with all
290409467b48Spatrick /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
290509467b48Spatrick /// We recognize several classes of intrinsics by their argument types and
2906097a140dSpatrick /// ModRefBehaviour and apply special instrumentation when we are reasonably
290709467b48Spatrick /// sure that we know what the intrinsic does.
290809467b48Spatrick ///
290909467b48Spatrick /// We special-case intrinsics where this approach fails. See llvm.bswap
291009467b48Spatrick /// handling as an example of that.
handleUnknownIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor291109467b48Spatrick bool handleUnknownIntrinsic(IntrinsicInst &I) {
2912*d415bd75Srobert unsigned NumArgOperands = I.arg_size();
291309467b48Spatrick if (NumArgOperands == 0)
291409467b48Spatrick return false;
291509467b48Spatrick
2916*d415bd75Srobert if (NumArgOperands == 2 && I.getArgOperand(0)->getType()->isPointerTy() &&
291709467b48Spatrick I.getArgOperand(1)->getType()->isVectorTy() &&
2918*d415bd75Srobert I.getType()->isVoidTy() && !I.onlyReadsMemory()) {
291909467b48Spatrick // This looks like a vector store.
292009467b48Spatrick return handleVectorStoreIntrinsic(I);
292109467b48Spatrick }
292209467b48Spatrick
2923*d415bd75Srobert if (NumArgOperands == 1 && I.getArgOperand(0)->getType()->isPointerTy() &&
2924*d415bd75Srobert I.getType()->isVectorTy() && I.onlyReadsMemory()) {
292509467b48Spatrick // This looks like a vector load.
292609467b48Spatrick return handleVectorLoadIntrinsic(I);
292709467b48Spatrick }
292809467b48Spatrick
292909467b48Spatrick if (I.doesNotAccessMemory())
293009467b48Spatrick if (maybeHandleSimpleNomemIntrinsic(I))
293109467b48Spatrick return true;
293209467b48Spatrick
293309467b48Spatrick // FIXME: detect and handle SSE maskstore/maskload
293409467b48Spatrick return false;
293509467b48Spatrick }
293609467b48Spatrick
handleInvariantGroup__anon9f4f4c900811::MemorySanitizerVisitor293709467b48Spatrick void handleInvariantGroup(IntrinsicInst &I) {
293809467b48Spatrick setShadow(&I, getShadow(&I, 0));
293909467b48Spatrick setOrigin(&I, getOrigin(&I, 0));
294009467b48Spatrick }
294109467b48Spatrick
handleLifetimeStart__anon9f4f4c900811::MemorySanitizerVisitor294209467b48Spatrick void handleLifetimeStart(IntrinsicInst &I) {
294309467b48Spatrick if (!PoisonStack)
294409467b48Spatrick return;
294573471bf0Spatrick AllocaInst *AI = llvm::findAllocaForValue(I.getArgOperand(1));
294609467b48Spatrick if (!AI)
294709467b48Spatrick InstrumentLifetimeStart = false;
294809467b48Spatrick LifetimeStartList.push_back(std::make_pair(&I, AI));
294909467b48Spatrick }
295009467b48Spatrick
handleBswap__anon9f4f4c900811::MemorySanitizerVisitor295109467b48Spatrick void handleBswap(IntrinsicInst &I) {
295209467b48Spatrick IRBuilder<> IRB(&I);
295309467b48Spatrick Value *Op = I.getArgOperand(0);
295409467b48Spatrick Type *OpType = Op->getType();
295509467b48Spatrick Function *BswapFunc = Intrinsic::getDeclaration(
2956*d415bd75Srobert F.getParent(), Intrinsic::bswap, ArrayRef(&OpType, 1));
295709467b48Spatrick setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
295809467b48Spatrick setOrigin(&I, getOrigin(Op));
295909467b48Spatrick }
296009467b48Spatrick
handleCountZeroes__anon9f4f4c900811::MemorySanitizerVisitor2961*d415bd75Srobert void handleCountZeroes(IntrinsicInst &I) {
2962*d415bd75Srobert IRBuilder<> IRB(&I);
2963*d415bd75Srobert Value *Src = I.getArgOperand(0);
2964*d415bd75Srobert
2965*d415bd75Srobert // Set the Output shadow based on input Shadow
2966*d415bd75Srobert Value *BoolShadow = IRB.CreateIsNotNull(getShadow(Src), "_mscz_bs");
2967*d415bd75Srobert
2968*d415bd75Srobert // If zero poison is requested, mix in with the shadow
2969*d415bd75Srobert Constant *IsZeroPoison = cast<Constant>(I.getOperand(1));
2970*d415bd75Srobert if (!IsZeroPoison->isZeroValue()) {
2971*d415bd75Srobert Value *BoolZeroPoison = IRB.CreateIsNull(Src, "_mscz_bzp");
2972*d415bd75Srobert BoolShadow = IRB.CreateOr(BoolShadow, BoolZeroPoison, "_mscz_bs");
2973*d415bd75Srobert }
2974*d415bd75Srobert
2975*d415bd75Srobert Value *OutputShadow =
2976*d415bd75Srobert IRB.CreateSExt(BoolShadow, getShadowTy(Src), "_mscz_os");
2977*d415bd75Srobert
2978*d415bd75Srobert setShadow(&I, OutputShadow);
2979*d415bd75Srobert setOriginForNaryOp(I);
2980*d415bd75Srobert }
2981*d415bd75Srobert
2982097a140dSpatrick // Instrument vector convert intrinsic.
298309467b48Spatrick //
298409467b48Spatrick // This function instruments intrinsics like cvtsi2ss:
298509467b48Spatrick // %Out = int_xxx_cvtyyy(%ConvertOp)
298609467b48Spatrick // or
298709467b48Spatrick // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
298809467b48Spatrick // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
298909467b48Spatrick // number \p Out elements, and (if has 2 arguments) copies the rest of the
299009467b48Spatrick // elements from \p CopyOp.
299109467b48Spatrick // In most cases conversion involves floating-point value which may trigger a
299209467b48Spatrick // hardware exception when not fully initialized. For this reason we require
299309467b48Spatrick // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
299409467b48Spatrick // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
299509467b48Spatrick // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
299609467b48Spatrick // return a fully initialized value.
handleVectorConvertIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor299773471bf0Spatrick void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
299873471bf0Spatrick bool HasRoundingMode = false) {
299909467b48Spatrick IRBuilder<> IRB(&I);
300009467b48Spatrick Value *CopyOp, *ConvertOp;
300109467b48Spatrick
300273471bf0Spatrick assert((!HasRoundingMode ||
3003*d415bd75Srobert isa<ConstantInt>(I.getArgOperand(I.arg_size() - 1))) &&
300473471bf0Spatrick "Invalid rounding mode");
300573471bf0Spatrick
3006*d415bd75Srobert switch (I.arg_size() - HasRoundingMode) {
300709467b48Spatrick case 2:
300809467b48Spatrick CopyOp = I.getArgOperand(0);
300909467b48Spatrick ConvertOp = I.getArgOperand(1);
301009467b48Spatrick break;
301109467b48Spatrick case 1:
301209467b48Spatrick ConvertOp = I.getArgOperand(0);
301309467b48Spatrick CopyOp = nullptr;
301409467b48Spatrick break;
301509467b48Spatrick default:
301609467b48Spatrick llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
301709467b48Spatrick }
301809467b48Spatrick
301909467b48Spatrick // The first *NumUsedElements* elements of ConvertOp are converted to the
302009467b48Spatrick // same number of output elements. The rest of the output is copied from
302109467b48Spatrick // CopyOp, or (if not available) filled with zeroes.
302209467b48Spatrick // Combine shadow for elements of ConvertOp that are used in this operation,
302309467b48Spatrick // and insert a check.
302409467b48Spatrick // FIXME: consider propagating shadow of ConvertOp, at least in the case of
302509467b48Spatrick // int->any conversion.
302609467b48Spatrick Value *ConvertShadow = getShadow(ConvertOp);
302709467b48Spatrick Value *AggShadow = nullptr;
302809467b48Spatrick if (ConvertOp->getType()->isVectorTy()) {
302909467b48Spatrick AggShadow = IRB.CreateExtractElement(
303009467b48Spatrick ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
303109467b48Spatrick for (int i = 1; i < NumUsedElements; ++i) {
303209467b48Spatrick Value *MoreShadow = IRB.CreateExtractElement(
303309467b48Spatrick ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
303409467b48Spatrick AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
303509467b48Spatrick }
303609467b48Spatrick } else {
303709467b48Spatrick AggShadow = ConvertShadow;
303809467b48Spatrick }
303909467b48Spatrick assert(AggShadow->getType()->isIntegerTy());
304009467b48Spatrick insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
304109467b48Spatrick
304209467b48Spatrick // Build result shadow by zero-filling parts of CopyOp shadow that come from
304309467b48Spatrick // ConvertOp.
304409467b48Spatrick if (CopyOp) {
304509467b48Spatrick assert(CopyOp->getType() == I.getType());
304609467b48Spatrick assert(CopyOp->getType()->isVectorTy());
304709467b48Spatrick Value *ResultShadow = getShadow(CopyOp);
3048097a140dSpatrick Type *EltTy = cast<VectorType>(ResultShadow->getType())->getElementType();
304909467b48Spatrick for (int i = 0; i < NumUsedElements; ++i) {
305009467b48Spatrick ResultShadow = IRB.CreateInsertElement(
305109467b48Spatrick ResultShadow, ConstantInt::getNullValue(EltTy),
305209467b48Spatrick ConstantInt::get(IRB.getInt32Ty(), i));
305309467b48Spatrick }
305409467b48Spatrick setShadow(&I, ResultShadow);
305509467b48Spatrick setOrigin(&I, getOrigin(CopyOp));
305609467b48Spatrick } else {
305709467b48Spatrick setShadow(&I, getCleanShadow(&I));
305809467b48Spatrick setOrigin(&I, getCleanOrigin());
305909467b48Spatrick }
306009467b48Spatrick }
306109467b48Spatrick
306209467b48Spatrick // Given a scalar or vector, extract lower 64 bits (or less), and return all
306309467b48Spatrick // zeroes if it is zero, and all ones otherwise.
Lower64ShadowExtend__anon9f4f4c900811::MemorySanitizerVisitor306409467b48Spatrick Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
306509467b48Spatrick if (S->getType()->isVectorTy())
306609467b48Spatrick S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
306709467b48Spatrick assert(S->getType()->getPrimitiveSizeInBits() <= 64);
306809467b48Spatrick Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
306909467b48Spatrick return CreateShadowCast(IRB, S2, T, /* Signed */ true);
307009467b48Spatrick }
307109467b48Spatrick
307209467b48Spatrick // Given a vector, extract its first element, and return all
307309467b48Spatrick // zeroes if it is zero, and all ones otherwise.
LowerElementShadowExtend__anon9f4f4c900811::MemorySanitizerVisitor307409467b48Spatrick Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
307509467b48Spatrick Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
307609467b48Spatrick Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
307709467b48Spatrick return CreateShadowCast(IRB, S2, T, /* Signed */ true);
307809467b48Spatrick }
307909467b48Spatrick
VariableShadowExtend__anon9f4f4c900811::MemorySanitizerVisitor308009467b48Spatrick Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
308109467b48Spatrick Type *T = S->getType();
308209467b48Spatrick assert(T->isVectorTy());
308309467b48Spatrick Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
308409467b48Spatrick return IRB.CreateSExt(S2, T);
308509467b48Spatrick }
308609467b48Spatrick
3087097a140dSpatrick // Instrument vector shift intrinsic.
308809467b48Spatrick //
308909467b48Spatrick // This function instruments intrinsics like int_x86_avx2_psll_w.
309009467b48Spatrick // Intrinsic shifts %In by %ShiftSize bits.
309109467b48Spatrick // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
309209467b48Spatrick // size, and the rest is ignored. Behavior is defined even if shift size is
309309467b48Spatrick // greater than register (or field) width.
handleVectorShiftIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor309409467b48Spatrick void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
3095*d415bd75Srobert assert(I.arg_size() == 2);
309609467b48Spatrick IRBuilder<> IRB(&I);
309709467b48Spatrick // If any of the S2 bits are poisoned, the whole thing is poisoned.
309809467b48Spatrick // Otherwise perform the same shift on S1.
309909467b48Spatrick Value *S1 = getShadow(&I, 0);
310009467b48Spatrick Value *S2 = getShadow(&I, 1);
310109467b48Spatrick Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
310209467b48Spatrick : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
310309467b48Spatrick Value *V1 = I.getOperand(0);
310409467b48Spatrick Value *V2 = I.getOperand(1);
3105097a140dSpatrick Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledOperand(),
310609467b48Spatrick {IRB.CreateBitCast(S1, V1->getType()), V2});
310709467b48Spatrick Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
310809467b48Spatrick setShadow(&I, IRB.CreateOr(Shift, S2Conv));
310909467b48Spatrick setOriginForNaryOp(I);
311009467b48Spatrick }
311109467b48Spatrick
311209467b48Spatrick // Get an X86_MMX-sized vector type.
getMMXVectorTy__anon9f4f4c900811::MemorySanitizerVisitor311309467b48Spatrick Type *getMMXVectorTy(unsigned EltSizeInBits) {
311409467b48Spatrick const unsigned X86_MMXSizeInBits = 64;
311509467b48Spatrick assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
311609467b48Spatrick "Illegal MMX vector element size");
3117097a140dSpatrick return FixedVectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
311809467b48Spatrick X86_MMXSizeInBits / EltSizeInBits);
311909467b48Spatrick }
312009467b48Spatrick
312109467b48Spatrick // Returns a signed counterpart for an (un)signed-saturate-and-pack
312209467b48Spatrick // intrinsic.
getSignedPackIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor312309467b48Spatrick Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
312409467b48Spatrick switch (id) {
312509467b48Spatrick case Intrinsic::x86_sse2_packsswb_128:
312609467b48Spatrick case Intrinsic::x86_sse2_packuswb_128:
312709467b48Spatrick return Intrinsic::x86_sse2_packsswb_128;
312809467b48Spatrick
312909467b48Spatrick case Intrinsic::x86_sse2_packssdw_128:
313009467b48Spatrick case Intrinsic::x86_sse41_packusdw:
313109467b48Spatrick return Intrinsic::x86_sse2_packssdw_128;
313209467b48Spatrick
313309467b48Spatrick case Intrinsic::x86_avx2_packsswb:
313409467b48Spatrick case Intrinsic::x86_avx2_packuswb:
313509467b48Spatrick return Intrinsic::x86_avx2_packsswb;
313609467b48Spatrick
313709467b48Spatrick case Intrinsic::x86_avx2_packssdw:
313809467b48Spatrick case Intrinsic::x86_avx2_packusdw:
313909467b48Spatrick return Intrinsic::x86_avx2_packssdw;
314009467b48Spatrick
314109467b48Spatrick case Intrinsic::x86_mmx_packsswb:
314209467b48Spatrick case Intrinsic::x86_mmx_packuswb:
314309467b48Spatrick return Intrinsic::x86_mmx_packsswb;
314409467b48Spatrick
314509467b48Spatrick case Intrinsic::x86_mmx_packssdw:
314609467b48Spatrick return Intrinsic::x86_mmx_packssdw;
314709467b48Spatrick default:
314809467b48Spatrick llvm_unreachable("unexpected intrinsic id");
314909467b48Spatrick }
315009467b48Spatrick }
315109467b48Spatrick
3152097a140dSpatrick // Instrument vector pack intrinsic.
315309467b48Spatrick //
315409467b48Spatrick // This function instruments intrinsics like x86_mmx_packsswb, that
315509467b48Spatrick // packs elements of 2 input vectors into half as many bits with saturation.
315609467b48Spatrick // Shadow is propagated with the signed variant of the same intrinsic applied
315709467b48Spatrick // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
315809467b48Spatrick // EltSizeInBits is used only for x86mmx arguments.
handleVectorPackIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor315909467b48Spatrick void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
3160*d415bd75Srobert assert(I.arg_size() == 2);
316109467b48Spatrick bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
316209467b48Spatrick IRBuilder<> IRB(&I);
316309467b48Spatrick Value *S1 = getShadow(&I, 0);
316409467b48Spatrick Value *S2 = getShadow(&I, 1);
316509467b48Spatrick assert(isX86_MMX || S1->getType()->isVectorTy());
316609467b48Spatrick
316709467b48Spatrick // SExt and ICmpNE below must apply to individual elements of input vectors.
316809467b48Spatrick // In case of x86mmx arguments, cast them to appropriate vector types and
316909467b48Spatrick // back.
317009467b48Spatrick Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
317109467b48Spatrick if (isX86_MMX) {
317209467b48Spatrick S1 = IRB.CreateBitCast(S1, T);
317309467b48Spatrick S2 = IRB.CreateBitCast(S2, T);
317409467b48Spatrick }
3175*d415bd75Srobert Value *S1_ext =
3176*d415bd75Srobert IRB.CreateSExt(IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
3177*d415bd75Srobert Value *S2_ext =
3178*d415bd75Srobert IRB.CreateSExt(IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
317909467b48Spatrick if (isX86_MMX) {
318009467b48Spatrick Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
318109467b48Spatrick S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
318209467b48Spatrick S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
318309467b48Spatrick }
318409467b48Spatrick
318509467b48Spatrick Function *ShadowFn = Intrinsic::getDeclaration(
318609467b48Spatrick F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
318709467b48Spatrick
318809467b48Spatrick Value *S =
318909467b48Spatrick IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
3190*d415bd75Srobert if (isX86_MMX)
3191*d415bd75Srobert S = IRB.CreateBitCast(S, getShadowTy(&I));
319209467b48Spatrick setShadow(&I, S);
319309467b48Spatrick setOriginForNaryOp(I);
319409467b48Spatrick }
319509467b48Spatrick
3196097a140dSpatrick // Instrument sum-of-absolute-differences intrinsic.
handleVectorSadIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor319709467b48Spatrick void handleVectorSadIntrinsic(IntrinsicInst &I) {
319809467b48Spatrick const unsigned SignificantBitsPerResultElement = 16;
319909467b48Spatrick bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
320009467b48Spatrick Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType();
320109467b48Spatrick unsigned ZeroBitsPerResultElement =
320209467b48Spatrick ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
320309467b48Spatrick
320409467b48Spatrick IRBuilder<> IRB(&I);
3205*d415bd75Srobert auto *Shadow0 = getShadow(&I, 0);
3206*d415bd75Srobert auto *Shadow1 = getShadow(&I, 1);
3207*d415bd75Srobert Value *S = IRB.CreateOr(Shadow0, Shadow1);
320809467b48Spatrick S = IRB.CreateBitCast(S, ResTy);
320909467b48Spatrick S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
321009467b48Spatrick ResTy);
321109467b48Spatrick S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
321209467b48Spatrick S = IRB.CreateBitCast(S, getShadowTy(&I));
321309467b48Spatrick setShadow(&I, S);
321409467b48Spatrick setOriginForNaryOp(I);
321509467b48Spatrick }
321609467b48Spatrick
321709467b48Spatrick // Instrument multiply-add intrinsic.
handleVectorPmaddIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor321809467b48Spatrick void handleVectorPmaddIntrinsic(IntrinsicInst &I,
321909467b48Spatrick unsigned EltSizeInBits = 0) {
322009467b48Spatrick bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
322109467b48Spatrick Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType();
322209467b48Spatrick IRBuilder<> IRB(&I);
3223*d415bd75Srobert auto *Shadow0 = getShadow(&I, 0);
3224*d415bd75Srobert auto *Shadow1 = getShadow(&I, 1);
3225*d415bd75Srobert Value *S = IRB.CreateOr(Shadow0, Shadow1);
322609467b48Spatrick S = IRB.CreateBitCast(S, ResTy);
322709467b48Spatrick S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
322809467b48Spatrick ResTy);
322909467b48Spatrick S = IRB.CreateBitCast(S, getShadowTy(&I));
323009467b48Spatrick setShadow(&I, S);
323109467b48Spatrick setOriginForNaryOp(I);
323209467b48Spatrick }
323309467b48Spatrick
323409467b48Spatrick // Instrument compare-packed intrinsic.
323509467b48Spatrick // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
323609467b48Spatrick // all-ones shadow.
handleVectorComparePackedIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor323709467b48Spatrick void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
323809467b48Spatrick IRBuilder<> IRB(&I);
323909467b48Spatrick Type *ResTy = getShadowTy(&I);
3240*d415bd75Srobert auto *Shadow0 = getShadow(&I, 0);
3241*d415bd75Srobert auto *Shadow1 = getShadow(&I, 1);
3242*d415bd75Srobert Value *S0 = IRB.CreateOr(Shadow0, Shadow1);
324309467b48Spatrick Value *S = IRB.CreateSExt(
324409467b48Spatrick IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
324509467b48Spatrick setShadow(&I, S);
324609467b48Spatrick setOriginForNaryOp(I);
324709467b48Spatrick }
324809467b48Spatrick
324909467b48Spatrick // Instrument compare-scalar intrinsic.
325009467b48Spatrick // This handles both cmp* intrinsics which return the result in the first
325109467b48Spatrick // element of a vector, and comi* which return the result as i32.
handleVectorCompareScalarIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor325209467b48Spatrick void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
325309467b48Spatrick IRBuilder<> IRB(&I);
3254*d415bd75Srobert auto *Shadow0 = getShadow(&I, 0);
3255*d415bd75Srobert auto *Shadow1 = getShadow(&I, 1);
3256*d415bd75Srobert Value *S0 = IRB.CreateOr(Shadow0, Shadow1);
325709467b48Spatrick Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
325809467b48Spatrick setShadow(&I, S);
325909467b48Spatrick setOriginForNaryOp(I);
326009467b48Spatrick }
326109467b48Spatrick
3262097a140dSpatrick // Instrument generic vector reduction intrinsics
3263097a140dSpatrick // by ORing together all their fields.
handleVectorReduceIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor3264097a140dSpatrick void handleVectorReduceIntrinsic(IntrinsicInst &I) {
3265097a140dSpatrick IRBuilder<> IRB(&I);
3266097a140dSpatrick Value *S = IRB.CreateOrReduce(getShadow(&I, 0));
3267097a140dSpatrick setShadow(&I, S);
3268097a140dSpatrick setOrigin(&I, getOrigin(&I, 0));
3269097a140dSpatrick }
3270097a140dSpatrick
327173471bf0Spatrick // Instrument vector.reduce.or intrinsic.
3272097a140dSpatrick // Valid (non-poisoned) set bits in the operand pull low the
3273097a140dSpatrick // corresponding shadow bits.
handleVectorReduceOrIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor3274097a140dSpatrick void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
3275097a140dSpatrick IRBuilder<> IRB(&I);
3276097a140dSpatrick Value *OperandShadow = getShadow(&I, 0);
3277097a140dSpatrick Value *OperandUnsetBits = IRB.CreateNot(I.getOperand(0));
3278097a140dSpatrick Value *OperandUnsetOrPoison = IRB.CreateOr(OperandUnsetBits, OperandShadow);
3279097a140dSpatrick // Bit N is clean if any field's bit N is 1 and unpoison
3280097a140dSpatrick Value *OutShadowMask = IRB.CreateAndReduce(OperandUnsetOrPoison);
3281097a140dSpatrick // Otherwise, it is clean if every field's bit N is unpoison
3282097a140dSpatrick Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3283097a140dSpatrick Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3284097a140dSpatrick
3285097a140dSpatrick setShadow(&I, S);
3286097a140dSpatrick setOrigin(&I, getOrigin(&I, 0));
3287097a140dSpatrick }
3288097a140dSpatrick
328973471bf0Spatrick // Instrument vector.reduce.and intrinsic.
3290097a140dSpatrick // Valid (non-poisoned) unset bits in the operand pull down the
3291097a140dSpatrick // corresponding shadow bits.
handleVectorReduceAndIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor3292097a140dSpatrick void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
3293097a140dSpatrick IRBuilder<> IRB(&I);
3294097a140dSpatrick Value *OperandShadow = getShadow(&I, 0);
3295097a140dSpatrick Value *OperandSetOrPoison = IRB.CreateOr(I.getOperand(0), OperandShadow);
3296097a140dSpatrick // Bit N is clean if any field's bit N is 0 and unpoison
3297097a140dSpatrick Value *OutShadowMask = IRB.CreateAndReduce(OperandSetOrPoison);
3298097a140dSpatrick // Otherwise, it is clean if every field's bit N is unpoison
3299097a140dSpatrick Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3300097a140dSpatrick Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3301097a140dSpatrick
3302097a140dSpatrick setShadow(&I, S);
3303097a140dSpatrick setOrigin(&I, getOrigin(&I, 0));
3304097a140dSpatrick }
3305097a140dSpatrick
handleStmxcsr__anon9f4f4c900811::MemorySanitizerVisitor330609467b48Spatrick void handleStmxcsr(IntrinsicInst &I) {
330709467b48Spatrick IRBuilder<> IRB(&I);
330809467b48Spatrick Value *Addr = I.getArgOperand(0);
330909467b48Spatrick Type *Ty = IRB.getInt32Ty();
331009467b48Spatrick Value *ShadowPtr =
3311097a140dSpatrick getShadowOriginPtr(Addr, IRB, Ty, Align(1), /*isStore*/ true).first;
331209467b48Spatrick
331309467b48Spatrick IRB.CreateStore(getCleanShadow(Ty),
331409467b48Spatrick IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
331509467b48Spatrick
331609467b48Spatrick if (ClCheckAccessAddress)
331709467b48Spatrick insertShadowCheck(Addr, &I);
331809467b48Spatrick }
331909467b48Spatrick
handleLdmxcsr__anon9f4f4c900811::MemorySanitizerVisitor332009467b48Spatrick void handleLdmxcsr(IntrinsicInst &I) {
3321*d415bd75Srobert if (!InsertChecks)
3322*d415bd75Srobert return;
332309467b48Spatrick
332409467b48Spatrick IRBuilder<> IRB(&I);
332509467b48Spatrick Value *Addr = I.getArgOperand(0);
332609467b48Spatrick Type *Ty = IRB.getInt32Ty();
3327097a140dSpatrick const Align Alignment = Align(1);
332809467b48Spatrick Value *ShadowPtr, *OriginPtr;
332909467b48Spatrick std::tie(ShadowPtr, OriginPtr) =
333009467b48Spatrick getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
333109467b48Spatrick
333209467b48Spatrick if (ClCheckAccessAddress)
333309467b48Spatrick insertShadowCheck(Addr, &I);
333409467b48Spatrick
3335097a140dSpatrick Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr");
333609467b48Spatrick Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
333709467b48Spatrick : getCleanOrigin();
333809467b48Spatrick insertShadowCheck(Shadow, Origin, &I);
333909467b48Spatrick }
334009467b48Spatrick
handleMaskedExpandLoad__anon9f4f4c900811::MemorySanitizerVisitor3341*d415bd75Srobert void handleMaskedExpandLoad(IntrinsicInst &I) {
334209467b48Spatrick IRBuilder<> IRB(&I);
3343*d415bd75Srobert Value *Ptr = I.getArgOperand(0);
3344*d415bd75Srobert Value *Mask = I.getArgOperand(1);
3345*d415bd75Srobert Value *PassThru = I.getArgOperand(2);
334609467b48Spatrick
334709467b48Spatrick if (ClCheckAccessAddress) {
3348*d415bd75Srobert insertShadowCheck(Ptr, &I);
334909467b48Spatrick insertShadowCheck(Mask, &I);
335009467b48Spatrick }
335109467b48Spatrick
3352*d415bd75Srobert if (!PropagateShadow) {
3353*d415bd75Srobert setShadow(&I, getCleanShadow(&I));
3354*d415bd75Srobert setOrigin(&I, getCleanOrigin());
3355*d415bd75Srobert return;
335609467b48Spatrick }
335709467b48Spatrick
3358*d415bd75Srobert Type *ShadowTy = getShadowTy(&I);
3359*d415bd75Srobert Type *ElementShadowTy = cast<FixedVectorType>(ShadowTy)->getElementType();
3360*d415bd75Srobert auto [ShadowPtr, OriginPtr] =
3361*d415bd75Srobert getShadowOriginPtr(Ptr, IRB, ElementShadowTy, {}, /*isStore*/ false);
3362*d415bd75Srobert
3363*d415bd75Srobert Value *Shadow = IRB.CreateMaskedExpandLoad(
3364*d415bd75Srobert ShadowTy, ShadowPtr, Mask, getShadow(PassThru), "_msmaskedexpload");
3365*d415bd75Srobert
3366*d415bd75Srobert setShadow(&I, Shadow);
3367*d415bd75Srobert
3368*d415bd75Srobert // TODO: Store origins.
3369*d415bd75Srobert setOrigin(&I, getCleanOrigin());
3370*d415bd75Srobert }
3371*d415bd75Srobert
handleMaskedCompressStore__anon9f4f4c900811::MemorySanitizerVisitor3372*d415bd75Srobert void handleMaskedCompressStore(IntrinsicInst &I) {
337309467b48Spatrick IRBuilder<> IRB(&I);
3374*d415bd75Srobert Value *Values = I.getArgOperand(0);
3375*d415bd75Srobert Value *Ptr = I.getArgOperand(1);
3376*d415bd75Srobert Value *Mask = I.getArgOperand(2);
3377*d415bd75Srobert
3378*d415bd75Srobert if (ClCheckAccessAddress) {
3379*d415bd75Srobert insertShadowCheck(Ptr, &I);
3380*d415bd75Srobert insertShadowCheck(Mask, &I);
3381*d415bd75Srobert }
3382*d415bd75Srobert
3383*d415bd75Srobert Value *Shadow = getShadow(Values);
3384*d415bd75Srobert Type *ElementShadowTy =
3385*d415bd75Srobert getShadowTy(cast<FixedVectorType>(Values->getType())->getElementType());
3386*d415bd75Srobert auto [ShadowPtr, OriginPtrs] =
3387*d415bd75Srobert getShadowOriginPtr(Ptr, IRB, ElementShadowTy, {}, /*isStore*/ true);
3388*d415bd75Srobert
3389*d415bd75Srobert IRB.CreateMaskedCompressStore(Shadow, ShadowPtr, Mask);
3390*d415bd75Srobert
3391*d415bd75Srobert // TODO: Store origins.
3392*d415bd75Srobert }
3393*d415bd75Srobert
handleMaskedGather__anon9f4f4c900811::MemorySanitizerVisitor3394*d415bd75Srobert void handleMaskedGather(IntrinsicInst &I) {
3395*d415bd75Srobert IRBuilder<> IRB(&I);
3396*d415bd75Srobert Value *Ptrs = I.getArgOperand(0);
3397097a140dSpatrick const Align Alignment(
339809467b48Spatrick cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
339909467b48Spatrick Value *Mask = I.getArgOperand(2);
340009467b48Spatrick Value *PassThru = I.getArgOperand(3);
340109467b48Spatrick
3402*d415bd75Srobert Type *PtrsShadowTy = getShadowTy(Ptrs);
3403*d415bd75Srobert if (ClCheckAccessAddress) {
3404*d415bd75Srobert insertShadowCheck(Mask, &I);
3405*d415bd75Srobert Value *MaskedPtrShadow = IRB.CreateSelect(
3406*d415bd75Srobert Mask, getShadow(Ptrs), Constant::getNullValue((PtrsShadowTy)),
3407*d415bd75Srobert "_msmaskedptrs");
3408*d415bd75Srobert insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &I);
340909467b48Spatrick }
341009467b48Spatrick
3411*d415bd75Srobert if (!PropagateShadow) {
3412*d415bd75Srobert setShadow(&I, getCleanShadow(&I));
3413*d415bd75Srobert setOrigin(&I, getCleanOrigin());
3414*d415bd75Srobert return;
3415*d415bd75Srobert }
3416*d415bd75Srobert
3417*d415bd75Srobert Type *ShadowTy = getShadowTy(&I);
3418*d415bd75Srobert Type *ElementShadowTy = cast<FixedVectorType>(ShadowTy)->getElementType();
3419*d415bd75Srobert auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3420*d415bd75Srobert Ptrs, IRB, ElementShadowTy, Alignment, /*isStore*/ false);
3421*d415bd75Srobert
3422*d415bd75Srobert Value *Shadow =
3423*d415bd75Srobert IRB.CreateMaskedGather(ShadowTy, ShadowPtrs, Alignment, Mask,
3424*d415bd75Srobert getShadow(PassThru), "_msmaskedgather");
3425*d415bd75Srobert
3426*d415bd75Srobert setShadow(&I, Shadow);
3427*d415bd75Srobert
3428*d415bd75Srobert // TODO: Store origins.
3429*d415bd75Srobert setOrigin(&I, getCleanOrigin());
3430*d415bd75Srobert }
3431*d415bd75Srobert
handleMaskedScatter__anon9f4f4c900811::MemorySanitizerVisitor3432*d415bd75Srobert void handleMaskedScatter(IntrinsicInst &I) {
3433*d415bd75Srobert IRBuilder<> IRB(&I);
3434*d415bd75Srobert Value *Values = I.getArgOperand(0);
3435*d415bd75Srobert Value *Ptrs = I.getArgOperand(1);
3436*d415bd75Srobert const Align Alignment(
3437*d415bd75Srobert cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
3438*d415bd75Srobert Value *Mask = I.getArgOperand(3);
3439*d415bd75Srobert
3440*d415bd75Srobert Type *PtrsShadowTy = getShadowTy(Ptrs);
344109467b48Spatrick if (ClCheckAccessAddress) {
3442*d415bd75Srobert insertShadowCheck(Mask, &I);
3443*d415bd75Srobert Value *MaskedPtrShadow = IRB.CreateSelect(
3444*d415bd75Srobert Mask, getShadow(Ptrs), Constant::getNullValue((PtrsShadowTy)),
3445*d415bd75Srobert "_msmaskedptrs");
3446*d415bd75Srobert insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &I);
3447*d415bd75Srobert }
3448*d415bd75Srobert
3449*d415bd75Srobert Value *Shadow = getShadow(Values);
3450*d415bd75Srobert Type *ElementShadowTy =
3451*d415bd75Srobert getShadowTy(cast<FixedVectorType>(Values->getType())->getElementType());
3452*d415bd75Srobert auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3453*d415bd75Srobert Ptrs, IRB, ElementShadowTy, Alignment, /*isStore*/ true);
3454*d415bd75Srobert
3455*d415bd75Srobert IRB.CreateMaskedScatter(Shadow, ShadowPtrs, Alignment, Mask);
3456*d415bd75Srobert
3457*d415bd75Srobert // TODO: Store origin.
3458*d415bd75Srobert }
3459*d415bd75Srobert
handleMaskedStore__anon9f4f4c900811::MemorySanitizerVisitor3460*d415bd75Srobert void handleMaskedStore(IntrinsicInst &I) {
3461*d415bd75Srobert IRBuilder<> IRB(&I);
3462*d415bd75Srobert Value *V = I.getArgOperand(0);
3463*d415bd75Srobert Value *Ptr = I.getArgOperand(1);
3464*d415bd75Srobert const Align Alignment(
3465*d415bd75Srobert cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
3466*d415bd75Srobert Value *Mask = I.getArgOperand(3);
3467*d415bd75Srobert Value *Shadow = getShadow(V);
3468*d415bd75Srobert
3469*d415bd75Srobert if (ClCheckAccessAddress) {
3470*d415bd75Srobert insertShadowCheck(Ptr, &I);
347109467b48Spatrick insertShadowCheck(Mask, &I);
347209467b48Spatrick }
347309467b48Spatrick
3474*d415bd75Srobert Value *ShadowPtr;
3475*d415bd75Srobert Value *OriginPtr;
3476*d415bd75Srobert std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3477*d415bd75Srobert Ptr, IRB, Shadow->getType(), Alignment, /*isStore*/ true);
3478*d415bd75Srobert
3479*d415bd75Srobert IRB.CreateMaskedStore(Shadow, ShadowPtr, Alignment, Mask);
3480*d415bd75Srobert
3481*d415bd75Srobert if (!MS.TrackOrigins)
3482*d415bd75Srobert return;
3483*d415bd75Srobert
3484*d415bd75Srobert auto &DL = F.getParent()->getDataLayout();
3485*d415bd75Srobert paintOrigin(IRB, getOrigin(V), OriginPtr,
3486*d415bd75Srobert DL.getTypeStoreSize(Shadow->getType()),
3487*d415bd75Srobert std::max(Alignment, kMinOriginAlignment));
3488*d415bd75Srobert }
3489*d415bd75Srobert
handleMaskedLoad__anon9f4f4c900811::MemorySanitizerVisitor3490*d415bd75Srobert void handleMaskedLoad(IntrinsicInst &I) {
3491*d415bd75Srobert IRBuilder<> IRB(&I);
3492*d415bd75Srobert Value *Ptr = I.getArgOperand(0);
3493*d415bd75Srobert const Align Alignment(
3494*d415bd75Srobert cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
3495*d415bd75Srobert Value *Mask = I.getArgOperand(2);
3496*d415bd75Srobert Value *PassThru = I.getArgOperand(3);
3497*d415bd75Srobert
3498*d415bd75Srobert if (ClCheckAccessAddress) {
3499*d415bd75Srobert insertShadowCheck(Ptr, &I);
3500*d415bd75Srobert insertShadowCheck(Mask, &I);
3501*d415bd75Srobert }
3502*d415bd75Srobert
3503*d415bd75Srobert if (!PropagateShadow) {
3504*d415bd75Srobert setShadow(&I, getCleanShadow(&I));
3505*d415bd75Srobert setOrigin(&I, getCleanOrigin());
3506*d415bd75Srobert return;
3507*d415bd75Srobert }
3508*d415bd75Srobert
3509*d415bd75Srobert Type *ShadowTy = getShadowTy(&I);
3510*d415bd75Srobert Value *ShadowPtr, *OriginPtr;
3511*d415bd75Srobert std::tie(ShadowPtr, OriginPtr) =
3512*d415bd75Srobert getShadowOriginPtr(Ptr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3513*d415bd75Srobert setShadow(&I, IRB.CreateMaskedLoad(ShadowTy, ShadowPtr, Alignment, Mask,
3514*d415bd75Srobert getShadow(PassThru), "_msmaskedld"));
3515*d415bd75Srobert
3516*d415bd75Srobert if (!MS.TrackOrigins)
3517*d415bd75Srobert return;
3518*d415bd75Srobert
351909467b48Spatrick // Choose between PassThru's and the loaded value's origins.
352009467b48Spatrick Value *MaskedPassThruShadow = IRB.CreateAnd(
352109467b48Spatrick getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
352209467b48Spatrick
3523*d415bd75Srobert Value *ConvertedShadow = convertShadowToScalar(MaskedPassThruShadow, IRB);
3524*d415bd75Srobert Value *NotNull = convertToBool(ConvertedShadow, IRB, "_mscmp");
352509467b48Spatrick
3526*d415bd75Srobert Value *PtrOrigin = IRB.CreateLoad(MS.OriginTy, OriginPtr);
3527*d415bd75Srobert Value *Origin = IRB.CreateSelect(NotNull, getOrigin(PassThru), PtrOrigin);
352809467b48Spatrick
352909467b48Spatrick setOrigin(&I, Origin);
353009467b48Spatrick }
353109467b48Spatrick
353209467b48Spatrick // Instrument BMI / BMI2 intrinsics.
353309467b48Spatrick // All of these intrinsics are Z = I(X, Y)
3534*d415bd75Srobert // where the types of all operands and the result match, and are either i32 or
3535*d415bd75Srobert // i64. The following instrumentation happens to work for all of them:
353609467b48Spatrick // Sz = I(Sx, Y) | (sext (Sy != 0))
handleBmiIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor353709467b48Spatrick void handleBmiIntrinsic(IntrinsicInst &I) {
353809467b48Spatrick IRBuilder<> IRB(&I);
353909467b48Spatrick Type *ShadowTy = getShadowTy(&I);
354009467b48Spatrick
354109467b48Spatrick // If any bit of the mask operand is poisoned, then the whole thing is.
354209467b48Spatrick Value *SMask = getShadow(&I, 1);
354309467b48Spatrick SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)),
354409467b48Spatrick ShadowTy);
354509467b48Spatrick // Apply the same intrinsic to the shadow of the first operand.
354609467b48Spatrick Value *S = IRB.CreateCall(I.getCalledFunction(),
354709467b48Spatrick {getShadow(&I, 0), I.getOperand(1)});
354809467b48Spatrick S = IRB.CreateOr(SMask, S);
354909467b48Spatrick setShadow(&I, S);
355009467b48Spatrick setOriginForNaryOp(I);
355109467b48Spatrick }
355209467b48Spatrick
getPclmulMask__anon9f4f4c900811::MemorySanitizerVisitor3553097a140dSpatrick SmallVector<int, 8> getPclmulMask(unsigned Width, bool OddElements) {
3554097a140dSpatrick SmallVector<int, 8> Mask;
355509467b48Spatrick for (unsigned X = OddElements ? 1 : 0; X < Width; X += 2) {
3556097a140dSpatrick Mask.append(2, X);
355709467b48Spatrick }
3558097a140dSpatrick return Mask;
355909467b48Spatrick }
356009467b48Spatrick
356109467b48Spatrick // Instrument pclmul intrinsics.
356209467b48Spatrick // These intrinsics operate either on odd or on even elements of the input
356309467b48Spatrick // vectors, depending on the constant in the 3rd argument, ignoring the rest.
356409467b48Spatrick // Replace the unused elements with copies of the used ones, ex:
356509467b48Spatrick // (0, 1, 2, 3) -> (0, 0, 2, 2) (even case)
356609467b48Spatrick // or
356709467b48Spatrick // (0, 1, 2, 3) -> (1, 1, 3, 3) (odd case)
356809467b48Spatrick // and then apply the usual shadow combining logic.
handlePclmulIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor356909467b48Spatrick void handlePclmulIntrinsic(IntrinsicInst &I) {
357009467b48Spatrick IRBuilder<> IRB(&I);
3571097a140dSpatrick unsigned Width =
3572097a140dSpatrick cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
357309467b48Spatrick assert(isa<ConstantInt>(I.getArgOperand(2)) &&
357409467b48Spatrick "pclmul 3rd operand must be a constant");
357509467b48Spatrick unsigned Imm = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
357673471bf0Spatrick Value *Shuf0 = IRB.CreateShuffleVector(getShadow(&I, 0),
3577097a140dSpatrick getPclmulMask(Width, Imm & 0x01));
357873471bf0Spatrick Value *Shuf1 = IRB.CreateShuffleVector(getShadow(&I, 1),
3579097a140dSpatrick getPclmulMask(Width, Imm & 0x10));
358009467b48Spatrick ShadowAndOriginCombiner SOC(this, IRB);
358109467b48Spatrick SOC.Add(Shuf0, getOrigin(&I, 0));
358209467b48Spatrick SOC.Add(Shuf1, getOrigin(&I, 1));
358309467b48Spatrick SOC.Done(&I);
358409467b48Spatrick }
358509467b48Spatrick
3586*d415bd75Srobert // Instrument _mm_*_sd|ss intrinsics
handleUnarySdSsIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor3587*d415bd75Srobert void handleUnarySdSsIntrinsic(IntrinsicInst &I) {
3588097a140dSpatrick IRBuilder<> IRB(&I);
3589*d415bd75Srobert unsigned Width =
3590*d415bd75Srobert cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
3591097a140dSpatrick Value *First = getShadow(&I, 0);
3592097a140dSpatrick Value *Second = getShadow(&I, 1);
3593*d415bd75Srobert // First element of second operand, remaining elements of first operand
3594*d415bd75Srobert SmallVector<int, 16> Mask;
3595*d415bd75Srobert Mask.push_back(Width);
3596*d415bd75Srobert for (unsigned i = 1; i < Width; i++)
3597*d415bd75Srobert Mask.push_back(i);
3598*d415bd75Srobert Value *Shadow = IRB.CreateShuffleVector(First, Second, Mask);
3599097a140dSpatrick
3600097a140dSpatrick setShadow(&I, Shadow);
3601097a140dSpatrick setOriginForNaryOp(I);
3602097a140dSpatrick }
3603097a140dSpatrick
handleVtestIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor3604*d415bd75Srobert void handleVtestIntrinsic(IntrinsicInst &I) {
3605097a140dSpatrick IRBuilder<> IRB(&I);
3606*d415bd75Srobert Value *Shadow0 = getShadow(&I, 0);
3607*d415bd75Srobert Value *Shadow1 = getShadow(&I, 1);
3608*d415bd75Srobert Value *Or = IRB.CreateOr(Shadow0, Shadow1);
3609*d415bd75Srobert Value *NZ = IRB.CreateICmpNE(Or, Constant::getNullValue(Or->getType()));
3610*d415bd75Srobert Value *Scalar = convertShadowToScalar(NZ, IRB);
3611*d415bd75Srobert Value *Shadow = IRB.CreateZExt(Scalar, getShadowTy(&I));
3612*d415bd75Srobert
3613*d415bd75Srobert setShadow(&I, Shadow);
3614*d415bd75Srobert setOriginForNaryOp(I);
3615*d415bd75Srobert }
3616*d415bd75Srobert
handleBinarySdSsIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor3617*d415bd75Srobert void handleBinarySdSsIntrinsic(IntrinsicInst &I) {
3618*d415bd75Srobert IRBuilder<> IRB(&I);
3619*d415bd75Srobert unsigned Width =
3620*d415bd75Srobert cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
3621097a140dSpatrick Value *First = getShadow(&I, 0);
3622097a140dSpatrick Value *Second = getShadow(&I, 1);
3623097a140dSpatrick Value *OrShadow = IRB.CreateOr(First, Second);
3624*d415bd75Srobert // First element of both OR'd together, remaining elements of first operand
3625*d415bd75Srobert SmallVector<int, 16> Mask;
3626*d415bd75Srobert Mask.push_back(Width);
3627*d415bd75Srobert for (unsigned i = 1; i < Width; i++)
3628*d415bd75Srobert Mask.push_back(i);
3629*d415bd75Srobert Value *Shadow = IRB.CreateShuffleVector(First, OrShadow, Mask);
3630097a140dSpatrick
3631097a140dSpatrick setShadow(&I, Shadow);
3632097a140dSpatrick setOriginForNaryOp(I);
3633097a140dSpatrick }
3634097a140dSpatrick
363573471bf0Spatrick // Instrument abs intrinsic.
363673471bf0Spatrick // handleUnknownIntrinsic can't handle it because of the last
363773471bf0Spatrick // is_int_min_poison argument which does not match the result type.
handleAbsIntrinsic__anon9f4f4c900811::MemorySanitizerVisitor363873471bf0Spatrick void handleAbsIntrinsic(IntrinsicInst &I) {
363973471bf0Spatrick assert(I.getType()->isIntOrIntVectorTy());
364073471bf0Spatrick assert(I.getArgOperand(0)->getType() == I.getType());
364173471bf0Spatrick
364273471bf0Spatrick // FIXME: Handle is_int_min_poison.
364373471bf0Spatrick IRBuilder<> IRB(&I);
364473471bf0Spatrick setShadow(&I, getShadow(&I, 0));
364573471bf0Spatrick setOrigin(&I, getOrigin(&I, 0));
364673471bf0Spatrick }
364773471bf0Spatrick
visitIntrinsicInst__anon9f4f4c900811::MemorySanitizerVisitor364809467b48Spatrick void visitIntrinsicInst(IntrinsicInst &I) {
364909467b48Spatrick switch (I.getIntrinsicID()) {
365073471bf0Spatrick case Intrinsic::abs:
365173471bf0Spatrick handleAbsIntrinsic(I);
365273471bf0Spatrick break;
365309467b48Spatrick case Intrinsic::lifetime_start:
365409467b48Spatrick handleLifetimeStart(I);
365509467b48Spatrick break;
365609467b48Spatrick case Intrinsic::launder_invariant_group:
365709467b48Spatrick case Intrinsic::strip_invariant_group:
365809467b48Spatrick handleInvariantGroup(I);
365909467b48Spatrick break;
366009467b48Spatrick case Intrinsic::bswap:
366109467b48Spatrick handleBswap(I);
366209467b48Spatrick break;
3663*d415bd75Srobert case Intrinsic::ctlz:
3664*d415bd75Srobert case Intrinsic::cttz:
3665*d415bd75Srobert handleCountZeroes(I);
3666*d415bd75Srobert break;
3667*d415bd75Srobert case Intrinsic::masked_compressstore:
3668*d415bd75Srobert handleMaskedCompressStore(I);
3669*d415bd75Srobert break;
3670*d415bd75Srobert case Intrinsic::masked_expandload:
3671*d415bd75Srobert handleMaskedExpandLoad(I);
3672*d415bd75Srobert break;
3673*d415bd75Srobert case Intrinsic::masked_gather:
3674*d415bd75Srobert handleMaskedGather(I);
3675*d415bd75Srobert break;
3676*d415bd75Srobert case Intrinsic::masked_scatter:
3677*d415bd75Srobert handleMaskedScatter(I);
3678*d415bd75Srobert break;
367909467b48Spatrick case Intrinsic::masked_store:
368009467b48Spatrick handleMaskedStore(I);
368109467b48Spatrick break;
368209467b48Spatrick case Intrinsic::masked_load:
368309467b48Spatrick handleMaskedLoad(I);
368409467b48Spatrick break;
368573471bf0Spatrick case Intrinsic::vector_reduce_and:
3686097a140dSpatrick handleVectorReduceAndIntrinsic(I);
3687097a140dSpatrick break;
368873471bf0Spatrick case Intrinsic::vector_reduce_or:
3689097a140dSpatrick handleVectorReduceOrIntrinsic(I);
3690097a140dSpatrick break;
369173471bf0Spatrick case Intrinsic::vector_reduce_add:
369273471bf0Spatrick case Intrinsic::vector_reduce_xor:
369373471bf0Spatrick case Intrinsic::vector_reduce_mul:
3694097a140dSpatrick handleVectorReduceIntrinsic(I);
3695097a140dSpatrick break;
369609467b48Spatrick case Intrinsic::x86_sse_stmxcsr:
369709467b48Spatrick handleStmxcsr(I);
369809467b48Spatrick break;
369909467b48Spatrick case Intrinsic::x86_sse_ldmxcsr:
370009467b48Spatrick handleLdmxcsr(I);
370109467b48Spatrick break;
370209467b48Spatrick case Intrinsic::x86_avx512_vcvtsd2usi64:
370309467b48Spatrick case Intrinsic::x86_avx512_vcvtsd2usi32:
370409467b48Spatrick case Intrinsic::x86_avx512_vcvtss2usi64:
370509467b48Spatrick case Intrinsic::x86_avx512_vcvtss2usi32:
370609467b48Spatrick case Intrinsic::x86_avx512_cvttss2usi64:
370709467b48Spatrick case Intrinsic::x86_avx512_cvttss2usi:
370809467b48Spatrick case Intrinsic::x86_avx512_cvttsd2usi64:
370909467b48Spatrick case Intrinsic::x86_avx512_cvttsd2usi:
371009467b48Spatrick case Intrinsic::x86_avx512_cvtusi2ss:
371109467b48Spatrick case Intrinsic::x86_avx512_cvtusi642sd:
371209467b48Spatrick case Intrinsic::x86_avx512_cvtusi642ss:
371373471bf0Spatrick handleVectorConvertIntrinsic(I, 1, true);
371473471bf0Spatrick break;
371509467b48Spatrick case Intrinsic::x86_sse2_cvtsd2si64:
371609467b48Spatrick case Intrinsic::x86_sse2_cvtsd2si:
371709467b48Spatrick case Intrinsic::x86_sse2_cvtsd2ss:
371809467b48Spatrick case Intrinsic::x86_sse2_cvttsd2si64:
371909467b48Spatrick case Intrinsic::x86_sse2_cvttsd2si:
372009467b48Spatrick case Intrinsic::x86_sse_cvtss2si64:
372109467b48Spatrick case Intrinsic::x86_sse_cvtss2si:
372209467b48Spatrick case Intrinsic::x86_sse_cvttss2si64:
372309467b48Spatrick case Intrinsic::x86_sse_cvttss2si:
372409467b48Spatrick handleVectorConvertIntrinsic(I, 1);
372509467b48Spatrick break;
372609467b48Spatrick case Intrinsic::x86_sse_cvtps2pi:
372709467b48Spatrick case Intrinsic::x86_sse_cvttps2pi:
372809467b48Spatrick handleVectorConvertIntrinsic(I, 2);
372909467b48Spatrick break;
373009467b48Spatrick
373109467b48Spatrick case Intrinsic::x86_avx512_psll_w_512:
373209467b48Spatrick case Intrinsic::x86_avx512_psll_d_512:
373309467b48Spatrick case Intrinsic::x86_avx512_psll_q_512:
373409467b48Spatrick case Intrinsic::x86_avx512_pslli_w_512:
373509467b48Spatrick case Intrinsic::x86_avx512_pslli_d_512:
373609467b48Spatrick case Intrinsic::x86_avx512_pslli_q_512:
373709467b48Spatrick case Intrinsic::x86_avx512_psrl_w_512:
373809467b48Spatrick case Intrinsic::x86_avx512_psrl_d_512:
373909467b48Spatrick case Intrinsic::x86_avx512_psrl_q_512:
374009467b48Spatrick case Intrinsic::x86_avx512_psra_w_512:
374109467b48Spatrick case Intrinsic::x86_avx512_psra_d_512:
374209467b48Spatrick case Intrinsic::x86_avx512_psra_q_512:
374309467b48Spatrick case Intrinsic::x86_avx512_psrli_w_512:
374409467b48Spatrick case Intrinsic::x86_avx512_psrli_d_512:
374509467b48Spatrick case Intrinsic::x86_avx512_psrli_q_512:
374609467b48Spatrick case Intrinsic::x86_avx512_psrai_w_512:
374709467b48Spatrick case Intrinsic::x86_avx512_psrai_d_512:
374809467b48Spatrick case Intrinsic::x86_avx512_psrai_q_512:
374909467b48Spatrick case Intrinsic::x86_avx512_psra_q_256:
375009467b48Spatrick case Intrinsic::x86_avx512_psra_q_128:
375109467b48Spatrick case Intrinsic::x86_avx512_psrai_q_256:
375209467b48Spatrick case Intrinsic::x86_avx512_psrai_q_128:
375309467b48Spatrick case Intrinsic::x86_avx2_psll_w:
375409467b48Spatrick case Intrinsic::x86_avx2_psll_d:
375509467b48Spatrick case Intrinsic::x86_avx2_psll_q:
375609467b48Spatrick case Intrinsic::x86_avx2_pslli_w:
375709467b48Spatrick case Intrinsic::x86_avx2_pslli_d:
375809467b48Spatrick case Intrinsic::x86_avx2_pslli_q:
375909467b48Spatrick case Intrinsic::x86_avx2_psrl_w:
376009467b48Spatrick case Intrinsic::x86_avx2_psrl_d:
376109467b48Spatrick case Intrinsic::x86_avx2_psrl_q:
376209467b48Spatrick case Intrinsic::x86_avx2_psra_w:
376309467b48Spatrick case Intrinsic::x86_avx2_psra_d:
376409467b48Spatrick case Intrinsic::x86_avx2_psrli_w:
376509467b48Spatrick case Intrinsic::x86_avx2_psrli_d:
376609467b48Spatrick case Intrinsic::x86_avx2_psrli_q:
376709467b48Spatrick case Intrinsic::x86_avx2_psrai_w:
376809467b48Spatrick case Intrinsic::x86_avx2_psrai_d:
376909467b48Spatrick case Intrinsic::x86_sse2_psll_w:
377009467b48Spatrick case Intrinsic::x86_sse2_psll_d:
377109467b48Spatrick case Intrinsic::x86_sse2_psll_q:
377209467b48Spatrick case Intrinsic::x86_sse2_pslli_w:
377309467b48Spatrick case Intrinsic::x86_sse2_pslli_d:
377409467b48Spatrick case Intrinsic::x86_sse2_pslli_q:
377509467b48Spatrick case Intrinsic::x86_sse2_psrl_w:
377609467b48Spatrick case Intrinsic::x86_sse2_psrl_d:
377709467b48Spatrick case Intrinsic::x86_sse2_psrl_q:
377809467b48Spatrick case Intrinsic::x86_sse2_psra_w:
377909467b48Spatrick case Intrinsic::x86_sse2_psra_d:
378009467b48Spatrick case Intrinsic::x86_sse2_psrli_w:
378109467b48Spatrick case Intrinsic::x86_sse2_psrli_d:
378209467b48Spatrick case Intrinsic::x86_sse2_psrli_q:
378309467b48Spatrick case Intrinsic::x86_sse2_psrai_w:
378409467b48Spatrick case Intrinsic::x86_sse2_psrai_d:
378509467b48Spatrick case Intrinsic::x86_mmx_psll_w:
378609467b48Spatrick case Intrinsic::x86_mmx_psll_d:
378709467b48Spatrick case Intrinsic::x86_mmx_psll_q:
378809467b48Spatrick case Intrinsic::x86_mmx_pslli_w:
378909467b48Spatrick case Intrinsic::x86_mmx_pslli_d:
379009467b48Spatrick case Intrinsic::x86_mmx_pslli_q:
379109467b48Spatrick case Intrinsic::x86_mmx_psrl_w:
379209467b48Spatrick case Intrinsic::x86_mmx_psrl_d:
379309467b48Spatrick case Intrinsic::x86_mmx_psrl_q:
379409467b48Spatrick case Intrinsic::x86_mmx_psra_w:
379509467b48Spatrick case Intrinsic::x86_mmx_psra_d:
379609467b48Spatrick case Intrinsic::x86_mmx_psrli_w:
379709467b48Spatrick case Intrinsic::x86_mmx_psrli_d:
379809467b48Spatrick case Intrinsic::x86_mmx_psrli_q:
379909467b48Spatrick case Intrinsic::x86_mmx_psrai_w:
380009467b48Spatrick case Intrinsic::x86_mmx_psrai_d:
380109467b48Spatrick handleVectorShiftIntrinsic(I, /* Variable */ false);
380209467b48Spatrick break;
380309467b48Spatrick case Intrinsic::x86_avx2_psllv_d:
380409467b48Spatrick case Intrinsic::x86_avx2_psllv_d_256:
380509467b48Spatrick case Intrinsic::x86_avx512_psllv_d_512:
380609467b48Spatrick case Intrinsic::x86_avx2_psllv_q:
380709467b48Spatrick case Intrinsic::x86_avx2_psllv_q_256:
380809467b48Spatrick case Intrinsic::x86_avx512_psllv_q_512:
380909467b48Spatrick case Intrinsic::x86_avx2_psrlv_d:
381009467b48Spatrick case Intrinsic::x86_avx2_psrlv_d_256:
381109467b48Spatrick case Intrinsic::x86_avx512_psrlv_d_512:
381209467b48Spatrick case Intrinsic::x86_avx2_psrlv_q:
381309467b48Spatrick case Intrinsic::x86_avx2_psrlv_q_256:
381409467b48Spatrick case Intrinsic::x86_avx512_psrlv_q_512:
381509467b48Spatrick case Intrinsic::x86_avx2_psrav_d:
381609467b48Spatrick case Intrinsic::x86_avx2_psrav_d_256:
381709467b48Spatrick case Intrinsic::x86_avx512_psrav_d_512:
381809467b48Spatrick case Intrinsic::x86_avx512_psrav_q_128:
381909467b48Spatrick case Intrinsic::x86_avx512_psrav_q_256:
382009467b48Spatrick case Intrinsic::x86_avx512_psrav_q_512:
382109467b48Spatrick handleVectorShiftIntrinsic(I, /* Variable */ true);
382209467b48Spatrick break;
382309467b48Spatrick
382409467b48Spatrick case Intrinsic::x86_sse2_packsswb_128:
382509467b48Spatrick case Intrinsic::x86_sse2_packssdw_128:
382609467b48Spatrick case Intrinsic::x86_sse2_packuswb_128:
382709467b48Spatrick case Intrinsic::x86_sse41_packusdw:
382809467b48Spatrick case Intrinsic::x86_avx2_packsswb:
382909467b48Spatrick case Intrinsic::x86_avx2_packssdw:
383009467b48Spatrick case Intrinsic::x86_avx2_packuswb:
383109467b48Spatrick case Intrinsic::x86_avx2_packusdw:
383209467b48Spatrick handleVectorPackIntrinsic(I);
383309467b48Spatrick break;
383409467b48Spatrick
383509467b48Spatrick case Intrinsic::x86_mmx_packsswb:
383609467b48Spatrick case Intrinsic::x86_mmx_packuswb:
383709467b48Spatrick handleVectorPackIntrinsic(I, 16);
383809467b48Spatrick break;
383909467b48Spatrick
384009467b48Spatrick case Intrinsic::x86_mmx_packssdw:
384109467b48Spatrick handleVectorPackIntrinsic(I, 32);
384209467b48Spatrick break;
384309467b48Spatrick
384409467b48Spatrick case Intrinsic::x86_mmx_psad_bw:
384509467b48Spatrick case Intrinsic::x86_sse2_psad_bw:
384609467b48Spatrick case Intrinsic::x86_avx2_psad_bw:
384709467b48Spatrick handleVectorSadIntrinsic(I);
384809467b48Spatrick break;
384909467b48Spatrick
385009467b48Spatrick case Intrinsic::x86_sse2_pmadd_wd:
385109467b48Spatrick case Intrinsic::x86_avx2_pmadd_wd:
385209467b48Spatrick case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
385309467b48Spatrick case Intrinsic::x86_avx2_pmadd_ub_sw:
385409467b48Spatrick handleVectorPmaddIntrinsic(I);
385509467b48Spatrick break;
385609467b48Spatrick
385709467b48Spatrick case Intrinsic::x86_ssse3_pmadd_ub_sw:
385809467b48Spatrick handleVectorPmaddIntrinsic(I, 8);
385909467b48Spatrick break;
386009467b48Spatrick
386109467b48Spatrick case Intrinsic::x86_mmx_pmadd_wd:
386209467b48Spatrick handleVectorPmaddIntrinsic(I, 16);
386309467b48Spatrick break;
386409467b48Spatrick
386509467b48Spatrick case Intrinsic::x86_sse_cmp_ss:
386609467b48Spatrick case Intrinsic::x86_sse2_cmp_sd:
386709467b48Spatrick case Intrinsic::x86_sse_comieq_ss:
386809467b48Spatrick case Intrinsic::x86_sse_comilt_ss:
386909467b48Spatrick case Intrinsic::x86_sse_comile_ss:
387009467b48Spatrick case Intrinsic::x86_sse_comigt_ss:
387109467b48Spatrick case Intrinsic::x86_sse_comige_ss:
387209467b48Spatrick case Intrinsic::x86_sse_comineq_ss:
387309467b48Spatrick case Intrinsic::x86_sse_ucomieq_ss:
387409467b48Spatrick case Intrinsic::x86_sse_ucomilt_ss:
387509467b48Spatrick case Intrinsic::x86_sse_ucomile_ss:
387609467b48Spatrick case Intrinsic::x86_sse_ucomigt_ss:
387709467b48Spatrick case Intrinsic::x86_sse_ucomige_ss:
387809467b48Spatrick case Intrinsic::x86_sse_ucomineq_ss:
387909467b48Spatrick case Intrinsic::x86_sse2_comieq_sd:
388009467b48Spatrick case Intrinsic::x86_sse2_comilt_sd:
388109467b48Spatrick case Intrinsic::x86_sse2_comile_sd:
388209467b48Spatrick case Intrinsic::x86_sse2_comigt_sd:
388309467b48Spatrick case Intrinsic::x86_sse2_comige_sd:
388409467b48Spatrick case Intrinsic::x86_sse2_comineq_sd:
388509467b48Spatrick case Intrinsic::x86_sse2_ucomieq_sd:
388609467b48Spatrick case Intrinsic::x86_sse2_ucomilt_sd:
388709467b48Spatrick case Intrinsic::x86_sse2_ucomile_sd:
388809467b48Spatrick case Intrinsic::x86_sse2_ucomigt_sd:
388909467b48Spatrick case Intrinsic::x86_sse2_ucomige_sd:
389009467b48Spatrick case Intrinsic::x86_sse2_ucomineq_sd:
389109467b48Spatrick handleVectorCompareScalarIntrinsic(I);
389209467b48Spatrick break;
389309467b48Spatrick
3894*d415bd75Srobert case Intrinsic::x86_avx_cmp_pd_256:
3895*d415bd75Srobert case Intrinsic::x86_avx_cmp_ps_256:
389609467b48Spatrick case Intrinsic::x86_sse2_cmp_pd:
3897*d415bd75Srobert case Intrinsic::x86_sse_cmp_ps:
389809467b48Spatrick handleVectorComparePackedIntrinsic(I);
389909467b48Spatrick break;
390009467b48Spatrick
390109467b48Spatrick case Intrinsic::x86_bmi_bextr_32:
390209467b48Spatrick case Intrinsic::x86_bmi_bextr_64:
390309467b48Spatrick case Intrinsic::x86_bmi_bzhi_32:
390409467b48Spatrick case Intrinsic::x86_bmi_bzhi_64:
390509467b48Spatrick case Intrinsic::x86_bmi_pdep_32:
390609467b48Spatrick case Intrinsic::x86_bmi_pdep_64:
390709467b48Spatrick case Intrinsic::x86_bmi_pext_32:
390809467b48Spatrick case Intrinsic::x86_bmi_pext_64:
390909467b48Spatrick handleBmiIntrinsic(I);
391009467b48Spatrick break;
391109467b48Spatrick
391209467b48Spatrick case Intrinsic::x86_pclmulqdq:
391309467b48Spatrick case Intrinsic::x86_pclmulqdq_256:
391409467b48Spatrick case Intrinsic::x86_pclmulqdq_512:
391509467b48Spatrick handlePclmulIntrinsic(I);
391609467b48Spatrick break;
391709467b48Spatrick
3918097a140dSpatrick case Intrinsic::x86_sse41_round_sd:
3919*d415bd75Srobert case Intrinsic::x86_sse41_round_ss:
3920*d415bd75Srobert handleUnarySdSsIntrinsic(I);
3921097a140dSpatrick break;
3922097a140dSpatrick case Intrinsic::x86_sse2_max_sd:
3923*d415bd75Srobert case Intrinsic::x86_sse_max_ss:
3924097a140dSpatrick case Intrinsic::x86_sse2_min_sd:
3925*d415bd75Srobert case Intrinsic::x86_sse_min_ss:
3926*d415bd75Srobert handleBinarySdSsIntrinsic(I);
3927*d415bd75Srobert break;
3928*d415bd75Srobert
3929*d415bd75Srobert case Intrinsic::x86_avx_vtestc_pd:
3930*d415bd75Srobert case Intrinsic::x86_avx_vtestc_pd_256:
3931*d415bd75Srobert case Intrinsic::x86_avx_vtestc_ps:
3932*d415bd75Srobert case Intrinsic::x86_avx_vtestc_ps_256:
3933*d415bd75Srobert case Intrinsic::x86_avx_vtestnzc_pd:
3934*d415bd75Srobert case Intrinsic::x86_avx_vtestnzc_pd_256:
3935*d415bd75Srobert case Intrinsic::x86_avx_vtestnzc_ps:
3936*d415bd75Srobert case Intrinsic::x86_avx_vtestnzc_ps_256:
3937*d415bd75Srobert case Intrinsic::x86_avx_vtestz_pd:
3938*d415bd75Srobert case Intrinsic::x86_avx_vtestz_pd_256:
3939*d415bd75Srobert case Intrinsic::x86_avx_vtestz_ps:
3940*d415bd75Srobert case Intrinsic::x86_avx_vtestz_ps_256:
3941*d415bd75Srobert case Intrinsic::x86_avx_ptestc_256:
3942*d415bd75Srobert case Intrinsic::x86_avx_ptestnzc_256:
3943*d415bd75Srobert case Intrinsic::x86_avx_ptestz_256:
3944*d415bd75Srobert case Intrinsic::x86_sse41_ptestc:
3945*d415bd75Srobert case Intrinsic::x86_sse41_ptestnzc:
3946*d415bd75Srobert case Intrinsic::x86_sse41_ptestz:
3947*d415bd75Srobert handleVtestIntrinsic(I);
3948097a140dSpatrick break;
3949097a140dSpatrick
395073471bf0Spatrick case Intrinsic::fshl:
395173471bf0Spatrick case Intrinsic::fshr:
395273471bf0Spatrick handleFunnelShift(I);
395373471bf0Spatrick break;
395473471bf0Spatrick
395509467b48Spatrick case Intrinsic::is_constant:
395609467b48Spatrick // The result of llvm.is.constant() is always defined.
395709467b48Spatrick setShadow(&I, getCleanShadow(&I));
395809467b48Spatrick setOrigin(&I, getCleanOrigin());
395909467b48Spatrick break;
396009467b48Spatrick
396109467b48Spatrick default:
396209467b48Spatrick if (!handleUnknownIntrinsic(I))
396309467b48Spatrick visitInstruction(I);
396409467b48Spatrick break;
396509467b48Spatrick }
396609467b48Spatrick }
396709467b48Spatrick
visitLibAtomicLoad__anon9f4f4c900811::MemorySanitizerVisitor396873471bf0Spatrick void visitLibAtomicLoad(CallBase &CB) {
396973471bf0Spatrick // Since we use getNextNode here, we can't have CB terminate the BB.
397073471bf0Spatrick assert(isa<CallInst>(CB));
397173471bf0Spatrick
397273471bf0Spatrick IRBuilder<> IRB(&CB);
397373471bf0Spatrick Value *Size = CB.getArgOperand(0);
397473471bf0Spatrick Value *SrcPtr = CB.getArgOperand(1);
397573471bf0Spatrick Value *DstPtr = CB.getArgOperand(2);
397673471bf0Spatrick Value *Ordering = CB.getArgOperand(3);
397773471bf0Spatrick // Convert the call to have at least Acquire ordering to make sure
397873471bf0Spatrick // the shadow operations aren't reordered before it.
397973471bf0Spatrick Value *NewOrdering =
398073471bf0Spatrick IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
398173471bf0Spatrick CB.setArgOperand(3, NewOrdering);
398273471bf0Spatrick
3983*d415bd75Srobert NextNodeIRBuilder NextIRB(&CB);
398473471bf0Spatrick Value *SrcShadowPtr, *SrcOriginPtr;
398573471bf0Spatrick std::tie(SrcShadowPtr, SrcOriginPtr) =
398673471bf0Spatrick getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
398773471bf0Spatrick /*isStore*/ false);
398873471bf0Spatrick Value *DstShadowPtr =
398973471bf0Spatrick getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
399073471bf0Spatrick /*isStore*/ true)
399173471bf0Spatrick .first;
399273471bf0Spatrick
399373471bf0Spatrick NextIRB.CreateMemCpy(DstShadowPtr, Align(1), SrcShadowPtr, Align(1), Size);
399473471bf0Spatrick if (MS.TrackOrigins) {
399573471bf0Spatrick Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
399673471bf0Spatrick kMinOriginAlignment);
399773471bf0Spatrick Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
399873471bf0Spatrick NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
399973471bf0Spatrick }
400073471bf0Spatrick }
400173471bf0Spatrick
visitLibAtomicStore__anon9f4f4c900811::MemorySanitizerVisitor400273471bf0Spatrick void visitLibAtomicStore(CallBase &CB) {
400373471bf0Spatrick IRBuilder<> IRB(&CB);
400473471bf0Spatrick Value *Size = CB.getArgOperand(0);
400573471bf0Spatrick Value *DstPtr = CB.getArgOperand(2);
400673471bf0Spatrick Value *Ordering = CB.getArgOperand(3);
400773471bf0Spatrick // Convert the call to have at least Release ordering to make sure
400873471bf0Spatrick // the shadow operations aren't reordered after it.
400973471bf0Spatrick Value *NewOrdering =
401073471bf0Spatrick IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
401173471bf0Spatrick CB.setArgOperand(3, NewOrdering);
401273471bf0Spatrick
401373471bf0Spatrick Value *DstShadowPtr =
401473471bf0Spatrick getShadowOriginPtr(DstPtr, IRB, IRB.getInt8Ty(), Align(1),
401573471bf0Spatrick /*isStore*/ true)
401673471bf0Spatrick .first;
401773471bf0Spatrick
401873471bf0Spatrick // Atomic store always paints clean shadow/origin. See file header.
401973471bf0Spatrick IRB.CreateMemSet(DstShadowPtr, getCleanShadow(IRB.getInt8Ty()), Size,
402073471bf0Spatrick Align(1));
402173471bf0Spatrick }
402273471bf0Spatrick
visitCallBase__anon9f4f4c900811::MemorySanitizerVisitor4023097a140dSpatrick void visitCallBase(CallBase &CB) {
4024*d415bd75Srobert assert(!CB.getMetadata(LLVMContext::MD_nosanitize));
4025097a140dSpatrick if (CB.isInlineAsm()) {
402609467b48Spatrick // For inline asm (either a call to asm function, or callbr instruction),
402709467b48Spatrick // do the usual thing: check argument shadow and mark all outputs as
402809467b48Spatrick // clean. Note that any side effects of the inline asm that are not
402909467b48Spatrick // immediately visible in its constraints are not handled.
403009467b48Spatrick if (ClHandleAsmConservative && MS.CompileKernel)
4031097a140dSpatrick visitAsmInstruction(CB);
403209467b48Spatrick else
4033097a140dSpatrick visitInstruction(CB);
403409467b48Spatrick return;
403509467b48Spatrick }
403673471bf0Spatrick LibFunc LF;
403773471bf0Spatrick if (TLI->getLibFunc(CB, LF)) {
403873471bf0Spatrick // libatomic.a functions need to have special handling because there isn't
403973471bf0Spatrick // a good way to intercept them or compile the library with
404073471bf0Spatrick // instrumentation.
404173471bf0Spatrick switch (LF) {
404273471bf0Spatrick case LibFunc_atomic_load:
404373471bf0Spatrick if (!isa<CallInst>(CB)) {
404473471bf0Spatrick llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
404573471bf0Spatrick "Ignoring!\n";
404673471bf0Spatrick break;
404773471bf0Spatrick }
404873471bf0Spatrick visitLibAtomicLoad(CB);
404973471bf0Spatrick return;
405073471bf0Spatrick case LibFunc_atomic_store:
405173471bf0Spatrick visitLibAtomicStore(CB);
405273471bf0Spatrick return;
405373471bf0Spatrick default:
405473471bf0Spatrick break;
405573471bf0Spatrick }
405673471bf0Spatrick }
405773471bf0Spatrick
4058097a140dSpatrick if (auto *Call = dyn_cast<CallInst>(&CB)) {
4059097a140dSpatrick assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere");
406009467b48Spatrick
406109467b48Spatrick // We are going to insert code that relies on the fact that the callee
406209467b48Spatrick // will become a non-readonly function after it is instrumented by us. To
406309467b48Spatrick // prevent this code from being optimized out, mark that function
406409467b48Spatrick // non-readonly in advance.
4065*d415bd75Srobert // TODO: We can likely do better than dropping memory() completely here.
4066*d415bd75Srobert AttributeMask B;
4067*d415bd75Srobert B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
406873471bf0Spatrick
4069*d415bd75Srobert Call->removeFnAttrs(B);
407073471bf0Spatrick if (Function *Func = Call->getCalledFunction()) {
4071*d415bd75Srobert Func->removeFnAttrs(B);
407209467b48Spatrick }
407309467b48Spatrick
407409467b48Spatrick maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
407509467b48Spatrick }
4076097a140dSpatrick IRBuilder<> IRB(&CB);
4077*d415bd75Srobert bool MayCheckCall = MS.EagerChecks;
407873471bf0Spatrick if (Function *Func = CB.getCalledFunction()) {
407973471bf0Spatrick // __sanitizer_unaligned_{load,store} functions may be called by users
408073471bf0Spatrick // and always expects shadows in the TLS. So don't check them.
408173471bf0Spatrick MayCheckCall &= !Func->getName().startswith("__sanitizer_unaligned_");
408273471bf0Spatrick }
408309467b48Spatrick
408409467b48Spatrick unsigned ArgOffset = 0;
4085097a140dSpatrick LLVM_DEBUG(dbgs() << " CallSite: " << CB << "\n");
4086*d415bd75Srobert for (const auto &[i, A] : llvm::enumerate(CB.args())) {
408709467b48Spatrick if (!A->getType()->isSized()) {
4088097a140dSpatrick LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << CB << "\n");
408909467b48Spatrick continue;
409009467b48Spatrick }
409109467b48Spatrick unsigned Size = 0;
409209467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
4093097a140dSpatrick
4094097a140dSpatrick bool ByVal = CB.paramHasAttr(i, Attribute::ByVal);
4095097a140dSpatrick bool NoUndef = CB.paramHasAttr(i, Attribute::NoUndef);
409673471bf0Spatrick bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
4097097a140dSpatrick
4098097a140dSpatrick if (EagerCheck) {
4099097a140dSpatrick insertShadowCheck(A, &CB);
4100*d415bd75Srobert Size = DL.getTypeAllocSize(A->getType());
4101*d415bd75Srobert } else {
4102*d415bd75Srobert Value *Store = nullptr;
4103*d415bd75Srobert // Compute the Shadow for arg even if it is ByVal, because
4104*d415bd75Srobert // in that case getShadow() will copy the actual arg shadow to
4105*d415bd75Srobert // __msan_param_tls.
4106*d415bd75Srobert Value *ArgShadow = getShadow(A);
4107*d415bd75Srobert Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
4108*d415bd75Srobert LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *A
4109*d415bd75Srobert << " Shadow: " << *ArgShadow << "\n");
4110097a140dSpatrick if (ByVal) {
4111097a140dSpatrick // ByVal requires some special handling as it's too big for a single
4112097a140dSpatrick // load
411309467b48Spatrick assert(A->getType()->isPointerTy() &&
411409467b48Spatrick "ByVal argument is not a pointer!");
4115097a140dSpatrick Size = DL.getTypeAllocSize(CB.getParamByValType(i));
4116*d415bd75Srobert if (ArgOffset + Size > kParamTLSSize)
4117*d415bd75Srobert break;
4118097a140dSpatrick const MaybeAlign ParamAlignment(CB.getParamAlign(i));
4119*d415bd75Srobert MaybeAlign Alignment = std::nullopt;
412009467b48Spatrick if (ParamAlignment)
412109467b48Spatrick Alignment = std::min(*ParamAlignment, kShadowTLSAlignment);
4122*d415bd75Srobert Value *AShadowPtr, *AOriginPtr;
4123*d415bd75Srobert std::tie(AShadowPtr, AOriginPtr) =
412409467b48Spatrick getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment,
4125*d415bd75Srobert /*isStore*/ false);
4126*d415bd75Srobert if (!PropagateShadow) {
4127*d415bd75Srobert Store = IRB.CreateMemSet(ArgShadowBase,
4128*d415bd75Srobert Constant::getNullValue(IRB.getInt8Ty()),
4129*d415bd75Srobert Size, Alignment);
4130*d415bd75Srobert } else {
413109467b48Spatrick Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
413209467b48Spatrick Alignment, Size);
4133*d415bd75Srobert if (MS.TrackOrigins) {
4134*d415bd75Srobert Value *ArgOriginBase = getOriginPtrForArgument(A, IRB, ArgOffset);
4135*d415bd75Srobert // FIXME: OriginSize should be:
4136*d415bd75Srobert // alignTo(A % kMinOriginAlignment + Size, kMinOriginAlignment)
4137*d415bd75Srobert unsigned OriginSize = alignTo(Size, kMinOriginAlignment);
4138*d415bd75Srobert IRB.CreateMemCpy(
4139*d415bd75Srobert ArgOriginBase,
4140*d415bd75Srobert /* by origin_tls[ArgOffset] */ kMinOriginAlignment,
4141*d415bd75Srobert AOriginPtr,
4142*d415bd75Srobert /* by getShadowOriginPtr */ kMinOriginAlignment, OriginSize);
4143*d415bd75Srobert }
4144*d415bd75Srobert }
414509467b48Spatrick } else {
4146*d415bd75Srobert // Any other parameters mean we need bit-grained tracking of uninit
4147*d415bd75Srobert // data
414809467b48Spatrick Size = DL.getTypeAllocSize(A->getType());
4149*d415bd75Srobert if (ArgOffset + Size > kParamTLSSize)
4150*d415bd75Srobert break;
415109467b48Spatrick Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
4152097a140dSpatrick kShadowTLSAlignment);
415309467b48Spatrick Constant *Cst = dyn_cast<Constant>(ArgShadow);
4154*d415bd75Srobert if (MS.TrackOrigins && !(Cst && Cst->isNullValue())) {
415509467b48Spatrick IRB.CreateStore(getOrigin(A),
415609467b48Spatrick getOriginPtrForArgument(A, IRB, ArgOffset));
4157*d415bd75Srobert }
4158*d415bd75Srobert }
415909467b48Spatrick (void)Store;
4160*d415bd75Srobert assert(Store != nullptr);
416109467b48Spatrick LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
4162*d415bd75Srobert }
4163*d415bd75Srobert assert(Size != 0);
416473471bf0Spatrick ArgOffset += alignTo(Size, kShadowTLSAlignment);
416509467b48Spatrick }
416609467b48Spatrick LLVM_DEBUG(dbgs() << " done with call args\n");
416709467b48Spatrick
4168097a140dSpatrick FunctionType *FT = CB.getFunctionType();
416909467b48Spatrick if (FT->isVarArg()) {
4170097a140dSpatrick VAHelper->visitCallBase(CB, IRB);
417109467b48Spatrick }
417209467b48Spatrick
417309467b48Spatrick // Now, get the shadow for the RetVal.
4174097a140dSpatrick if (!CB.getType()->isSized())
4175097a140dSpatrick return;
417609467b48Spatrick // Don't emit the epilogue for musttail call returns.
4177097a140dSpatrick if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
4178097a140dSpatrick return;
4179097a140dSpatrick
418073471bf0Spatrick if (MayCheckCall && CB.hasRetAttr(Attribute::NoUndef)) {
4181097a140dSpatrick setShadow(&CB, getCleanShadow(&CB));
4182097a140dSpatrick setOrigin(&CB, getCleanOrigin());
4183097a140dSpatrick return;
4184097a140dSpatrick }
4185097a140dSpatrick
4186097a140dSpatrick IRBuilder<> IRBBefore(&CB);
418709467b48Spatrick // Until we have full dynamic coverage, make sure the retval shadow is 0.
4188097a140dSpatrick Value *Base = getShadowPtrForRetval(&CB, IRBBefore);
4189097a140dSpatrick IRBBefore.CreateAlignedStore(getCleanShadow(&CB), Base,
4190097a140dSpatrick kShadowTLSAlignment);
419109467b48Spatrick BasicBlock::iterator NextInsn;
4192097a140dSpatrick if (isa<CallInst>(CB)) {
4193097a140dSpatrick NextInsn = ++CB.getIterator();
4194097a140dSpatrick assert(NextInsn != CB.getParent()->end());
419509467b48Spatrick } else {
4196097a140dSpatrick BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
419709467b48Spatrick if (!NormalDest->getSinglePredecessor()) {
419809467b48Spatrick // FIXME: this case is tricky, so we are just conservative here.
419909467b48Spatrick // Perhaps we need to split the edge between this BB and NormalDest,
420009467b48Spatrick // but a naive attempt to use SplitEdge leads to a crash.
4201097a140dSpatrick setShadow(&CB, getCleanShadow(&CB));
4202097a140dSpatrick setOrigin(&CB, getCleanOrigin());
420309467b48Spatrick return;
420409467b48Spatrick }
4205*d415bd75Srobert // FIXME: NextInsn is likely in a basic block that has not been visited
4206*d415bd75Srobert // yet. Anything inserted there will be instrumented by MSan later!
420709467b48Spatrick NextInsn = NormalDest->getFirstInsertionPt();
420809467b48Spatrick assert(NextInsn != NormalDest->end() &&
420909467b48Spatrick "Could not find insertion point for retval shadow load");
421009467b48Spatrick }
421109467b48Spatrick IRBuilder<> IRBAfter(&*NextInsn);
421209467b48Spatrick Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
4213097a140dSpatrick getShadowTy(&CB), getShadowPtrForRetval(&CB, IRBAfter),
4214097a140dSpatrick kShadowTLSAlignment, "_msret");
4215097a140dSpatrick setShadow(&CB, RetvalShadow);
421609467b48Spatrick if (MS.TrackOrigins)
4217097a140dSpatrick setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
421809467b48Spatrick getOriginPtrForRetval(IRBAfter)));
421909467b48Spatrick }
422009467b48Spatrick
isAMustTailRetVal__anon9f4f4c900811::MemorySanitizerVisitor422109467b48Spatrick bool isAMustTailRetVal(Value *RetVal) {
422209467b48Spatrick if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
422309467b48Spatrick RetVal = I->getOperand(0);
422409467b48Spatrick }
422509467b48Spatrick if (auto *I = dyn_cast<CallInst>(RetVal)) {
422609467b48Spatrick return I->isMustTailCall();
422709467b48Spatrick }
422809467b48Spatrick return false;
422909467b48Spatrick }
423009467b48Spatrick
visitReturnInst__anon9f4f4c900811::MemorySanitizerVisitor423109467b48Spatrick void visitReturnInst(ReturnInst &I) {
423209467b48Spatrick IRBuilder<> IRB(&I);
423309467b48Spatrick Value *RetVal = I.getReturnValue();
4234*d415bd75Srobert if (!RetVal)
4235*d415bd75Srobert return;
423609467b48Spatrick // Don't emit the epilogue for musttail call returns.
4237*d415bd75Srobert if (isAMustTailRetVal(RetVal))
4238*d415bd75Srobert return;
423909467b48Spatrick Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
4240*d415bd75Srobert bool HasNoUndef = F.hasRetAttribute(Attribute::NoUndef);
4241*d415bd75Srobert bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
4242097a140dSpatrick // FIXME: Consider using SpecialCaseList to specify a list of functions that
4243097a140dSpatrick // must always return fully initialized values. For now, we hardcode "main".
4244*d415bd75Srobert bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (F.getName() == "main");
4245097a140dSpatrick
424609467b48Spatrick Value *Shadow = getShadow(RetVal);
4247097a140dSpatrick bool StoreOrigin = true;
4248097a140dSpatrick if (EagerCheck) {
4249097a140dSpatrick insertShadowCheck(RetVal, &I);
4250097a140dSpatrick Shadow = getCleanShadow(RetVal);
4251097a140dSpatrick StoreOrigin = false;
4252097a140dSpatrick }
4253097a140dSpatrick
4254097a140dSpatrick // The caller may still expect information passed over TLS if we pass our
4255097a140dSpatrick // check
4256097a140dSpatrick if (StoreShadow) {
4257097a140dSpatrick IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
4258097a140dSpatrick if (MS.TrackOrigins && StoreOrigin)
425909467b48Spatrick IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
426009467b48Spatrick }
426109467b48Spatrick }
426209467b48Spatrick
visitPHINode__anon9f4f4c900811::MemorySanitizerVisitor426309467b48Spatrick void visitPHINode(PHINode &I) {
426409467b48Spatrick IRBuilder<> IRB(&I);
426509467b48Spatrick if (!PropagateShadow) {
426609467b48Spatrick setShadow(&I, getCleanShadow(&I));
426709467b48Spatrick setOrigin(&I, getCleanOrigin());
426809467b48Spatrick return;
426909467b48Spatrick }
427009467b48Spatrick
427109467b48Spatrick ShadowPHINodes.push_back(&I);
427209467b48Spatrick setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
427309467b48Spatrick "_msphi_s"));
427409467b48Spatrick if (MS.TrackOrigins)
4275*d415bd75Srobert setOrigin(
4276*d415bd75Srobert &I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(), "_msphi_o"));
4277*d415bd75Srobert }
4278*d415bd75Srobert
getLocalVarIdptr__anon9f4f4c900811::MemorySanitizerVisitor4279*d415bd75Srobert Value *getLocalVarIdptr(AllocaInst &I) {
4280*d415bd75Srobert ConstantInt *IntConst =
4281*d415bd75Srobert ConstantInt::get(Type::getInt32Ty((*F.getParent()).getContext()), 0);
4282*d415bd75Srobert return new GlobalVariable(*F.getParent(), IntConst->getType(),
4283*d415bd75Srobert /*isConstant=*/false, GlobalValue::PrivateLinkage,
4284*d415bd75Srobert IntConst);
428509467b48Spatrick }
428609467b48Spatrick
getLocalVarDescription__anon9f4f4c900811::MemorySanitizerVisitor428709467b48Spatrick Value *getLocalVarDescription(AllocaInst &I) {
4288*d415bd75Srobert return createPrivateConstGlobalForString(*F.getParent(), I.getName());
428909467b48Spatrick }
429009467b48Spatrick
poisonAllocaUserspace__anon9f4f4c900811::MemorySanitizerVisitor429109467b48Spatrick void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
429209467b48Spatrick if (PoisonStack && ClPoisonStackWithCall) {
429309467b48Spatrick IRB.CreateCall(MS.MsanPoisonStackFn,
429409467b48Spatrick {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
429509467b48Spatrick } else {
429609467b48Spatrick Value *ShadowBase, *OriginBase;
429709467b48Spatrick std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
4298097a140dSpatrick &I, IRB, IRB.getInt8Ty(), Align(1), /*isStore*/ true);
429909467b48Spatrick
430009467b48Spatrick Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
4301*d415bd75Srobert IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlign());
430209467b48Spatrick }
430309467b48Spatrick
430409467b48Spatrick if (PoisonStack && MS.TrackOrigins) {
4305*d415bd75Srobert Value *Idptr = getLocalVarIdptr(I);
4306*d415bd75Srobert if (ClPrintStackNames) {
430709467b48Spatrick Value *Descr = getLocalVarDescription(I);
4308*d415bd75Srobert IRB.CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
430909467b48Spatrick {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
4310*d415bd75Srobert IRB.CreatePointerCast(Idptr, IRB.getInt8PtrTy()),
4311*d415bd75Srobert IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
4312*d415bd75Srobert } else {
4313*d415bd75Srobert IRB.CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn,
4314*d415bd75Srobert {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
4315*d415bd75Srobert IRB.CreatePointerCast(Idptr, IRB.getInt8PtrTy())});
4316*d415bd75Srobert }
431709467b48Spatrick }
431809467b48Spatrick }
431909467b48Spatrick
poisonAllocaKmsan__anon9f4f4c900811::MemorySanitizerVisitor432009467b48Spatrick void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
432109467b48Spatrick Value *Descr = getLocalVarDescription(I);
432209467b48Spatrick if (PoisonStack) {
432309467b48Spatrick IRB.CreateCall(MS.MsanPoisonAllocaFn,
432409467b48Spatrick {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
432509467b48Spatrick IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
432609467b48Spatrick } else {
432709467b48Spatrick IRB.CreateCall(MS.MsanUnpoisonAllocaFn,
432809467b48Spatrick {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
432909467b48Spatrick }
433009467b48Spatrick }
433109467b48Spatrick
instrumentAlloca__anon9f4f4c900811::MemorySanitizerVisitor433209467b48Spatrick void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
433309467b48Spatrick if (!InsPoint)
433409467b48Spatrick InsPoint = &I;
4335*d415bd75Srobert NextNodeIRBuilder IRB(InsPoint);
433609467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
433709467b48Spatrick uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
433809467b48Spatrick Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
433909467b48Spatrick if (I.isArrayAllocation())
4340*d415bd75Srobert Len = IRB.CreateMul(Len,
4341*d415bd75Srobert IRB.CreateZExtOrTrunc(I.getArraySize(), MS.IntptrTy));
434209467b48Spatrick
434309467b48Spatrick if (MS.CompileKernel)
434409467b48Spatrick poisonAllocaKmsan(I, IRB, Len);
434509467b48Spatrick else
434609467b48Spatrick poisonAllocaUserspace(I, IRB, Len);
434709467b48Spatrick }
434809467b48Spatrick
visitAllocaInst__anon9f4f4c900811::MemorySanitizerVisitor434909467b48Spatrick void visitAllocaInst(AllocaInst &I) {
435009467b48Spatrick setShadow(&I, getCleanShadow(&I));
435109467b48Spatrick setOrigin(&I, getCleanOrigin());
435209467b48Spatrick // We'll get to this alloca later unless it's poisoned at the corresponding
435309467b48Spatrick // llvm.lifetime.start.
435409467b48Spatrick AllocaSet.insert(&I);
435509467b48Spatrick }
435609467b48Spatrick
visitSelectInst__anon9f4f4c900811::MemorySanitizerVisitor435709467b48Spatrick void visitSelectInst(SelectInst &I) {
435809467b48Spatrick IRBuilder<> IRB(&I);
435909467b48Spatrick // a = select b, c, d
436009467b48Spatrick Value *B = I.getCondition();
436109467b48Spatrick Value *C = I.getTrueValue();
436209467b48Spatrick Value *D = I.getFalseValue();
436309467b48Spatrick Value *Sb = getShadow(B);
436409467b48Spatrick Value *Sc = getShadow(C);
436509467b48Spatrick Value *Sd = getShadow(D);
436609467b48Spatrick
436709467b48Spatrick // Result shadow if condition shadow is 0.
436809467b48Spatrick Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
436909467b48Spatrick Value *Sa1;
437009467b48Spatrick if (I.getType()->isAggregateType()) {
437109467b48Spatrick // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
437209467b48Spatrick // an extra "select". This results in much more compact IR.
437309467b48Spatrick // Sa = select Sb, poisoned, (select b, Sc, Sd)
437409467b48Spatrick Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
437509467b48Spatrick } else {
437609467b48Spatrick // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
437709467b48Spatrick // If Sb (condition is poisoned), look for bits in c and d that are equal
437809467b48Spatrick // and both unpoisoned.
437909467b48Spatrick // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
438009467b48Spatrick
438109467b48Spatrick // Cast arguments to shadow-compatible type.
438209467b48Spatrick C = CreateAppToShadowCast(IRB, C);
438309467b48Spatrick D = CreateAppToShadowCast(IRB, D);
438409467b48Spatrick
438509467b48Spatrick // Result shadow if condition shadow is 1.
438609467b48Spatrick Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd});
438709467b48Spatrick }
438809467b48Spatrick Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
438909467b48Spatrick setShadow(&I, Sa);
439009467b48Spatrick if (MS.TrackOrigins) {
439109467b48Spatrick // Origins are always i32, so any vector conditions must be flattened.
439209467b48Spatrick // FIXME: consider tracking vector origins for app vectors?
439309467b48Spatrick if (B->getType()->isVectorTy()) {
439409467b48Spatrick Type *FlatTy = getShadowTyNoVec(B->getType());
439509467b48Spatrick B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
439609467b48Spatrick ConstantInt::getNullValue(FlatTy));
439709467b48Spatrick Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
439809467b48Spatrick ConstantInt::getNullValue(FlatTy));
439909467b48Spatrick }
440009467b48Spatrick // a = select b, c, d
440109467b48Spatrick // Oa = Sb ? Ob : (b ? Oc : Od)
440209467b48Spatrick setOrigin(
440309467b48Spatrick &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
440409467b48Spatrick IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
440509467b48Spatrick getOrigin(I.getFalseValue()))));
440609467b48Spatrick }
440709467b48Spatrick }
440809467b48Spatrick
visitLandingPadInst__anon9f4f4c900811::MemorySanitizerVisitor440909467b48Spatrick void visitLandingPadInst(LandingPadInst &I) {
441009467b48Spatrick // Do nothing.
441109467b48Spatrick // See https://github.com/google/sanitizers/issues/504
441209467b48Spatrick setShadow(&I, getCleanShadow(&I));
441309467b48Spatrick setOrigin(&I, getCleanOrigin());
441409467b48Spatrick }
441509467b48Spatrick
visitCatchSwitchInst__anon9f4f4c900811::MemorySanitizerVisitor441609467b48Spatrick void visitCatchSwitchInst(CatchSwitchInst &I) {
441709467b48Spatrick setShadow(&I, getCleanShadow(&I));
441809467b48Spatrick setOrigin(&I, getCleanOrigin());
441909467b48Spatrick }
442009467b48Spatrick
visitFuncletPadInst__anon9f4f4c900811::MemorySanitizerVisitor442109467b48Spatrick void visitFuncletPadInst(FuncletPadInst &I) {
442209467b48Spatrick setShadow(&I, getCleanShadow(&I));
442309467b48Spatrick setOrigin(&I, getCleanOrigin());
442409467b48Spatrick }
442509467b48Spatrick
visitGetElementPtrInst__anon9f4f4c900811::MemorySanitizerVisitor4426*d415bd75Srobert void visitGetElementPtrInst(GetElementPtrInst &I) { handleShadowOr(I); }
442709467b48Spatrick
visitExtractValueInst__anon9f4f4c900811::MemorySanitizerVisitor442809467b48Spatrick void visitExtractValueInst(ExtractValueInst &I) {
442909467b48Spatrick IRBuilder<> IRB(&I);
443009467b48Spatrick Value *Agg = I.getAggregateOperand();
443109467b48Spatrick LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n");
443209467b48Spatrick Value *AggShadow = getShadow(Agg);
443309467b48Spatrick LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
443409467b48Spatrick Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
443509467b48Spatrick LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
443609467b48Spatrick setShadow(&I, ResShadow);
443709467b48Spatrick setOriginForNaryOp(I);
443809467b48Spatrick }
443909467b48Spatrick
visitInsertValueInst__anon9f4f4c900811::MemorySanitizerVisitor444009467b48Spatrick void visitInsertValueInst(InsertValueInst &I) {
444109467b48Spatrick IRBuilder<> IRB(&I);
444209467b48Spatrick LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n");
444309467b48Spatrick Value *AggShadow = getShadow(I.getAggregateOperand());
444409467b48Spatrick Value *InsShadow = getShadow(I.getInsertedValueOperand());
444509467b48Spatrick LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
444609467b48Spatrick LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
444709467b48Spatrick Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
444809467b48Spatrick LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n");
444909467b48Spatrick setShadow(&I, Res);
445009467b48Spatrick setOriginForNaryOp(I);
445109467b48Spatrick }
445209467b48Spatrick
dumpInst__anon9f4f4c900811::MemorySanitizerVisitor445309467b48Spatrick void dumpInst(Instruction &I) {
445409467b48Spatrick if (CallInst *CI = dyn_cast<CallInst>(&I)) {
445509467b48Spatrick errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
445609467b48Spatrick } else {
445709467b48Spatrick errs() << "ZZZ " << I.getOpcodeName() << "\n";
445809467b48Spatrick }
445909467b48Spatrick errs() << "QQQ " << I << "\n";
446009467b48Spatrick }
446109467b48Spatrick
visitResumeInst__anon9f4f4c900811::MemorySanitizerVisitor446209467b48Spatrick void visitResumeInst(ResumeInst &I) {
446309467b48Spatrick LLVM_DEBUG(dbgs() << "Resume: " << I << "\n");
446409467b48Spatrick // Nothing to do here.
446509467b48Spatrick }
446609467b48Spatrick
visitCleanupReturnInst__anon9f4f4c900811::MemorySanitizerVisitor446709467b48Spatrick void visitCleanupReturnInst(CleanupReturnInst &CRI) {
446809467b48Spatrick LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
446909467b48Spatrick // Nothing to do here.
447009467b48Spatrick }
447109467b48Spatrick
visitCatchReturnInst__anon9f4f4c900811::MemorySanitizerVisitor447209467b48Spatrick void visitCatchReturnInst(CatchReturnInst &CRI) {
447309467b48Spatrick LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
447409467b48Spatrick // Nothing to do here.
447509467b48Spatrick }
447609467b48Spatrick
instrumentAsmArgument__anon9f4f4c900811::MemorySanitizerVisitor4477*d415bd75Srobert void instrumentAsmArgument(Value *Operand, Type *ElemTy, Instruction &I,
4478*d415bd75Srobert IRBuilder<> &IRB, const DataLayout &DL,
4479*d415bd75Srobert bool isOutput) {
448009467b48Spatrick // For each assembly argument, we check its value for being initialized.
448109467b48Spatrick // If the argument is a pointer, we assume it points to a single element
448209467b48Spatrick // of the corresponding type (or to a 8-byte word, if the type is unsized).
448309467b48Spatrick // Each such pointer is instrumented with a call to the runtime library.
448409467b48Spatrick Type *OpType = Operand->getType();
448509467b48Spatrick // Check the operand value itself.
448609467b48Spatrick insertShadowCheck(Operand, &I);
448709467b48Spatrick if (!OpType->isPointerTy() || !isOutput) {
448809467b48Spatrick assert(!isOutput);
448909467b48Spatrick return;
449009467b48Spatrick }
4491*d415bd75Srobert if (!ElemTy->isSized())
449209467b48Spatrick return;
4493*d415bd75Srobert int Size = DL.getTypeStoreSize(ElemTy);
449409467b48Spatrick Value *Ptr = IRB.CreatePointerCast(Operand, IRB.getInt8PtrTy());
449509467b48Spatrick Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
449609467b48Spatrick IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Ptr, SizeVal});
449709467b48Spatrick }
449809467b48Spatrick
449909467b48Spatrick /// Get the number of output arguments returned by pointers.
getNumOutputArgs__anon9f4f4c900811::MemorySanitizerVisitor450009467b48Spatrick int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
450109467b48Spatrick int NumRetOutputs = 0;
450209467b48Spatrick int NumOutputs = 0;
450309467b48Spatrick Type *RetTy = cast<Value>(CB)->getType();
450409467b48Spatrick if (!RetTy->isVoidTy()) {
450509467b48Spatrick // Register outputs are returned via the CallInst return value.
450609467b48Spatrick auto *ST = dyn_cast<StructType>(RetTy);
450709467b48Spatrick if (ST)
450809467b48Spatrick NumRetOutputs = ST->getNumElements();
450909467b48Spatrick else
451009467b48Spatrick NumRetOutputs = 1;
451109467b48Spatrick }
451209467b48Spatrick InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
451373471bf0Spatrick for (const InlineAsm::ConstraintInfo &Info : Constraints) {
451409467b48Spatrick switch (Info.Type) {
451509467b48Spatrick case InlineAsm::isOutput:
451609467b48Spatrick NumOutputs++;
451709467b48Spatrick break;
451809467b48Spatrick default:
451909467b48Spatrick break;
452009467b48Spatrick }
452109467b48Spatrick }
452209467b48Spatrick return NumOutputs - NumRetOutputs;
452309467b48Spatrick }
452409467b48Spatrick
visitAsmInstruction__anon9f4f4c900811::MemorySanitizerVisitor452509467b48Spatrick void visitAsmInstruction(Instruction &I) {
452609467b48Spatrick // Conservative inline assembly handling: check for poisoned shadow of
452709467b48Spatrick // asm() arguments, then unpoison the result and all the memory locations
452809467b48Spatrick // pointed to by those arguments.
452909467b48Spatrick // An inline asm() statement in C++ contains lists of input and output
453009467b48Spatrick // arguments used by the assembly code. These are mapped to operands of the
453109467b48Spatrick // CallInst as follows:
453209467b48Spatrick // - nR register outputs ("=r) are returned by value in a single structure
453309467b48Spatrick // (SSA value of the CallInst);
453409467b48Spatrick // - nO other outputs ("=m" and others) are returned by pointer as first
453509467b48Spatrick // nO operands of the CallInst;
453609467b48Spatrick // - nI inputs ("r", "m" and others) are passed to CallInst as the
453709467b48Spatrick // remaining nI operands.
453809467b48Spatrick // The total number of asm() arguments in the source is nR+nO+nI, and the
453909467b48Spatrick // corresponding CallInst has nO+nI+1 operands (the last operand is the
454009467b48Spatrick // function to be called).
454109467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
454209467b48Spatrick CallBase *CB = cast<CallBase>(&I);
454309467b48Spatrick IRBuilder<> IRB(&I);
4544097a140dSpatrick InlineAsm *IA = cast<InlineAsm>(CB->getCalledOperand());
454509467b48Spatrick int OutputArgs = getNumOutputArgs(IA, CB);
454609467b48Spatrick // The last operand of a CallInst is the function itself.
454709467b48Spatrick int NumOperands = CB->getNumOperands() - 1;
454809467b48Spatrick
454909467b48Spatrick // Check input arguments. Doing so before unpoisoning output arguments, so
455009467b48Spatrick // that we won't overwrite uninit values before checking them.
455109467b48Spatrick for (int i = OutputArgs; i < NumOperands; i++) {
455209467b48Spatrick Value *Operand = CB->getOperand(i);
4553*d415bd75Srobert instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL,
4554*d415bd75Srobert /*isOutput*/ false);
455509467b48Spatrick }
455609467b48Spatrick // Unpoison output arguments. This must happen before the actual InlineAsm
455709467b48Spatrick // call, so that the shadow for memory published in the asm() statement
455809467b48Spatrick // remains valid.
455909467b48Spatrick for (int i = 0; i < OutputArgs; i++) {
456009467b48Spatrick Value *Operand = CB->getOperand(i);
4561*d415bd75Srobert instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL,
4562*d415bd75Srobert /*isOutput*/ true);
456309467b48Spatrick }
456409467b48Spatrick
456509467b48Spatrick setShadow(&I, getCleanShadow(&I));
456609467b48Spatrick setOrigin(&I, getCleanOrigin());
456709467b48Spatrick }
456809467b48Spatrick
visitFreezeInst__anon9f4f4c900811::MemorySanitizerVisitor456973471bf0Spatrick void visitFreezeInst(FreezeInst &I) {
457073471bf0Spatrick // Freeze always returns a fully defined value.
457173471bf0Spatrick setShadow(&I, getCleanShadow(&I));
457273471bf0Spatrick setOrigin(&I, getCleanOrigin());
457373471bf0Spatrick }
457473471bf0Spatrick
visitInstruction__anon9f4f4c900811::MemorySanitizerVisitor457509467b48Spatrick void visitInstruction(Instruction &I) {
457609467b48Spatrick // Everything else: stop propagating and check for poisoned shadow.
457709467b48Spatrick if (ClDumpStrictInstructions)
457809467b48Spatrick dumpInst(I);
457909467b48Spatrick LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n");
458009467b48Spatrick for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
458109467b48Spatrick Value *Operand = I.getOperand(i);
458209467b48Spatrick if (Operand->getType()->isSized())
458309467b48Spatrick insertShadowCheck(Operand, &I);
458409467b48Spatrick }
458509467b48Spatrick setShadow(&I, getCleanShadow(&I));
458609467b48Spatrick setOrigin(&I, getCleanOrigin());
458709467b48Spatrick }
458809467b48Spatrick };
458909467b48Spatrick
459009467b48Spatrick /// AMD64-specific implementation of VarArgHelper.
459109467b48Spatrick struct VarArgAMD64Helper : public VarArgHelper {
459209467b48Spatrick // An unfortunate workaround for asymmetric lowering of va_arg stuff.
4593097a140dSpatrick // See a comment in visitCallBase for more details.
459409467b48Spatrick static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
459509467b48Spatrick static const unsigned AMD64FpEndOffsetSSE = 176;
459609467b48Spatrick // If SSE is disabled, fp_offset in va_list is zero.
459709467b48Spatrick static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
459809467b48Spatrick
459909467b48Spatrick unsigned AMD64FpEndOffset;
460009467b48Spatrick Function &F;
460109467b48Spatrick MemorySanitizer &MS;
460209467b48Spatrick MemorySanitizerVisitor &MSV;
460309467b48Spatrick Value *VAArgTLSCopy = nullptr;
460409467b48Spatrick Value *VAArgTLSOriginCopy = nullptr;
460509467b48Spatrick Value *VAArgOverflowSize = nullptr;
460609467b48Spatrick
460709467b48Spatrick SmallVector<CallInst *, 16> VAStartInstrumentationList;
460809467b48Spatrick
460909467b48Spatrick enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
461009467b48Spatrick
VarArgAMD64Helper__anon9f4f4c900811::VarArgAMD64Helper461109467b48Spatrick VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
461209467b48Spatrick MemorySanitizerVisitor &MSV)
461309467b48Spatrick : F(F), MS(MS), MSV(MSV) {
461409467b48Spatrick AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4615*d415bd75Srobert for (const auto &Attr : F.getAttributes().getFnAttrs()) {
461609467b48Spatrick if (Attr.isStringAttribute() &&
461709467b48Spatrick (Attr.getKindAsString() == "target-features")) {
461809467b48Spatrick if (Attr.getValueAsString().contains("-sse"))
461909467b48Spatrick AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
462009467b48Spatrick break;
462109467b48Spatrick }
462209467b48Spatrick }
462309467b48Spatrick }
462409467b48Spatrick
classifyArgument__anon9f4f4c900811::VarArgAMD64Helper462509467b48Spatrick ArgKind classifyArgument(Value *arg) {
462609467b48Spatrick // A very rough approximation of X86_64 argument classification rules.
462709467b48Spatrick Type *T = arg->getType();
462809467b48Spatrick if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
462909467b48Spatrick return AK_FloatingPoint;
463009467b48Spatrick if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
463109467b48Spatrick return AK_GeneralPurpose;
463209467b48Spatrick if (T->isPointerTy())
463309467b48Spatrick return AK_GeneralPurpose;
463409467b48Spatrick return AK_Memory;
463509467b48Spatrick }
463609467b48Spatrick
463709467b48Spatrick // For VarArg functions, store the argument shadow in an ABI-specific format
463809467b48Spatrick // that corresponds to va_list layout.
463909467b48Spatrick // We do this because Clang lowers va_arg in the frontend, and this pass
464009467b48Spatrick // only sees the low level code that deals with va_list internals.
464109467b48Spatrick // A much easier alternative (provided that Clang emits va_arg instructions)
464209467b48Spatrick // would have been to associate each live instance of va_list with a copy of
464309467b48Spatrick // MSanParamTLS, and extract shadow on va_arg() call in the argument list
464409467b48Spatrick // order.
visitCallBase__anon9f4f4c900811::VarArgAMD64Helper4645097a140dSpatrick void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
464609467b48Spatrick unsigned GpOffset = 0;
464709467b48Spatrick unsigned FpOffset = AMD64GpEndOffset;
464809467b48Spatrick unsigned OverflowOffset = AMD64FpEndOffset;
464909467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
4650*d415bd75Srobert for (const auto &[ArgNo, A] : llvm::enumerate(CB.args())) {
4651097a140dSpatrick bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4652097a140dSpatrick bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
465309467b48Spatrick if (IsByVal) {
465409467b48Spatrick // ByVal arguments always go to the overflow area.
465509467b48Spatrick // Fixed arguments passed through the overflow area will be stepped
465609467b48Spatrick // over by va_start, so don't count them towards the offset.
465709467b48Spatrick if (IsFixed)
465809467b48Spatrick continue;
465909467b48Spatrick assert(A->getType()->isPointerTy());
4660097a140dSpatrick Type *RealTy = CB.getParamByValType(ArgNo);
466109467b48Spatrick uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
466209467b48Spatrick Value *ShadowBase = getShadowPtrForVAArgument(
466309467b48Spatrick RealTy, IRB, OverflowOffset, alignTo(ArgSize, 8));
466409467b48Spatrick Value *OriginBase = nullptr;
466509467b48Spatrick if (MS.TrackOrigins)
466609467b48Spatrick OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset);
466709467b48Spatrick OverflowOffset += alignTo(ArgSize, 8);
466809467b48Spatrick if (!ShadowBase)
466909467b48Spatrick continue;
467009467b48Spatrick Value *ShadowPtr, *OriginPtr;
467109467b48Spatrick std::tie(ShadowPtr, OriginPtr) =
467209467b48Spatrick MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
467309467b48Spatrick /*isStore*/ false);
467409467b48Spatrick
467509467b48Spatrick IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
467609467b48Spatrick kShadowTLSAlignment, ArgSize);
467709467b48Spatrick if (MS.TrackOrigins)
467809467b48Spatrick IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr,
467909467b48Spatrick kShadowTLSAlignment, ArgSize);
468009467b48Spatrick } else {
468109467b48Spatrick ArgKind AK = classifyArgument(A);
468209467b48Spatrick if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
468309467b48Spatrick AK = AK_Memory;
468409467b48Spatrick if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
468509467b48Spatrick AK = AK_Memory;
468609467b48Spatrick Value *ShadowBase, *OriginBase = nullptr;
468709467b48Spatrick switch (AK) {
468809467b48Spatrick case AK_GeneralPurpose:
468909467b48Spatrick ShadowBase =
469009467b48Spatrick getShadowPtrForVAArgument(A->getType(), IRB, GpOffset, 8);
469109467b48Spatrick if (MS.TrackOrigins)
4692*d415bd75Srobert OriginBase = getOriginPtrForVAArgument(A->getType(), IRB, GpOffset);
469309467b48Spatrick GpOffset += 8;
469409467b48Spatrick break;
469509467b48Spatrick case AK_FloatingPoint:
469609467b48Spatrick ShadowBase =
469709467b48Spatrick getShadowPtrForVAArgument(A->getType(), IRB, FpOffset, 16);
469809467b48Spatrick if (MS.TrackOrigins)
4699*d415bd75Srobert OriginBase = getOriginPtrForVAArgument(A->getType(), IRB, FpOffset);
470009467b48Spatrick FpOffset += 16;
470109467b48Spatrick break;
470209467b48Spatrick case AK_Memory:
470309467b48Spatrick if (IsFixed)
470409467b48Spatrick continue;
470509467b48Spatrick uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
470609467b48Spatrick ShadowBase =
470709467b48Spatrick getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset, 8);
470809467b48Spatrick if (MS.TrackOrigins)
470909467b48Spatrick OriginBase =
471009467b48Spatrick getOriginPtrForVAArgument(A->getType(), IRB, OverflowOffset);
471109467b48Spatrick OverflowOffset += alignTo(ArgSize, 8);
471209467b48Spatrick }
471309467b48Spatrick // Take fixed arguments into account for GpOffset and FpOffset,
471409467b48Spatrick // but don't actually store shadows for them.
471509467b48Spatrick // TODO(glider): don't call get*PtrForVAArgument() for them.
471609467b48Spatrick if (IsFixed)
471709467b48Spatrick continue;
471809467b48Spatrick if (!ShadowBase)
471909467b48Spatrick continue;
472009467b48Spatrick Value *Shadow = MSV.getShadow(A);
4721097a140dSpatrick IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment);
472209467b48Spatrick if (MS.TrackOrigins) {
472309467b48Spatrick Value *Origin = MSV.getOrigin(A);
472409467b48Spatrick unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
472509467b48Spatrick MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
472609467b48Spatrick std::max(kShadowTLSAlignment, kMinOriginAlignment));
472709467b48Spatrick }
472809467b48Spatrick }
472909467b48Spatrick }
473009467b48Spatrick Constant *OverflowSize =
473109467b48Spatrick ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
473209467b48Spatrick IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
473309467b48Spatrick }
473409467b48Spatrick
473509467b48Spatrick /// Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anon9f4f4c900811::VarArgAMD64Helper473609467b48Spatrick Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
473709467b48Spatrick unsigned ArgOffset, unsigned ArgSize) {
473809467b48Spatrick // Make sure we don't overflow __msan_va_arg_tls.
473909467b48Spatrick if (ArgOffset + ArgSize > kParamTLSSize)
474009467b48Spatrick return nullptr;
474109467b48Spatrick Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
474209467b48Spatrick Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
474309467b48Spatrick return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
474409467b48Spatrick "_msarg_va_s");
474509467b48Spatrick }
474609467b48Spatrick
474709467b48Spatrick /// Compute the origin address for a given va_arg.
getOriginPtrForVAArgument__anon9f4f4c900811::VarArgAMD64Helper474809467b48Spatrick Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) {
474909467b48Spatrick Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
475009467b48Spatrick // getOriginPtrForVAArgument() is always called after
475109467b48Spatrick // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
475209467b48Spatrick // overflow.
475309467b48Spatrick Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
475409467b48Spatrick return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
475509467b48Spatrick "_msarg_va_o");
475609467b48Spatrick }
475709467b48Spatrick
unpoisonVAListTagForInst__anon9f4f4c900811::VarArgAMD64Helper475809467b48Spatrick void unpoisonVAListTagForInst(IntrinsicInst &I) {
475909467b48Spatrick IRBuilder<> IRB(&I);
476009467b48Spatrick Value *VAListTag = I.getArgOperand(0);
476109467b48Spatrick Value *ShadowPtr, *OriginPtr;
476209467b48Spatrick const Align Alignment = Align(8);
476309467b48Spatrick std::tie(ShadowPtr, OriginPtr) =
476409467b48Spatrick MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
476509467b48Spatrick /*isStore*/ true);
476609467b48Spatrick
476709467b48Spatrick // Unpoison the whole __va_list_tag.
476809467b48Spatrick // FIXME: magic ABI constants.
476909467b48Spatrick IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
477009467b48Spatrick /* size */ 24, Alignment, false);
477109467b48Spatrick // We shouldn't need to zero out the origins, as they're only checked for
477209467b48Spatrick // nonzero shadow.
477309467b48Spatrick }
477409467b48Spatrick
visitVAStartInst__anon9f4f4c900811::VarArgAMD64Helper477509467b48Spatrick void visitVAStartInst(VAStartInst &I) override {
477609467b48Spatrick if (F.getCallingConv() == CallingConv::Win64)
477709467b48Spatrick return;
477809467b48Spatrick VAStartInstrumentationList.push_back(&I);
477909467b48Spatrick unpoisonVAListTagForInst(I);
478009467b48Spatrick }
478109467b48Spatrick
visitVACopyInst__anon9f4f4c900811::VarArgAMD64Helper478209467b48Spatrick void visitVACopyInst(VACopyInst &I) override {
4783*d415bd75Srobert if (F.getCallingConv() == CallingConv::Win64)
4784*d415bd75Srobert return;
478509467b48Spatrick unpoisonVAListTagForInst(I);
478609467b48Spatrick }
478709467b48Spatrick
finalizeInstrumentation__anon9f4f4c900811::VarArgAMD64Helper478809467b48Spatrick void finalizeInstrumentation() override {
478909467b48Spatrick assert(!VAArgOverflowSize && !VAArgTLSCopy &&
479009467b48Spatrick "finalizeInstrumentation called twice");
479109467b48Spatrick if (!VAStartInstrumentationList.empty()) {
479209467b48Spatrick // If there is a va_start in this function, make a backup copy of
479309467b48Spatrick // va_arg_tls somewhere in the function entry block.
479473471bf0Spatrick IRBuilder<> IRB(MSV.FnPrologueEnd);
479509467b48Spatrick VAArgOverflowSize =
479609467b48Spatrick IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4797*d415bd75Srobert Value *CopySize = IRB.CreateAdd(
4798*d415bd75Srobert ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize);
479909467b48Spatrick VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
480009467b48Spatrick IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
480109467b48Spatrick if (MS.TrackOrigins) {
480209467b48Spatrick VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
480309467b48Spatrick IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS,
480409467b48Spatrick Align(8), CopySize);
480509467b48Spatrick }
480609467b48Spatrick }
480709467b48Spatrick
480809467b48Spatrick // Instrument va_start.
480909467b48Spatrick // Copy va_list shadow from the backup copy of the TLS contents.
481009467b48Spatrick for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
481109467b48Spatrick CallInst *OrigInst = VAStartInstrumentationList[i];
4812*d415bd75Srobert NextNodeIRBuilder IRB(OrigInst);
481309467b48Spatrick Value *VAListTag = OrigInst->getArgOperand(0);
481409467b48Spatrick
481509467b48Spatrick Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
481609467b48Spatrick Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
481709467b48Spatrick IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
481809467b48Spatrick ConstantInt::get(MS.IntptrTy, 16)),
481909467b48Spatrick PointerType::get(RegSaveAreaPtrTy, 0));
482009467b48Spatrick Value *RegSaveAreaPtr =
482109467b48Spatrick IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
482209467b48Spatrick Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
482309467b48Spatrick const Align Alignment = Align(16);
482409467b48Spatrick std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
482509467b48Spatrick MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
482609467b48Spatrick Alignment, /*isStore*/ true);
482709467b48Spatrick IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
482809467b48Spatrick AMD64FpEndOffset);
482909467b48Spatrick if (MS.TrackOrigins)
483009467b48Spatrick IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
483109467b48Spatrick Alignment, AMD64FpEndOffset);
483209467b48Spatrick Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
483309467b48Spatrick Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
483409467b48Spatrick IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
483509467b48Spatrick ConstantInt::get(MS.IntptrTy, 8)),
483609467b48Spatrick PointerType::get(OverflowArgAreaPtrTy, 0));
483709467b48Spatrick Value *OverflowArgAreaPtr =
483809467b48Spatrick IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
483909467b48Spatrick Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
484009467b48Spatrick std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
484109467b48Spatrick MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
484209467b48Spatrick Alignment, /*isStore*/ true);
484309467b48Spatrick Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
484409467b48Spatrick AMD64FpEndOffset);
484509467b48Spatrick IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
484609467b48Spatrick VAArgOverflowSize);
484709467b48Spatrick if (MS.TrackOrigins) {
484809467b48Spatrick SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
484909467b48Spatrick AMD64FpEndOffset);
485009467b48Spatrick IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
485109467b48Spatrick VAArgOverflowSize);
485209467b48Spatrick }
485309467b48Spatrick }
485409467b48Spatrick }
485509467b48Spatrick };
485609467b48Spatrick
485709467b48Spatrick /// MIPS64-specific implementation of VarArgHelper.
485809467b48Spatrick struct VarArgMIPS64Helper : public VarArgHelper {
485909467b48Spatrick Function &F;
486009467b48Spatrick MemorySanitizer &MS;
486109467b48Spatrick MemorySanitizerVisitor &MSV;
486209467b48Spatrick Value *VAArgTLSCopy = nullptr;
486309467b48Spatrick Value *VAArgSize = nullptr;
486409467b48Spatrick
486509467b48Spatrick SmallVector<CallInst *, 16> VAStartInstrumentationList;
486609467b48Spatrick
VarArgMIPS64Helper__anon9f4f4c900811::VarArgMIPS64Helper486709467b48Spatrick VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
4868*d415bd75Srobert MemorySanitizerVisitor &MSV)
4869*d415bd75Srobert : F(F), MS(MS), MSV(MSV) {}
487009467b48Spatrick
visitCallBase__anon9f4f4c900811::VarArgMIPS64Helper4871097a140dSpatrick void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
487209467b48Spatrick unsigned VAArgOffset = 0;
487309467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
4874*d415bd75Srobert for (Value *A :
4875*d415bd75Srobert llvm::drop_begin(CB.args(), CB.getFunctionType()->getNumParams())) {
487609467b48Spatrick Triple TargetTriple(F.getParent()->getTargetTriple());
487709467b48Spatrick Value *Base;
487809467b48Spatrick uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
487909467b48Spatrick if (TargetTriple.getArch() == Triple::mips64) {
4880*d415bd75Srobert // Adjusting the shadow for argument with size < 8 to match the
4881*d415bd75Srobert // placement of bits in big endian system
488209467b48Spatrick if (ArgSize < 8)
488309467b48Spatrick VAArgOffset += (8 - ArgSize);
488409467b48Spatrick }
488509467b48Spatrick Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset, ArgSize);
488609467b48Spatrick VAArgOffset += ArgSize;
488709467b48Spatrick VAArgOffset = alignTo(VAArgOffset, 8);
488809467b48Spatrick if (!Base)
488909467b48Spatrick continue;
4890097a140dSpatrick IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
489109467b48Spatrick }
489209467b48Spatrick
489309467b48Spatrick Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
489409467b48Spatrick // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
489509467b48Spatrick // a new class member i.e. it is the total size of all VarArgs.
489609467b48Spatrick IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
489709467b48Spatrick }
489809467b48Spatrick
489909467b48Spatrick /// Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anon9f4f4c900811::VarArgMIPS64Helper490009467b48Spatrick Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
490109467b48Spatrick unsigned ArgOffset, unsigned ArgSize) {
490209467b48Spatrick // Make sure we don't overflow __msan_va_arg_tls.
490309467b48Spatrick if (ArgOffset + ArgSize > kParamTLSSize)
490409467b48Spatrick return nullptr;
490509467b48Spatrick Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
490609467b48Spatrick Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
490709467b48Spatrick return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
490809467b48Spatrick "_msarg");
490909467b48Spatrick }
491009467b48Spatrick
visitVAStartInst__anon9f4f4c900811::VarArgMIPS64Helper491109467b48Spatrick void visitVAStartInst(VAStartInst &I) override {
491209467b48Spatrick IRBuilder<> IRB(&I);
491309467b48Spatrick VAStartInstrumentationList.push_back(&I);
491409467b48Spatrick Value *VAListTag = I.getArgOperand(0);
491509467b48Spatrick Value *ShadowPtr, *OriginPtr;
491609467b48Spatrick const Align Alignment = Align(8);
491709467b48Spatrick std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
491809467b48Spatrick VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
491909467b48Spatrick IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
492009467b48Spatrick /* size */ 8, Alignment, false);
492109467b48Spatrick }
492209467b48Spatrick
visitVACopyInst__anon9f4f4c900811::VarArgMIPS64Helper492309467b48Spatrick void visitVACopyInst(VACopyInst &I) override {
492409467b48Spatrick IRBuilder<> IRB(&I);
492509467b48Spatrick VAStartInstrumentationList.push_back(&I);
492609467b48Spatrick Value *VAListTag = I.getArgOperand(0);
492709467b48Spatrick Value *ShadowPtr, *OriginPtr;
492809467b48Spatrick const Align Alignment = Align(8);
492909467b48Spatrick std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
493009467b48Spatrick VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
493109467b48Spatrick IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
493209467b48Spatrick /* size */ 8, Alignment, false);
493309467b48Spatrick }
493409467b48Spatrick
finalizeInstrumentation__anon9f4f4c900811::VarArgMIPS64Helper493509467b48Spatrick void finalizeInstrumentation() override {
493609467b48Spatrick assert(!VAArgSize && !VAArgTLSCopy &&
493709467b48Spatrick "finalizeInstrumentation called twice");
493873471bf0Spatrick IRBuilder<> IRB(MSV.FnPrologueEnd);
493909467b48Spatrick VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4940*d415bd75Srobert Value *CopySize =
4941*d415bd75Srobert IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
494209467b48Spatrick
494309467b48Spatrick if (!VAStartInstrumentationList.empty()) {
494409467b48Spatrick // If there is a va_start in this function, make a backup copy of
494509467b48Spatrick // va_arg_tls somewhere in the function entry block.
494609467b48Spatrick VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
494709467b48Spatrick IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
494809467b48Spatrick }
494909467b48Spatrick
495009467b48Spatrick // Instrument va_start.
495109467b48Spatrick // Copy va_list shadow from the backup copy of the TLS contents.
495209467b48Spatrick for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
495309467b48Spatrick CallInst *OrigInst = VAStartInstrumentationList[i];
4954*d415bd75Srobert NextNodeIRBuilder IRB(OrigInst);
495509467b48Spatrick Value *VAListTag = OrigInst->getArgOperand(0);
495609467b48Spatrick Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
495709467b48Spatrick Value *RegSaveAreaPtrPtr =
495809467b48Spatrick IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
495909467b48Spatrick PointerType::get(RegSaveAreaPtrTy, 0));
496009467b48Spatrick Value *RegSaveAreaPtr =
496109467b48Spatrick IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
496209467b48Spatrick Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
496309467b48Spatrick const Align Alignment = Align(8);
496409467b48Spatrick std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
496509467b48Spatrick MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
496609467b48Spatrick Alignment, /*isStore*/ true);
496709467b48Spatrick IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
496809467b48Spatrick CopySize);
496909467b48Spatrick }
497009467b48Spatrick }
497109467b48Spatrick };
497209467b48Spatrick
497309467b48Spatrick /// AArch64-specific implementation of VarArgHelper.
497409467b48Spatrick struct VarArgAArch64Helper : public VarArgHelper {
497509467b48Spatrick static const unsigned kAArch64GrArgSize = 64;
497609467b48Spatrick static const unsigned kAArch64VrArgSize = 128;
497709467b48Spatrick
497809467b48Spatrick static const unsigned AArch64GrBegOffset = 0;
497909467b48Spatrick static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
498009467b48Spatrick // Make VR space aligned to 16 bytes.
498109467b48Spatrick static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
4982*d415bd75Srobert static const unsigned AArch64VrEndOffset =
4983*d415bd75Srobert AArch64VrBegOffset + kAArch64VrArgSize;
498409467b48Spatrick static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
498509467b48Spatrick
498609467b48Spatrick Function &F;
498709467b48Spatrick MemorySanitizer &MS;
498809467b48Spatrick MemorySanitizerVisitor &MSV;
498909467b48Spatrick Value *VAArgTLSCopy = nullptr;
499009467b48Spatrick Value *VAArgOverflowSize = nullptr;
499109467b48Spatrick
499209467b48Spatrick SmallVector<CallInst *, 16> VAStartInstrumentationList;
499309467b48Spatrick
499409467b48Spatrick enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
499509467b48Spatrick
VarArgAArch64Helper__anon9f4f4c900811::VarArgAArch64Helper499609467b48Spatrick VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
4997*d415bd75Srobert MemorySanitizerVisitor &MSV)
4998*d415bd75Srobert : F(F), MS(MS), MSV(MSV) {}
499909467b48Spatrick
classifyArgument__anon9f4f4c900811::VarArgAArch64Helper500009467b48Spatrick ArgKind classifyArgument(Value *arg) {
500109467b48Spatrick Type *T = arg->getType();
500209467b48Spatrick if (T->isFPOrFPVectorTy())
500309467b48Spatrick return AK_FloatingPoint;
5004*d415bd75Srobert if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64) ||
5005*d415bd75Srobert (T->isPointerTy()))
500609467b48Spatrick return AK_GeneralPurpose;
500709467b48Spatrick return AK_Memory;
500809467b48Spatrick }
500909467b48Spatrick
501009467b48Spatrick // The instrumentation stores the argument shadow in a non ABI-specific
501109467b48Spatrick // format because it does not know which argument is named (since Clang,
501209467b48Spatrick // like x86_64 case, lowers the va_args in the frontend and this pass only
501309467b48Spatrick // sees the low level code that deals with va_list internals).
501409467b48Spatrick // The first seven GR registers are saved in the first 56 bytes of the
501509467b48Spatrick // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
501609467b48Spatrick // the remaining arguments.
501709467b48Spatrick // Using constant offset within the va_arg TLS array allows fast copy
501809467b48Spatrick // in the finalize instrumentation.
visitCallBase__anon9f4f4c900811::VarArgAArch64Helper5019097a140dSpatrick void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
502009467b48Spatrick unsigned GrOffset = AArch64GrBegOffset;
502109467b48Spatrick unsigned VrOffset = AArch64VrBegOffset;
502209467b48Spatrick unsigned OverflowOffset = AArch64VAEndOffset;
502309467b48Spatrick
502409467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
5025*d415bd75Srobert for (const auto &[ArgNo, A] : llvm::enumerate(CB.args())) {
5026097a140dSpatrick bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
502709467b48Spatrick ArgKind AK = classifyArgument(A);
502809467b48Spatrick if (AK == AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset)
502909467b48Spatrick AK = AK_Memory;
503009467b48Spatrick if (AK == AK_FloatingPoint && VrOffset >= AArch64VrEndOffset)
503109467b48Spatrick AK = AK_Memory;
503209467b48Spatrick Value *Base;
503309467b48Spatrick switch (AK) {
503409467b48Spatrick case AK_GeneralPurpose:
503509467b48Spatrick Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset, 8);
503609467b48Spatrick GrOffset += 8;
503709467b48Spatrick break;
503809467b48Spatrick case AK_FloatingPoint:
503909467b48Spatrick Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset, 8);
504009467b48Spatrick VrOffset += 16;
504109467b48Spatrick break;
504209467b48Spatrick case AK_Memory:
504309467b48Spatrick // Don't count fixed arguments in the overflow area - va_start will
504409467b48Spatrick // skip right over them.
504509467b48Spatrick if (IsFixed)
504609467b48Spatrick continue;
504709467b48Spatrick uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
504809467b48Spatrick Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset,
504909467b48Spatrick alignTo(ArgSize, 8));
505009467b48Spatrick OverflowOffset += alignTo(ArgSize, 8);
505109467b48Spatrick break;
505209467b48Spatrick }
505309467b48Spatrick // Count Gp/Vr fixed arguments to their respective offsets, but don't
505409467b48Spatrick // bother to actually store a shadow.
505509467b48Spatrick if (IsFixed)
505609467b48Spatrick continue;
505709467b48Spatrick if (!Base)
505809467b48Spatrick continue;
5059097a140dSpatrick IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
506009467b48Spatrick }
506109467b48Spatrick Constant *OverflowSize =
506209467b48Spatrick ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
506309467b48Spatrick IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
506409467b48Spatrick }
506509467b48Spatrick
506609467b48Spatrick /// Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anon9f4f4c900811::VarArgAArch64Helper506709467b48Spatrick Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
506809467b48Spatrick unsigned ArgOffset, unsigned ArgSize) {
506909467b48Spatrick // Make sure we don't overflow __msan_va_arg_tls.
507009467b48Spatrick if (ArgOffset + ArgSize > kParamTLSSize)
507109467b48Spatrick return nullptr;
507209467b48Spatrick Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
507309467b48Spatrick Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
507409467b48Spatrick return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
507509467b48Spatrick "_msarg");
507609467b48Spatrick }
507709467b48Spatrick
visitVAStartInst__anon9f4f4c900811::VarArgAArch64Helper507809467b48Spatrick void visitVAStartInst(VAStartInst &I) override {
507909467b48Spatrick IRBuilder<> IRB(&I);
508009467b48Spatrick VAStartInstrumentationList.push_back(&I);
508109467b48Spatrick Value *VAListTag = I.getArgOperand(0);
508209467b48Spatrick Value *ShadowPtr, *OriginPtr;
508309467b48Spatrick const Align Alignment = Align(8);
508409467b48Spatrick std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
508509467b48Spatrick VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
508609467b48Spatrick IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
508709467b48Spatrick /* size */ 32, Alignment, false);
508809467b48Spatrick }
508909467b48Spatrick
visitVACopyInst__anon9f4f4c900811::VarArgAArch64Helper509009467b48Spatrick void visitVACopyInst(VACopyInst &I) override {
509109467b48Spatrick IRBuilder<> IRB(&I);
509209467b48Spatrick VAStartInstrumentationList.push_back(&I);
509309467b48Spatrick Value *VAListTag = I.getArgOperand(0);
509409467b48Spatrick Value *ShadowPtr, *OriginPtr;
509509467b48Spatrick const Align Alignment = Align(8);
509609467b48Spatrick std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
509709467b48Spatrick VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
509809467b48Spatrick IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
509909467b48Spatrick /* size */ 32, Alignment, false);
510009467b48Spatrick }
510109467b48Spatrick
510209467b48Spatrick // Retrieve a va_list field of 'void*' size.
getVAField64__anon9f4f4c900811::VarArgAArch64Helper510309467b48Spatrick Value *getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
5104*d415bd75Srobert Value *SaveAreaPtrPtr = IRB.CreateIntToPtr(
510509467b48Spatrick IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
510609467b48Spatrick ConstantInt::get(MS.IntptrTy, offset)),
510709467b48Spatrick Type::getInt64PtrTy(*MS.C));
510809467b48Spatrick return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr);
510909467b48Spatrick }
511009467b48Spatrick
511109467b48Spatrick // Retrieve a va_list field of 'int' size.
getVAField32__anon9f4f4c900811::VarArgAArch64Helper511209467b48Spatrick Value *getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
5113*d415bd75Srobert Value *SaveAreaPtr = IRB.CreateIntToPtr(
511409467b48Spatrick IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
511509467b48Spatrick ConstantInt::get(MS.IntptrTy, offset)),
511609467b48Spatrick Type::getInt32PtrTy(*MS.C));
511709467b48Spatrick Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr);
511809467b48Spatrick return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
511909467b48Spatrick }
512009467b48Spatrick
finalizeInstrumentation__anon9f4f4c900811::VarArgAArch64Helper512109467b48Spatrick void finalizeInstrumentation() override {
512209467b48Spatrick assert(!VAArgOverflowSize && !VAArgTLSCopy &&
512309467b48Spatrick "finalizeInstrumentation called twice");
512409467b48Spatrick if (!VAStartInstrumentationList.empty()) {
512509467b48Spatrick // If there is a va_start in this function, make a backup copy of
512609467b48Spatrick // va_arg_tls somewhere in the function entry block.
512773471bf0Spatrick IRBuilder<> IRB(MSV.FnPrologueEnd);
512809467b48Spatrick VAArgOverflowSize =
512909467b48Spatrick IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5130*d415bd75Srobert Value *CopySize = IRB.CreateAdd(
5131*d415bd75Srobert ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize);
513209467b48Spatrick VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
513309467b48Spatrick IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
513409467b48Spatrick }
513509467b48Spatrick
513609467b48Spatrick Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
513709467b48Spatrick Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
513809467b48Spatrick
513909467b48Spatrick // Instrument va_start, copy va_list shadow from the backup copy of
514009467b48Spatrick // the TLS contents.
514109467b48Spatrick for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
514209467b48Spatrick CallInst *OrigInst = VAStartInstrumentationList[i];
5143*d415bd75Srobert NextNodeIRBuilder IRB(OrigInst);
514409467b48Spatrick
514509467b48Spatrick Value *VAListTag = OrigInst->getArgOperand(0);
514609467b48Spatrick
514709467b48Spatrick // The variadic ABI for AArch64 creates two areas to save the incoming
514809467b48Spatrick // argument registers (one for 64-bit general register xn-x7 and another
514909467b48Spatrick // for 128-bit FP/SIMD vn-v7).
515009467b48Spatrick // We need then to propagate the shadow arguments on both regions
515109467b48Spatrick // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
5152097a140dSpatrick // The remaining arguments are saved on shadow for 'va::stack'.
515309467b48Spatrick // One caveat is it requires only to propagate the non-named arguments,
515409467b48Spatrick // however on the call site instrumentation 'all' the arguments are
515509467b48Spatrick // saved. So to copy the shadow values from the va_arg TLS array
515609467b48Spatrick // we need to adjust the offset for both GR and VR fields based on
515709467b48Spatrick // the __{gr,vr}_offs value (since they are stores based on incoming
515809467b48Spatrick // named arguments).
515909467b48Spatrick
516009467b48Spatrick // Read the stack pointer from the va_list.
516109467b48Spatrick Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
516209467b48Spatrick
516309467b48Spatrick // Read both the __gr_top and __gr_off and add them up.
516409467b48Spatrick Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
516509467b48Spatrick Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
516609467b48Spatrick
516709467b48Spatrick Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
516809467b48Spatrick
516909467b48Spatrick // Read both the __vr_top and __vr_off and add them up.
517009467b48Spatrick Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
517109467b48Spatrick Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
517209467b48Spatrick
517309467b48Spatrick Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
517409467b48Spatrick
517509467b48Spatrick // It does not know how many named arguments is being used and, on the
517609467b48Spatrick // callsite all the arguments were saved. Since __gr_off is defined as
517709467b48Spatrick // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
517809467b48Spatrick // argument by ignoring the bytes of shadow from named arguments.
517909467b48Spatrick Value *GrRegSaveAreaShadowPtrOff =
518009467b48Spatrick IRB.CreateAdd(GrArgSize, GrOffSaveArea);
518109467b48Spatrick
518209467b48Spatrick Value *GrRegSaveAreaShadowPtr =
518309467b48Spatrick MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
518409467b48Spatrick Align(8), /*isStore*/ true)
518509467b48Spatrick .first;
518609467b48Spatrick
518709467b48Spatrick Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
518809467b48Spatrick GrRegSaveAreaShadowPtrOff);
518909467b48Spatrick Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
519009467b48Spatrick
519109467b48Spatrick IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, Align(8), GrSrcPtr, Align(8),
519209467b48Spatrick GrCopySize);
519309467b48Spatrick
519409467b48Spatrick // Again, but for FP/SIMD values.
519509467b48Spatrick Value *VrRegSaveAreaShadowPtrOff =
519609467b48Spatrick IRB.CreateAdd(VrArgSize, VrOffSaveArea);
519709467b48Spatrick
519809467b48Spatrick Value *VrRegSaveAreaShadowPtr =
519909467b48Spatrick MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
520009467b48Spatrick Align(8), /*isStore*/ true)
520109467b48Spatrick .first;
520209467b48Spatrick
520309467b48Spatrick Value *VrSrcPtr = IRB.CreateInBoundsGEP(
520409467b48Spatrick IRB.getInt8Ty(),
520509467b48Spatrick IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
520609467b48Spatrick IRB.getInt32(AArch64VrBegOffset)),
520709467b48Spatrick VrRegSaveAreaShadowPtrOff);
520809467b48Spatrick Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
520909467b48Spatrick
521009467b48Spatrick IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, Align(8), VrSrcPtr, Align(8),
521109467b48Spatrick VrCopySize);
521209467b48Spatrick
521309467b48Spatrick // And finally for remaining arguments.
521409467b48Spatrick Value *StackSaveAreaShadowPtr =
521509467b48Spatrick MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
521609467b48Spatrick Align(16), /*isStore*/ true)
521709467b48Spatrick .first;
521809467b48Spatrick
5219*d415bd75Srobert Value *StackSrcPtr = IRB.CreateInBoundsGEP(
5220*d415bd75Srobert IRB.getInt8Ty(), VAArgTLSCopy, IRB.getInt32(AArch64VAEndOffset));
522109467b48Spatrick
522209467b48Spatrick IRB.CreateMemCpy(StackSaveAreaShadowPtr, Align(16), StackSrcPtr,
522309467b48Spatrick Align(16), VAArgOverflowSize);
522409467b48Spatrick }
522509467b48Spatrick }
522609467b48Spatrick };
522709467b48Spatrick
522809467b48Spatrick /// PowerPC64-specific implementation of VarArgHelper.
522909467b48Spatrick struct VarArgPowerPC64Helper : public VarArgHelper {
523009467b48Spatrick Function &F;
523109467b48Spatrick MemorySanitizer &MS;
523209467b48Spatrick MemorySanitizerVisitor &MSV;
523309467b48Spatrick Value *VAArgTLSCopy = nullptr;
523409467b48Spatrick Value *VAArgSize = nullptr;
523509467b48Spatrick
523609467b48Spatrick SmallVector<CallInst *, 16> VAStartInstrumentationList;
523709467b48Spatrick
VarArgPowerPC64Helper__anon9f4f4c900811::VarArgPowerPC64Helper523809467b48Spatrick VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
5239*d415bd75Srobert MemorySanitizerVisitor &MSV)
5240*d415bd75Srobert : F(F), MS(MS), MSV(MSV) {}
524109467b48Spatrick
visitCallBase__anon9f4f4c900811::VarArgPowerPC64Helper5242097a140dSpatrick void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
524309467b48Spatrick // For PowerPC, we need to deal with alignment of stack arguments -
524409467b48Spatrick // they are mostly aligned to 8 bytes, but vectors and i128 arrays
524509467b48Spatrick // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
524673471bf0Spatrick // For that reason, we compute current offset from stack pointer (which is
524773471bf0Spatrick // always properly aligned), and offset for the first vararg, then subtract
524873471bf0Spatrick // them.
524909467b48Spatrick unsigned VAArgBase;
525009467b48Spatrick Triple TargetTriple(F.getParent()->getTargetTriple());
525109467b48Spatrick // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
525209467b48Spatrick // and 32 bytes for ABIv2. This is usually determined by target
5253097a140dSpatrick // endianness, but in theory could be overridden by function attribute.
525409467b48Spatrick if (TargetTriple.getArch() == Triple::ppc64)
525509467b48Spatrick VAArgBase = 48;
525609467b48Spatrick else
525709467b48Spatrick VAArgBase = 32;
525809467b48Spatrick unsigned VAArgOffset = VAArgBase;
525909467b48Spatrick const DataLayout &DL = F.getParent()->getDataLayout();
5260*d415bd75Srobert for (const auto &[ArgNo, A] : llvm::enumerate(CB.args())) {
5261097a140dSpatrick bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
5262097a140dSpatrick bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
526309467b48Spatrick if (IsByVal) {
526409467b48Spatrick assert(A->getType()->isPointerTy());
5265097a140dSpatrick Type *RealTy = CB.getParamByValType(ArgNo);
526609467b48Spatrick uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
5267*d415bd75Srobert Align ArgAlign = CB.getParamAlign(ArgNo).value_or(Align(8));
5268*d415bd75Srobert if (ArgAlign < 8)
5269097a140dSpatrick ArgAlign = Align(8);
527009467b48Spatrick VAArgOffset = alignTo(VAArgOffset, ArgAlign);
527109467b48Spatrick if (!IsFixed) {
527209467b48Spatrick Value *Base = getShadowPtrForVAArgument(
527309467b48Spatrick RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
527409467b48Spatrick if (Base) {
527509467b48Spatrick Value *AShadowPtr, *AOriginPtr;
527609467b48Spatrick std::tie(AShadowPtr, AOriginPtr) =
527709467b48Spatrick MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
527809467b48Spatrick kShadowTLSAlignment, /*isStore*/ false);
527909467b48Spatrick
528009467b48Spatrick IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
528109467b48Spatrick kShadowTLSAlignment, ArgSize);
528209467b48Spatrick }
528309467b48Spatrick }
5284*d415bd75Srobert VAArgOffset += alignTo(ArgSize, Align(8));
528509467b48Spatrick } else {
528609467b48Spatrick Value *Base;
528709467b48Spatrick uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
5288*d415bd75Srobert Align ArgAlign = Align(8);
528909467b48Spatrick if (A->getType()->isArrayTy()) {
529009467b48Spatrick // Arrays are aligned to element size, except for long double
529109467b48Spatrick // arrays, which are aligned to 8 bytes.
529209467b48Spatrick Type *ElementTy = A->getType()->getArrayElementType();
529309467b48Spatrick if (!ElementTy->isPPC_FP128Ty())
5294*d415bd75Srobert ArgAlign = Align(DL.getTypeAllocSize(ElementTy));
529509467b48Spatrick } else if (A->getType()->isVectorTy()) {
529609467b48Spatrick // Vectors are naturally aligned.
5297*d415bd75Srobert ArgAlign = Align(ArgSize);
529809467b48Spatrick }
529909467b48Spatrick if (ArgAlign < 8)
5300*d415bd75Srobert ArgAlign = Align(8);
530109467b48Spatrick VAArgOffset = alignTo(VAArgOffset, ArgAlign);
530209467b48Spatrick if (DL.isBigEndian()) {
5303*d415bd75Srobert // Adjusting the shadow for argument with size < 8 to match the
5304*d415bd75Srobert // placement of bits in big endian system
530509467b48Spatrick if (ArgSize < 8)
530609467b48Spatrick VAArgOffset += (8 - ArgSize);
530709467b48Spatrick }
530809467b48Spatrick if (!IsFixed) {
530909467b48Spatrick Base = getShadowPtrForVAArgument(A->getType(), IRB,
531009467b48Spatrick VAArgOffset - VAArgBase, ArgSize);
531109467b48Spatrick if (Base)
5312097a140dSpatrick IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
531309467b48Spatrick }
531409467b48Spatrick VAArgOffset += ArgSize;
5315*d415bd75Srobert VAArgOffset = alignTo(VAArgOffset, Align(8));
531609467b48Spatrick }
531709467b48Spatrick if (IsFixed)
531809467b48Spatrick VAArgBase = VAArgOffset;
531909467b48Spatrick }
532009467b48Spatrick
5321*d415bd75Srobert Constant *TotalVAArgSize =
5322*d415bd75Srobert ConstantInt::get(IRB.getInt64Ty(), VAArgOffset - VAArgBase);
532309467b48Spatrick // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
532409467b48Spatrick // a new class member i.e. it is the total size of all VarArgs.
532509467b48Spatrick IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
532609467b48Spatrick }
532709467b48Spatrick
532809467b48Spatrick /// Compute the shadow address for a given va_arg.
getShadowPtrForVAArgument__anon9f4f4c900811::VarArgPowerPC64Helper532909467b48Spatrick Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
533009467b48Spatrick unsigned ArgOffset, unsigned ArgSize) {
533109467b48Spatrick // Make sure we don't overflow __msan_va_arg_tls.
533209467b48Spatrick if (ArgOffset + ArgSize > kParamTLSSize)
533309467b48Spatrick return nullptr;
533409467b48Spatrick Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
533509467b48Spatrick Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
533609467b48Spatrick return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
533709467b48Spatrick "_msarg");
533809467b48Spatrick }
533909467b48Spatrick
visitVAStartInst__anon9f4f4c900811::VarArgPowerPC64Helper534009467b48Spatrick void visitVAStartInst(VAStartInst &I) override {
534109467b48Spatrick IRBuilder<> IRB(&I);
534209467b48Spatrick VAStartInstrumentationList.push_back(&I);
534309467b48Spatrick Value *VAListTag = I.getArgOperand(0);
534409467b48Spatrick Value *ShadowPtr, *OriginPtr;
534509467b48Spatrick const Align Alignment = Align(8);
534609467b48Spatrick std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
534709467b48Spatrick VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
534809467b48Spatrick IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
534909467b48Spatrick /* size */ 8, Alignment, false);
535009467b48Spatrick }
535109467b48Spatrick
visitVACopyInst__anon9f4f4c900811::VarArgPowerPC64Helper535209467b48Spatrick void visitVACopyInst(VACopyInst &I) override {
535309467b48Spatrick IRBuilder<> IRB(&I);
535409467b48Spatrick Value *VAListTag = I.getArgOperand(0);
535509467b48Spatrick Value *ShadowPtr, *OriginPtr;
535609467b48Spatrick const Align Alignment = Align(8);
535709467b48Spatrick std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
535809467b48Spatrick VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
535909467b48Spatrick // Unpoison the whole __va_list_tag.
536009467b48Spatrick // FIXME: magic ABI constants.
536109467b48Spatrick IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
536209467b48Spatrick /* size */ 8, Alignment, false);
536309467b48Spatrick }
536409467b48Spatrick
finalizeInstrumentation__anon9f4f4c900811::VarArgPowerPC64Helper536509467b48Spatrick void finalizeInstrumentation() override {
536609467b48Spatrick assert(!VAArgSize && !VAArgTLSCopy &&
536709467b48Spatrick "finalizeInstrumentation called twice");
536873471bf0Spatrick IRBuilder<> IRB(MSV.FnPrologueEnd);
536909467b48Spatrick VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5370*d415bd75Srobert Value *CopySize =
5371*d415bd75Srobert IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
537209467b48Spatrick
537309467b48Spatrick if (!VAStartInstrumentationList.empty()) {
537409467b48Spatrick // If there is a va_start in this function, make a backup copy of
537509467b48Spatrick // va_arg_tls somewhere in the function entry block.
537609467b48Spatrick VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
537709467b48Spatrick IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
537809467b48Spatrick }
537909467b48Spatrick
538009467b48Spatrick // Instrument va_start.
538109467b48Spatrick // Copy va_list shadow from the backup copy of the TLS contents.
538209467b48Spatrick for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
538309467b48Spatrick CallInst *OrigInst = VAStartInstrumentationList[i];
5384*d415bd75Srobert NextNodeIRBuilder IRB(OrigInst);
538509467b48Spatrick Value *VAListTag = OrigInst->getArgOperand(0);
538609467b48Spatrick Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
538709467b48Spatrick Value *RegSaveAreaPtrPtr =
538809467b48Spatrick IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
538909467b48Spatrick PointerType::get(RegSaveAreaPtrTy, 0));
539009467b48Spatrick Value *RegSaveAreaPtr =
539109467b48Spatrick IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
539209467b48Spatrick Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
539309467b48Spatrick const Align Alignment = Align(8);
539409467b48Spatrick std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
539509467b48Spatrick MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
539609467b48Spatrick Alignment, /*isStore*/ true);
539709467b48Spatrick IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
539809467b48Spatrick CopySize);
539909467b48Spatrick }
540009467b48Spatrick }
540109467b48Spatrick };
540209467b48Spatrick
5403097a140dSpatrick /// SystemZ-specific implementation of VarArgHelper.
5404097a140dSpatrick struct VarArgSystemZHelper : public VarArgHelper {
5405097a140dSpatrick static const unsigned SystemZGpOffset = 16;
5406097a140dSpatrick static const unsigned SystemZGpEndOffset = 56;
5407097a140dSpatrick static const unsigned SystemZFpOffset = 128;
5408097a140dSpatrick static const unsigned SystemZFpEndOffset = 160;
5409097a140dSpatrick static const unsigned SystemZMaxVrArgs = 8;
5410097a140dSpatrick static const unsigned SystemZRegSaveAreaSize = 160;
5411097a140dSpatrick static const unsigned SystemZOverflowOffset = 160;
5412097a140dSpatrick static const unsigned SystemZVAListTagSize = 32;
5413097a140dSpatrick static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
5414097a140dSpatrick static const unsigned SystemZRegSaveAreaPtrOffset = 24;
5415097a140dSpatrick
5416097a140dSpatrick Function &F;
5417097a140dSpatrick MemorySanitizer &MS;
5418097a140dSpatrick MemorySanitizerVisitor &MSV;
5419097a140dSpatrick Value *VAArgTLSCopy = nullptr;
5420097a140dSpatrick Value *VAArgTLSOriginCopy = nullptr;
5421097a140dSpatrick Value *VAArgOverflowSize = nullptr;
5422097a140dSpatrick
5423097a140dSpatrick SmallVector<CallInst *, 16> VAStartInstrumentationList;
5424097a140dSpatrick
5425097a140dSpatrick enum class ArgKind {
5426097a140dSpatrick GeneralPurpose,
5427097a140dSpatrick FloatingPoint,
5428097a140dSpatrick Vector,
5429097a140dSpatrick Memory,
5430097a140dSpatrick Indirect,
5431097a140dSpatrick };
5432097a140dSpatrick
5433097a140dSpatrick enum class ShadowExtension { None, Zero, Sign };
5434097a140dSpatrick
VarArgSystemZHelper__anon9f4f4c900811::VarArgSystemZHelper5435097a140dSpatrick VarArgSystemZHelper(Function &F, MemorySanitizer &MS,
5436097a140dSpatrick MemorySanitizerVisitor &MSV)
5437097a140dSpatrick : F(F), MS(MS), MSV(MSV) {}
5438097a140dSpatrick
classifyArgument__anon9f4f4c900811::VarArgSystemZHelper5439097a140dSpatrick ArgKind classifyArgument(Type *T, bool IsSoftFloatABI) {
5440097a140dSpatrick // T is a SystemZABIInfo::classifyArgumentType() output, and there are
5441097a140dSpatrick // only a few possibilities of what it can be. In particular, enums, single
5442097a140dSpatrick // element structs and large types have already been taken care of.
5443097a140dSpatrick
5444097a140dSpatrick // Some i128 and fp128 arguments are converted to pointers only in the
5445097a140dSpatrick // back end.
5446097a140dSpatrick if (T->isIntegerTy(128) || T->isFP128Ty())
5447097a140dSpatrick return ArgKind::Indirect;
5448097a140dSpatrick if (T->isFloatingPointTy())
5449097a140dSpatrick return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5450097a140dSpatrick if (T->isIntegerTy() || T->isPointerTy())
5451097a140dSpatrick return ArgKind::GeneralPurpose;
5452097a140dSpatrick if (T->isVectorTy())
5453097a140dSpatrick return ArgKind::Vector;
5454097a140dSpatrick return ArgKind::Memory;
5455097a140dSpatrick }
5456097a140dSpatrick
getShadowExtension__anon9f4f4c900811::VarArgSystemZHelper5457097a140dSpatrick ShadowExtension getShadowExtension(const CallBase &CB, unsigned ArgNo) {
5458097a140dSpatrick // ABI says: "One of the simple integer types no more than 64 bits wide.
5459097a140dSpatrick // ... If such an argument is shorter than 64 bits, replace it by a full
5460097a140dSpatrick // 64-bit integer representing the same number, using sign or zero
5461097a140dSpatrick // extension". Shadow for an integer argument has the same type as the
5462097a140dSpatrick // argument itself, so it can be sign or zero extended as well.
5463097a140dSpatrick bool ZExt = CB.paramHasAttr(ArgNo, Attribute::ZExt);
5464097a140dSpatrick bool SExt = CB.paramHasAttr(ArgNo, Attribute::SExt);
5465097a140dSpatrick if (ZExt) {
5466097a140dSpatrick assert(!SExt);
5467097a140dSpatrick return ShadowExtension::Zero;
5468097a140dSpatrick }
5469097a140dSpatrick if (SExt) {
5470097a140dSpatrick assert(!ZExt);
5471097a140dSpatrick return ShadowExtension::Sign;
5472097a140dSpatrick }
5473097a140dSpatrick return ShadowExtension::None;
5474097a140dSpatrick }
5475097a140dSpatrick
visitCallBase__anon9f4f4c900811::VarArgSystemZHelper5476097a140dSpatrick void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5477097a140dSpatrick bool IsSoftFloatABI = CB.getCalledFunction()
5478097a140dSpatrick ->getFnAttribute("use-soft-float")
547973471bf0Spatrick .getValueAsBool();
5480097a140dSpatrick unsigned GpOffset = SystemZGpOffset;
5481097a140dSpatrick unsigned FpOffset = SystemZFpOffset;
5482097a140dSpatrick unsigned VrIndex = 0;
5483097a140dSpatrick unsigned OverflowOffset = SystemZOverflowOffset;
5484097a140dSpatrick const DataLayout &DL = F.getParent()->getDataLayout();
5485*d415bd75Srobert for (const auto &[ArgNo, A] : llvm::enumerate(CB.args())) {
5486097a140dSpatrick bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
5487097a140dSpatrick // SystemZABIInfo does not produce ByVal parameters.
5488097a140dSpatrick assert(!CB.paramHasAttr(ArgNo, Attribute::ByVal));
5489097a140dSpatrick Type *T = A->getType();
5490097a140dSpatrick ArgKind AK = classifyArgument(T, IsSoftFloatABI);
5491097a140dSpatrick if (AK == ArgKind::Indirect) {
5492097a140dSpatrick T = PointerType::get(T, 0);
5493097a140dSpatrick AK = ArgKind::GeneralPurpose;
5494097a140dSpatrick }
5495097a140dSpatrick if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5496097a140dSpatrick AK = ArgKind::Memory;
5497097a140dSpatrick if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5498097a140dSpatrick AK = ArgKind::Memory;
5499097a140dSpatrick if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5500097a140dSpatrick AK = ArgKind::Memory;
5501097a140dSpatrick Value *ShadowBase = nullptr;
5502097a140dSpatrick Value *OriginBase = nullptr;
5503097a140dSpatrick ShadowExtension SE = ShadowExtension::None;
5504097a140dSpatrick switch (AK) {
5505097a140dSpatrick case ArgKind::GeneralPurpose: {
5506097a140dSpatrick // Always keep track of GpOffset, but store shadow only for varargs.
5507097a140dSpatrick uint64_t ArgSize = 8;
5508097a140dSpatrick if (GpOffset + ArgSize <= kParamTLSSize) {
5509097a140dSpatrick if (!IsFixed) {
5510097a140dSpatrick SE = getShadowExtension(CB, ArgNo);
5511097a140dSpatrick uint64_t GapSize = 0;
5512097a140dSpatrick if (SE == ShadowExtension::None) {
5513097a140dSpatrick uint64_t ArgAllocSize = DL.getTypeAllocSize(T);
5514097a140dSpatrick assert(ArgAllocSize <= ArgSize);
5515097a140dSpatrick GapSize = ArgSize - ArgAllocSize;
5516097a140dSpatrick }
5517097a140dSpatrick ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5518097a140dSpatrick if (MS.TrackOrigins)
5519097a140dSpatrick OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5520097a140dSpatrick }
5521097a140dSpatrick GpOffset += ArgSize;
5522097a140dSpatrick } else {
5523097a140dSpatrick GpOffset = kParamTLSSize;
5524097a140dSpatrick }
5525097a140dSpatrick break;
5526097a140dSpatrick }
5527097a140dSpatrick case ArgKind::FloatingPoint: {
5528097a140dSpatrick // Always keep track of FpOffset, but store shadow only for varargs.
5529097a140dSpatrick uint64_t ArgSize = 8;
5530097a140dSpatrick if (FpOffset + ArgSize <= kParamTLSSize) {
5531097a140dSpatrick if (!IsFixed) {
5532097a140dSpatrick // PoP says: "A short floating-point datum requires only the
5533097a140dSpatrick // left-most 32 bit positions of a floating-point register".
5534097a140dSpatrick // Therefore, in contrast to AK_GeneralPurpose and AK_Memory,
5535097a140dSpatrick // don't extend shadow and don't mind the gap.
5536097a140dSpatrick ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5537097a140dSpatrick if (MS.TrackOrigins)
5538097a140dSpatrick OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5539097a140dSpatrick }
5540097a140dSpatrick FpOffset += ArgSize;
5541097a140dSpatrick } else {
5542097a140dSpatrick FpOffset = kParamTLSSize;
5543097a140dSpatrick }
5544097a140dSpatrick break;
5545097a140dSpatrick }
5546097a140dSpatrick case ArgKind::Vector: {
5547097a140dSpatrick // Keep track of VrIndex. No need to store shadow, since vector varargs
5548097a140dSpatrick // go through AK_Memory.
5549097a140dSpatrick assert(IsFixed);
5550097a140dSpatrick VrIndex++;
5551097a140dSpatrick break;
5552097a140dSpatrick }
5553097a140dSpatrick case ArgKind::Memory: {
5554097a140dSpatrick // Keep track of OverflowOffset and store shadow only for varargs.
5555097a140dSpatrick // Ignore fixed args, since we need to copy only the vararg portion of
5556097a140dSpatrick // the overflow area shadow.
5557097a140dSpatrick if (!IsFixed) {
5558097a140dSpatrick uint64_t ArgAllocSize = DL.getTypeAllocSize(T);
5559097a140dSpatrick uint64_t ArgSize = alignTo(ArgAllocSize, 8);
5560097a140dSpatrick if (OverflowOffset + ArgSize <= kParamTLSSize) {
5561097a140dSpatrick SE = getShadowExtension(CB, ArgNo);
5562097a140dSpatrick uint64_t GapSize =
5563097a140dSpatrick SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5564097a140dSpatrick ShadowBase =
5565097a140dSpatrick getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5566097a140dSpatrick if (MS.TrackOrigins)
5567097a140dSpatrick OriginBase =
5568097a140dSpatrick getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5569097a140dSpatrick OverflowOffset += ArgSize;
5570097a140dSpatrick } else {
5571097a140dSpatrick OverflowOffset = kParamTLSSize;
5572097a140dSpatrick }
5573097a140dSpatrick }
5574097a140dSpatrick break;
5575097a140dSpatrick }
5576097a140dSpatrick case ArgKind::Indirect:
5577097a140dSpatrick llvm_unreachable("Indirect must be converted to GeneralPurpose");
5578097a140dSpatrick }
5579097a140dSpatrick if (ShadowBase == nullptr)
5580097a140dSpatrick continue;
5581097a140dSpatrick Value *Shadow = MSV.getShadow(A);
5582097a140dSpatrick if (SE != ShadowExtension::None)
5583097a140dSpatrick Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.getInt64Ty(),
5584097a140dSpatrick /*Signed*/ SE == ShadowExtension::Sign);
5585097a140dSpatrick ShadowBase = IRB.CreateIntToPtr(
5586097a140dSpatrick ShadowBase, PointerType::get(Shadow->getType(), 0), "_msarg_va_s");
5587097a140dSpatrick IRB.CreateStore(Shadow, ShadowBase);
5588097a140dSpatrick if (MS.TrackOrigins) {
5589097a140dSpatrick Value *Origin = MSV.getOrigin(A);
5590097a140dSpatrick unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
5591097a140dSpatrick MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5592097a140dSpatrick kMinOriginAlignment);
5593097a140dSpatrick }
5594097a140dSpatrick }
5595097a140dSpatrick Constant *OverflowSize = ConstantInt::get(
5596097a140dSpatrick IRB.getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
5597097a140dSpatrick IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5598097a140dSpatrick }
5599097a140dSpatrick
getShadowAddrForVAArgument__anon9f4f4c900811::VarArgSystemZHelper5600097a140dSpatrick Value *getShadowAddrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
5601097a140dSpatrick Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
5602097a140dSpatrick return IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
5603097a140dSpatrick }
5604097a140dSpatrick
getOriginPtrForVAArgument__anon9f4f4c900811::VarArgSystemZHelper5605097a140dSpatrick Value *getOriginPtrForVAArgument(IRBuilder<> &IRB, int ArgOffset) {
5606097a140dSpatrick Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
5607097a140dSpatrick Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
5608097a140dSpatrick return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
5609097a140dSpatrick "_msarg_va_o");
5610097a140dSpatrick }
5611097a140dSpatrick
unpoisonVAListTagForInst__anon9f4f4c900811::VarArgSystemZHelper5612097a140dSpatrick void unpoisonVAListTagForInst(IntrinsicInst &I) {
5613097a140dSpatrick IRBuilder<> IRB(&I);
5614097a140dSpatrick Value *VAListTag = I.getArgOperand(0);
5615097a140dSpatrick Value *ShadowPtr, *OriginPtr;
5616097a140dSpatrick const Align Alignment = Align(8);
5617097a140dSpatrick std::tie(ShadowPtr, OriginPtr) =
5618097a140dSpatrick MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
5619097a140dSpatrick /*isStore*/ true);
5620097a140dSpatrick IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
5621097a140dSpatrick SystemZVAListTagSize, Alignment, false);
5622097a140dSpatrick }
5623097a140dSpatrick
visitVAStartInst__anon9f4f4c900811::VarArgSystemZHelper5624097a140dSpatrick void visitVAStartInst(VAStartInst &I) override {
5625097a140dSpatrick VAStartInstrumentationList.push_back(&I);
5626097a140dSpatrick unpoisonVAListTagForInst(I);
5627097a140dSpatrick }
5628097a140dSpatrick
visitVACopyInst__anon9f4f4c900811::VarArgSystemZHelper5629097a140dSpatrick void visitVACopyInst(VACopyInst &I) override { unpoisonVAListTagForInst(I); }
5630097a140dSpatrick
copyRegSaveArea__anon9f4f4c900811::VarArgSystemZHelper5631097a140dSpatrick void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) {
5632097a140dSpatrick Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
5633097a140dSpatrick Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
5634097a140dSpatrick IRB.CreateAdd(
5635097a140dSpatrick IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5636097a140dSpatrick ConstantInt::get(MS.IntptrTy, SystemZRegSaveAreaPtrOffset)),
5637097a140dSpatrick PointerType::get(RegSaveAreaPtrTy, 0));
5638097a140dSpatrick Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5639097a140dSpatrick Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5640097a140dSpatrick const Align Alignment = Align(8);
5641097a140dSpatrick std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5642097a140dSpatrick MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment,
5643097a140dSpatrick /*isStore*/ true);
5644097a140dSpatrick // TODO(iii): copy only fragments filled by visitCallBase()
5645097a140dSpatrick IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5646097a140dSpatrick SystemZRegSaveAreaSize);
5647097a140dSpatrick if (MS.TrackOrigins)
5648097a140dSpatrick IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5649097a140dSpatrick Alignment, SystemZRegSaveAreaSize);
5650097a140dSpatrick }
5651097a140dSpatrick
copyOverflowArea__anon9f4f4c900811::VarArgSystemZHelper5652097a140dSpatrick void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) {
5653097a140dSpatrick Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
5654097a140dSpatrick Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
5655097a140dSpatrick IRB.CreateAdd(
5656097a140dSpatrick IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5657097a140dSpatrick ConstantInt::get(MS.IntptrTy, SystemZOverflowArgAreaPtrOffset)),
5658097a140dSpatrick PointerType::get(OverflowArgAreaPtrTy, 0));
5659097a140dSpatrick Value *OverflowArgAreaPtr =
5660097a140dSpatrick IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5661097a140dSpatrick Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5662097a140dSpatrick const Align Alignment = Align(8);
5663097a140dSpatrick std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5664097a140dSpatrick MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
5665097a140dSpatrick Alignment, /*isStore*/ true);
5666097a140dSpatrick Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
5667097a140dSpatrick SystemZOverflowOffset);
5668097a140dSpatrick IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5669097a140dSpatrick VAArgOverflowSize);
5670097a140dSpatrick if (MS.TrackOrigins) {
5671097a140dSpatrick SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
5672097a140dSpatrick SystemZOverflowOffset);
5673097a140dSpatrick IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5674097a140dSpatrick VAArgOverflowSize);
5675097a140dSpatrick }
5676097a140dSpatrick }
5677097a140dSpatrick
finalizeInstrumentation__anon9f4f4c900811::VarArgSystemZHelper5678097a140dSpatrick void finalizeInstrumentation() override {
5679097a140dSpatrick assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5680097a140dSpatrick "finalizeInstrumentation called twice");
5681097a140dSpatrick if (!VAStartInstrumentationList.empty()) {
5682097a140dSpatrick // If there is a va_start in this function, make a backup copy of
5683097a140dSpatrick // va_arg_tls somewhere in the function entry block.
568473471bf0Spatrick IRBuilder<> IRB(MSV.FnPrologueEnd);
5685097a140dSpatrick VAArgOverflowSize =
5686097a140dSpatrick IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5687097a140dSpatrick Value *CopySize =
5688097a140dSpatrick IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
5689097a140dSpatrick VAArgOverflowSize);
5690097a140dSpatrick VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5691097a140dSpatrick IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
5692097a140dSpatrick if (MS.TrackOrigins) {
5693097a140dSpatrick VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5694097a140dSpatrick IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS,
5695097a140dSpatrick Align(8), CopySize);
5696097a140dSpatrick }
5697097a140dSpatrick }
5698097a140dSpatrick
5699097a140dSpatrick // Instrument va_start.
5700097a140dSpatrick // Copy va_list shadow from the backup copy of the TLS contents.
5701097a140dSpatrick for (size_t VaStartNo = 0, VaStartNum = VAStartInstrumentationList.size();
5702097a140dSpatrick VaStartNo < VaStartNum; VaStartNo++) {
5703097a140dSpatrick CallInst *OrigInst = VAStartInstrumentationList[VaStartNo];
5704*d415bd75Srobert NextNodeIRBuilder IRB(OrigInst);
5705097a140dSpatrick Value *VAListTag = OrigInst->getArgOperand(0);
5706097a140dSpatrick copyRegSaveArea(IRB, VAListTag);
5707097a140dSpatrick copyOverflowArea(IRB, VAListTag);
5708097a140dSpatrick }
5709097a140dSpatrick }
5710097a140dSpatrick };
5711097a140dSpatrick
571209467b48Spatrick /// A no-op implementation of VarArgHelper.
571309467b48Spatrick struct VarArgNoOpHelper : public VarArgHelper {
VarArgNoOpHelper__anon9f4f4c900811::VarArgNoOpHelper571409467b48Spatrick VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
571509467b48Spatrick MemorySanitizerVisitor &MSV) {}
571609467b48Spatrick
visitCallBase__anon9f4f4c900811::VarArgNoOpHelper5717097a140dSpatrick void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {}
571809467b48Spatrick
visitVAStartInst__anon9f4f4c900811::VarArgNoOpHelper571909467b48Spatrick void visitVAStartInst(VAStartInst &I) override {}
572009467b48Spatrick
visitVACopyInst__anon9f4f4c900811::VarArgNoOpHelper572109467b48Spatrick void visitVACopyInst(VACopyInst &I) override {}
572209467b48Spatrick
finalizeInstrumentation__anon9f4f4c900811::VarArgNoOpHelper572309467b48Spatrick void finalizeInstrumentation() override {}
572409467b48Spatrick };
572509467b48Spatrick
572609467b48Spatrick } // end anonymous namespace
572709467b48Spatrick
CreateVarArgHelper(Function & Func,MemorySanitizer & Msan,MemorySanitizerVisitor & Visitor)572809467b48Spatrick static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
572909467b48Spatrick MemorySanitizerVisitor &Visitor) {
573009467b48Spatrick // VarArg handling is only implemented on AMD64. False positives are possible
573109467b48Spatrick // on other platforms.
573209467b48Spatrick Triple TargetTriple(Func.getParent()->getTargetTriple());
573309467b48Spatrick if (TargetTriple.getArch() == Triple::x86_64)
573409467b48Spatrick return new VarArgAMD64Helper(Func, Msan, Visitor);
573509467b48Spatrick else if (TargetTriple.isMIPS64())
573609467b48Spatrick return new VarArgMIPS64Helper(Func, Msan, Visitor);
573709467b48Spatrick else if (TargetTriple.getArch() == Triple::aarch64)
573809467b48Spatrick return new VarArgAArch64Helper(Func, Msan, Visitor);
573909467b48Spatrick else if (TargetTriple.getArch() == Triple::ppc64 ||
574009467b48Spatrick TargetTriple.getArch() == Triple::ppc64le)
574109467b48Spatrick return new VarArgPowerPC64Helper(Func, Msan, Visitor);
5742097a140dSpatrick else if (TargetTriple.getArch() == Triple::systemz)
5743097a140dSpatrick return new VarArgSystemZHelper(Func, Msan, Visitor);
574409467b48Spatrick else
574509467b48Spatrick return new VarArgNoOpHelper(Func, Msan, Visitor);
574609467b48Spatrick }
574709467b48Spatrick
sanitizeFunction(Function & F,TargetLibraryInfo & TLI)574809467b48Spatrick bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
574909467b48Spatrick if (!CompileKernel && F.getName() == kMsanModuleCtorName)
575009467b48Spatrick return false;
575109467b48Spatrick
5752*d415bd75Srobert if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
5753*d415bd75Srobert return false;
5754*d415bd75Srobert
575509467b48Spatrick MemorySanitizerVisitor Visitor(F, *this, TLI);
575609467b48Spatrick
5757*d415bd75Srobert // Clear out memory attributes.
5758*d415bd75Srobert AttributeMask B;
5759*d415bd75Srobert B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
5760*d415bd75Srobert F.removeFnAttrs(B);
576109467b48Spatrick
576209467b48Spatrick return Visitor.runOnFunction();
576309467b48Spatrick }
5764