10b57cec5SDimitry Andric //===- DataFlowSanitizer.cpp - dynamic data flow analysis -----------------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric /// \file
100b57cec5SDimitry Andric /// This file is a part of DataFlowSanitizer, a generalised dynamic data flow
110b57cec5SDimitry Andric /// analysis.
120b57cec5SDimitry Andric ///
130b57cec5SDimitry Andric /// Unlike other Sanitizer tools, this tool is not designed to detect a specific
140b57cec5SDimitry Andric /// class of bugs on its own. Instead, it provides a generic dynamic data flow
150b57cec5SDimitry Andric /// analysis framework to be used by clients to help detect application-specific
160b57cec5SDimitry Andric /// issues within their own code.
170b57cec5SDimitry Andric ///
180b57cec5SDimitry Andric /// The analysis is based on automatic propagation of data flow labels (also
19fe6060f1SDimitry Andric /// known as taint labels) through a program as it performs computation.
20fe6060f1SDimitry Andric ///
21349cc55cSDimitry Andric /// Argument and return value labels are passed through TLS variables
22349cc55cSDimitry Andric /// __dfsan_arg_tls and __dfsan_retval_tls.
23349cc55cSDimitry Andric ///
24fe6060f1SDimitry Andric /// Each byte of application memory is backed by a shadow memory byte. The
25fe6060f1SDimitry Andric /// shadow byte can represent up to 8 labels. On Linux/x86_64, memory is then
26fe6060f1SDimitry Andric /// laid out as follows:
270b57cec5SDimitry Andric ///
280b57cec5SDimitry Andric /// +--------------------+ 0x800000000000 (top of memory)
29fe6060f1SDimitry Andric /// | application 3 |
30fe6060f1SDimitry Andric /// +--------------------+ 0x700000000000
31fe6060f1SDimitry Andric /// | invalid |
32fe6060f1SDimitry Andric /// +--------------------+ 0x610000000000
33fe6060f1SDimitry Andric /// | origin 1 |
34fe6060f1SDimitry Andric /// +--------------------+ 0x600000000000
35fe6060f1SDimitry Andric /// | application 2 |
36fe6060f1SDimitry Andric /// +--------------------+ 0x510000000000
37fe6060f1SDimitry Andric /// | shadow 1 |
38fe6060f1SDimitry Andric /// +--------------------+ 0x500000000000
39fe6060f1SDimitry Andric /// | invalid |
40fe6060f1SDimitry Andric /// +--------------------+ 0x400000000000
41fe6060f1SDimitry Andric /// | origin 3 |
42fe6060f1SDimitry Andric /// +--------------------+ 0x300000000000
43fe6060f1SDimitry Andric /// | shadow 3 |
44fe6060f1SDimitry Andric /// +--------------------+ 0x200000000000
45fe6060f1SDimitry Andric /// | origin 2 |
46fe6060f1SDimitry Andric /// +--------------------+ 0x110000000000
47fe6060f1SDimitry Andric /// | invalid |
48fe6060f1SDimitry Andric /// +--------------------+ 0x100000000000
49fe6060f1SDimitry Andric /// | shadow 2 |
50fe6060f1SDimitry Andric /// +--------------------+ 0x010000000000
51fe6060f1SDimitry Andric /// | application 1 |
520b57cec5SDimitry Andric /// +--------------------+ 0x000000000000
530b57cec5SDimitry Andric ///
54fe6060f1SDimitry Andric /// MEM_TO_SHADOW(mem) = mem ^ 0x500000000000
55fe6060f1SDimitry Andric /// SHADOW_TO_ORIGIN(shadow) = shadow + 0x100000000000
560b57cec5SDimitry Andric ///
570b57cec5SDimitry Andric /// For more information, please refer to the design document:
580b57cec5SDimitry Andric /// http://clang.llvm.org/docs/DataFlowSanitizerDesign.html
590b57cec5SDimitry Andric //
600b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
610b57cec5SDimitry Andric
62e8d8bef9SDimitry Andric #include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
630b57cec5SDimitry Andric #include "llvm/ADT/DenseMap.h"
640b57cec5SDimitry Andric #include "llvm/ADT/DenseSet.h"
650b57cec5SDimitry Andric #include "llvm/ADT/DepthFirstIterator.h"
660b57cec5SDimitry Andric #include "llvm/ADT/SmallPtrSet.h"
670b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h"
680b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h"
6981ad6265SDimitry Andric #include "llvm/ADT/StringSet.h"
70fe6060f1SDimitry Andric #include "llvm/ADT/iterator.h"
7106c3fb27SDimitry Andric #include "llvm/Analysis/DomTreeUpdater.h"
72bdd1243dSDimitry Andric #include "llvm/Analysis/GlobalsModRef.h"
73bdd1243dSDimitry Andric #include "llvm/Analysis/TargetLibraryInfo.h"
740b57cec5SDimitry Andric #include "llvm/Analysis/ValueTracking.h"
750b57cec5SDimitry Andric #include "llvm/IR/Argument.h"
7606c3fb27SDimitry Andric #include "llvm/IR/AttributeMask.h"
770b57cec5SDimitry Andric #include "llvm/IR/Attributes.h"
780b57cec5SDimitry Andric #include "llvm/IR/BasicBlock.h"
790b57cec5SDimitry Andric #include "llvm/IR/Constant.h"
800b57cec5SDimitry Andric #include "llvm/IR/Constants.h"
810b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h"
820b57cec5SDimitry Andric #include "llvm/IR/DerivedTypes.h"
830b57cec5SDimitry Andric #include "llvm/IR/Dominators.h"
840b57cec5SDimitry Andric #include "llvm/IR/Function.h"
850b57cec5SDimitry Andric #include "llvm/IR/GlobalAlias.h"
860b57cec5SDimitry Andric #include "llvm/IR/GlobalValue.h"
870b57cec5SDimitry Andric #include "llvm/IR/GlobalVariable.h"
880b57cec5SDimitry Andric #include "llvm/IR/IRBuilder.h"
890b57cec5SDimitry Andric #include "llvm/IR/InstVisitor.h"
900b57cec5SDimitry Andric #include "llvm/IR/InstrTypes.h"
910b57cec5SDimitry Andric #include "llvm/IR/Instruction.h"
920b57cec5SDimitry Andric #include "llvm/IR/Instructions.h"
930b57cec5SDimitry Andric #include "llvm/IR/IntrinsicInst.h"
940b57cec5SDimitry Andric #include "llvm/IR/MDBuilder.h"
950b57cec5SDimitry Andric #include "llvm/IR/Module.h"
96e8d8bef9SDimitry Andric #include "llvm/IR/PassManager.h"
970b57cec5SDimitry Andric #include "llvm/IR/Type.h"
980b57cec5SDimitry Andric #include "llvm/IR/User.h"
990b57cec5SDimitry Andric #include "llvm/IR/Value.h"
100fe6060f1SDimitry Andric #include "llvm/Support/Alignment.h"
1010b57cec5SDimitry Andric #include "llvm/Support/Casting.h"
1020b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h"
1030b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h"
1040b57cec5SDimitry Andric #include "llvm/Support/SpecialCaseList.h"
105480093f4SDimitry Andric #include "llvm/Support/VirtualFileSystem.h"
10606c3fb27SDimitry Andric #include "llvm/TargetParser/Triple.h"
1070b57cec5SDimitry Andric #include "llvm/Transforms/Instrumentation.h"
1080b57cec5SDimitry Andric #include "llvm/Transforms/Utils/BasicBlockUtils.h"
109480093f4SDimitry Andric #include "llvm/Transforms/Utils/Local.h"
1100b57cec5SDimitry Andric #include <algorithm>
1110b57cec5SDimitry Andric #include <cassert>
1120b57cec5SDimitry Andric #include <cstddef>
1130b57cec5SDimitry Andric #include <cstdint>
1140b57cec5SDimitry Andric #include <memory>
1150b57cec5SDimitry Andric #include <set>
1160b57cec5SDimitry Andric #include <string>
1170b57cec5SDimitry Andric #include <utility>
1180b57cec5SDimitry Andric #include <vector>
1190b57cec5SDimitry Andric
1200b57cec5SDimitry Andric using namespace llvm;
1210b57cec5SDimitry Andric
122e8d8bef9SDimitry Andric // This must be consistent with ShadowWidthBits.
123fe6060f1SDimitry Andric static const Align ShadowTLSAlignment = Align(2);
124fe6060f1SDimitry Andric
125fe6060f1SDimitry Andric static const Align MinOriginAlignment = Align(4);
126e8d8bef9SDimitry Andric
127e8d8bef9SDimitry Andric // The size of TLS variables. These constants must be kept in sync with the ones
128e8d8bef9SDimitry Andric // in dfsan.cpp.
129fe6060f1SDimitry Andric static const unsigned ArgTLSSize = 800;
130fe6060f1SDimitry Andric static const unsigned RetvalTLSSize = 800;
1310b57cec5SDimitry Andric
1320b57cec5SDimitry Andric // The -dfsan-preserve-alignment flag controls whether this pass assumes that
1330b57cec5SDimitry Andric // alignment requirements provided by the input IR are correct. For example,
1340b57cec5SDimitry Andric // if the input IR contains a load with alignment 8, this flag will cause
1350b57cec5SDimitry Andric // the shadow load to have alignment 16. This flag is disabled by default as
1360b57cec5SDimitry Andric // we have unfortunately encountered too much code (including Clang itself;
1370b57cec5SDimitry Andric // see PR14291) which performs misaligned access.
1380b57cec5SDimitry Andric static cl::opt<bool> ClPreserveAlignment(
1390b57cec5SDimitry Andric "dfsan-preserve-alignment",
1400b57cec5SDimitry Andric cl::desc("respect alignment requirements provided by input IR"), cl::Hidden,
1410b57cec5SDimitry Andric cl::init(false));
1420b57cec5SDimitry Andric
1430b57cec5SDimitry Andric // The ABI list files control how shadow parameters are passed. The pass treats
1440b57cec5SDimitry Andric // every function labelled "uninstrumented" in the ABI list file as conforming
1450b57cec5SDimitry Andric // to the "native" (i.e. unsanitized) ABI. Unless the ABI list contains
1460b57cec5SDimitry Andric // additional annotations for those functions, a call to one of those functions
1470b57cec5SDimitry Andric // will produce a warning message, as the labelling behaviour of the function is
148349cc55cSDimitry Andric // unknown. The other supported annotations for uninstrumented functions are
149349cc55cSDimitry Andric // "functional" and "discard", which are described below under
150349cc55cSDimitry Andric // DataFlowSanitizer::WrapperKind.
151349cc55cSDimitry Andric // Functions will often be labelled with both "uninstrumented" and one of
152349cc55cSDimitry Andric // "functional" or "discard". This will leave the function unchanged by this
153349cc55cSDimitry Andric // pass, and create a wrapper function that will call the original.
154349cc55cSDimitry Andric //
155349cc55cSDimitry Andric // Instrumented functions can also be annotated as "force_zero_labels", which
156349cc55cSDimitry Andric // will make all shadow and return values set zero labels.
157349cc55cSDimitry Andric // Functions should never be labelled with both "force_zero_labels" and
158349cc55cSDimitry Andric // "uninstrumented" or any of the unistrumented wrapper kinds.
1590b57cec5SDimitry Andric static cl::list<std::string> ClABIListFiles(
1600b57cec5SDimitry Andric "dfsan-abilist",
1610b57cec5SDimitry Andric cl::desc("File listing native ABI functions and how the pass treats them"),
1620b57cec5SDimitry Andric cl::Hidden);
1630b57cec5SDimitry Andric
1640b57cec5SDimitry Andric // Controls whether the pass includes or ignores the labels of pointers in load
1650b57cec5SDimitry Andric // instructions.
1660b57cec5SDimitry Andric static cl::opt<bool> ClCombinePointerLabelsOnLoad(
1670b57cec5SDimitry Andric "dfsan-combine-pointer-labels-on-load",
1680b57cec5SDimitry Andric cl::desc("Combine the label of the pointer with the label of the data when "
1690b57cec5SDimitry Andric "loading from memory."),
1700b57cec5SDimitry Andric cl::Hidden, cl::init(true));
1710b57cec5SDimitry Andric
1720b57cec5SDimitry Andric // Controls whether the pass includes or ignores the labels of pointers in
1730b57cec5SDimitry Andric // stores instructions.
1740b57cec5SDimitry Andric static cl::opt<bool> ClCombinePointerLabelsOnStore(
1750b57cec5SDimitry Andric "dfsan-combine-pointer-labels-on-store",
1760b57cec5SDimitry Andric cl::desc("Combine the label of the pointer with the label of the data when "
1770b57cec5SDimitry Andric "storing in memory."),
1780b57cec5SDimitry Andric cl::Hidden, cl::init(false));
1790b57cec5SDimitry Andric
180fe6060f1SDimitry Andric // Controls whether the pass propagates labels of offsets in GEP instructions.
181fe6060f1SDimitry Andric static cl::opt<bool> ClCombineOffsetLabelsOnGEP(
182fe6060f1SDimitry Andric "dfsan-combine-offset-labels-on-gep",
183fe6060f1SDimitry Andric cl::desc(
184fe6060f1SDimitry Andric "Combine the label of the offset with the label of the pointer when "
185fe6060f1SDimitry Andric "doing pointer arithmetic."),
186fe6060f1SDimitry Andric cl::Hidden, cl::init(true));
187fe6060f1SDimitry Andric
18881ad6265SDimitry Andric static cl::list<std::string> ClCombineTaintLookupTables(
18981ad6265SDimitry Andric "dfsan-combine-taint-lookup-table",
19081ad6265SDimitry Andric cl::desc(
19181ad6265SDimitry Andric "When dfsan-combine-offset-labels-on-gep and/or "
19281ad6265SDimitry Andric "dfsan-combine-pointer-labels-on-load are false, this flag can "
19381ad6265SDimitry Andric "be used to re-enable combining offset and/or pointer taint when "
19481ad6265SDimitry Andric "loading specific constant global variables (i.e. lookup tables)."),
19581ad6265SDimitry Andric cl::Hidden);
19681ad6265SDimitry Andric
1970b57cec5SDimitry Andric static cl::opt<bool> ClDebugNonzeroLabels(
1980b57cec5SDimitry Andric "dfsan-debug-nonzero-labels",
1990b57cec5SDimitry Andric cl::desc("Insert calls to __dfsan_nonzero_label on observing a parameter, "
2000b57cec5SDimitry Andric "load or return with a nonzero label"),
2010b57cec5SDimitry Andric cl::Hidden);
2020b57cec5SDimitry Andric
2035ffd83dbSDimitry Andric // Experimental feature that inserts callbacks for certain data events.
2045ffd83dbSDimitry Andric // Currently callbacks are only inserted for loads, stores, memory transfers
2055ffd83dbSDimitry Andric // (i.e. memcpy and memmove), and comparisons.
2065ffd83dbSDimitry Andric //
2075ffd83dbSDimitry Andric // If this flag is set to true, the user must provide definitions for the
2085ffd83dbSDimitry Andric // following callback functions:
209e8d8bef9SDimitry Andric // void __dfsan_load_callback(dfsan_label Label, void* addr);
210e8d8bef9SDimitry Andric // void __dfsan_store_callback(dfsan_label Label, void* addr);
2115ffd83dbSDimitry Andric // void __dfsan_mem_transfer_callback(dfsan_label *Start, size_t Len);
2125ffd83dbSDimitry Andric // void __dfsan_cmp_callback(dfsan_label CombinedLabel);
2135ffd83dbSDimitry Andric static cl::opt<bool> ClEventCallbacks(
2145ffd83dbSDimitry Andric "dfsan-event-callbacks",
2155ffd83dbSDimitry Andric cl::desc("Insert calls to __dfsan_*_callback functions on data events."),
2165ffd83dbSDimitry Andric cl::Hidden, cl::init(false));
2175ffd83dbSDimitry Andric
21804eeddc0SDimitry Andric // Experimental feature that inserts callbacks for conditionals, including:
21904eeddc0SDimitry Andric // conditional branch, switch, select.
22004eeddc0SDimitry Andric // This must be true for dfsan_set_conditional_callback() to have effect.
22104eeddc0SDimitry Andric static cl::opt<bool> ClConditionalCallbacks(
22204eeddc0SDimitry Andric "dfsan-conditional-callbacks",
22304eeddc0SDimitry Andric cl::desc("Insert calls to callback functions on conditionals."), cl::Hidden,
22404eeddc0SDimitry Andric cl::init(false));
22504eeddc0SDimitry Andric
226bdd1243dSDimitry Andric // Experimental feature that inserts callbacks for data reaching a function,
227bdd1243dSDimitry Andric // either via function arguments and loads.
228bdd1243dSDimitry Andric // This must be true for dfsan_set_reaches_function_callback() to have effect.
229bdd1243dSDimitry Andric static cl::opt<bool> ClReachesFunctionCallbacks(
230bdd1243dSDimitry Andric "dfsan-reaches-function-callbacks",
231bdd1243dSDimitry Andric cl::desc("Insert calls to callback functions on data reaching a function."),
232bdd1243dSDimitry Andric cl::Hidden, cl::init(false));
233bdd1243dSDimitry Andric
234e8d8bef9SDimitry Andric // Controls whether the pass tracks the control flow of select instructions.
235e8d8bef9SDimitry Andric static cl::opt<bool> ClTrackSelectControlFlow(
236e8d8bef9SDimitry Andric "dfsan-track-select-control-flow",
237e8d8bef9SDimitry Andric cl::desc("Propagate labels from condition values of select instructions "
238e8d8bef9SDimitry Andric "to results."),
239e8d8bef9SDimitry Andric cl::Hidden, cl::init(true));
240e8d8bef9SDimitry Andric
241fe6060f1SDimitry Andric // TODO: This default value follows MSan. DFSan may use a different value.
242fe6060f1SDimitry Andric static cl::opt<int> ClInstrumentWithCallThreshold(
243fe6060f1SDimitry Andric "dfsan-instrument-with-call-threshold",
244fe6060f1SDimitry Andric cl::desc("If the function being instrumented requires more than "
245fe6060f1SDimitry Andric "this number of origin stores, use callbacks instead of "
246fe6060f1SDimitry Andric "inline checks (-1 means never use callbacks)."),
247fe6060f1SDimitry Andric cl::Hidden, cl::init(3500));
248fe6060f1SDimitry Andric
249fe6060f1SDimitry Andric // Controls how to track origins.
250fe6060f1SDimitry Andric // * 0: do not track origins.
251fe6060f1SDimitry Andric // * 1: track origins at memory store operations.
252fe6060f1SDimitry Andric // * 2: track origins at memory load and store operations.
253fe6060f1SDimitry Andric // TODO: track callsites.
254fe6060f1SDimitry Andric static cl::opt<int> ClTrackOrigins("dfsan-track-origins",
255fe6060f1SDimitry Andric cl::desc("Track origins of labels"),
256fe6060f1SDimitry Andric cl::Hidden, cl::init(0));
257fe6060f1SDimitry Andric
2580eae32dcSDimitry Andric static cl::opt<bool> ClIgnorePersonalityRoutine(
2590eae32dcSDimitry Andric "dfsan-ignore-personality-routine",
2600eae32dcSDimitry Andric cl::desc("If a personality routine is marked uninstrumented from the ABI "
2610eae32dcSDimitry Andric "list, do not create a wrapper for it."),
2620eae32dcSDimitry Andric cl::Hidden, cl::init(false));
2630eae32dcSDimitry Andric
getGlobalTypeString(const GlobalValue & G)264fe6060f1SDimitry Andric static StringRef getGlobalTypeString(const GlobalValue &G) {
2650b57cec5SDimitry Andric // Types of GlobalVariables are always pointer types.
2660b57cec5SDimitry Andric Type *GType = G.getValueType();
2675ffd83dbSDimitry Andric // For now we support excluding struct types only.
2680b57cec5SDimitry Andric if (StructType *SGType = dyn_cast<StructType>(GType)) {
2690b57cec5SDimitry Andric if (!SGType->isLiteral())
2700b57cec5SDimitry Andric return SGType->getName();
2710b57cec5SDimitry Andric }
2720b57cec5SDimitry Andric return "<unknown type>";
2730b57cec5SDimitry Andric }
2740b57cec5SDimitry Andric
2750b57cec5SDimitry Andric namespace {
2760b57cec5SDimitry Andric
277fe6060f1SDimitry Andric // Memory map parameters used in application-to-shadow address calculation.
278fe6060f1SDimitry Andric // Offset = (Addr & ~AndMask) ^ XorMask
279fe6060f1SDimitry Andric // Shadow = ShadowBase + Offset
280fe6060f1SDimitry Andric // Origin = (OriginBase + Offset) & ~3ULL
281fe6060f1SDimitry Andric struct MemoryMapParams {
282fe6060f1SDimitry Andric uint64_t AndMask;
283fe6060f1SDimitry Andric uint64_t XorMask;
284fe6060f1SDimitry Andric uint64_t ShadowBase;
285fe6060f1SDimitry Andric uint64_t OriginBase;
286fe6060f1SDimitry Andric };
287fe6060f1SDimitry Andric
288fe6060f1SDimitry Andric } // end anonymous namespace
289fe6060f1SDimitry Andric
290bdd1243dSDimitry Andric // NOLINTBEGIN(readability-identifier-naming)
291bdd1243dSDimitry Andric // aarch64 Linux
292bdd1243dSDimitry Andric const MemoryMapParams Linux_AArch64_MemoryMapParams = {
293bdd1243dSDimitry Andric 0, // AndMask (not used)
294bdd1243dSDimitry Andric 0x0B00000000000, // XorMask
295bdd1243dSDimitry Andric 0, // ShadowBase (not used)
296bdd1243dSDimitry Andric 0x0200000000000, // OriginBase
297bdd1243dSDimitry Andric };
298bdd1243dSDimitry Andric
299fe6060f1SDimitry Andric // x86_64 Linux
300bdd1243dSDimitry Andric const MemoryMapParams Linux_X86_64_MemoryMapParams = {
301fe6060f1SDimitry Andric 0, // AndMask (not used)
302fe6060f1SDimitry Andric 0x500000000000, // XorMask
303fe6060f1SDimitry Andric 0, // ShadowBase (not used)
304fe6060f1SDimitry Andric 0x100000000000, // OriginBase
305fe6060f1SDimitry Andric };
306bdd1243dSDimitry Andric // NOLINTEND(readability-identifier-naming)
307fe6060f1SDimitry Andric
30806c3fb27SDimitry Andric // loongarch64 Linux
30906c3fb27SDimitry Andric const MemoryMapParams Linux_LoongArch64_MemoryMapParams = {
31006c3fb27SDimitry Andric 0, // AndMask (not used)
31106c3fb27SDimitry Andric 0x500000000000, // XorMask
31206c3fb27SDimitry Andric 0, // ShadowBase (not used)
31306c3fb27SDimitry Andric 0x100000000000, // OriginBase
31406c3fb27SDimitry Andric };
31506c3fb27SDimitry Andric
316fe6060f1SDimitry Andric namespace {
317fe6060f1SDimitry Andric
3180b57cec5SDimitry Andric class DFSanABIList {
3190b57cec5SDimitry Andric std::unique_ptr<SpecialCaseList> SCL;
3200b57cec5SDimitry Andric
3210b57cec5SDimitry Andric public:
3220b57cec5SDimitry Andric DFSanABIList() = default;
3230b57cec5SDimitry Andric
set(std::unique_ptr<SpecialCaseList> List)3240b57cec5SDimitry Andric void set(std::unique_ptr<SpecialCaseList> List) { SCL = std::move(List); }
3250b57cec5SDimitry Andric
3260b57cec5SDimitry Andric /// Returns whether either this function or its source file are listed in the
3270b57cec5SDimitry Andric /// given category.
isIn(const Function & F,StringRef Category) const3280b57cec5SDimitry Andric bool isIn(const Function &F, StringRef Category) const {
3290b57cec5SDimitry Andric return isIn(*F.getParent(), Category) ||
3300b57cec5SDimitry Andric SCL->inSection("dataflow", "fun", F.getName(), Category);
3310b57cec5SDimitry Andric }
3320b57cec5SDimitry Andric
3330b57cec5SDimitry Andric /// Returns whether this global alias is listed in the given category.
3340b57cec5SDimitry Andric ///
3350b57cec5SDimitry Andric /// If GA aliases a function, the alias's name is matched as a function name
3360b57cec5SDimitry Andric /// would be. Similarly, aliases of globals are matched like globals.
isIn(const GlobalAlias & GA,StringRef Category) const3370b57cec5SDimitry Andric bool isIn(const GlobalAlias &GA, StringRef Category) const {
3380b57cec5SDimitry Andric if (isIn(*GA.getParent(), Category))
3390b57cec5SDimitry Andric return true;
3400b57cec5SDimitry Andric
3410b57cec5SDimitry Andric if (isa<FunctionType>(GA.getValueType()))
3420b57cec5SDimitry Andric return SCL->inSection("dataflow", "fun", GA.getName(), Category);
3430b57cec5SDimitry Andric
3440b57cec5SDimitry Andric return SCL->inSection("dataflow", "global", GA.getName(), Category) ||
345fe6060f1SDimitry Andric SCL->inSection("dataflow", "type", getGlobalTypeString(GA),
3460b57cec5SDimitry Andric Category);
3470b57cec5SDimitry Andric }
3480b57cec5SDimitry Andric
3490b57cec5SDimitry Andric /// Returns whether this module is listed in the given category.
isIn(const Module & M,StringRef Category) const3500b57cec5SDimitry Andric bool isIn(const Module &M, StringRef Category) const {
3510b57cec5SDimitry Andric return SCL->inSection("dataflow", "src", M.getModuleIdentifier(), Category);
3520b57cec5SDimitry Andric }
3530b57cec5SDimitry Andric };
3540b57cec5SDimitry Andric
3550b57cec5SDimitry Andric /// TransformedFunction is used to express the result of transforming one
3560b57cec5SDimitry Andric /// function type into another. This struct is immutable. It holds metadata
3570b57cec5SDimitry Andric /// useful for updating calls of the old function to the new type.
3580b57cec5SDimitry Andric struct TransformedFunction {
TransformedFunction__anon7eade2240211::TransformedFunction359fe6060f1SDimitry Andric TransformedFunction(FunctionType *OriginalType, FunctionType *TransformedType,
3600b57cec5SDimitry Andric std::vector<unsigned> ArgumentIndexMapping)
361fe6060f1SDimitry Andric : OriginalType(OriginalType), TransformedType(TransformedType),
3620b57cec5SDimitry Andric ArgumentIndexMapping(ArgumentIndexMapping) {}
3630b57cec5SDimitry Andric
3640b57cec5SDimitry Andric // Disallow copies.
3650b57cec5SDimitry Andric TransformedFunction(const TransformedFunction &) = delete;
3660b57cec5SDimitry Andric TransformedFunction &operator=(const TransformedFunction &) = delete;
3670b57cec5SDimitry Andric
3680b57cec5SDimitry Andric // Allow moves.
3690b57cec5SDimitry Andric TransformedFunction(TransformedFunction &&) = default;
3700b57cec5SDimitry Andric TransformedFunction &operator=(TransformedFunction &&) = default;
3710b57cec5SDimitry Andric
3720b57cec5SDimitry Andric /// Type of the function before the transformation.
3730b57cec5SDimitry Andric FunctionType *OriginalType;
3740b57cec5SDimitry Andric
3750b57cec5SDimitry Andric /// Type of the function after the transformation.
3760b57cec5SDimitry Andric FunctionType *TransformedType;
3770b57cec5SDimitry Andric
3780b57cec5SDimitry Andric /// Transforming a function may change the position of arguments. This
3790b57cec5SDimitry Andric /// member records the mapping from each argument's old position to its new
3800b57cec5SDimitry Andric /// position. Argument positions are zero-indexed. If the transformation
3810b57cec5SDimitry Andric /// from F to F' made the first argument of F into the third argument of F',
3820b57cec5SDimitry Andric /// then ArgumentIndexMapping[0] will equal 2.
3830b57cec5SDimitry Andric std::vector<unsigned> ArgumentIndexMapping;
3840b57cec5SDimitry Andric };
3850b57cec5SDimitry Andric
3860b57cec5SDimitry Andric /// Given function attributes from a call site for the original function,
3870b57cec5SDimitry Andric /// return function attributes appropriate for a call to the transformed
3880b57cec5SDimitry Andric /// function.
389fe6060f1SDimitry Andric AttributeList
transformFunctionAttributes(const TransformedFunction & TransformedFunction,LLVMContext & Ctx,AttributeList CallSiteAttrs)390fe6060f1SDimitry Andric transformFunctionAttributes(const TransformedFunction &TransformedFunction,
3910b57cec5SDimitry Andric LLVMContext &Ctx, AttributeList CallSiteAttrs) {
3920b57cec5SDimitry Andric
3930b57cec5SDimitry Andric // Construct a vector of AttributeSet for each function argument.
3940b57cec5SDimitry Andric std::vector<llvm::AttributeSet> ArgumentAttributes(
3950b57cec5SDimitry Andric TransformedFunction.TransformedType->getNumParams());
3960b57cec5SDimitry Andric
3970b57cec5SDimitry Andric // Copy attributes from the parameter of the original function to the
3980b57cec5SDimitry Andric // transformed version. 'ArgumentIndexMapping' holds the mapping from
3990b57cec5SDimitry Andric // old argument position to new.
400fe6060f1SDimitry Andric for (unsigned I = 0, IE = TransformedFunction.ArgumentIndexMapping.size();
401fe6060f1SDimitry Andric I < IE; ++I) {
402fe6060f1SDimitry Andric unsigned TransformedIndex = TransformedFunction.ArgumentIndexMapping[I];
403349cc55cSDimitry Andric ArgumentAttributes[TransformedIndex] = CallSiteAttrs.getParamAttrs(I);
4040b57cec5SDimitry Andric }
4050b57cec5SDimitry Andric
4060b57cec5SDimitry Andric // Copy annotations on varargs arguments.
407fe6060f1SDimitry Andric for (unsigned I = TransformedFunction.OriginalType->getNumParams(),
408fe6060f1SDimitry Andric IE = CallSiteAttrs.getNumAttrSets();
409fe6060f1SDimitry Andric I < IE; ++I) {
410349cc55cSDimitry Andric ArgumentAttributes.push_back(CallSiteAttrs.getParamAttrs(I));
4110b57cec5SDimitry Andric }
4120b57cec5SDimitry Andric
413349cc55cSDimitry Andric return AttributeList::get(Ctx, CallSiteAttrs.getFnAttrs(),
414349cc55cSDimitry Andric CallSiteAttrs.getRetAttrs(),
415bdd1243dSDimitry Andric llvm::ArrayRef(ArgumentAttributes));
4160b57cec5SDimitry Andric }
4170b57cec5SDimitry Andric
418e8d8bef9SDimitry Andric class DataFlowSanitizer {
4190b57cec5SDimitry Andric friend struct DFSanFunction;
4200b57cec5SDimitry Andric friend class DFSanVisitor;
4210b57cec5SDimitry Andric
422fe6060f1SDimitry Andric enum { ShadowWidthBits = 8, ShadowWidthBytes = ShadowWidthBits / 8 };
423fe6060f1SDimitry Andric
424fe6060f1SDimitry Andric enum { OriginWidthBits = 32, OriginWidthBytes = OriginWidthBits / 8 };
4250b57cec5SDimitry Andric
4260b57cec5SDimitry Andric /// How should calls to uninstrumented functions be handled?
4270b57cec5SDimitry Andric enum WrapperKind {
4280b57cec5SDimitry Andric /// This function is present in an uninstrumented form but we don't know
4290b57cec5SDimitry Andric /// how it should be handled. Print a warning and call the function anyway.
4300b57cec5SDimitry Andric /// Don't label the return value.
4310b57cec5SDimitry Andric WK_Warning,
4320b57cec5SDimitry Andric
4330b57cec5SDimitry Andric /// This function does not write to (user-accessible) memory, and its return
4340b57cec5SDimitry Andric /// value is unlabelled.
4350b57cec5SDimitry Andric WK_Discard,
4360b57cec5SDimitry Andric
4370b57cec5SDimitry Andric /// This function does not write to (user-accessible) memory, and the label
4380b57cec5SDimitry Andric /// of its return value is the union of the label of its arguments.
4390b57cec5SDimitry Andric WK_Functional,
4400b57cec5SDimitry Andric
4410b57cec5SDimitry Andric /// Instead of calling the function, a custom wrapper __dfsw_F is called,
4420b57cec5SDimitry Andric /// where F is the name of the function. This function may wrap the
443349cc55cSDimitry Andric /// original function or provide its own implementation. WK_Custom uses an
4440b57cec5SDimitry Andric /// extra pointer argument to return the shadow. This allows the wrapped
4450b57cec5SDimitry Andric /// form of the function type to be expressed in C.
4460b57cec5SDimitry Andric WK_Custom
4470b57cec5SDimitry Andric };
4480b57cec5SDimitry Andric
4490b57cec5SDimitry Andric Module *Mod;
4500b57cec5SDimitry Andric LLVMContext *Ctx;
451e8d8bef9SDimitry Andric Type *Int8Ptr;
452fe6060f1SDimitry Andric IntegerType *OriginTy;
453fe6060f1SDimitry Andric PointerType *OriginPtrTy;
454fe6060f1SDimitry Andric ConstantInt *ZeroOrigin;
455e8d8bef9SDimitry Andric /// The shadow type for all primitive types and vector types.
456e8d8bef9SDimitry Andric IntegerType *PrimitiveShadowTy;
457e8d8bef9SDimitry Andric PointerType *PrimitiveShadowPtrTy;
4580b57cec5SDimitry Andric IntegerType *IntptrTy;
459e8d8bef9SDimitry Andric ConstantInt *ZeroPrimitiveShadow;
4600b57cec5SDimitry Andric Constant *ArgTLS;
461fe6060f1SDimitry Andric ArrayType *ArgOriginTLSTy;
462fe6060f1SDimitry Andric Constant *ArgOriginTLS;
4630b57cec5SDimitry Andric Constant *RetvalTLS;
464fe6060f1SDimitry Andric Constant *RetvalOriginTLS;
4650b57cec5SDimitry Andric FunctionType *DFSanUnionLoadFnTy;
466fe6060f1SDimitry Andric FunctionType *DFSanLoadLabelAndOriginFnTy;
4670b57cec5SDimitry Andric FunctionType *DFSanUnimplementedFnTy;
46881ad6265SDimitry Andric FunctionType *DFSanWrapperExternWeakNullFnTy;
4690b57cec5SDimitry Andric FunctionType *DFSanSetLabelFnTy;
4700b57cec5SDimitry Andric FunctionType *DFSanNonzeroLabelFnTy;
4710b57cec5SDimitry Andric FunctionType *DFSanVarargWrapperFnTy;
47204eeddc0SDimitry Andric FunctionType *DFSanConditionalCallbackFnTy;
47304eeddc0SDimitry Andric FunctionType *DFSanConditionalCallbackOriginFnTy;
474bdd1243dSDimitry Andric FunctionType *DFSanReachesFunctionCallbackFnTy;
475bdd1243dSDimitry Andric FunctionType *DFSanReachesFunctionCallbackOriginFnTy;
476e8d8bef9SDimitry Andric FunctionType *DFSanCmpCallbackFnTy;
477e8d8bef9SDimitry Andric FunctionType *DFSanLoadStoreCallbackFnTy;
4785ffd83dbSDimitry Andric FunctionType *DFSanMemTransferCallbackFnTy;
479fe6060f1SDimitry Andric FunctionType *DFSanChainOriginFnTy;
480fe6060f1SDimitry Andric FunctionType *DFSanChainOriginIfTaintedFnTy;
481fe6060f1SDimitry Andric FunctionType *DFSanMemOriginTransferFnTy;
482bdd1243dSDimitry Andric FunctionType *DFSanMemShadowOriginTransferFnTy;
483bdd1243dSDimitry Andric FunctionType *DFSanMemShadowOriginConditionalExchangeFnTy;
484fe6060f1SDimitry Andric FunctionType *DFSanMaybeStoreOriginFnTy;
4850b57cec5SDimitry Andric FunctionCallee DFSanUnionLoadFn;
486fe6060f1SDimitry Andric FunctionCallee DFSanLoadLabelAndOriginFn;
4870b57cec5SDimitry Andric FunctionCallee DFSanUnimplementedFn;
48881ad6265SDimitry Andric FunctionCallee DFSanWrapperExternWeakNullFn;
4890b57cec5SDimitry Andric FunctionCallee DFSanSetLabelFn;
4900b57cec5SDimitry Andric FunctionCallee DFSanNonzeroLabelFn;
4910b57cec5SDimitry Andric FunctionCallee DFSanVarargWrapperFn;
4925ffd83dbSDimitry Andric FunctionCallee DFSanLoadCallbackFn;
4935ffd83dbSDimitry Andric FunctionCallee DFSanStoreCallbackFn;
4945ffd83dbSDimitry Andric FunctionCallee DFSanMemTransferCallbackFn;
49504eeddc0SDimitry Andric FunctionCallee DFSanConditionalCallbackFn;
49604eeddc0SDimitry Andric FunctionCallee DFSanConditionalCallbackOriginFn;
497bdd1243dSDimitry Andric FunctionCallee DFSanReachesFunctionCallbackFn;
498bdd1243dSDimitry Andric FunctionCallee DFSanReachesFunctionCallbackOriginFn;
4995ffd83dbSDimitry Andric FunctionCallee DFSanCmpCallbackFn;
500fe6060f1SDimitry Andric FunctionCallee DFSanChainOriginFn;
501fe6060f1SDimitry Andric FunctionCallee DFSanChainOriginIfTaintedFn;
502fe6060f1SDimitry Andric FunctionCallee DFSanMemOriginTransferFn;
503bdd1243dSDimitry Andric FunctionCallee DFSanMemShadowOriginTransferFn;
504bdd1243dSDimitry Andric FunctionCallee DFSanMemShadowOriginConditionalExchangeFn;
505fe6060f1SDimitry Andric FunctionCallee DFSanMaybeStoreOriginFn;
506fe6060f1SDimitry Andric SmallPtrSet<Value *, 16> DFSanRuntimeFunctions;
5070b57cec5SDimitry Andric MDNode *ColdCallWeights;
508fe6060f1SDimitry Andric MDNode *OriginStoreWeights;
5090b57cec5SDimitry Andric DFSanABIList ABIList;
5100b57cec5SDimitry Andric DenseMap<Value *, Function *> UnwrappedFnMap;
51104eeddc0SDimitry Andric AttributeMask ReadOnlyNoneAttrs;
51281ad6265SDimitry Andric StringSet<> CombineTaintLookupTableNames;
5130b57cec5SDimitry Andric
514fe6060f1SDimitry Andric /// Memory map parameters used in calculation mapping application addresses
515fe6060f1SDimitry Andric /// to shadow addresses and origin addresses.
516fe6060f1SDimitry Andric const MemoryMapParams *MapParams;
517fe6060f1SDimitry Andric
518fe6060f1SDimitry Andric Value *getShadowOffset(Value *Addr, IRBuilder<> &IRB);
5190b57cec5SDimitry Andric Value *getShadowAddress(Value *Addr, Instruction *Pos);
520fe6060f1SDimitry Andric Value *getShadowAddress(Value *Addr, Instruction *Pos, Value *ShadowOffset);
521fe6060f1SDimitry Andric std::pair<Value *, Value *>
522fe6060f1SDimitry Andric getShadowOriginAddress(Value *Addr, Align InstAlignment, Instruction *Pos);
5230b57cec5SDimitry Andric bool isInstrumented(const Function *F);
5240b57cec5SDimitry Andric bool isInstrumented(const GlobalAlias *GA);
525349cc55cSDimitry Andric bool isForceZeroLabels(const Function *F);
5260b57cec5SDimitry Andric TransformedFunction getCustomFunctionType(FunctionType *T);
5270b57cec5SDimitry Andric WrapperKind getWrapperKind(Function *F);
528fe6060f1SDimitry Andric void addGlobalNameSuffix(GlobalValue *GV);
52981ad6265SDimitry Andric void buildExternWeakCheckIfNeeded(IRBuilder<> &IRB, Function *F);
5300b57cec5SDimitry Andric Function *buildWrapperFunction(Function *F, StringRef NewFName,
5310b57cec5SDimitry Andric GlobalValue::LinkageTypes NewFLink,
5320b57cec5SDimitry Andric FunctionType *NewFT);
5335ffd83dbSDimitry Andric void initializeCallbackFunctions(Module &M);
5345ffd83dbSDimitry Andric void initializeRuntimeFunctions(Module &M);
535fe6060f1SDimitry Andric bool initializeModule(Module &M);
5360b57cec5SDimitry Andric
537fe6060f1SDimitry Andric /// Advances \p OriginAddr to point to the next 32-bit origin and then loads
538fe6060f1SDimitry Andric /// from it. Returns the origin's loaded value.
539fe6060f1SDimitry Andric Value *loadNextOrigin(Instruction *Pos, Align OriginAlign,
540fe6060f1SDimitry Andric Value **OriginAddr);
541fe6060f1SDimitry Andric
542fe6060f1SDimitry Andric /// Returns whether the given load byte size is amenable to inlined
543fe6060f1SDimitry Andric /// optimization patterns.
544fe6060f1SDimitry Andric bool hasLoadSizeForFastPath(uint64_t Size);
545fe6060f1SDimitry Andric
546fe6060f1SDimitry Andric /// Returns whether the pass tracks origins. Supports only TLS ABI mode.
547fe6060f1SDimitry Andric bool shouldTrackOrigins();
548e8d8bef9SDimitry Andric
549e8d8bef9SDimitry Andric /// Returns a zero constant with the shadow type of OrigTy.
550e8d8bef9SDimitry Andric ///
551e8d8bef9SDimitry Andric /// getZeroShadow({T1,T2,...}) = {getZeroShadow(T1),getZeroShadow(T2,...}
552e8d8bef9SDimitry Andric /// getZeroShadow([n x T]) = [n x getZeroShadow(T)]
553e8d8bef9SDimitry Andric /// getZeroShadow(other type) = i16(0)
554e8d8bef9SDimitry Andric Constant *getZeroShadow(Type *OrigTy);
555e8d8bef9SDimitry Andric /// Returns a zero constant with the shadow type of V's type.
556e8d8bef9SDimitry Andric Constant *getZeroShadow(Value *V);
557e8d8bef9SDimitry Andric
558e8d8bef9SDimitry Andric /// Checks if V is a zero shadow.
559e8d8bef9SDimitry Andric bool isZeroShadow(Value *V);
560e8d8bef9SDimitry Andric
561e8d8bef9SDimitry Andric /// Returns the shadow type of OrigTy.
562e8d8bef9SDimitry Andric ///
563e8d8bef9SDimitry Andric /// getShadowTy({T1,T2,...}) = {getShadowTy(T1),getShadowTy(T2),...}
564e8d8bef9SDimitry Andric /// getShadowTy([n x T]) = [n x getShadowTy(T)]
565e8d8bef9SDimitry Andric /// getShadowTy(other type) = i16
566e8d8bef9SDimitry Andric Type *getShadowTy(Type *OrigTy);
5675f757f3fSDimitry Andric /// Returns the shadow type of V's type.
568e8d8bef9SDimitry Andric Type *getShadowTy(Value *V);
569e8d8bef9SDimitry Andric
570fe6060f1SDimitry Andric const uint64_t NumOfElementsInArgOrgTLS = ArgTLSSize / OriginWidthBytes;
571fe6060f1SDimitry Andric
5720b57cec5SDimitry Andric public:
573e8d8bef9SDimitry Andric DataFlowSanitizer(const std::vector<std::string> &ABIListFiles);
5740b57cec5SDimitry Andric
575bdd1243dSDimitry Andric bool runImpl(Module &M,
576bdd1243dSDimitry Andric llvm::function_ref<TargetLibraryInfo &(Function &)> GetTLI);
5770b57cec5SDimitry Andric };
5780b57cec5SDimitry Andric
5790b57cec5SDimitry Andric struct DFSanFunction {
5800b57cec5SDimitry Andric DataFlowSanitizer &DFS;
5810b57cec5SDimitry Andric Function *F;
5820b57cec5SDimitry Andric DominatorTree DT;
5830b57cec5SDimitry Andric bool IsNativeABI;
584349cc55cSDimitry Andric bool IsForceZeroLabels;
585bdd1243dSDimitry Andric TargetLibraryInfo &TLI;
5860b57cec5SDimitry Andric AllocaInst *LabelReturnAlloca = nullptr;
587fe6060f1SDimitry Andric AllocaInst *OriginReturnAlloca = nullptr;
5880b57cec5SDimitry Andric DenseMap<Value *, Value *> ValShadowMap;
589fe6060f1SDimitry Andric DenseMap<Value *, Value *> ValOriginMap;
5900b57cec5SDimitry Andric DenseMap<AllocaInst *, AllocaInst *> AllocaShadowMap;
591fe6060f1SDimitry Andric DenseMap<AllocaInst *, AllocaInst *> AllocaOriginMap;
592fe6060f1SDimitry Andric
593fe6060f1SDimitry Andric struct PHIFixupElement {
594fe6060f1SDimitry Andric PHINode *Phi;
595fe6060f1SDimitry Andric PHINode *ShadowPhi;
596fe6060f1SDimitry Andric PHINode *OriginPhi;
597fe6060f1SDimitry Andric };
598fe6060f1SDimitry Andric std::vector<PHIFixupElement> PHIFixups;
599fe6060f1SDimitry Andric
6000b57cec5SDimitry Andric DenseSet<Instruction *> SkipInsts;
6010b57cec5SDimitry Andric std::vector<Value *> NonZeroChecks;
6020b57cec5SDimitry Andric
603e8d8bef9SDimitry Andric struct CachedShadow {
604e8d8bef9SDimitry Andric BasicBlock *Block; // The block where Shadow is defined.
6050b57cec5SDimitry Andric Value *Shadow;
6060b57cec5SDimitry Andric };
607e8d8bef9SDimitry Andric /// Maps a value to its latest shadow value in terms of domination tree.
608e8d8bef9SDimitry Andric DenseMap<std::pair<Value *, Value *>, CachedShadow> CachedShadows;
609e8d8bef9SDimitry Andric /// Maps a value to its latest collapsed shadow value it was converted to in
610e8d8bef9SDimitry Andric /// terms of domination tree. When ClDebugNonzeroLabels is on, this cache is
611e8d8bef9SDimitry Andric /// used at a post process where CFG blocks are split. So it does not cache
612e8d8bef9SDimitry Andric /// BasicBlock like CachedShadows, but uses domination between values.
613e8d8bef9SDimitry Andric DenseMap<Value *, Value *> CachedCollapsedShadows;
6140b57cec5SDimitry Andric DenseMap<Value *, std::set<Value *>> ShadowElements;
6150b57cec5SDimitry Andric
DFSanFunction__anon7eade2240211::DFSanFunction616349cc55cSDimitry Andric DFSanFunction(DataFlowSanitizer &DFS, Function *F, bool IsNativeABI,
617bdd1243dSDimitry Andric bool IsForceZeroLabels, TargetLibraryInfo &TLI)
618349cc55cSDimitry Andric : DFS(DFS), F(F), IsNativeABI(IsNativeABI),
619bdd1243dSDimitry Andric IsForceZeroLabels(IsForceZeroLabels), TLI(TLI) {
6200b57cec5SDimitry Andric DT.recalculate(*F);
6210b57cec5SDimitry Andric }
6220b57cec5SDimitry Andric
623e8d8bef9SDimitry Andric /// Computes the shadow address for a given function argument.
624e8d8bef9SDimitry Andric ///
625e8d8bef9SDimitry Andric /// Shadow = ArgTLS+ArgOffset.
626e8d8bef9SDimitry Andric Value *getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB);
627e8d8bef9SDimitry Andric
628fe6060f1SDimitry Andric /// Computes the shadow address for a return value.
629e8d8bef9SDimitry Andric Value *getRetvalTLS(Type *T, IRBuilder<> &IRB);
630e8d8bef9SDimitry Andric
631fe6060f1SDimitry Andric /// Computes the origin address for a given function argument.
632fe6060f1SDimitry Andric ///
633fe6060f1SDimitry Andric /// Origin = ArgOriginTLS[ArgNo].
634fe6060f1SDimitry Andric Value *getArgOriginTLS(unsigned ArgNo, IRBuilder<> &IRB);
635fe6060f1SDimitry Andric
636fe6060f1SDimitry Andric /// Computes the origin address for a return value.
637fe6060f1SDimitry Andric Value *getRetvalOriginTLS();
638fe6060f1SDimitry Andric
639fe6060f1SDimitry Andric Value *getOrigin(Value *V);
640fe6060f1SDimitry Andric void setOrigin(Instruction *I, Value *Origin);
641fe6060f1SDimitry Andric /// Generates IR to compute the origin of the last operand with a taint label.
642fe6060f1SDimitry Andric Value *combineOperandOrigins(Instruction *Inst);
643fe6060f1SDimitry Andric /// Before the instruction Pos, generates IR to compute the last origin with a
644fe6060f1SDimitry Andric /// taint label. Labels and origins are from vectors Shadows and Origins
645fe6060f1SDimitry Andric /// correspondingly. The generated IR is like
646fe6060f1SDimitry Andric /// Sn-1 != Zero ? On-1: ... S2 != Zero ? O2: S1 != Zero ? O1: O0
647fe6060f1SDimitry Andric /// When Zero is nullptr, it uses ZeroPrimitiveShadow. Otherwise it can be
648fe6060f1SDimitry Andric /// zeros with other bitwidths.
649fe6060f1SDimitry Andric Value *combineOrigins(const std::vector<Value *> &Shadows,
650fe6060f1SDimitry Andric const std::vector<Value *> &Origins, Instruction *Pos,
651fe6060f1SDimitry Andric ConstantInt *Zero = nullptr);
652fe6060f1SDimitry Andric
6530b57cec5SDimitry Andric Value *getShadow(Value *V);
6540b57cec5SDimitry Andric void setShadow(Instruction *I, Value *Shadow);
655e8d8bef9SDimitry Andric /// Generates IR to compute the union of the two given shadows, inserting it
656e8d8bef9SDimitry Andric /// before Pos. The combined value is with primitive type.
6570b57cec5SDimitry Andric Value *combineShadows(Value *V1, Value *V2, Instruction *Pos);
658e8d8bef9SDimitry Andric /// Combines the shadow values of V1 and V2, then converts the combined value
659e8d8bef9SDimitry Andric /// with primitive type into a shadow value with the original type T.
660e8d8bef9SDimitry Andric Value *combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
661e8d8bef9SDimitry Andric Instruction *Pos);
6620b57cec5SDimitry Andric Value *combineOperandShadows(Instruction *Inst);
663fe6060f1SDimitry Andric
664fe6060f1SDimitry Andric /// Generates IR to load shadow and origin corresponding to bytes [\p
665fe6060f1SDimitry Andric /// Addr, \p Addr + \p Size), where addr has alignment \p
666fe6060f1SDimitry Andric /// InstAlignment, and take the union of each of those shadows. The returned
667fe6060f1SDimitry Andric /// shadow always has primitive type.
668fe6060f1SDimitry Andric ///
669fe6060f1SDimitry Andric /// When tracking loads is enabled, the returned origin is a chain at the
670fe6060f1SDimitry Andric /// current stack if the returned shadow is tainted.
671fe6060f1SDimitry Andric std::pair<Value *, Value *> loadShadowOrigin(Value *Addr, uint64_t Size,
672fe6060f1SDimitry Andric Align InstAlignment,
6730b57cec5SDimitry Andric Instruction *Pos);
674fe6060f1SDimitry Andric
675fe6060f1SDimitry Andric void storePrimitiveShadowOrigin(Value *Addr, uint64_t Size,
676fe6060f1SDimitry Andric Align InstAlignment, Value *PrimitiveShadow,
677fe6060f1SDimitry Andric Value *Origin, Instruction *Pos);
678e8d8bef9SDimitry Andric /// Applies PrimitiveShadow to all primitive subtypes of T, returning
679e8d8bef9SDimitry Andric /// the expanded shadow value.
680e8d8bef9SDimitry Andric ///
681e8d8bef9SDimitry Andric /// EFP({T1,T2, ...}, PS) = {EFP(T1,PS),EFP(T2,PS),...}
682e8d8bef9SDimitry Andric /// EFP([n x T], PS) = [n x EFP(T,PS)]
683e8d8bef9SDimitry Andric /// EFP(other types, PS) = PS
684e8d8bef9SDimitry Andric Value *expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
6850b57cec5SDimitry Andric Instruction *Pos);
686e8d8bef9SDimitry Andric /// Collapses Shadow into a single primitive shadow value, unioning all
687e8d8bef9SDimitry Andric /// primitive shadow values in the process. Returns the final primitive
688e8d8bef9SDimitry Andric /// shadow value.
689e8d8bef9SDimitry Andric ///
690e8d8bef9SDimitry Andric /// CTP({V1,V2, ...}) = UNION(CFP(V1,PS),CFP(V2,PS),...)
691e8d8bef9SDimitry Andric /// CTP([V1,V2,...]) = UNION(CFP(V1,PS),CFP(V2,PS),...)
692e8d8bef9SDimitry Andric /// CTP(other types, PS) = PS
693e8d8bef9SDimitry Andric Value *collapseToPrimitiveShadow(Value *Shadow, Instruction *Pos);
694e8d8bef9SDimitry Andric
695fe6060f1SDimitry Andric void storeZeroPrimitiveShadow(Value *Addr, uint64_t Size, Align ShadowAlign,
696fe6060f1SDimitry Andric Instruction *Pos);
697fe6060f1SDimitry Andric
698fe6060f1SDimitry Andric Align getShadowAlign(Align InstAlignment);
699fe6060f1SDimitry Andric
70004eeddc0SDimitry Andric // If ClConditionalCallbacks is enabled, insert a callback after a given
70104eeddc0SDimitry Andric // branch instruction using the given conditional expression.
70204eeddc0SDimitry Andric void addConditionalCallbacksIfEnabled(Instruction &I, Value *Condition);
70304eeddc0SDimitry Andric
704bdd1243dSDimitry Andric // If ClReachesFunctionCallbacks is enabled, insert a callback for each
705bdd1243dSDimitry Andric // argument and load instruction.
706bdd1243dSDimitry Andric void addReachesFunctionCallbacksIfEnabled(IRBuilder<> &IRB, Instruction &I,
707bdd1243dSDimitry Andric Value *Data);
708bdd1243dSDimitry Andric
70981ad6265SDimitry Andric bool isLookupTableConstant(Value *P);
71081ad6265SDimitry Andric
711e8d8bef9SDimitry Andric private:
712e8d8bef9SDimitry Andric /// Collapses the shadow with aggregate type into a single primitive shadow
713e8d8bef9SDimitry Andric /// value.
714e8d8bef9SDimitry Andric template <class AggregateType>
715e8d8bef9SDimitry Andric Value *collapseAggregateShadow(AggregateType *AT, Value *Shadow,
716e8d8bef9SDimitry Andric IRBuilder<> &IRB);
717e8d8bef9SDimitry Andric
718e8d8bef9SDimitry Andric Value *collapseToPrimitiveShadow(Value *Shadow, IRBuilder<> &IRB);
719e8d8bef9SDimitry Andric
720e8d8bef9SDimitry Andric /// Returns the shadow value of an argument A.
721e8d8bef9SDimitry Andric Value *getShadowForTLSArgument(Argument *A);
722fe6060f1SDimitry Andric
723fe6060f1SDimitry Andric /// The fast path of loading shadows.
724fe6060f1SDimitry Andric std::pair<Value *, Value *>
725fe6060f1SDimitry Andric loadShadowFast(Value *ShadowAddr, Value *OriginAddr, uint64_t Size,
726fe6060f1SDimitry Andric Align ShadowAlign, Align OriginAlign, Value *FirstOrigin,
727fe6060f1SDimitry Andric Instruction *Pos);
728fe6060f1SDimitry Andric
729fe6060f1SDimitry Andric Align getOriginAlign(Align InstAlignment);
730fe6060f1SDimitry Andric
731fe6060f1SDimitry Andric /// Because 4 contiguous bytes share one 4-byte origin, the most accurate load
732fe6060f1SDimitry Andric /// is __dfsan_load_label_and_origin. This function returns the union of all
733fe6060f1SDimitry Andric /// labels and the origin of the first taint label. However this is an
734fe6060f1SDimitry Andric /// additional call with many instructions. To ensure common cases are fast,
735fe6060f1SDimitry Andric /// checks if it is possible to load labels and origins without using the
736fe6060f1SDimitry Andric /// callback function.
737fe6060f1SDimitry Andric ///
738fe6060f1SDimitry Andric /// When enabling tracking load instructions, we always use
739fe6060f1SDimitry Andric /// __dfsan_load_label_and_origin to reduce code size.
740fe6060f1SDimitry Andric bool useCallbackLoadLabelAndOrigin(uint64_t Size, Align InstAlignment);
741fe6060f1SDimitry Andric
742fe6060f1SDimitry Andric /// Returns a chain at the current stack with previous origin V.
743fe6060f1SDimitry Andric Value *updateOrigin(Value *V, IRBuilder<> &IRB);
744fe6060f1SDimitry Andric
745fe6060f1SDimitry Andric /// Returns a chain at the current stack with previous origin V if Shadow is
746fe6060f1SDimitry Andric /// tainted.
747fe6060f1SDimitry Andric Value *updateOriginIfTainted(Value *Shadow, Value *Origin, IRBuilder<> &IRB);
748fe6060f1SDimitry Andric
749fe6060f1SDimitry Andric /// Creates an Intptr = Origin | Origin << 32 if Intptr's size is 64. Returns
750fe6060f1SDimitry Andric /// Origin otherwise.
751fe6060f1SDimitry Andric Value *originToIntptr(IRBuilder<> &IRB, Value *Origin);
752fe6060f1SDimitry Andric
753fe6060f1SDimitry Andric /// Stores Origin into the address range [StoreOriginAddr, StoreOriginAddr +
754fe6060f1SDimitry Andric /// Size).
755fe6060f1SDimitry Andric void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *StoreOriginAddr,
756fe6060f1SDimitry Andric uint64_t StoreOriginSize, Align Alignment);
757fe6060f1SDimitry Andric
758fe6060f1SDimitry Andric /// Stores Origin in terms of its Shadow value.
759fe6060f1SDimitry Andric /// * Do not write origins for zero shadows because we do not trace origins
760fe6060f1SDimitry Andric /// for untainted sinks.
761fe6060f1SDimitry Andric /// * Use __dfsan_maybe_store_origin if there are too many origin store
762fe6060f1SDimitry Andric /// instrumentations.
763fe6060f1SDimitry Andric void storeOrigin(Instruction *Pos, Value *Addr, uint64_t Size, Value *Shadow,
764fe6060f1SDimitry Andric Value *Origin, Value *StoreOriginAddr, Align InstAlignment);
765fe6060f1SDimitry Andric
766fe6060f1SDimitry Andric /// Convert a scalar value to an i1 by comparing with 0.
767fe6060f1SDimitry Andric Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &Name = "");
768fe6060f1SDimitry Andric
769fe6060f1SDimitry Andric bool shouldInstrumentWithCall();
770fe6060f1SDimitry Andric
771fe6060f1SDimitry Andric /// Generates IR to load shadow and origin corresponding to bytes [\p
772fe6060f1SDimitry Andric /// Addr, \p Addr + \p Size), where addr has alignment \p
773fe6060f1SDimitry Andric /// InstAlignment, and take the union of each of those shadows. The returned
774fe6060f1SDimitry Andric /// shadow always has primitive type.
775fe6060f1SDimitry Andric std::pair<Value *, Value *>
776fe6060f1SDimitry Andric loadShadowOriginSansLoadTracking(Value *Addr, uint64_t Size,
777fe6060f1SDimitry Andric Align InstAlignment, Instruction *Pos);
778fe6060f1SDimitry Andric int NumOriginStores = 0;
7790b57cec5SDimitry Andric };
7800b57cec5SDimitry Andric
7810b57cec5SDimitry Andric class DFSanVisitor : public InstVisitor<DFSanVisitor> {
7820b57cec5SDimitry Andric public:
7830b57cec5SDimitry Andric DFSanFunction &DFSF;
7840b57cec5SDimitry Andric
DFSanVisitor(DFSanFunction & DFSF)7850b57cec5SDimitry Andric DFSanVisitor(DFSanFunction &DFSF) : DFSF(DFSF) {}
7860b57cec5SDimitry Andric
getDataLayout() const7870b57cec5SDimitry Andric const DataLayout &getDataLayout() const {
7880b57cec5SDimitry Andric return DFSF.F->getParent()->getDataLayout();
7890b57cec5SDimitry Andric }
7900b57cec5SDimitry Andric
791fe6060f1SDimitry Andric // Combines shadow values and origins for all of I's operands.
792fe6060f1SDimitry Andric void visitInstOperands(Instruction &I);
7935ffd83dbSDimitry Andric
7940b57cec5SDimitry Andric void visitUnaryOperator(UnaryOperator &UO);
7950b57cec5SDimitry Andric void visitBinaryOperator(BinaryOperator &BO);
796fe6060f1SDimitry Andric void visitBitCastInst(BitCastInst &BCI);
7970b57cec5SDimitry Andric void visitCastInst(CastInst &CI);
7980b57cec5SDimitry Andric void visitCmpInst(CmpInst &CI);
799fe6060f1SDimitry Andric void visitLandingPadInst(LandingPadInst &LPI);
8000b57cec5SDimitry Andric void visitGetElementPtrInst(GetElementPtrInst &GEPI);
8010b57cec5SDimitry Andric void visitLoadInst(LoadInst &LI);
8020b57cec5SDimitry Andric void visitStoreInst(StoreInst &SI);
803fe6060f1SDimitry Andric void visitAtomicRMWInst(AtomicRMWInst &I);
804fe6060f1SDimitry Andric void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
8050b57cec5SDimitry Andric void visitReturnInst(ReturnInst &RI);
806bdd1243dSDimitry Andric void visitLibAtomicLoad(CallBase &CB);
807bdd1243dSDimitry Andric void visitLibAtomicStore(CallBase &CB);
808bdd1243dSDimitry Andric void visitLibAtomicExchange(CallBase &CB);
809bdd1243dSDimitry Andric void visitLibAtomicCompareExchange(CallBase &CB);
8105ffd83dbSDimitry Andric void visitCallBase(CallBase &CB);
8110b57cec5SDimitry Andric void visitPHINode(PHINode &PN);
8120b57cec5SDimitry Andric void visitExtractElementInst(ExtractElementInst &I);
8130b57cec5SDimitry Andric void visitInsertElementInst(InsertElementInst &I);
8140b57cec5SDimitry Andric void visitShuffleVectorInst(ShuffleVectorInst &I);
8150b57cec5SDimitry Andric void visitExtractValueInst(ExtractValueInst &I);
8160b57cec5SDimitry Andric void visitInsertValueInst(InsertValueInst &I);
8170b57cec5SDimitry Andric void visitAllocaInst(AllocaInst &I);
8180b57cec5SDimitry Andric void visitSelectInst(SelectInst &I);
8190b57cec5SDimitry Andric void visitMemSetInst(MemSetInst &I);
8200b57cec5SDimitry Andric void visitMemTransferInst(MemTransferInst &I);
82104eeddc0SDimitry Andric void visitBranchInst(BranchInst &BR);
82204eeddc0SDimitry Andric void visitSwitchInst(SwitchInst &SW);
823fe6060f1SDimitry Andric
824fe6060f1SDimitry Andric private:
825fe6060f1SDimitry Andric void visitCASOrRMW(Align InstAlignment, Instruction &I);
826fe6060f1SDimitry Andric
827fe6060f1SDimitry Andric // Returns false when this is an invoke of a custom function.
828fe6060f1SDimitry Andric bool visitWrappedCallBase(Function &F, CallBase &CB);
829fe6060f1SDimitry Andric
830fe6060f1SDimitry Andric // Combines origins for all of I's operands.
831fe6060f1SDimitry Andric void visitInstOperandOrigins(Instruction &I);
832fe6060f1SDimitry Andric
833fe6060f1SDimitry Andric void addShadowArguments(Function &F, CallBase &CB, std::vector<Value *> &Args,
834fe6060f1SDimitry Andric IRBuilder<> &IRB);
835fe6060f1SDimitry Andric
836fe6060f1SDimitry Andric void addOriginArguments(Function &F, CallBase &CB, std::vector<Value *> &Args,
837fe6060f1SDimitry Andric IRBuilder<> &IRB);
838bdd1243dSDimitry Andric
839bdd1243dSDimitry Andric Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB);
840bdd1243dSDimitry Andric Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB);
8410b57cec5SDimitry Andric };
8420b57cec5SDimitry Andric
LibAtomicFunction(const Function & F)843bdd1243dSDimitry Andric bool LibAtomicFunction(const Function &F) {
844bdd1243dSDimitry Andric // This is a bit of a hack because TargetLibraryInfo is a function pass.
845bdd1243dSDimitry Andric // The DFSan pass would need to be refactored to be function pass oriented
846bdd1243dSDimitry Andric // (like MSan is) in order to fit together nicely with TargetLibraryInfo.
847bdd1243dSDimitry Andric // We need this check to prevent them from being instrumented, or wrapped.
848bdd1243dSDimitry Andric // Match on name and number of arguments.
849bdd1243dSDimitry Andric if (!F.hasName() || F.isVarArg())
850bdd1243dSDimitry Andric return false;
851bdd1243dSDimitry Andric switch (F.arg_size()) {
852bdd1243dSDimitry Andric case 4:
853bdd1243dSDimitry Andric return F.getName() == "__atomic_load" || F.getName() == "__atomic_store";
854bdd1243dSDimitry Andric case 5:
855bdd1243dSDimitry Andric return F.getName() == "__atomic_exchange";
856bdd1243dSDimitry Andric case 6:
857bdd1243dSDimitry Andric return F.getName() == "__atomic_compare_exchange";
858bdd1243dSDimitry Andric default:
859bdd1243dSDimitry Andric return false;
860bdd1243dSDimitry Andric }
861bdd1243dSDimitry Andric }
862bdd1243dSDimitry Andric
8630b57cec5SDimitry Andric } // end anonymous namespace
8640b57cec5SDimitry Andric
DataFlowSanitizer(const std::vector<std::string> & ABIListFiles)8650b57cec5SDimitry Andric DataFlowSanitizer::DataFlowSanitizer(
866e8d8bef9SDimitry Andric const std::vector<std::string> &ABIListFiles) {
8670b57cec5SDimitry Andric std::vector<std::string> AllABIListFiles(std::move(ABIListFiles));
868e8d8bef9SDimitry Andric llvm::append_range(AllABIListFiles, ClABIListFiles);
869480093f4SDimitry Andric // FIXME: should we propagate vfs::FileSystem to this constructor?
870480093f4SDimitry Andric ABIList.set(
871480093f4SDimitry Andric SpecialCaseList::createOrDie(AllABIListFiles, *vfs::getRealFileSystem()));
8720b57cec5SDimitry Andric
87381ad6265SDimitry Andric for (StringRef v : ClCombineTaintLookupTables)
87481ad6265SDimitry Andric CombineTaintLookupTableNames.insert(v);
8750b57cec5SDimitry Andric }
8760b57cec5SDimitry Andric
getCustomFunctionType(FunctionType * T)8770b57cec5SDimitry Andric TransformedFunction DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
8780b57cec5SDimitry Andric SmallVector<Type *, 4> ArgTypes;
8790b57cec5SDimitry Andric
8800b57cec5SDimitry Andric // Some parameters of the custom function being constructed are
8810b57cec5SDimitry Andric // parameters of T. Record the mapping from parameters of T to
8820b57cec5SDimitry Andric // parameters of the custom function, so that parameter attributes
8830b57cec5SDimitry Andric // at call sites can be updated.
8840b57cec5SDimitry Andric std::vector<unsigned> ArgumentIndexMapping;
885fe6060f1SDimitry Andric for (unsigned I = 0, E = T->getNumParams(); I != E; ++I) {
886fe6060f1SDimitry Andric Type *ParamType = T->getParamType(I);
8870b57cec5SDimitry Andric ArgumentIndexMapping.push_back(ArgTypes.size());
888fe6060f1SDimitry Andric ArgTypes.push_back(ParamType);
8890b57cec5SDimitry Andric }
890fe6060f1SDimitry Andric for (unsigned I = 0, E = T->getNumParams(); I != E; ++I)
891e8d8bef9SDimitry Andric ArgTypes.push_back(PrimitiveShadowTy);
8920b57cec5SDimitry Andric if (T->isVarArg())
893e8d8bef9SDimitry Andric ArgTypes.push_back(PrimitiveShadowPtrTy);
8940b57cec5SDimitry Andric Type *RetType = T->getReturnType();
8950b57cec5SDimitry Andric if (!RetType->isVoidTy())
896e8d8bef9SDimitry Andric ArgTypes.push_back(PrimitiveShadowPtrTy);
897fe6060f1SDimitry Andric
898fe6060f1SDimitry Andric if (shouldTrackOrigins()) {
899fe6060f1SDimitry Andric for (unsigned I = 0, E = T->getNumParams(); I != E; ++I)
900fe6060f1SDimitry Andric ArgTypes.push_back(OriginTy);
901fe6060f1SDimitry Andric if (T->isVarArg())
902fe6060f1SDimitry Andric ArgTypes.push_back(OriginPtrTy);
903fe6060f1SDimitry Andric if (!RetType->isVoidTy())
904fe6060f1SDimitry Andric ArgTypes.push_back(OriginPtrTy);
905fe6060f1SDimitry Andric }
906fe6060f1SDimitry Andric
9070b57cec5SDimitry Andric return TransformedFunction(
9080b57cec5SDimitry Andric T, FunctionType::get(T->getReturnType(), ArgTypes, T->isVarArg()),
9090b57cec5SDimitry Andric ArgumentIndexMapping);
9100b57cec5SDimitry Andric }
9110b57cec5SDimitry Andric
isZeroShadow(Value * V)912e8d8bef9SDimitry Andric bool DataFlowSanitizer::isZeroShadow(Value *V) {
913e8d8bef9SDimitry Andric Type *T = V->getType();
914e8d8bef9SDimitry Andric if (!isa<ArrayType>(T) && !isa<StructType>(T)) {
915e8d8bef9SDimitry Andric if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
916e8d8bef9SDimitry Andric return CI->isZero();
917e8d8bef9SDimitry Andric return false;
918e8d8bef9SDimitry Andric }
919e8d8bef9SDimitry Andric
920e8d8bef9SDimitry Andric return isa<ConstantAggregateZero>(V);
921e8d8bef9SDimitry Andric }
922e8d8bef9SDimitry Andric
hasLoadSizeForFastPath(uint64_t Size)923fe6060f1SDimitry Andric bool DataFlowSanitizer::hasLoadSizeForFastPath(uint64_t Size) {
924fe6060f1SDimitry Andric uint64_t ShadowSize = Size * ShadowWidthBytes;
925fe6060f1SDimitry Andric return ShadowSize % 8 == 0 || ShadowSize == 4;
926fe6060f1SDimitry Andric }
927fe6060f1SDimitry Andric
shouldTrackOrigins()928fe6060f1SDimitry Andric bool DataFlowSanitizer::shouldTrackOrigins() {
929349cc55cSDimitry Andric static const bool ShouldTrackOrigins = ClTrackOrigins;
930fe6060f1SDimitry Andric return ShouldTrackOrigins;
931fe6060f1SDimitry Andric }
932fe6060f1SDimitry Andric
getZeroShadow(Type * OrigTy)933e8d8bef9SDimitry Andric Constant *DataFlowSanitizer::getZeroShadow(Type *OrigTy) {
934e8d8bef9SDimitry Andric if (!isa<ArrayType>(OrigTy) && !isa<StructType>(OrigTy))
935e8d8bef9SDimitry Andric return ZeroPrimitiveShadow;
936e8d8bef9SDimitry Andric Type *ShadowTy = getShadowTy(OrigTy);
937e8d8bef9SDimitry Andric return ConstantAggregateZero::get(ShadowTy);
938e8d8bef9SDimitry Andric }
939e8d8bef9SDimitry Andric
getZeroShadow(Value * V)940e8d8bef9SDimitry Andric Constant *DataFlowSanitizer::getZeroShadow(Value *V) {
941e8d8bef9SDimitry Andric return getZeroShadow(V->getType());
942e8d8bef9SDimitry Andric }
943e8d8bef9SDimitry Andric
expandFromPrimitiveShadowRecursive(Value * Shadow,SmallVector<unsigned,4> & Indices,Type * SubShadowTy,Value * PrimitiveShadow,IRBuilder<> & IRB)944e8d8bef9SDimitry Andric static Value *expandFromPrimitiveShadowRecursive(
945e8d8bef9SDimitry Andric Value *Shadow, SmallVector<unsigned, 4> &Indices, Type *SubShadowTy,
946e8d8bef9SDimitry Andric Value *PrimitiveShadow, IRBuilder<> &IRB) {
947e8d8bef9SDimitry Andric if (!isa<ArrayType>(SubShadowTy) && !isa<StructType>(SubShadowTy))
948e8d8bef9SDimitry Andric return IRB.CreateInsertValue(Shadow, PrimitiveShadow, Indices);
949e8d8bef9SDimitry Andric
950e8d8bef9SDimitry Andric if (ArrayType *AT = dyn_cast<ArrayType>(SubShadowTy)) {
951e8d8bef9SDimitry Andric for (unsigned Idx = 0; Idx < AT->getNumElements(); Idx++) {
952e8d8bef9SDimitry Andric Indices.push_back(Idx);
953e8d8bef9SDimitry Andric Shadow = expandFromPrimitiveShadowRecursive(
954e8d8bef9SDimitry Andric Shadow, Indices, AT->getElementType(), PrimitiveShadow, IRB);
955e8d8bef9SDimitry Andric Indices.pop_back();
956e8d8bef9SDimitry Andric }
957e8d8bef9SDimitry Andric return Shadow;
958e8d8bef9SDimitry Andric }
959e8d8bef9SDimitry Andric
960e8d8bef9SDimitry Andric if (StructType *ST = dyn_cast<StructType>(SubShadowTy)) {
961e8d8bef9SDimitry Andric for (unsigned Idx = 0; Idx < ST->getNumElements(); Idx++) {
962e8d8bef9SDimitry Andric Indices.push_back(Idx);
963e8d8bef9SDimitry Andric Shadow = expandFromPrimitiveShadowRecursive(
964e8d8bef9SDimitry Andric Shadow, Indices, ST->getElementType(Idx), PrimitiveShadow, IRB);
965e8d8bef9SDimitry Andric Indices.pop_back();
966e8d8bef9SDimitry Andric }
967e8d8bef9SDimitry Andric return Shadow;
968e8d8bef9SDimitry Andric }
969e8d8bef9SDimitry Andric llvm_unreachable("Unexpected shadow type");
970e8d8bef9SDimitry Andric }
971e8d8bef9SDimitry Andric
shouldInstrumentWithCall()972fe6060f1SDimitry Andric bool DFSanFunction::shouldInstrumentWithCall() {
973fe6060f1SDimitry Andric return ClInstrumentWithCallThreshold >= 0 &&
974fe6060f1SDimitry Andric NumOriginStores >= ClInstrumentWithCallThreshold;
975fe6060f1SDimitry Andric }
976fe6060f1SDimitry Andric
expandFromPrimitiveShadow(Type * T,Value * PrimitiveShadow,Instruction * Pos)977e8d8bef9SDimitry Andric Value *DFSanFunction::expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
978e8d8bef9SDimitry Andric Instruction *Pos) {
979e8d8bef9SDimitry Andric Type *ShadowTy = DFS.getShadowTy(T);
980e8d8bef9SDimitry Andric
981e8d8bef9SDimitry Andric if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
982e8d8bef9SDimitry Andric return PrimitiveShadow;
983e8d8bef9SDimitry Andric
984e8d8bef9SDimitry Andric if (DFS.isZeroShadow(PrimitiveShadow))
985e8d8bef9SDimitry Andric return DFS.getZeroShadow(ShadowTy);
986e8d8bef9SDimitry Andric
987e8d8bef9SDimitry Andric IRBuilder<> IRB(Pos);
988e8d8bef9SDimitry Andric SmallVector<unsigned, 4> Indices;
989e8d8bef9SDimitry Andric Value *Shadow = UndefValue::get(ShadowTy);
990e8d8bef9SDimitry Andric Shadow = expandFromPrimitiveShadowRecursive(Shadow, Indices, ShadowTy,
991e8d8bef9SDimitry Andric PrimitiveShadow, IRB);
992e8d8bef9SDimitry Andric
993e8d8bef9SDimitry Andric // Caches the primitive shadow value that built the shadow value.
994e8d8bef9SDimitry Andric CachedCollapsedShadows[Shadow] = PrimitiveShadow;
995e8d8bef9SDimitry Andric return Shadow;
996e8d8bef9SDimitry Andric }
997e8d8bef9SDimitry Andric
998e8d8bef9SDimitry Andric template <class AggregateType>
collapseAggregateShadow(AggregateType * AT,Value * Shadow,IRBuilder<> & IRB)999e8d8bef9SDimitry Andric Value *DFSanFunction::collapseAggregateShadow(AggregateType *AT, Value *Shadow,
1000e8d8bef9SDimitry Andric IRBuilder<> &IRB) {
1001e8d8bef9SDimitry Andric if (!AT->getNumElements())
1002e8d8bef9SDimitry Andric return DFS.ZeroPrimitiveShadow;
1003e8d8bef9SDimitry Andric
1004e8d8bef9SDimitry Andric Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
1005e8d8bef9SDimitry Andric Value *Aggregator = collapseToPrimitiveShadow(FirstItem, IRB);
1006e8d8bef9SDimitry Andric
1007e8d8bef9SDimitry Andric for (unsigned Idx = 1; Idx < AT->getNumElements(); Idx++) {
1008e8d8bef9SDimitry Andric Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1009e8d8bef9SDimitry Andric Value *ShadowInner = collapseToPrimitiveShadow(ShadowItem, IRB);
1010e8d8bef9SDimitry Andric Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
1011e8d8bef9SDimitry Andric }
1012e8d8bef9SDimitry Andric return Aggregator;
1013e8d8bef9SDimitry Andric }
1014e8d8bef9SDimitry Andric
collapseToPrimitiveShadow(Value * Shadow,IRBuilder<> & IRB)1015e8d8bef9SDimitry Andric Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
1016e8d8bef9SDimitry Andric IRBuilder<> &IRB) {
1017e8d8bef9SDimitry Andric Type *ShadowTy = Shadow->getType();
1018e8d8bef9SDimitry Andric if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
1019e8d8bef9SDimitry Andric return Shadow;
1020e8d8bef9SDimitry Andric if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy))
1021e8d8bef9SDimitry Andric return collapseAggregateShadow<>(AT, Shadow, IRB);
1022e8d8bef9SDimitry Andric if (StructType *ST = dyn_cast<StructType>(ShadowTy))
1023e8d8bef9SDimitry Andric return collapseAggregateShadow<>(ST, Shadow, IRB);
1024e8d8bef9SDimitry Andric llvm_unreachable("Unexpected shadow type");
1025e8d8bef9SDimitry Andric }
1026e8d8bef9SDimitry Andric
collapseToPrimitiveShadow(Value * Shadow,Instruction * Pos)1027e8d8bef9SDimitry Andric Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
1028e8d8bef9SDimitry Andric Instruction *Pos) {
1029e8d8bef9SDimitry Andric Type *ShadowTy = Shadow->getType();
1030e8d8bef9SDimitry Andric if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
1031e8d8bef9SDimitry Andric return Shadow;
1032e8d8bef9SDimitry Andric
1033e8d8bef9SDimitry Andric // Checks if the cached collapsed shadow value dominates Pos.
1034e8d8bef9SDimitry Andric Value *&CS = CachedCollapsedShadows[Shadow];
1035e8d8bef9SDimitry Andric if (CS && DT.dominates(CS, Pos))
1036e8d8bef9SDimitry Andric return CS;
1037e8d8bef9SDimitry Andric
1038e8d8bef9SDimitry Andric IRBuilder<> IRB(Pos);
1039e8d8bef9SDimitry Andric Value *PrimitiveShadow = collapseToPrimitiveShadow(Shadow, IRB);
1040e8d8bef9SDimitry Andric // Caches the converted primitive shadow value.
1041e8d8bef9SDimitry Andric CS = PrimitiveShadow;
1042e8d8bef9SDimitry Andric return PrimitiveShadow;
1043e8d8bef9SDimitry Andric }
1044e8d8bef9SDimitry Andric
addConditionalCallbacksIfEnabled(Instruction & I,Value * Condition)104504eeddc0SDimitry Andric void DFSanFunction::addConditionalCallbacksIfEnabled(Instruction &I,
104604eeddc0SDimitry Andric Value *Condition) {
104704eeddc0SDimitry Andric if (!ClConditionalCallbacks) {
104804eeddc0SDimitry Andric return;
104904eeddc0SDimitry Andric }
105004eeddc0SDimitry Andric IRBuilder<> IRB(&I);
105104eeddc0SDimitry Andric Value *CondShadow = getShadow(Condition);
1052bdd1243dSDimitry Andric CallInst *CI;
105304eeddc0SDimitry Andric if (DFS.shouldTrackOrigins()) {
105404eeddc0SDimitry Andric Value *CondOrigin = getOrigin(Condition);
1055bdd1243dSDimitry Andric CI = IRB.CreateCall(DFS.DFSanConditionalCallbackOriginFn,
105604eeddc0SDimitry Andric {CondShadow, CondOrigin});
105704eeddc0SDimitry Andric } else {
1058bdd1243dSDimitry Andric CI = IRB.CreateCall(DFS.DFSanConditionalCallbackFn, {CondShadow});
105904eeddc0SDimitry Andric }
1060bdd1243dSDimitry Andric CI->addParamAttr(0, Attribute::ZExt);
1061bdd1243dSDimitry Andric }
1062bdd1243dSDimitry Andric
addReachesFunctionCallbacksIfEnabled(IRBuilder<> & IRB,Instruction & I,Value * Data)1063bdd1243dSDimitry Andric void DFSanFunction::addReachesFunctionCallbacksIfEnabled(IRBuilder<> &IRB,
1064bdd1243dSDimitry Andric Instruction &I,
1065bdd1243dSDimitry Andric Value *Data) {
1066bdd1243dSDimitry Andric if (!ClReachesFunctionCallbacks) {
1067bdd1243dSDimitry Andric return;
1068bdd1243dSDimitry Andric }
1069bdd1243dSDimitry Andric const DebugLoc &dbgloc = I.getDebugLoc();
1070bdd1243dSDimitry Andric Value *DataShadow = collapseToPrimitiveShadow(getShadow(Data), IRB);
1071bdd1243dSDimitry Andric ConstantInt *CILine;
1072bdd1243dSDimitry Andric llvm::Value *FilePathPtr;
1073bdd1243dSDimitry Andric
1074bdd1243dSDimitry Andric if (dbgloc.get() == nullptr) {
1075bdd1243dSDimitry Andric CILine = llvm::ConstantInt::get(I.getContext(), llvm::APInt(32, 0));
1076bdd1243dSDimitry Andric FilePathPtr = IRB.CreateGlobalStringPtr(
1077bdd1243dSDimitry Andric I.getFunction()->getParent()->getSourceFileName());
1078bdd1243dSDimitry Andric } else {
1079bdd1243dSDimitry Andric CILine = llvm::ConstantInt::get(I.getContext(),
1080bdd1243dSDimitry Andric llvm::APInt(32, dbgloc.getLine()));
1081bdd1243dSDimitry Andric FilePathPtr =
1082bdd1243dSDimitry Andric IRB.CreateGlobalStringPtr(dbgloc->getFilename());
1083bdd1243dSDimitry Andric }
1084bdd1243dSDimitry Andric
1085bdd1243dSDimitry Andric llvm::Value *FunctionNamePtr =
1086bdd1243dSDimitry Andric IRB.CreateGlobalStringPtr(I.getFunction()->getName());
1087bdd1243dSDimitry Andric
1088bdd1243dSDimitry Andric CallInst *CB;
1089bdd1243dSDimitry Andric std::vector<Value *> args;
1090bdd1243dSDimitry Andric
1091bdd1243dSDimitry Andric if (DFS.shouldTrackOrigins()) {
1092bdd1243dSDimitry Andric Value *DataOrigin = getOrigin(Data);
1093bdd1243dSDimitry Andric args = { DataShadow, DataOrigin, FilePathPtr, CILine, FunctionNamePtr };
1094bdd1243dSDimitry Andric CB = IRB.CreateCall(DFS.DFSanReachesFunctionCallbackOriginFn, args);
1095bdd1243dSDimitry Andric } else {
1096bdd1243dSDimitry Andric args = { DataShadow, FilePathPtr, CILine, FunctionNamePtr };
1097bdd1243dSDimitry Andric CB = IRB.CreateCall(DFS.DFSanReachesFunctionCallbackFn, args);
1098bdd1243dSDimitry Andric }
1099bdd1243dSDimitry Andric CB->addParamAttr(0, Attribute::ZExt);
1100bdd1243dSDimitry Andric CB->setDebugLoc(dbgloc);
110104eeddc0SDimitry Andric }
110204eeddc0SDimitry Andric
getShadowTy(Type * OrigTy)1103e8d8bef9SDimitry Andric Type *DataFlowSanitizer::getShadowTy(Type *OrigTy) {
1104e8d8bef9SDimitry Andric if (!OrigTy->isSized())
1105e8d8bef9SDimitry Andric return PrimitiveShadowTy;
1106e8d8bef9SDimitry Andric if (isa<IntegerType>(OrigTy))
1107e8d8bef9SDimitry Andric return PrimitiveShadowTy;
1108e8d8bef9SDimitry Andric if (isa<VectorType>(OrigTy))
1109e8d8bef9SDimitry Andric return PrimitiveShadowTy;
1110e8d8bef9SDimitry Andric if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy))
1111e8d8bef9SDimitry Andric return ArrayType::get(getShadowTy(AT->getElementType()),
1112e8d8bef9SDimitry Andric AT->getNumElements());
1113e8d8bef9SDimitry Andric if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1114e8d8bef9SDimitry Andric SmallVector<Type *, 4> Elements;
1115e8d8bef9SDimitry Andric for (unsigned I = 0, N = ST->getNumElements(); I < N; ++I)
1116e8d8bef9SDimitry Andric Elements.push_back(getShadowTy(ST->getElementType(I)));
1117e8d8bef9SDimitry Andric return StructType::get(*Ctx, Elements);
1118e8d8bef9SDimitry Andric }
1119e8d8bef9SDimitry Andric return PrimitiveShadowTy;
1120e8d8bef9SDimitry Andric }
1121e8d8bef9SDimitry Andric
getShadowTy(Value * V)1122e8d8bef9SDimitry Andric Type *DataFlowSanitizer::getShadowTy(Value *V) {
1123e8d8bef9SDimitry Andric return getShadowTy(V->getType());
1124e8d8bef9SDimitry Andric }
1125e8d8bef9SDimitry Andric
initializeModule(Module & M)1126fe6060f1SDimitry Andric bool DataFlowSanitizer::initializeModule(Module &M) {
11270b57cec5SDimitry Andric Triple TargetTriple(M.getTargetTriple());
11280b57cec5SDimitry Andric const DataLayout &DL = M.getDataLayout();
11290b57cec5SDimitry Andric
1130fe6060f1SDimitry Andric if (TargetTriple.getOS() != Triple::Linux)
1131fe6060f1SDimitry Andric report_fatal_error("unsupported operating system");
1132bdd1243dSDimitry Andric switch (TargetTriple.getArch()) {
1133bdd1243dSDimitry Andric case Triple::aarch64:
1134bdd1243dSDimitry Andric MapParams = &Linux_AArch64_MemoryMapParams;
1135bdd1243dSDimitry Andric break;
1136bdd1243dSDimitry Andric case Triple::x86_64:
1137fe6060f1SDimitry Andric MapParams = &Linux_X86_64_MemoryMapParams;
1138bdd1243dSDimitry Andric break;
113906c3fb27SDimitry Andric case Triple::loongarch64:
114006c3fb27SDimitry Andric MapParams = &Linux_LoongArch64_MemoryMapParams;
114106c3fb27SDimitry Andric break;
1142bdd1243dSDimitry Andric default:
1143bdd1243dSDimitry Andric report_fatal_error("unsupported architecture");
1144bdd1243dSDimitry Andric }
1145fe6060f1SDimitry Andric
11460b57cec5SDimitry Andric Mod = &M;
11470b57cec5SDimitry Andric Ctx = &M.getContext();
11485f757f3fSDimitry Andric Int8Ptr = PointerType::getUnqual(*Ctx);
1149fe6060f1SDimitry Andric OriginTy = IntegerType::get(*Ctx, OriginWidthBits);
1150fe6060f1SDimitry Andric OriginPtrTy = PointerType::getUnqual(OriginTy);
1151e8d8bef9SDimitry Andric PrimitiveShadowTy = IntegerType::get(*Ctx, ShadowWidthBits);
1152e8d8bef9SDimitry Andric PrimitiveShadowPtrTy = PointerType::getUnqual(PrimitiveShadowTy);
11530b57cec5SDimitry Andric IntptrTy = DL.getIntPtrType(*Ctx);
1154e8d8bef9SDimitry Andric ZeroPrimitiveShadow = ConstantInt::getSigned(PrimitiveShadowTy, 0);
1155fe6060f1SDimitry Andric ZeroOrigin = ConstantInt::getSigned(OriginTy, 0);
11560b57cec5SDimitry Andric
1157e8d8bef9SDimitry Andric Type *DFSanUnionLoadArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
1158e8d8bef9SDimitry Andric DFSanUnionLoadFnTy = FunctionType::get(PrimitiveShadowTy, DFSanUnionLoadArgs,
1159e8d8bef9SDimitry Andric /*isVarArg=*/false);
1160fe6060f1SDimitry Andric Type *DFSanLoadLabelAndOriginArgs[2] = {Int8Ptr, IntptrTy};
1161fe6060f1SDimitry Andric DFSanLoadLabelAndOriginFnTy =
1162fe6060f1SDimitry Andric FunctionType::get(IntegerType::get(*Ctx, 64), DFSanLoadLabelAndOriginArgs,
1163fe6060f1SDimitry Andric /*isVarArg=*/false);
11640b57cec5SDimitry Andric DFSanUnimplementedFnTy = FunctionType::get(
11655f757f3fSDimitry Andric Type::getVoidTy(*Ctx), PointerType::getUnqual(*Ctx), /*isVarArg=*/false);
116681ad6265SDimitry Andric Type *DFSanWrapperExternWeakNullArgs[2] = {Int8Ptr, Int8Ptr};
116781ad6265SDimitry Andric DFSanWrapperExternWeakNullFnTy =
116881ad6265SDimitry Andric FunctionType::get(Type::getVoidTy(*Ctx), DFSanWrapperExternWeakNullArgs,
116981ad6265SDimitry Andric /*isVarArg=*/false);
1170fe6060f1SDimitry Andric Type *DFSanSetLabelArgs[4] = {PrimitiveShadowTy, OriginTy,
11715f757f3fSDimitry Andric PointerType::getUnqual(*Ctx), IntptrTy};
11720b57cec5SDimitry Andric DFSanSetLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx),
11730b57cec5SDimitry Andric DFSanSetLabelArgs, /*isVarArg=*/false);
1174bdd1243dSDimitry Andric DFSanNonzeroLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx), std::nullopt,
1175bdd1243dSDimitry Andric /*isVarArg=*/false);
11760b57cec5SDimitry Andric DFSanVarargWrapperFnTy = FunctionType::get(
11775f757f3fSDimitry Andric Type::getVoidTy(*Ctx), PointerType::getUnqual(*Ctx), /*isVarArg=*/false);
117804eeddc0SDimitry Andric DFSanConditionalCallbackFnTy =
117904eeddc0SDimitry Andric FunctionType::get(Type::getVoidTy(*Ctx), PrimitiveShadowTy,
118004eeddc0SDimitry Andric /*isVarArg=*/false);
118104eeddc0SDimitry Andric Type *DFSanConditionalCallbackOriginArgs[2] = {PrimitiveShadowTy, OriginTy};
118204eeddc0SDimitry Andric DFSanConditionalCallbackOriginFnTy = FunctionType::get(
118304eeddc0SDimitry Andric Type::getVoidTy(*Ctx), DFSanConditionalCallbackOriginArgs,
118404eeddc0SDimitry Andric /*isVarArg=*/false);
1185bdd1243dSDimitry Andric Type *DFSanReachesFunctionCallbackArgs[4] = {PrimitiveShadowTy, Int8Ptr,
1186bdd1243dSDimitry Andric OriginTy, Int8Ptr};
1187bdd1243dSDimitry Andric DFSanReachesFunctionCallbackFnTy =
1188bdd1243dSDimitry Andric FunctionType::get(Type::getVoidTy(*Ctx), DFSanReachesFunctionCallbackArgs,
1189bdd1243dSDimitry Andric /*isVarArg=*/false);
1190bdd1243dSDimitry Andric Type *DFSanReachesFunctionCallbackOriginArgs[5] = {
1191bdd1243dSDimitry Andric PrimitiveShadowTy, OriginTy, Int8Ptr, OriginTy, Int8Ptr};
1192bdd1243dSDimitry Andric DFSanReachesFunctionCallbackOriginFnTy = FunctionType::get(
1193bdd1243dSDimitry Andric Type::getVoidTy(*Ctx), DFSanReachesFunctionCallbackOriginArgs,
1194bdd1243dSDimitry Andric /*isVarArg=*/false);
1195e8d8bef9SDimitry Andric DFSanCmpCallbackFnTy =
1196e8d8bef9SDimitry Andric FunctionType::get(Type::getVoidTy(*Ctx), PrimitiveShadowTy,
1197e8d8bef9SDimitry Andric /*isVarArg=*/false);
1198fe6060f1SDimitry Andric DFSanChainOriginFnTy =
1199fe6060f1SDimitry Andric FunctionType::get(OriginTy, OriginTy, /*isVarArg=*/false);
1200fe6060f1SDimitry Andric Type *DFSanChainOriginIfTaintedArgs[2] = {PrimitiveShadowTy, OriginTy};
1201fe6060f1SDimitry Andric DFSanChainOriginIfTaintedFnTy = FunctionType::get(
1202fe6060f1SDimitry Andric OriginTy, DFSanChainOriginIfTaintedArgs, /*isVarArg=*/false);
1203fe6060f1SDimitry Andric Type *DFSanMaybeStoreOriginArgs[4] = {IntegerType::get(*Ctx, ShadowWidthBits),
1204fe6060f1SDimitry Andric Int8Ptr, IntptrTy, OriginTy};
1205fe6060f1SDimitry Andric DFSanMaybeStoreOriginFnTy = FunctionType::get(
1206fe6060f1SDimitry Andric Type::getVoidTy(*Ctx), DFSanMaybeStoreOriginArgs, /*isVarArg=*/false);
1207fe6060f1SDimitry Andric Type *DFSanMemOriginTransferArgs[3] = {Int8Ptr, Int8Ptr, IntptrTy};
1208fe6060f1SDimitry Andric DFSanMemOriginTransferFnTy = FunctionType::get(
1209fe6060f1SDimitry Andric Type::getVoidTy(*Ctx), DFSanMemOriginTransferArgs, /*isVarArg=*/false);
1210bdd1243dSDimitry Andric Type *DFSanMemShadowOriginTransferArgs[3] = {Int8Ptr, Int8Ptr, IntptrTy};
1211bdd1243dSDimitry Andric DFSanMemShadowOriginTransferFnTy =
1212bdd1243dSDimitry Andric FunctionType::get(Type::getVoidTy(*Ctx), DFSanMemShadowOriginTransferArgs,
1213bdd1243dSDimitry Andric /*isVarArg=*/false);
1214bdd1243dSDimitry Andric Type *DFSanMemShadowOriginConditionalExchangeArgs[5] = {
1215bdd1243dSDimitry Andric IntegerType::get(*Ctx, 8), Int8Ptr, Int8Ptr, Int8Ptr, IntptrTy};
1216bdd1243dSDimitry Andric DFSanMemShadowOriginConditionalExchangeFnTy = FunctionType::get(
1217bdd1243dSDimitry Andric Type::getVoidTy(*Ctx), DFSanMemShadowOriginConditionalExchangeArgs,
1218bdd1243dSDimitry Andric /*isVarArg=*/false);
1219e8d8bef9SDimitry Andric Type *DFSanLoadStoreCallbackArgs[2] = {PrimitiveShadowTy, Int8Ptr};
1220e8d8bef9SDimitry Andric DFSanLoadStoreCallbackFnTy =
1221e8d8bef9SDimitry Andric FunctionType::get(Type::getVoidTy(*Ctx), DFSanLoadStoreCallbackArgs,
1222e8d8bef9SDimitry Andric /*isVarArg=*/false);
1223e8d8bef9SDimitry Andric Type *DFSanMemTransferCallbackArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
12245ffd83dbSDimitry Andric DFSanMemTransferCallbackFnTy =
12255ffd83dbSDimitry Andric FunctionType::get(Type::getVoidTy(*Ctx), DFSanMemTransferCallbackArgs,
12265ffd83dbSDimitry Andric /*isVarArg=*/false);
12270b57cec5SDimitry Andric
12280b57cec5SDimitry Andric ColdCallWeights = MDBuilder(*Ctx).createBranchWeights(1, 1000);
1229fe6060f1SDimitry Andric OriginStoreWeights = MDBuilder(*Ctx).createBranchWeights(1, 1000);
12300b57cec5SDimitry Andric return true;
12310b57cec5SDimitry Andric }
12320b57cec5SDimitry Andric
isInstrumented(const Function * F)12330b57cec5SDimitry Andric bool DataFlowSanitizer::isInstrumented(const Function *F) {
12340b57cec5SDimitry Andric return !ABIList.isIn(*F, "uninstrumented");
12350b57cec5SDimitry Andric }
12360b57cec5SDimitry Andric
isInstrumented(const GlobalAlias * GA)12370b57cec5SDimitry Andric bool DataFlowSanitizer::isInstrumented(const GlobalAlias *GA) {
12380b57cec5SDimitry Andric return !ABIList.isIn(*GA, "uninstrumented");
12390b57cec5SDimitry Andric }
12400b57cec5SDimitry Andric
isForceZeroLabels(const Function * F)1241349cc55cSDimitry Andric bool DataFlowSanitizer::isForceZeroLabels(const Function *F) {
1242349cc55cSDimitry Andric return ABIList.isIn(*F, "force_zero_labels");
12430b57cec5SDimitry Andric }
12440b57cec5SDimitry Andric
getWrapperKind(Function * F)12450b57cec5SDimitry Andric DataFlowSanitizer::WrapperKind DataFlowSanitizer::getWrapperKind(Function *F) {
12460b57cec5SDimitry Andric if (ABIList.isIn(*F, "functional"))
12470b57cec5SDimitry Andric return WK_Functional;
12480b57cec5SDimitry Andric if (ABIList.isIn(*F, "discard"))
12490b57cec5SDimitry Andric return WK_Discard;
12500b57cec5SDimitry Andric if (ABIList.isIn(*F, "custom"))
12510b57cec5SDimitry Andric return WK_Custom;
12520b57cec5SDimitry Andric
12530b57cec5SDimitry Andric return WK_Warning;
12540b57cec5SDimitry Andric }
12550b57cec5SDimitry Andric
addGlobalNameSuffix(GlobalValue * GV)1256fe6060f1SDimitry Andric void DataFlowSanitizer::addGlobalNameSuffix(GlobalValue *GV) {
1257fe6060f1SDimitry Andric std::string GVName = std::string(GV->getName()), Suffix = ".dfsan";
1258fe6060f1SDimitry Andric GV->setName(GVName + Suffix);
12590b57cec5SDimitry Andric
12600b57cec5SDimitry Andric // Try to change the name of the function in module inline asm. We only do
12610b57cec5SDimitry Andric // this for specific asm directives, currently only ".symver", to try to avoid
12620b57cec5SDimitry Andric // corrupting asm which happens to contain the symbol name as a substring.
12630b57cec5SDimitry Andric // Note that the substitution for .symver assumes that the versioned symbol
12640b57cec5SDimitry Andric // also has an instrumented name.
12650b57cec5SDimitry Andric std::string Asm = GV->getParent()->getModuleInlineAsm();
12660b57cec5SDimitry Andric std::string SearchStr = ".symver " + GVName + ",";
12670b57cec5SDimitry Andric size_t Pos = Asm.find(SearchStr);
12680b57cec5SDimitry Andric if (Pos != std::string::npos) {
1269fe6060f1SDimitry Andric Asm.replace(Pos, SearchStr.size(), ".symver " + GVName + Suffix + ",");
127006c3fb27SDimitry Andric Pos = Asm.find('@');
1271fe6060f1SDimitry Andric
1272fe6060f1SDimitry Andric if (Pos == std::string::npos)
1273349cc55cSDimitry Andric report_fatal_error(Twine("unsupported .symver: ", Asm));
1274fe6060f1SDimitry Andric
1275fe6060f1SDimitry Andric Asm.replace(Pos, 1, Suffix + "@");
12760b57cec5SDimitry Andric GV->getParent()->setModuleInlineAsm(Asm);
12770b57cec5SDimitry Andric }
12780b57cec5SDimitry Andric }
12790b57cec5SDimitry Andric
buildExternWeakCheckIfNeeded(IRBuilder<> & IRB,Function * F)128081ad6265SDimitry Andric void DataFlowSanitizer::buildExternWeakCheckIfNeeded(IRBuilder<> &IRB,
128181ad6265SDimitry Andric Function *F) {
128281ad6265SDimitry Andric // If the function we are wrapping was ExternWeak, it may be null.
128381ad6265SDimitry Andric // The original code before calling this wrapper may have checked for null,
128481ad6265SDimitry Andric // but replacing with a known-to-not-be-null wrapper can break this check.
128581ad6265SDimitry Andric // When replacing uses of the extern weak function with the wrapper we try
128681ad6265SDimitry Andric // to avoid replacing uses in conditionals, but this is not perfect.
1287bdd1243dSDimitry Andric // In the case where we fail, and accidentally optimize out a null check
128881ad6265SDimitry Andric // for a extern weak function, add a check here to help identify the issue.
128981ad6265SDimitry Andric if (GlobalValue::isExternalWeakLinkage(F->getLinkage())) {
129081ad6265SDimitry Andric std::vector<Value *> Args;
12915f757f3fSDimitry Andric Args.push_back(F);
129281ad6265SDimitry Andric Args.push_back(IRB.CreateGlobalStringPtr(F->getName()));
129381ad6265SDimitry Andric IRB.CreateCall(DFSanWrapperExternWeakNullFn, Args);
129481ad6265SDimitry Andric }
129581ad6265SDimitry Andric }
129681ad6265SDimitry Andric
12970b57cec5SDimitry Andric Function *
buildWrapperFunction(Function * F,StringRef NewFName,GlobalValue::LinkageTypes NewFLink,FunctionType * NewFT)12980b57cec5SDimitry Andric DataFlowSanitizer::buildWrapperFunction(Function *F, StringRef NewFName,
12990b57cec5SDimitry Andric GlobalValue::LinkageTypes NewFLink,
13000b57cec5SDimitry Andric FunctionType *NewFT) {
13010b57cec5SDimitry Andric FunctionType *FT = F->getFunctionType();
13020b57cec5SDimitry Andric Function *NewF = Function::Create(NewFT, NewFLink, F->getAddressSpace(),
13030b57cec5SDimitry Andric NewFName, F->getParent());
13040b57cec5SDimitry Andric NewF->copyAttributesFrom(F);
1305349cc55cSDimitry Andric NewF->removeRetAttrs(
13060b57cec5SDimitry Andric AttributeFuncs::typeIncompatible(NewFT->getReturnType()));
13070b57cec5SDimitry Andric
13080b57cec5SDimitry Andric BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", NewF);
13090b57cec5SDimitry Andric if (F->isVarArg()) {
13100eae32dcSDimitry Andric NewF->removeFnAttr("split-stack");
13110b57cec5SDimitry Andric CallInst::Create(DFSanVarargWrapperFn,
13120b57cec5SDimitry Andric IRBuilder<>(BB).CreateGlobalStringPtr(F->getName()), "",
13130b57cec5SDimitry Andric BB);
13140b57cec5SDimitry Andric new UnreachableInst(*Ctx, BB);
13150b57cec5SDimitry Andric } else {
1316fe6060f1SDimitry Andric auto ArgIt = pointer_iterator<Argument *>(NewF->arg_begin());
1317fe6060f1SDimitry Andric std::vector<Value *> Args(ArgIt, ArgIt + FT->getNumParams());
1318fe6060f1SDimitry Andric
13190b57cec5SDimitry Andric CallInst *CI = CallInst::Create(F, Args, "", BB);
13200b57cec5SDimitry Andric if (FT->getReturnType()->isVoidTy())
13210b57cec5SDimitry Andric ReturnInst::Create(*Ctx, BB);
13220b57cec5SDimitry Andric else
13230b57cec5SDimitry Andric ReturnInst::Create(*Ctx, CI, BB);
13240b57cec5SDimitry Andric }
13250b57cec5SDimitry Andric
13260b57cec5SDimitry Andric return NewF;
13270b57cec5SDimitry Andric }
13280b57cec5SDimitry Andric
13295ffd83dbSDimitry Andric // Initialize DataFlowSanitizer runtime functions and declare them in the module
initializeRuntimeFunctions(Module & M)13305ffd83dbSDimitry Andric void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
1331bdd1243dSDimitry Andric LLVMContext &C = M.getContext();
13320b57cec5SDimitry Andric {
13330b57cec5SDimitry Andric AttributeList AL;
1334bdd1243dSDimitry Andric AL = AL.addFnAttribute(C, Attribute::NoUnwind);
1335bdd1243dSDimitry Andric AL = AL.addFnAttribute(
1336bdd1243dSDimitry Andric C, Attribute::getWithMemoryEffects(C, MemoryEffects::readOnly()));
1337bdd1243dSDimitry Andric AL = AL.addRetAttribute(C, Attribute::ZExt);
13380b57cec5SDimitry Andric DFSanUnionLoadFn =
13390b57cec5SDimitry Andric Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL);
13400b57cec5SDimitry Andric }
1341e8d8bef9SDimitry Andric {
1342e8d8bef9SDimitry Andric AttributeList AL;
1343bdd1243dSDimitry Andric AL = AL.addFnAttribute(C, Attribute::NoUnwind);
1344bdd1243dSDimitry Andric AL = AL.addFnAttribute(
1345bdd1243dSDimitry Andric C, Attribute::getWithMemoryEffects(C, MemoryEffects::readOnly()));
1346bdd1243dSDimitry Andric AL = AL.addRetAttribute(C, Attribute::ZExt);
1347fe6060f1SDimitry Andric DFSanLoadLabelAndOriginFn = Mod->getOrInsertFunction(
1348fe6060f1SDimitry Andric "__dfsan_load_label_and_origin", DFSanLoadLabelAndOriginFnTy, AL);
1349e8d8bef9SDimitry Andric }
13500b57cec5SDimitry Andric DFSanUnimplementedFn =
13510b57cec5SDimitry Andric Mod->getOrInsertFunction("__dfsan_unimplemented", DFSanUnimplementedFnTy);
135281ad6265SDimitry Andric DFSanWrapperExternWeakNullFn = Mod->getOrInsertFunction(
135381ad6265SDimitry Andric "__dfsan_wrapper_extern_weak_null", DFSanWrapperExternWeakNullFnTy);
13540b57cec5SDimitry Andric {
13550b57cec5SDimitry Andric AttributeList AL;
13560b57cec5SDimitry Andric AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1357fe6060f1SDimitry Andric AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
13580b57cec5SDimitry Andric DFSanSetLabelFn =
13590b57cec5SDimitry Andric Mod->getOrInsertFunction("__dfsan_set_label", DFSanSetLabelFnTy, AL);
13600b57cec5SDimitry Andric }
13610b57cec5SDimitry Andric DFSanNonzeroLabelFn =
13620b57cec5SDimitry Andric Mod->getOrInsertFunction("__dfsan_nonzero_label", DFSanNonzeroLabelFnTy);
13630b57cec5SDimitry Andric DFSanVarargWrapperFn = Mod->getOrInsertFunction("__dfsan_vararg_wrapper",
13640b57cec5SDimitry Andric DFSanVarargWrapperFnTy);
1365fe6060f1SDimitry Andric {
1366fe6060f1SDimitry Andric AttributeList AL;
1367fe6060f1SDimitry Andric AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1368349cc55cSDimitry Andric AL = AL.addRetAttribute(M.getContext(), Attribute::ZExt);
1369fe6060f1SDimitry Andric DFSanChainOriginFn = Mod->getOrInsertFunction("__dfsan_chain_origin",
1370fe6060f1SDimitry Andric DFSanChainOriginFnTy, AL);
1371fe6060f1SDimitry Andric }
1372fe6060f1SDimitry Andric {
1373fe6060f1SDimitry Andric AttributeList AL;
1374fe6060f1SDimitry Andric AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1375fe6060f1SDimitry Andric AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
1376349cc55cSDimitry Andric AL = AL.addRetAttribute(M.getContext(), Attribute::ZExt);
1377fe6060f1SDimitry Andric DFSanChainOriginIfTaintedFn = Mod->getOrInsertFunction(
1378fe6060f1SDimitry Andric "__dfsan_chain_origin_if_tainted", DFSanChainOriginIfTaintedFnTy, AL);
1379fe6060f1SDimitry Andric }
1380fe6060f1SDimitry Andric DFSanMemOriginTransferFn = Mod->getOrInsertFunction(
1381fe6060f1SDimitry Andric "__dfsan_mem_origin_transfer", DFSanMemOriginTransferFnTy);
1382fe6060f1SDimitry Andric
1383bdd1243dSDimitry Andric DFSanMemShadowOriginTransferFn = Mod->getOrInsertFunction(
1384bdd1243dSDimitry Andric "__dfsan_mem_shadow_origin_transfer", DFSanMemShadowOriginTransferFnTy);
1385bdd1243dSDimitry Andric
1386bdd1243dSDimitry Andric DFSanMemShadowOriginConditionalExchangeFn =
1387bdd1243dSDimitry Andric Mod->getOrInsertFunction("__dfsan_mem_shadow_origin_conditional_exchange",
1388bdd1243dSDimitry Andric DFSanMemShadowOriginConditionalExchangeFnTy);
1389bdd1243dSDimitry Andric
1390fe6060f1SDimitry Andric {
1391fe6060f1SDimitry Andric AttributeList AL;
1392fe6060f1SDimitry Andric AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1393fe6060f1SDimitry Andric AL = AL.addParamAttribute(M.getContext(), 3, Attribute::ZExt);
1394fe6060f1SDimitry Andric DFSanMaybeStoreOriginFn = Mod->getOrInsertFunction(
1395fe6060f1SDimitry Andric "__dfsan_maybe_store_origin", DFSanMaybeStoreOriginFnTy, AL);
1396fe6060f1SDimitry Andric }
1397fe6060f1SDimitry Andric
1398fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1399fe6060f1SDimitry Andric DFSanUnionLoadFn.getCallee()->stripPointerCasts());
1400fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1401fe6060f1SDimitry Andric DFSanLoadLabelAndOriginFn.getCallee()->stripPointerCasts());
1402fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1403fe6060f1SDimitry Andric DFSanUnimplementedFn.getCallee()->stripPointerCasts());
1404fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
140581ad6265SDimitry Andric DFSanWrapperExternWeakNullFn.getCallee()->stripPointerCasts());
140681ad6265SDimitry Andric DFSanRuntimeFunctions.insert(
1407fe6060f1SDimitry Andric DFSanSetLabelFn.getCallee()->stripPointerCasts());
1408fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1409fe6060f1SDimitry Andric DFSanNonzeroLabelFn.getCallee()->stripPointerCasts());
1410fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1411fe6060f1SDimitry Andric DFSanVarargWrapperFn.getCallee()->stripPointerCasts());
1412fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1413fe6060f1SDimitry Andric DFSanLoadCallbackFn.getCallee()->stripPointerCasts());
1414fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1415fe6060f1SDimitry Andric DFSanStoreCallbackFn.getCallee()->stripPointerCasts());
1416fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1417fe6060f1SDimitry Andric DFSanMemTransferCallbackFn.getCallee()->stripPointerCasts());
1418fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
141904eeddc0SDimitry Andric DFSanConditionalCallbackFn.getCallee()->stripPointerCasts());
142004eeddc0SDimitry Andric DFSanRuntimeFunctions.insert(
142104eeddc0SDimitry Andric DFSanConditionalCallbackOriginFn.getCallee()->stripPointerCasts());
142204eeddc0SDimitry Andric DFSanRuntimeFunctions.insert(
1423bdd1243dSDimitry Andric DFSanReachesFunctionCallbackFn.getCallee()->stripPointerCasts());
1424bdd1243dSDimitry Andric DFSanRuntimeFunctions.insert(
1425bdd1243dSDimitry Andric DFSanReachesFunctionCallbackOriginFn.getCallee()->stripPointerCasts());
1426bdd1243dSDimitry Andric DFSanRuntimeFunctions.insert(
1427fe6060f1SDimitry Andric DFSanCmpCallbackFn.getCallee()->stripPointerCasts());
1428fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1429fe6060f1SDimitry Andric DFSanChainOriginFn.getCallee()->stripPointerCasts());
1430fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1431fe6060f1SDimitry Andric DFSanChainOriginIfTaintedFn.getCallee()->stripPointerCasts());
1432fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1433fe6060f1SDimitry Andric DFSanMemOriginTransferFn.getCallee()->stripPointerCasts());
1434fe6060f1SDimitry Andric DFSanRuntimeFunctions.insert(
1435bdd1243dSDimitry Andric DFSanMemShadowOriginTransferFn.getCallee()->stripPointerCasts());
1436bdd1243dSDimitry Andric DFSanRuntimeFunctions.insert(
1437bdd1243dSDimitry Andric DFSanMemShadowOriginConditionalExchangeFn.getCallee()
1438bdd1243dSDimitry Andric ->stripPointerCasts());
1439bdd1243dSDimitry Andric DFSanRuntimeFunctions.insert(
1440fe6060f1SDimitry Andric DFSanMaybeStoreOriginFn.getCallee()->stripPointerCasts());
14415ffd83dbSDimitry Andric }
14425ffd83dbSDimitry Andric
14435ffd83dbSDimitry Andric // Initializes event callback functions and declare them in the module
initializeCallbackFunctions(Module & M)14445ffd83dbSDimitry Andric void DataFlowSanitizer::initializeCallbackFunctions(Module &M) {
1445bdd1243dSDimitry Andric {
1446bdd1243dSDimitry Andric AttributeList AL;
1447bdd1243dSDimitry Andric AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1448bdd1243dSDimitry Andric DFSanLoadCallbackFn = Mod->getOrInsertFunction(
1449bdd1243dSDimitry Andric "__dfsan_load_callback", DFSanLoadStoreCallbackFnTy, AL);
1450bdd1243dSDimitry Andric }
1451bdd1243dSDimitry Andric {
1452bdd1243dSDimitry Andric AttributeList AL;
1453bdd1243dSDimitry Andric AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1454bdd1243dSDimitry Andric DFSanStoreCallbackFn = Mod->getOrInsertFunction(
1455bdd1243dSDimitry Andric "__dfsan_store_callback", DFSanLoadStoreCallbackFnTy, AL);
1456bdd1243dSDimitry Andric }
14575ffd83dbSDimitry Andric DFSanMemTransferCallbackFn = Mod->getOrInsertFunction(
14585ffd83dbSDimitry Andric "__dfsan_mem_transfer_callback", DFSanMemTransferCallbackFnTy);
1459bdd1243dSDimitry Andric {
1460bdd1243dSDimitry Andric AttributeList AL;
1461bdd1243dSDimitry Andric AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1462bdd1243dSDimitry Andric DFSanCmpCallbackFn = Mod->getOrInsertFunction("__dfsan_cmp_callback",
1463bdd1243dSDimitry Andric DFSanCmpCallbackFnTy, AL);
1464bdd1243dSDimitry Andric }
1465bdd1243dSDimitry Andric {
1466bdd1243dSDimitry Andric AttributeList AL;
1467bdd1243dSDimitry Andric AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
146804eeddc0SDimitry Andric DFSanConditionalCallbackFn = Mod->getOrInsertFunction(
1469bdd1243dSDimitry Andric "__dfsan_conditional_callback", DFSanConditionalCallbackFnTy, AL);
1470bdd1243dSDimitry Andric }
1471bdd1243dSDimitry Andric {
1472bdd1243dSDimitry Andric AttributeList AL;
1473bdd1243dSDimitry Andric AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
147404eeddc0SDimitry Andric DFSanConditionalCallbackOriginFn =
147504eeddc0SDimitry Andric Mod->getOrInsertFunction("__dfsan_conditional_callback_origin",
1476bdd1243dSDimitry Andric DFSanConditionalCallbackOriginFnTy, AL);
1477bdd1243dSDimitry Andric }
1478bdd1243dSDimitry Andric {
1479bdd1243dSDimitry Andric AttributeList AL;
1480bdd1243dSDimitry Andric AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1481bdd1243dSDimitry Andric DFSanReachesFunctionCallbackFn =
1482bdd1243dSDimitry Andric Mod->getOrInsertFunction("__dfsan_reaches_function_callback",
1483bdd1243dSDimitry Andric DFSanReachesFunctionCallbackFnTy, AL);
1484bdd1243dSDimitry Andric }
1485bdd1243dSDimitry Andric {
1486bdd1243dSDimitry Andric AttributeList AL;
1487bdd1243dSDimitry Andric AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1488bdd1243dSDimitry Andric DFSanReachesFunctionCallbackOriginFn =
1489bdd1243dSDimitry Andric Mod->getOrInsertFunction("__dfsan_reaches_function_callback_origin",
1490bdd1243dSDimitry Andric DFSanReachesFunctionCallbackOriginFnTy, AL);
1491bdd1243dSDimitry Andric }
14925ffd83dbSDimitry Andric }
14935ffd83dbSDimitry Andric
runImpl(Module & M,llvm::function_ref<TargetLibraryInfo & (Function &)> GetTLI)1494bdd1243dSDimitry Andric bool DataFlowSanitizer::runImpl(
1495bdd1243dSDimitry Andric Module &M, llvm::function_ref<TargetLibraryInfo &(Function &)> GetTLI) {
1496fe6060f1SDimitry Andric initializeModule(M);
1497e8d8bef9SDimitry Andric
14985ffd83dbSDimitry Andric if (ABIList.isIn(M, "skip"))
14995ffd83dbSDimitry Andric return false;
15005ffd83dbSDimitry Andric
15015ffd83dbSDimitry Andric const unsigned InitialGlobalSize = M.global_size();
15025ffd83dbSDimitry Andric const unsigned InitialModuleSize = M.size();
15035ffd83dbSDimitry Andric
15045ffd83dbSDimitry Andric bool Changed = false;
15055ffd83dbSDimitry Andric
1506fe6060f1SDimitry Andric auto GetOrInsertGlobal = [this, &Changed](StringRef Name,
1507fe6060f1SDimitry Andric Type *Ty) -> Constant * {
1508fe6060f1SDimitry Andric Constant *C = Mod->getOrInsertGlobal(Name, Ty);
1509fe6060f1SDimitry Andric if (GlobalVariable *G = dyn_cast<GlobalVariable>(C)) {
15105ffd83dbSDimitry Andric Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
15115ffd83dbSDimitry Andric G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
15125ffd83dbSDimitry Andric }
1513fe6060f1SDimitry Andric return C;
1514fe6060f1SDimitry Andric };
15155ffd83dbSDimitry Andric
1516fe6060f1SDimitry Andric // These globals must be kept in sync with the ones in dfsan.cpp.
1517fe6060f1SDimitry Andric ArgTLS =
1518fe6060f1SDimitry Andric GetOrInsertGlobal("__dfsan_arg_tls",
1519fe6060f1SDimitry Andric ArrayType::get(Type::getInt64Ty(*Ctx), ArgTLSSize / 8));
1520fe6060f1SDimitry Andric RetvalTLS = GetOrInsertGlobal(
1521fe6060f1SDimitry Andric "__dfsan_retval_tls",
1522fe6060f1SDimitry Andric ArrayType::get(Type::getInt64Ty(*Ctx), RetvalTLSSize / 8));
1523fe6060f1SDimitry Andric ArgOriginTLSTy = ArrayType::get(OriginTy, NumOfElementsInArgOrgTLS);
1524fe6060f1SDimitry Andric ArgOriginTLS = GetOrInsertGlobal("__dfsan_arg_origin_tls", ArgOriginTLSTy);
1525fe6060f1SDimitry Andric RetvalOriginTLS = GetOrInsertGlobal("__dfsan_retval_origin_tls", OriginTy);
1526fe6060f1SDimitry Andric
1527fe6060f1SDimitry Andric (void)Mod->getOrInsertGlobal("__dfsan_track_origins", OriginTy, [&] {
1528fe6060f1SDimitry Andric Changed = true;
1529fe6060f1SDimitry Andric return new GlobalVariable(
1530fe6060f1SDimitry Andric M, OriginTy, true, GlobalValue::WeakODRLinkage,
1531fe6060f1SDimitry Andric ConstantInt::getSigned(OriginTy,
1532fe6060f1SDimitry Andric shouldTrackOrigins() ? ClTrackOrigins : 0),
1533fe6060f1SDimitry Andric "__dfsan_track_origins");
1534fe6060f1SDimitry Andric });
1535fe6060f1SDimitry Andric
15365ffd83dbSDimitry Andric initializeCallbackFunctions(M);
15375ffd83dbSDimitry Andric initializeRuntimeFunctions(M);
15380b57cec5SDimitry Andric
15390b57cec5SDimitry Andric std::vector<Function *> FnsToInstrument;
15400b57cec5SDimitry Andric SmallPtrSet<Function *, 2> FnsWithNativeABI;
1541349cc55cSDimitry Andric SmallPtrSet<Function *, 2> FnsWithForceZeroLabel;
15420eae32dcSDimitry Andric SmallPtrSet<Constant *, 1> PersonalityFns;
1543fe6060f1SDimitry Andric for (Function &F : M)
1544bdd1243dSDimitry Andric if (!F.isIntrinsic() && !DFSanRuntimeFunctions.contains(&F) &&
1545bdd1243dSDimitry Andric !LibAtomicFunction(F)) {
1546fe6060f1SDimitry Andric FnsToInstrument.push_back(&F);
15470eae32dcSDimitry Andric if (F.hasPersonalityFn())
15480eae32dcSDimitry Andric PersonalityFns.insert(F.getPersonalityFn()->stripPointerCasts());
15490eae32dcSDimitry Andric }
15500eae32dcSDimitry Andric
15510eae32dcSDimitry Andric if (ClIgnorePersonalityRoutine) {
15520eae32dcSDimitry Andric for (auto *C : PersonalityFns) {
15530eae32dcSDimitry Andric assert(isa<Function>(C) && "Personality routine is not a function!");
15540eae32dcSDimitry Andric Function *F = cast<Function>(C);
15550eae32dcSDimitry Andric if (!isInstrumented(F))
15565f757f3fSDimitry Andric llvm::erase(FnsToInstrument, F);
15570eae32dcSDimitry Andric }
15580eae32dcSDimitry Andric }
15590b57cec5SDimitry Andric
15600b57cec5SDimitry Andric // Give function aliases prefixes when necessary, and build wrappers where the
15610b57cec5SDimitry Andric // instrumentedness is inconsistent.
1562349cc55cSDimitry Andric for (GlobalAlias &GA : llvm::make_early_inc_range(M.aliases())) {
15630b57cec5SDimitry Andric // Don't stop on weak. We assume people aren't playing games with the
15640b57cec5SDimitry Andric // instrumentedness of overridden weak aliases.
1565349cc55cSDimitry Andric auto *F = dyn_cast<Function>(GA.getAliaseeObject());
1566fe6060f1SDimitry Andric if (!F)
1567fe6060f1SDimitry Andric continue;
1568fe6060f1SDimitry Andric
1569349cc55cSDimitry Andric bool GAInst = isInstrumented(&GA), FInst = isInstrumented(F);
15700b57cec5SDimitry Andric if (GAInst && FInst) {
1571349cc55cSDimitry Andric addGlobalNameSuffix(&GA);
15720b57cec5SDimitry Andric } else if (GAInst != FInst) {
15730b57cec5SDimitry Andric // Non-instrumented alias of an instrumented function, or vice versa.
15740b57cec5SDimitry Andric // Replace the alias with a native-ABI wrapper of the aliasee. The pass
15750b57cec5SDimitry Andric // below will take care of instrumenting it.
15760b57cec5SDimitry Andric Function *NewF =
1577349cc55cSDimitry Andric buildWrapperFunction(F, "", GA.getLinkage(), F->getFunctionType());
15785f757f3fSDimitry Andric GA.replaceAllUsesWith(NewF);
1579349cc55cSDimitry Andric NewF->takeName(&GA);
1580349cc55cSDimitry Andric GA.eraseFromParent();
15810b57cec5SDimitry Andric FnsToInstrument.push_back(NewF);
15820b57cec5SDimitry Andric }
15830b57cec5SDimitry Andric }
15840b57cec5SDimitry Andric
1585bdd1243dSDimitry Andric // TODO: This could be more precise.
1586bdd1243dSDimitry Andric ReadOnlyNoneAttrs.addAttribute(Attribute::Memory);
15870b57cec5SDimitry Andric
15880b57cec5SDimitry Andric // First, change the ABI of every function in the module. ABI-listed
15890b57cec5SDimitry Andric // functions keep their original ABI and get a wrapper function.
1590fe6060f1SDimitry Andric for (std::vector<Function *>::iterator FI = FnsToInstrument.begin(),
1591fe6060f1SDimitry Andric FE = FnsToInstrument.end();
1592fe6060f1SDimitry Andric FI != FE; ++FI) {
1593fe6060f1SDimitry Andric Function &F = **FI;
15940b57cec5SDimitry Andric FunctionType *FT = F.getFunctionType();
15950b57cec5SDimitry Andric
15960b57cec5SDimitry Andric bool IsZeroArgsVoidRet = (FT->getNumParams() == 0 && !FT->isVarArg() &&
15970b57cec5SDimitry Andric FT->getReturnType()->isVoidTy());
15980b57cec5SDimitry Andric
15990b57cec5SDimitry Andric if (isInstrumented(&F)) {
1600349cc55cSDimitry Andric if (isForceZeroLabels(&F))
1601349cc55cSDimitry Andric FnsWithForceZeroLabel.insert(&F);
1602349cc55cSDimitry Andric
1603fe6060f1SDimitry Andric // Instrumented functions get a '.dfsan' suffix. This allows us to more
1604fe6060f1SDimitry Andric // easily identify cases of mismatching ABIs. This naming scheme is
1605fe6060f1SDimitry Andric // mangling-compatible (see Itanium ABI), using a vendor-specific suffix.
1606fe6060f1SDimitry Andric addGlobalNameSuffix(&F);
16070b57cec5SDimitry Andric } else if (!IsZeroArgsVoidRet || getWrapperKind(&F) == WK_Custom) {
16080b57cec5SDimitry Andric // Build a wrapper function for F. The wrapper simply calls F, and is
16090b57cec5SDimitry Andric // added to FnsToInstrument so that any instrumentation according to its
16100b57cec5SDimitry Andric // WrapperKind is done in the second pass below.
16110b57cec5SDimitry Andric
16120b57cec5SDimitry Andric // If the function being wrapped has local linkage, then preserve the
16130b57cec5SDimitry Andric // function's linkage in the wrapper function.
1614fe6060f1SDimitry Andric GlobalValue::LinkageTypes WrapperLinkage =
1615fe6060f1SDimitry Andric F.hasLocalLinkage() ? F.getLinkage()
16160b57cec5SDimitry Andric : GlobalValue::LinkOnceODRLinkage;
16170b57cec5SDimitry Andric
16180b57cec5SDimitry Andric Function *NewF = buildWrapperFunction(
1619fe6060f1SDimitry Andric &F,
1620fe6060f1SDimitry Andric (shouldTrackOrigins() ? std::string("dfso$") : std::string("dfsw$")) +
1621fe6060f1SDimitry Andric std::string(F.getName()),
1622349cc55cSDimitry Andric WrapperLinkage, FT);
1623349cc55cSDimitry Andric NewF->removeFnAttrs(ReadOnlyNoneAttrs);
16240b57cec5SDimitry Andric
162581ad6265SDimitry Andric // Extern weak functions can sometimes be null at execution time.
162681ad6265SDimitry Andric // Code will sometimes check if an extern weak function is null.
162781ad6265SDimitry Andric // This could look something like:
162881ad6265SDimitry Andric // declare extern_weak i8 @my_func(i8)
162981ad6265SDimitry Andric // br i1 icmp ne (i8 (i8)* @my_func, i8 (i8)* null), label %use_my_func,
163081ad6265SDimitry Andric // label %avoid_my_func
163181ad6265SDimitry Andric // The @"dfsw$my_func" wrapper is never null, so if we replace this use
1632bdd1243dSDimitry Andric // in the comparison, the icmp will simplify to false and we have
1633bdd1243dSDimitry Andric // accidentally optimized away a null check that is necessary.
163481ad6265SDimitry Andric // This can lead to a crash when the null extern_weak my_func is called.
163581ad6265SDimitry Andric //
163681ad6265SDimitry Andric // To prevent (the most common pattern of) this problem,
163781ad6265SDimitry Andric // do not replace uses in comparisons with the wrapper.
163881ad6265SDimitry Andric // We definitely want to replace uses in call instructions.
163981ad6265SDimitry Andric // Other uses (e.g. store the function address somewhere) might be
164081ad6265SDimitry Andric // called or compared or both - this case may not be handled correctly.
164181ad6265SDimitry Andric // We will default to replacing with wrapper in cases we are unsure.
164281ad6265SDimitry Andric auto IsNotCmpUse = [](Use &U) -> bool {
164381ad6265SDimitry Andric User *Usr = U.getUser();
164481ad6265SDimitry Andric if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Usr)) {
164581ad6265SDimitry Andric // This is the most common case for icmp ne null
164681ad6265SDimitry Andric if (CE->getOpcode() == Instruction::ICmp) {
164781ad6265SDimitry Andric return false;
164881ad6265SDimitry Andric }
164981ad6265SDimitry Andric }
165081ad6265SDimitry Andric if (Instruction *I = dyn_cast<Instruction>(Usr)) {
165181ad6265SDimitry Andric if (I->getOpcode() == Instruction::ICmp) {
165281ad6265SDimitry Andric return false;
165381ad6265SDimitry Andric }
165481ad6265SDimitry Andric }
165581ad6265SDimitry Andric return true;
165681ad6265SDimitry Andric };
16575f757f3fSDimitry Andric F.replaceUsesWithIf(NewF, IsNotCmpUse);
16580b57cec5SDimitry Andric
16595f757f3fSDimitry Andric UnwrappedFnMap[NewF] = &F;
1660fe6060f1SDimitry Andric *FI = NewF;
16610b57cec5SDimitry Andric
16620b57cec5SDimitry Andric if (!F.isDeclaration()) {
16630b57cec5SDimitry Andric // This function is probably defining an interposition of an
16640b57cec5SDimitry Andric // uninstrumented function and hence needs to keep the original ABI.
16650b57cec5SDimitry Andric // But any functions it may call need to use the instrumented ABI, so
16660b57cec5SDimitry Andric // we instrument it in a mode which preserves the original ABI.
16670b57cec5SDimitry Andric FnsWithNativeABI.insert(&F);
16680b57cec5SDimitry Andric
16690b57cec5SDimitry Andric // This code needs to rebuild the iterators, as they may be invalidated
16700b57cec5SDimitry Andric // by the push_back, taking care that the new range does not include
16710b57cec5SDimitry Andric // any functions added by this code.
1672fe6060f1SDimitry Andric size_t N = FI - FnsToInstrument.begin(),
1673fe6060f1SDimitry Andric Count = FE - FnsToInstrument.begin();
16740b57cec5SDimitry Andric FnsToInstrument.push_back(&F);
1675fe6060f1SDimitry Andric FI = FnsToInstrument.begin() + N;
1676fe6060f1SDimitry Andric FE = FnsToInstrument.begin() + Count;
16770b57cec5SDimitry Andric }
16780b57cec5SDimitry Andric // Hopefully, nobody will try to indirectly call a vararg
16790b57cec5SDimitry Andric // function... yet.
16800b57cec5SDimitry Andric } else if (FT->isVarArg()) {
16810b57cec5SDimitry Andric UnwrappedFnMap[&F] = &F;
1682fe6060f1SDimitry Andric *FI = nullptr;
16830b57cec5SDimitry Andric }
16840b57cec5SDimitry Andric }
16850b57cec5SDimitry Andric
1686fe6060f1SDimitry Andric for (Function *F : FnsToInstrument) {
1687fe6060f1SDimitry Andric if (!F || F->isDeclaration())
16880b57cec5SDimitry Andric continue;
16890b57cec5SDimitry Andric
1690fe6060f1SDimitry Andric removeUnreachableBlocks(*F);
16910b57cec5SDimitry Andric
1692349cc55cSDimitry Andric DFSanFunction DFSF(*this, F, FnsWithNativeABI.count(F),
1693bdd1243dSDimitry Andric FnsWithForceZeroLabel.count(F), GetTLI(*F));
1694bdd1243dSDimitry Andric
1695bdd1243dSDimitry Andric if (ClReachesFunctionCallbacks) {
1696bdd1243dSDimitry Andric // Add callback for arguments reaching this function.
1697bdd1243dSDimitry Andric for (auto &FArg : F->args()) {
1698bdd1243dSDimitry Andric Instruction *Next = &F->getEntryBlock().front();
1699bdd1243dSDimitry Andric Value *FArgShadow = DFSF.getShadow(&FArg);
1700bdd1243dSDimitry Andric if (isZeroShadow(FArgShadow))
1701bdd1243dSDimitry Andric continue;
1702bdd1243dSDimitry Andric if (Instruction *FArgShadowInst = dyn_cast<Instruction>(FArgShadow)) {
1703bdd1243dSDimitry Andric Next = FArgShadowInst->getNextNode();
1704bdd1243dSDimitry Andric }
1705bdd1243dSDimitry Andric if (shouldTrackOrigins()) {
1706bdd1243dSDimitry Andric if (Instruction *Origin =
1707bdd1243dSDimitry Andric dyn_cast<Instruction>(DFSF.getOrigin(&FArg))) {
1708bdd1243dSDimitry Andric // Ensure IRB insertion point is after loads for shadow and origin.
1709bdd1243dSDimitry Andric Instruction *OriginNext = Origin->getNextNode();
1710bdd1243dSDimitry Andric if (Next->comesBefore(OriginNext)) {
1711bdd1243dSDimitry Andric Next = OriginNext;
1712bdd1243dSDimitry Andric }
1713bdd1243dSDimitry Andric }
1714bdd1243dSDimitry Andric }
1715bdd1243dSDimitry Andric IRBuilder<> IRB(Next);
1716bdd1243dSDimitry Andric DFSF.addReachesFunctionCallbacksIfEnabled(IRB, *Next, &FArg);
1717bdd1243dSDimitry Andric }
1718bdd1243dSDimitry Andric }
17190b57cec5SDimitry Andric
17200b57cec5SDimitry Andric // DFSanVisitor may create new basic blocks, which confuses df_iterator.
17210b57cec5SDimitry Andric // Build a copy of the list before iterating over it.
1722fe6060f1SDimitry Andric SmallVector<BasicBlock *, 4> BBList(depth_first(&F->getEntryBlock()));
17230b57cec5SDimitry Andric
1724fe6060f1SDimitry Andric for (BasicBlock *BB : BBList) {
1725fe6060f1SDimitry Andric Instruction *Inst = &BB->front();
17260b57cec5SDimitry Andric while (true) {
17270b57cec5SDimitry Andric // DFSanVisitor may split the current basic block, changing the current
17280b57cec5SDimitry Andric // instruction's next pointer and moving the next instruction to the
17290b57cec5SDimitry Andric // tail block from which we should continue.
17300b57cec5SDimitry Andric Instruction *Next = Inst->getNextNode();
17310b57cec5SDimitry Andric // DFSanVisitor may delete Inst, so keep track of whether it was a
17320b57cec5SDimitry Andric // terminator.
17330b57cec5SDimitry Andric bool IsTerminator = Inst->isTerminator();
17340b57cec5SDimitry Andric if (!DFSF.SkipInsts.count(Inst))
17350b57cec5SDimitry Andric DFSanVisitor(DFSF).visit(Inst);
17360b57cec5SDimitry Andric if (IsTerminator)
17370b57cec5SDimitry Andric break;
17380b57cec5SDimitry Andric Inst = Next;
17390b57cec5SDimitry Andric }
17400b57cec5SDimitry Andric }
17410b57cec5SDimitry Andric
17420b57cec5SDimitry Andric // We will not necessarily be able to compute the shadow for every phi node
17430b57cec5SDimitry Andric // until we have visited every block. Therefore, the code that handles phi
17440b57cec5SDimitry Andric // nodes adds them to the PHIFixups list so that they can be properly
17450b57cec5SDimitry Andric // handled here.
1746fe6060f1SDimitry Andric for (DFSanFunction::PHIFixupElement &P : DFSF.PHIFixups) {
1747fe6060f1SDimitry Andric for (unsigned Val = 0, N = P.Phi->getNumIncomingValues(); Val != N;
1748fe6060f1SDimitry Andric ++Val) {
1749fe6060f1SDimitry Andric P.ShadowPhi->setIncomingValue(
1750fe6060f1SDimitry Andric Val, DFSF.getShadow(P.Phi->getIncomingValue(Val)));
1751fe6060f1SDimitry Andric if (P.OriginPhi)
1752fe6060f1SDimitry Andric P.OriginPhi->setIncomingValue(
1753fe6060f1SDimitry Andric Val, DFSF.getOrigin(P.Phi->getIncomingValue(Val)));
17540b57cec5SDimitry Andric }
17550b57cec5SDimitry Andric }
17560b57cec5SDimitry Andric
17570b57cec5SDimitry Andric // -dfsan-debug-nonzero-labels will split the CFG in all kinds of crazy
17580b57cec5SDimitry Andric // places (i.e. instructions in basic blocks we haven't even begun visiting
17590b57cec5SDimitry Andric // yet). To make our life easier, do this work in a pass after the main
17600b57cec5SDimitry Andric // instrumentation.
17610b57cec5SDimitry Andric if (ClDebugNonzeroLabels) {
17620b57cec5SDimitry Andric for (Value *V : DFSF.NonZeroChecks) {
17630b57cec5SDimitry Andric Instruction *Pos;
17640b57cec5SDimitry Andric if (Instruction *I = dyn_cast<Instruction>(V))
17650b57cec5SDimitry Andric Pos = I->getNextNode();
17660b57cec5SDimitry Andric else
17670b57cec5SDimitry Andric Pos = &DFSF.F->getEntryBlock().front();
17680b57cec5SDimitry Andric while (isa<PHINode>(Pos) || isa<AllocaInst>(Pos))
17690b57cec5SDimitry Andric Pos = Pos->getNextNode();
17700b57cec5SDimitry Andric IRBuilder<> IRB(Pos);
1771e8d8bef9SDimitry Andric Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(V, Pos);
1772e8d8bef9SDimitry Andric Value *Ne =
1773e8d8bef9SDimitry Andric IRB.CreateICmpNE(PrimitiveShadow, DFSF.DFS.ZeroPrimitiveShadow);
17740b57cec5SDimitry Andric BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
17750b57cec5SDimitry Andric Ne, Pos, /*Unreachable=*/false, ColdCallWeights));
17760b57cec5SDimitry Andric IRBuilder<> ThenIRB(BI);
17770b57cec5SDimitry Andric ThenIRB.CreateCall(DFSF.DFS.DFSanNonzeroLabelFn, {});
17780b57cec5SDimitry Andric }
17790b57cec5SDimitry Andric }
17800b57cec5SDimitry Andric }
17810b57cec5SDimitry Andric
17825ffd83dbSDimitry Andric return Changed || !FnsToInstrument.empty() ||
17835ffd83dbSDimitry Andric M.global_size() != InitialGlobalSize || M.size() != InitialModuleSize;
17840b57cec5SDimitry Andric }
17850b57cec5SDimitry Andric
getArgTLS(Type * T,unsigned ArgOffset,IRBuilder<> & IRB)1786e8d8bef9SDimitry Andric Value *DFSanFunction::getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB) {
1787e8d8bef9SDimitry Andric Value *Base = IRB.CreatePointerCast(DFS.ArgTLS, DFS.IntptrTy);
1788e8d8bef9SDimitry Andric if (ArgOffset)
1789e8d8bef9SDimitry Andric Base = IRB.CreateAdd(Base, ConstantInt::get(DFS.IntptrTy, ArgOffset));
1790e8d8bef9SDimitry Andric return IRB.CreateIntToPtr(Base, PointerType::get(DFS.getShadowTy(T), 0),
1791e8d8bef9SDimitry Andric "_dfsarg");
17920b57cec5SDimitry Andric }
17930b57cec5SDimitry Andric
getRetvalTLS(Type * T,IRBuilder<> & IRB)1794e8d8bef9SDimitry Andric Value *DFSanFunction::getRetvalTLS(Type *T, IRBuilder<> &IRB) {
1795e8d8bef9SDimitry Andric return IRB.CreatePointerCast(
1796e8d8bef9SDimitry Andric DFS.RetvalTLS, PointerType::get(DFS.getShadowTy(T), 0), "_dfsret");
17970b57cec5SDimitry Andric }
17980b57cec5SDimitry Andric
getRetvalOriginTLS()1799fe6060f1SDimitry Andric Value *DFSanFunction::getRetvalOriginTLS() { return DFS.RetvalOriginTLS; }
1800fe6060f1SDimitry Andric
getArgOriginTLS(unsigned ArgNo,IRBuilder<> & IRB)1801fe6060f1SDimitry Andric Value *DFSanFunction::getArgOriginTLS(unsigned ArgNo, IRBuilder<> &IRB) {
1802fe6060f1SDimitry Andric return IRB.CreateConstGEP2_64(DFS.ArgOriginTLSTy, DFS.ArgOriginTLS, 0, ArgNo,
1803fe6060f1SDimitry Andric "_dfsarg_o");
1804fe6060f1SDimitry Andric }
1805fe6060f1SDimitry Andric
getOrigin(Value * V)1806fe6060f1SDimitry Andric Value *DFSanFunction::getOrigin(Value *V) {
1807fe6060f1SDimitry Andric assert(DFS.shouldTrackOrigins());
1808fe6060f1SDimitry Andric if (!isa<Argument>(V) && !isa<Instruction>(V))
1809fe6060f1SDimitry Andric return DFS.ZeroOrigin;
1810fe6060f1SDimitry Andric Value *&Origin = ValOriginMap[V];
1811fe6060f1SDimitry Andric if (!Origin) {
1812fe6060f1SDimitry Andric if (Argument *A = dyn_cast<Argument>(V)) {
1813fe6060f1SDimitry Andric if (IsNativeABI)
1814fe6060f1SDimitry Andric return DFS.ZeroOrigin;
1815fe6060f1SDimitry Andric if (A->getArgNo() < DFS.NumOfElementsInArgOrgTLS) {
1816fe6060f1SDimitry Andric Instruction *ArgOriginTLSPos = &*F->getEntryBlock().begin();
1817fe6060f1SDimitry Andric IRBuilder<> IRB(ArgOriginTLSPos);
1818fe6060f1SDimitry Andric Value *ArgOriginPtr = getArgOriginTLS(A->getArgNo(), IRB);
1819fe6060f1SDimitry Andric Origin = IRB.CreateLoad(DFS.OriginTy, ArgOriginPtr);
1820fe6060f1SDimitry Andric } else {
1821fe6060f1SDimitry Andric // Overflow
1822fe6060f1SDimitry Andric Origin = DFS.ZeroOrigin;
1823fe6060f1SDimitry Andric }
1824fe6060f1SDimitry Andric } else {
1825fe6060f1SDimitry Andric Origin = DFS.ZeroOrigin;
1826fe6060f1SDimitry Andric }
1827fe6060f1SDimitry Andric }
1828fe6060f1SDimitry Andric return Origin;
1829fe6060f1SDimitry Andric }
1830fe6060f1SDimitry Andric
setOrigin(Instruction * I,Value * Origin)1831fe6060f1SDimitry Andric void DFSanFunction::setOrigin(Instruction *I, Value *Origin) {
1832fe6060f1SDimitry Andric if (!DFS.shouldTrackOrigins())
1833fe6060f1SDimitry Andric return;
1834fe6060f1SDimitry Andric assert(!ValOriginMap.count(I));
1835fe6060f1SDimitry Andric assert(Origin->getType() == DFS.OriginTy);
1836fe6060f1SDimitry Andric ValOriginMap[I] = Origin;
1837fe6060f1SDimitry Andric }
1838fe6060f1SDimitry Andric
getShadowForTLSArgument(Argument * A)1839e8d8bef9SDimitry Andric Value *DFSanFunction::getShadowForTLSArgument(Argument *A) {
1840e8d8bef9SDimitry Andric unsigned ArgOffset = 0;
1841e8d8bef9SDimitry Andric const DataLayout &DL = F->getParent()->getDataLayout();
1842e8d8bef9SDimitry Andric for (auto &FArg : F->args()) {
1843e8d8bef9SDimitry Andric if (!FArg.getType()->isSized()) {
1844e8d8bef9SDimitry Andric if (A == &FArg)
1845e8d8bef9SDimitry Andric break;
1846e8d8bef9SDimitry Andric continue;
1847e8d8bef9SDimitry Andric }
1848e8d8bef9SDimitry Andric
1849e8d8bef9SDimitry Andric unsigned Size = DL.getTypeAllocSize(DFS.getShadowTy(&FArg));
1850e8d8bef9SDimitry Andric if (A != &FArg) {
1851fe6060f1SDimitry Andric ArgOffset += alignTo(Size, ShadowTLSAlignment);
1852fe6060f1SDimitry Andric if (ArgOffset > ArgTLSSize)
1853e8d8bef9SDimitry Andric break; // ArgTLS overflows, uses a zero shadow.
1854e8d8bef9SDimitry Andric continue;
1855e8d8bef9SDimitry Andric }
1856e8d8bef9SDimitry Andric
1857fe6060f1SDimitry Andric if (ArgOffset + Size > ArgTLSSize)
1858e8d8bef9SDimitry Andric break; // ArgTLS overflows, uses a zero shadow.
1859e8d8bef9SDimitry Andric
1860e8d8bef9SDimitry Andric Instruction *ArgTLSPos = &*F->getEntryBlock().begin();
1861e8d8bef9SDimitry Andric IRBuilder<> IRB(ArgTLSPos);
1862e8d8bef9SDimitry Andric Value *ArgShadowPtr = getArgTLS(FArg.getType(), ArgOffset, IRB);
1863e8d8bef9SDimitry Andric return IRB.CreateAlignedLoad(DFS.getShadowTy(&FArg), ArgShadowPtr,
1864fe6060f1SDimitry Andric ShadowTLSAlignment);
1865e8d8bef9SDimitry Andric }
1866e8d8bef9SDimitry Andric
1867e8d8bef9SDimitry Andric return DFS.getZeroShadow(A);
18680b57cec5SDimitry Andric }
18690b57cec5SDimitry Andric
getShadow(Value * V)18700b57cec5SDimitry Andric Value *DFSanFunction::getShadow(Value *V) {
18710b57cec5SDimitry Andric if (!isa<Argument>(V) && !isa<Instruction>(V))
1872e8d8bef9SDimitry Andric return DFS.getZeroShadow(V);
1873349cc55cSDimitry Andric if (IsForceZeroLabels)
1874349cc55cSDimitry Andric return DFS.getZeroShadow(V);
18750b57cec5SDimitry Andric Value *&Shadow = ValShadowMap[V];
18760b57cec5SDimitry Andric if (!Shadow) {
18770b57cec5SDimitry Andric if (Argument *A = dyn_cast<Argument>(V)) {
18780b57cec5SDimitry Andric if (IsNativeABI)
1879e8d8bef9SDimitry Andric return DFS.getZeroShadow(V);
1880e8d8bef9SDimitry Andric Shadow = getShadowForTLSArgument(A);
18810b57cec5SDimitry Andric NonZeroChecks.push_back(Shadow);
18820b57cec5SDimitry Andric } else {
1883e8d8bef9SDimitry Andric Shadow = DFS.getZeroShadow(V);
18840b57cec5SDimitry Andric }
18850b57cec5SDimitry Andric }
18860b57cec5SDimitry Andric return Shadow;
18870b57cec5SDimitry Andric }
18880b57cec5SDimitry Andric
setShadow(Instruction * I,Value * Shadow)18890b57cec5SDimitry Andric void DFSanFunction::setShadow(Instruction *I, Value *Shadow) {
18900b57cec5SDimitry Andric assert(!ValShadowMap.count(I));
18910b57cec5SDimitry Andric ValShadowMap[I] = Shadow;
18920b57cec5SDimitry Andric }
18930b57cec5SDimitry Andric
1894fe6060f1SDimitry Andric /// Compute the integer shadow offset that corresponds to a given
1895fe6060f1SDimitry Andric /// application address.
1896fe6060f1SDimitry Andric ///
1897fe6060f1SDimitry Andric /// Offset = (Addr & ~AndMask) ^ XorMask
getShadowOffset(Value * Addr,IRBuilder<> & IRB)1898fe6060f1SDimitry Andric Value *DataFlowSanitizer::getShadowOffset(Value *Addr, IRBuilder<> &IRB) {
18990b57cec5SDimitry Andric assert(Addr != RetvalTLS && "Reinstrumenting?");
1900fe6060f1SDimitry Andric Value *OffsetLong = IRB.CreatePointerCast(Addr, IntptrTy);
1901fe6060f1SDimitry Andric
1902fe6060f1SDimitry Andric uint64_t AndMask = MapParams->AndMask;
1903fe6060f1SDimitry Andric if (AndMask)
1904fe6060f1SDimitry Andric OffsetLong =
1905fe6060f1SDimitry Andric IRB.CreateAnd(OffsetLong, ConstantInt::get(IntptrTy, ~AndMask));
1906fe6060f1SDimitry Andric
1907fe6060f1SDimitry Andric uint64_t XorMask = MapParams->XorMask;
1908fe6060f1SDimitry Andric if (XorMask)
1909fe6060f1SDimitry Andric OffsetLong = IRB.CreateXor(OffsetLong, ConstantInt::get(IntptrTy, XorMask));
1910fe6060f1SDimitry Andric return OffsetLong;
1911fe6060f1SDimitry Andric }
1912fe6060f1SDimitry Andric
1913fe6060f1SDimitry Andric std::pair<Value *, Value *>
getShadowOriginAddress(Value * Addr,Align InstAlignment,Instruction * Pos)1914fe6060f1SDimitry Andric DataFlowSanitizer::getShadowOriginAddress(Value *Addr, Align InstAlignment,
1915fe6060f1SDimitry Andric Instruction *Pos) {
1916fe6060f1SDimitry Andric // Returns ((Addr & shadow_mask) + origin_base - shadow_base) & ~4UL
19170b57cec5SDimitry Andric IRBuilder<> IRB(Pos);
1918fe6060f1SDimitry Andric Value *ShadowOffset = getShadowOffset(Addr, IRB);
1919fe6060f1SDimitry Andric Value *ShadowLong = ShadowOffset;
1920fe6060f1SDimitry Andric uint64_t ShadowBase = MapParams->ShadowBase;
1921fe6060f1SDimitry Andric if (ShadowBase != 0) {
1922fe6060f1SDimitry Andric ShadowLong =
1923fe6060f1SDimitry Andric IRB.CreateAdd(ShadowLong, ConstantInt::get(IntptrTy, ShadowBase));
1924fe6060f1SDimitry Andric }
1925fe6060f1SDimitry Andric IntegerType *ShadowTy = IntegerType::get(*Ctx, ShadowWidthBits);
1926fe6060f1SDimitry Andric Value *ShadowPtr =
1927fe6060f1SDimitry Andric IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1928fe6060f1SDimitry Andric Value *OriginPtr = nullptr;
1929fe6060f1SDimitry Andric if (shouldTrackOrigins()) {
1930fe6060f1SDimitry Andric Value *OriginLong = ShadowOffset;
1931fe6060f1SDimitry Andric uint64_t OriginBase = MapParams->OriginBase;
1932fe6060f1SDimitry Andric if (OriginBase != 0)
1933fe6060f1SDimitry Andric OriginLong =
1934fe6060f1SDimitry Andric IRB.CreateAdd(OriginLong, ConstantInt::get(IntptrTy, OriginBase));
1935fe6060f1SDimitry Andric const Align Alignment = llvm::assumeAligned(InstAlignment.value());
1936fe6060f1SDimitry Andric // When alignment is >= 4, Addr must be aligned to 4, otherwise it is UB.
1937fe6060f1SDimitry Andric // So Mask is unnecessary.
1938fe6060f1SDimitry Andric if (Alignment < MinOriginAlignment) {
1939fe6060f1SDimitry Andric uint64_t Mask = MinOriginAlignment.value() - 1;
1940fe6060f1SDimitry Andric OriginLong = IRB.CreateAnd(OriginLong, ConstantInt::get(IntptrTy, ~Mask));
1941fe6060f1SDimitry Andric }
1942fe6060f1SDimitry Andric OriginPtr = IRB.CreateIntToPtr(OriginLong, OriginPtrTy);
1943fe6060f1SDimitry Andric }
1944fe6060f1SDimitry Andric return std::make_pair(ShadowPtr, OriginPtr);
1945fe6060f1SDimitry Andric }
1946fe6060f1SDimitry Andric
getShadowAddress(Value * Addr,Instruction * Pos,Value * ShadowOffset)1947fe6060f1SDimitry Andric Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos,
1948fe6060f1SDimitry Andric Value *ShadowOffset) {
1949fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
1950fe6060f1SDimitry Andric return IRB.CreateIntToPtr(ShadowOffset, PrimitiveShadowPtrTy);
1951fe6060f1SDimitry Andric }
1952fe6060f1SDimitry Andric
getShadowAddress(Value * Addr,Instruction * Pos)1953fe6060f1SDimitry Andric Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos) {
1954fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
1955fe6060f1SDimitry Andric Value *ShadowOffset = getShadowOffset(Addr, IRB);
1956fe6060f1SDimitry Andric return getShadowAddress(Addr, Pos, ShadowOffset);
1957e8d8bef9SDimitry Andric }
1958e8d8bef9SDimitry Andric
combineShadowsThenConvert(Type * T,Value * V1,Value * V2,Instruction * Pos)1959e8d8bef9SDimitry Andric Value *DFSanFunction::combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
1960e8d8bef9SDimitry Andric Instruction *Pos) {
1961e8d8bef9SDimitry Andric Value *PrimitiveValue = combineShadows(V1, V2, Pos);
1962e8d8bef9SDimitry Andric return expandFromPrimitiveShadow(T, PrimitiveValue, Pos);
19630b57cec5SDimitry Andric }
19640b57cec5SDimitry Andric
19650b57cec5SDimitry Andric // Generates IR to compute the union of the two given shadows, inserting it
1966e8d8bef9SDimitry Andric // before Pos. The combined value is with primitive type.
combineShadows(Value * V1,Value * V2,Instruction * Pos)19670b57cec5SDimitry Andric Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
1968e8d8bef9SDimitry Andric if (DFS.isZeroShadow(V1))
1969e8d8bef9SDimitry Andric return collapseToPrimitiveShadow(V2, Pos);
1970e8d8bef9SDimitry Andric if (DFS.isZeroShadow(V2))
1971e8d8bef9SDimitry Andric return collapseToPrimitiveShadow(V1, Pos);
19720b57cec5SDimitry Andric if (V1 == V2)
1973e8d8bef9SDimitry Andric return collapseToPrimitiveShadow(V1, Pos);
19740b57cec5SDimitry Andric
19750b57cec5SDimitry Andric auto V1Elems = ShadowElements.find(V1);
19760b57cec5SDimitry Andric auto V2Elems = ShadowElements.find(V2);
19770b57cec5SDimitry Andric if (V1Elems != ShadowElements.end() && V2Elems != ShadowElements.end()) {
19780b57cec5SDimitry Andric if (std::includes(V1Elems->second.begin(), V1Elems->second.end(),
19790b57cec5SDimitry Andric V2Elems->second.begin(), V2Elems->second.end())) {
1980e8d8bef9SDimitry Andric return collapseToPrimitiveShadow(V1, Pos);
1981fe6060f1SDimitry Andric }
1982fe6060f1SDimitry Andric if (std::includes(V2Elems->second.begin(), V2Elems->second.end(),
19830b57cec5SDimitry Andric V1Elems->second.begin(), V1Elems->second.end())) {
1984e8d8bef9SDimitry Andric return collapseToPrimitiveShadow(V2, Pos);
19850b57cec5SDimitry Andric }
19860b57cec5SDimitry Andric } else if (V1Elems != ShadowElements.end()) {
19870b57cec5SDimitry Andric if (V1Elems->second.count(V2))
1988e8d8bef9SDimitry Andric return collapseToPrimitiveShadow(V1, Pos);
19890b57cec5SDimitry Andric } else if (V2Elems != ShadowElements.end()) {
19900b57cec5SDimitry Andric if (V2Elems->second.count(V1))
1991e8d8bef9SDimitry Andric return collapseToPrimitiveShadow(V2, Pos);
19920b57cec5SDimitry Andric }
19930b57cec5SDimitry Andric
19940b57cec5SDimitry Andric auto Key = std::make_pair(V1, V2);
19950b57cec5SDimitry Andric if (V1 > V2)
19960b57cec5SDimitry Andric std::swap(Key.first, Key.second);
1997e8d8bef9SDimitry Andric CachedShadow &CCS = CachedShadows[Key];
19980b57cec5SDimitry Andric if (CCS.Block && DT.dominates(CCS.Block, Pos->getParent()))
19990b57cec5SDimitry Andric return CCS.Shadow;
20000b57cec5SDimitry Andric
2001e8d8bef9SDimitry Andric // Converts inputs shadows to shadows with primitive types.
2002e8d8bef9SDimitry Andric Value *PV1 = collapseToPrimitiveShadow(V1, Pos);
2003e8d8bef9SDimitry Andric Value *PV2 = collapseToPrimitiveShadow(V2, Pos);
2004e8d8bef9SDimitry Andric
20050b57cec5SDimitry Andric IRBuilder<> IRB(Pos);
2006e8d8bef9SDimitry Andric CCS.Block = Pos->getParent();
2007e8d8bef9SDimitry Andric CCS.Shadow = IRB.CreateOr(PV1, PV2);
20080b57cec5SDimitry Andric
20090b57cec5SDimitry Andric std::set<Value *> UnionElems;
20100b57cec5SDimitry Andric if (V1Elems != ShadowElements.end()) {
20110b57cec5SDimitry Andric UnionElems = V1Elems->second;
20120b57cec5SDimitry Andric } else {
20130b57cec5SDimitry Andric UnionElems.insert(V1);
20140b57cec5SDimitry Andric }
20150b57cec5SDimitry Andric if (V2Elems != ShadowElements.end()) {
20160b57cec5SDimitry Andric UnionElems.insert(V2Elems->second.begin(), V2Elems->second.end());
20170b57cec5SDimitry Andric } else {
20180b57cec5SDimitry Andric UnionElems.insert(V2);
20190b57cec5SDimitry Andric }
20200b57cec5SDimitry Andric ShadowElements[CCS.Shadow] = std::move(UnionElems);
20210b57cec5SDimitry Andric
20220b57cec5SDimitry Andric return CCS.Shadow;
20230b57cec5SDimitry Andric }
20240b57cec5SDimitry Andric
20250b57cec5SDimitry Andric // A convenience function which folds the shadows of each of the operands
20260b57cec5SDimitry Andric // of the provided instruction Inst, inserting the IR before Inst. Returns
20270b57cec5SDimitry Andric // the computed union Value.
combineOperandShadows(Instruction * Inst)20280b57cec5SDimitry Andric Value *DFSanFunction::combineOperandShadows(Instruction *Inst) {
20290b57cec5SDimitry Andric if (Inst->getNumOperands() == 0)
2030e8d8bef9SDimitry Andric return DFS.getZeroShadow(Inst);
20310b57cec5SDimitry Andric
20320b57cec5SDimitry Andric Value *Shadow = getShadow(Inst->getOperand(0));
2033fe6060f1SDimitry Andric for (unsigned I = 1, N = Inst->getNumOperands(); I < N; ++I)
2034fe6060f1SDimitry Andric Shadow = combineShadows(Shadow, getShadow(Inst->getOperand(I)), Inst);
2035fe6060f1SDimitry Andric
2036e8d8bef9SDimitry Andric return expandFromPrimitiveShadow(Inst->getType(), Shadow, Inst);
20370b57cec5SDimitry Andric }
20380b57cec5SDimitry Andric
visitInstOperands(Instruction & I)2039fe6060f1SDimitry Andric void DFSanVisitor::visitInstOperands(Instruction &I) {
20400b57cec5SDimitry Andric Value *CombinedShadow = DFSF.combineOperandShadows(&I);
20410b57cec5SDimitry Andric DFSF.setShadow(&I, CombinedShadow);
2042fe6060f1SDimitry Andric visitInstOperandOrigins(I);
20430b57cec5SDimitry Andric }
20440b57cec5SDimitry Andric
combineOrigins(const std::vector<Value * > & Shadows,const std::vector<Value * > & Origins,Instruction * Pos,ConstantInt * Zero)2045fe6060f1SDimitry Andric Value *DFSanFunction::combineOrigins(const std::vector<Value *> &Shadows,
2046fe6060f1SDimitry Andric const std::vector<Value *> &Origins,
2047fe6060f1SDimitry Andric Instruction *Pos, ConstantInt *Zero) {
2048fe6060f1SDimitry Andric assert(Shadows.size() == Origins.size());
2049fe6060f1SDimitry Andric size_t Size = Origins.size();
2050fe6060f1SDimitry Andric if (Size == 0)
2051fe6060f1SDimitry Andric return DFS.ZeroOrigin;
2052fe6060f1SDimitry Andric Value *Origin = nullptr;
2053fe6060f1SDimitry Andric if (!Zero)
2054fe6060f1SDimitry Andric Zero = DFS.ZeroPrimitiveShadow;
2055fe6060f1SDimitry Andric for (size_t I = 0; I != Size; ++I) {
2056fe6060f1SDimitry Andric Value *OpOrigin = Origins[I];
2057fe6060f1SDimitry Andric Constant *ConstOpOrigin = dyn_cast<Constant>(OpOrigin);
2058fe6060f1SDimitry Andric if (ConstOpOrigin && ConstOpOrigin->isNullValue())
2059fe6060f1SDimitry Andric continue;
2060fe6060f1SDimitry Andric if (!Origin) {
2061fe6060f1SDimitry Andric Origin = OpOrigin;
2062fe6060f1SDimitry Andric continue;
2063fe6060f1SDimitry Andric }
2064fe6060f1SDimitry Andric Value *OpShadow = Shadows[I];
2065fe6060f1SDimitry Andric Value *PrimitiveShadow = collapseToPrimitiveShadow(OpShadow, Pos);
20660b57cec5SDimitry Andric IRBuilder<> IRB(Pos);
2067fe6060f1SDimitry Andric Value *Cond = IRB.CreateICmpNE(PrimitiveShadow, Zero);
2068fe6060f1SDimitry Andric Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2069fe6060f1SDimitry Andric }
2070fe6060f1SDimitry Andric return Origin ? Origin : DFS.ZeroOrigin;
2071fe6060f1SDimitry Andric }
2072fe6060f1SDimitry Andric
combineOperandOrigins(Instruction * Inst)2073fe6060f1SDimitry Andric Value *DFSanFunction::combineOperandOrigins(Instruction *Inst) {
2074fe6060f1SDimitry Andric size_t Size = Inst->getNumOperands();
2075fe6060f1SDimitry Andric std::vector<Value *> Shadows(Size);
2076fe6060f1SDimitry Andric std::vector<Value *> Origins(Size);
2077fe6060f1SDimitry Andric for (unsigned I = 0; I != Size; ++I) {
2078fe6060f1SDimitry Andric Shadows[I] = getShadow(Inst->getOperand(I));
2079fe6060f1SDimitry Andric Origins[I] = getOrigin(Inst->getOperand(I));
2080fe6060f1SDimitry Andric }
2081fe6060f1SDimitry Andric return combineOrigins(Shadows, Origins, Inst);
2082fe6060f1SDimitry Andric }
2083fe6060f1SDimitry Andric
visitInstOperandOrigins(Instruction & I)2084fe6060f1SDimitry Andric void DFSanVisitor::visitInstOperandOrigins(Instruction &I) {
2085fe6060f1SDimitry Andric if (!DFSF.DFS.shouldTrackOrigins())
2086fe6060f1SDimitry Andric return;
2087fe6060f1SDimitry Andric Value *CombinedOrigin = DFSF.combineOperandOrigins(&I);
2088fe6060f1SDimitry Andric DFSF.setOrigin(&I, CombinedOrigin);
2089fe6060f1SDimitry Andric }
2090fe6060f1SDimitry Andric
getShadowAlign(Align InstAlignment)2091fe6060f1SDimitry Andric Align DFSanFunction::getShadowAlign(Align InstAlignment) {
2092fe6060f1SDimitry Andric const Align Alignment = ClPreserveAlignment ? InstAlignment : Align(1);
2093fe6060f1SDimitry Andric return Align(Alignment.value() * DFS.ShadowWidthBytes);
2094fe6060f1SDimitry Andric }
2095fe6060f1SDimitry Andric
getOriginAlign(Align InstAlignment)2096fe6060f1SDimitry Andric Align DFSanFunction::getOriginAlign(Align InstAlignment) {
2097fe6060f1SDimitry Andric const Align Alignment = llvm::assumeAligned(InstAlignment.value());
2098fe6060f1SDimitry Andric return Align(std::max(MinOriginAlignment, Alignment));
2099fe6060f1SDimitry Andric }
2100fe6060f1SDimitry Andric
isLookupTableConstant(Value * P)210181ad6265SDimitry Andric bool DFSanFunction::isLookupTableConstant(Value *P) {
210281ad6265SDimitry Andric if (GlobalVariable *GV = dyn_cast<GlobalVariable>(P->stripPointerCasts()))
210381ad6265SDimitry Andric if (GV->isConstant() && GV->hasName())
210481ad6265SDimitry Andric return DFS.CombineTaintLookupTableNames.count(GV->getName());
210581ad6265SDimitry Andric
210681ad6265SDimitry Andric return false;
210781ad6265SDimitry Andric }
210881ad6265SDimitry Andric
useCallbackLoadLabelAndOrigin(uint64_t Size,Align InstAlignment)2109fe6060f1SDimitry Andric bool DFSanFunction::useCallbackLoadLabelAndOrigin(uint64_t Size,
2110fe6060f1SDimitry Andric Align InstAlignment) {
2111fe6060f1SDimitry Andric // When enabling tracking load instructions, we always use
2112fe6060f1SDimitry Andric // __dfsan_load_label_and_origin to reduce code size.
2113fe6060f1SDimitry Andric if (ClTrackOrigins == 2)
2114fe6060f1SDimitry Andric return true;
2115fe6060f1SDimitry Andric
2116fe6060f1SDimitry Andric assert(Size != 0);
2117fe6060f1SDimitry Andric // * if Size == 1, it is sufficient to load its origin aligned at 4.
2118fe6060f1SDimitry Andric // * if Size == 2, we assume most cases Addr % 2 == 0, so it is sufficient to
2119fe6060f1SDimitry Andric // load its origin aligned at 4. If not, although origins may be lost, it
2120fe6060f1SDimitry Andric // should not happen very often.
2121fe6060f1SDimitry Andric // * if align >= 4, Addr must be aligned to 4, otherwise it is UB. When
2122fe6060f1SDimitry Andric // Size % 4 == 0, it is more efficient to load origins without callbacks.
2123fe6060f1SDimitry Andric // * Otherwise we use __dfsan_load_label_and_origin.
2124fe6060f1SDimitry Andric // This should ensure that common cases run efficiently.
2125fe6060f1SDimitry Andric if (Size <= 2)
2126fe6060f1SDimitry Andric return false;
2127fe6060f1SDimitry Andric
2128fe6060f1SDimitry Andric const Align Alignment = llvm::assumeAligned(InstAlignment.value());
2129fe6060f1SDimitry Andric return Alignment < MinOriginAlignment || !DFS.hasLoadSizeForFastPath(Size);
2130fe6060f1SDimitry Andric }
2131fe6060f1SDimitry Andric
loadNextOrigin(Instruction * Pos,Align OriginAlign,Value ** OriginAddr)2132fe6060f1SDimitry Andric Value *DataFlowSanitizer::loadNextOrigin(Instruction *Pos, Align OriginAlign,
2133fe6060f1SDimitry Andric Value **OriginAddr) {
2134fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
2135fe6060f1SDimitry Andric *OriginAddr =
2136fe6060f1SDimitry Andric IRB.CreateGEP(OriginTy, *OriginAddr, ConstantInt::get(IntptrTy, 1));
2137fe6060f1SDimitry Andric return IRB.CreateAlignedLoad(OriginTy, *OriginAddr, OriginAlign);
2138fe6060f1SDimitry Andric }
2139fe6060f1SDimitry Andric
loadShadowFast(Value * ShadowAddr,Value * OriginAddr,uint64_t Size,Align ShadowAlign,Align OriginAlign,Value * FirstOrigin,Instruction * Pos)2140fe6060f1SDimitry Andric std::pair<Value *, Value *> DFSanFunction::loadShadowFast(
2141fe6060f1SDimitry Andric Value *ShadowAddr, Value *OriginAddr, uint64_t Size, Align ShadowAlign,
2142fe6060f1SDimitry Andric Align OriginAlign, Value *FirstOrigin, Instruction *Pos) {
2143fe6060f1SDimitry Andric const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
2144fe6060f1SDimitry Andric const uint64_t ShadowSize = Size * DFS.ShadowWidthBytes;
2145fe6060f1SDimitry Andric
2146fe6060f1SDimitry Andric assert(Size >= 4 && "Not large enough load size for fast path!");
2147fe6060f1SDimitry Andric
2148fe6060f1SDimitry Andric // Used for origin tracking.
2149fe6060f1SDimitry Andric std::vector<Value *> Shadows;
2150fe6060f1SDimitry Andric std::vector<Value *> Origins;
2151fe6060f1SDimitry Andric
2152fe6060f1SDimitry Andric // Load instructions in LLVM can have arbitrary byte sizes (e.g., 3, 12, 20)
2153fe6060f1SDimitry Andric // but this function is only used in a subset of cases that make it possible
2154fe6060f1SDimitry Andric // to optimize the instrumentation.
2155fe6060f1SDimitry Andric //
2156fe6060f1SDimitry Andric // Specifically, when the shadow size in bytes (i.e., loaded bytes x shadow
2157fe6060f1SDimitry Andric // per byte) is either:
2158fe6060f1SDimitry Andric // - a multiple of 8 (common)
2159fe6060f1SDimitry Andric // - equal to 4 (only for load32)
2160fe6060f1SDimitry Andric //
2161fe6060f1SDimitry Andric // For the second case, we can fit the wide shadow in a 32-bit integer. In all
2162fe6060f1SDimitry Andric // other cases, we use a 64-bit integer to hold the wide shadow.
2163fe6060f1SDimitry Andric Type *WideShadowTy =
2164fe6060f1SDimitry Andric ShadowSize == 4 ? Type::getInt32Ty(*DFS.Ctx) : Type::getInt64Ty(*DFS.Ctx);
2165fe6060f1SDimitry Andric
2166fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
2167fe6060f1SDimitry Andric Value *CombinedWideShadow =
216806c3fb27SDimitry Andric IRB.CreateAlignedLoad(WideShadowTy, ShadowAddr, ShadowAlign);
2169fe6060f1SDimitry Andric
2170fe6060f1SDimitry Andric unsigned WideShadowBitWidth = WideShadowTy->getIntegerBitWidth();
2171fe6060f1SDimitry Andric const uint64_t BytesPerWideShadow = WideShadowBitWidth / DFS.ShadowWidthBits;
2172fe6060f1SDimitry Andric
2173fe6060f1SDimitry Andric auto AppendWideShadowAndOrigin = [&](Value *WideShadow, Value *Origin) {
2174fe6060f1SDimitry Andric if (BytesPerWideShadow > 4) {
2175fe6060f1SDimitry Andric assert(BytesPerWideShadow == 8);
2176fe6060f1SDimitry Andric // The wide shadow relates to two origin pointers: one for the first four
2177fe6060f1SDimitry Andric // application bytes, and one for the latest four. We use a left shift to
2178fe6060f1SDimitry Andric // get just the shadow bytes that correspond to the first origin pointer,
2179fe6060f1SDimitry Andric // and then the entire shadow for the second origin pointer (which will be
2180fe6060f1SDimitry Andric // chosen by combineOrigins() iff the least-significant half of the wide
2181fe6060f1SDimitry Andric // shadow was empty but the other half was not).
2182fe6060f1SDimitry Andric Value *WideShadowLo = IRB.CreateShl(
2183fe6060f1SDimitry Andric WideShadow, ConstantInt::get(WideShadowTy, WideShadowBitWidth / 2));
2184fe6060f1SDimitry Andric Shadows.push_back(WideShadow);
2185fe6060f1SDimitry Andric Origins.push_back(DFS.loadNextOrigin(Pos, OriginAlign, &OriginAddr));
2186fe6060f1SDimitry Andric
2187fe6060f1SDimitry Andric Shadows.push_back(WideShadowLo);
2188fe6060f1SDimitry Andric Origins.push_back(Origin);
2189fe6060f1SDimitry Andric } else {
2190fe6060f1SDimitry Andric Shadows.push_back(WideShadow);
2191fe6060f1SDimitry Andric Origins.push_back(Origin);
2192fe6060f1SDimitry Andric }
2193fe6060f1SDimitry Andric };
2194fe6060f1SDimitry Andric
2195fe6060f1SDimitry Andric if (ShouldTrackOrigins)
2196fe6060f1SDimitry Andric AppendWideShadowAndOrigin(CombinedWideShadow, FirstOrigin);
2197fe6060f1SDimitry Andric
2198fe6060f1SDimitry Andric // First OR all the WideShadows (i.e., 64bit or 32bit shadow chunks) linearly;
2199fe6060f1SDimitry Andric // then OR individual shadows within the combined WideShadow by binary ORing.
2200fe6060f1SDimitry Andric // This is fewer instructions than ORing shadows individually, since it
2201fe6060f1SDimitry Andric // needs logN shift/or instructions (N being the bytes of the combined wide
2202fe6060f1SDimitry Andric // shadow).
2203fe6060f1SDimitry Andric for (uint64_t ByteOfs = BytesPerWideShadow; ByteOfs < Size;
2204fe6060f1SDimitry Andric ByteOfs += BytesPerWideShadow) {
220506c3fb27SDimitry Andric ShadowAddr = IRB.CreateGEP(WideShadowTy, ShadowAddr,
2206fe6060f1SDimitry Andric ConstantInt::get(DFS.IntptrTy, 1));
2207fe6060f1SDimitry Andric Value *NextWideShadow =
220806c3fb27SDimitry Andric IRB.CreateAlignedLoad(WideShadowTy, ShadowAddr, ShadowAlign);
2209fe6060f1SDimitry Andric CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, NextWideShadow);
2210fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2211fe6060f1SDimitry Andric Value *NextOrigin = DFS.loadNextOrigin(Pos, OriginAlign, &OriginAddr);
2212fe6060f1SDimitry Andric AppendWideShadowAndOrigin(NextWideShadow, NextOrigin);
2213fe6060f1SDimitry Andric }
2214fe6060f1SDimitry Andric }
2215fe6060f1SDimitry Andric for (unsigned Width = WideShadowBitWidth / 2; Width >= DFS.ShadowWidthBits;
2216fe6060f1SDimitry Andric Width >>= 1) {
2217fe6060f1SDimitry Andric Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width);
2218fe6060f1SDimitry Andric CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow);
2219fe6060f1SDimitry Andric }
2220fe6060f1SDimitry Andric return {IRB.CreateTrunc(CombinedWideShadow, DFS.PrimitiveShadowTy),
2221fe6060f1SDimitry Andric ShouldTrackOrigins
2222fe6060f1SDimitry Andric ? combineOrigins(Shadows, Origins, Pos,
2223fe6060f1SDimitry Andric ConstantInt::getSigned(IRB.getInt64Ty(), 0))
2224fe6060f1SDimitry Andric : DFS.ZeroOrigin};
2225fe6060f1SDimitry Andric }
2226fe6060f1SDimitry Andric
loadShadowOriginSansLoadTracking(Value * Addr,uint64_t Size,Align InstAlignment,Instruction * Pos)2227fe6060f1SDimitry Andric std::pair<Value *, Value *> DFSanFunction::loadShadowOriginSansLoadTracking(
2228fe6060f1SDimitry Andric Value *Addr, uint64_t Size, Align InstAlignment, Instruction *Pos) {
2229fe6060f1SDimitry Andric const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
2230fe6060f1SDimitry Andric
2231fe6060f1SDimitry Andric // Non-escaped loads.
2232fe6060f1SDimitry Andric if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
2233fe6060f1SDimitry Andric const auto SI = AllocaShadowMap.find(AI);
2234fe6060f1SDimitry Andric if (SI != AllocaShadowMap.end()) {
2235fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
2236fe6060f1SDimitry Andric Value *ShadowLI = IRB.CreateLoad(DFS.PrimitiveShadowTy, SI->second);
2237fe6060f1SDimitry Andric const auto OI = AllocaOriginMap.find(AI);
2238fe6060f1SDimitry Andric assert(!ShouldTrackOrigins || OI != AllocaOriginMap.end());
2239fe6060f1SDimitry Andric return {ShadowLI, ShouldTrackOrigins
2240fe6060f1SDimitry Andric ? IRB.CreateLoad(DFS.OriginTy, OI->second)
2241fe6060f1SDimitry Andric : nullptr};
22420b57cec5SDimitry Andric }
22430b57cec5SDimitry Andric }
22440b57cec5SDimitry Andric
2245fe6060f1SDimitry Andric // Load from constant addresses.
22460b57cec5SDimitry Andric SmallVector<const Value *, 2> Objs;
2247e8d8bef9SDimitry Andric getUnderlyingObjects(Addr, Objs);
22480b57cec5SDimitry Andric bool AllConstants = true;
22490b57cec5SDimitry Andric for (const Value *Obj : Objs) {
22500b57cec5SDimitry Andric if (isa<Function>(Obj) || isa<BlockAddress>(Obj))
22510b57cec5SDimitry Andric continue;
22520b57cec5SDimitry Andric if (isa<GlobalVariable>(Obj) && cast<GlobalVariable>(Obj)->isConstant())
22530b57cec5SDimitry Andric continue;
22540b57cec5SDimitry Andric
22550b57cec5SDimitry Andric AllConstants = false;
22560b57cec5SDimitry Andric break;
22570b57cec5SDimitry Andric }
22580b57cec5SDimitry Andric if (AllConstants)
2259fe6060f1SDimitry Andric return {DFS.ZeroPrimitiveShadow,
2260fe6060f1SDimitry Andric ShouldTrackOrigins ? DFS.ZeroOrigin : nullptr};
22610b57cec5SDimitry Andric
2262fe6060f1SDimitry Andric if (Size == 0)
2263fe6060f1SDimitry Andric return {DFS.ZeroPrimitiveShadow,
2264fe6060f1SDimitry Andric ShouldTrackOrigins ? DFS.ZeroOrigin : nullptr};
2265fe6060f1SDimitry Andric
2266fe6060f1SDimitry Andric // Use callback to load if this is not an optimizable case for origin
2267fe6060f1SDimitry Andric // tracking.
2268fe6060f1SDimitry Andric if (ShouldTrackOrigins &&
2269fe6060f1SDimitry Andric useCallbackLoadLabelAndOrigin(Size, InstAlignment)) {
2270fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
2271fe6060f1SDimitry Andric CallInst *Call =
2272fe6060f1SDimitry Andric IRB.CreateCall(DFS.DFSanLoadLabelAndOriginFn,
22735f757f3fSDimitry Andric {Addr, ConstantInt::get(DFS.IntptrTy, Size)});
2274349cc55cSDimitry Andric Call->addRetAttr(Attribute::ZExt);
2275fe6060f1SDimitry Andric return {IRB.CreateTrunc(IRB.CreateLShr(Call, DFS.OriginWidthBits),
2276fe6060f1SDimitry Andric DFS.PrimitiveShadowTy),
2277fe6060f1SDimitry Andric IRB.CreateTrunc(Call, DFS.OriginTy)};
2278fe6060f1SDimitry Andric }
2279fe6060f1SDimitry Andric
2280fe6060f1SDimitry Andric // Other cases that support loading shadows or origins in a fast way.
2281fe6060f1SDimitry Andric Value *ShadowAddr, *OriginAddr;
2282fe6060f1SDimitry Andric std::tie(ShadowAddr, OriginAddr) =
2283fe6060f1SDimitry Andric DFS.getShadowOriginAddress(Addr, InstAlignment, Pos);
2284fe6060f1SDimitry Andric
2285fe6060f1SDimitry Andric const Align ShadowAlign = getShadowAlign(InstAlignment);
2286fe6060f1SDimitry Andric const Align OriginAlign = getOriginAlign(InstAlignment);
2287fe6060f1SDimitry Andric Value *Origin = nullptr;
2288fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2289fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
2290fe6060f1SDimitry Andric Origin = IRB.CreateAlignedLoad(DFS.OriginTy, OriginAddr, OriginAlign);
2291fe6060f1SDimitry Andric }
2292fe6060f1SDimitry Andric
2293fe6060f1SDimitry Andric // When the byte size is small enough, we can load the shadow directly with
2294fe6060f1SDimitry Andric // just a few instructions.
22950b57cec5SDimitry Andric switch (Size) {
22960b57cec5SDimitry Andric case 1: {
2297e8d8bef9SDimitry Andric LoadInst *LI = new LoadInst(DFS.PrimitiveShadowTy, ShadowAddr, "", Pos);
22985ffd83dbSDimitry Andric LI->setAlignment(ShadowAlign);
2299fe6060f1SDimitry Andric return {LI, Origin};
23000b57cec5SDimitry Andric }
23010b57cec5SDimitry Andric case 2: {
23020b57cec5SDimitry Andric IRBuilder<> IRB(Pos);
2303e8d8bef9SDimitry Andric Value *ShadowAddr1 = IRB.CreateGEP(DFS.PrimitiveShadowTy, ShadowAddr,
23040b57cec5SDimitry Andric ConstantInt::get(DFS.IntptrTy, 1));
2305fe6060f1SDimitry Andric Value *Load =
2306fe6060f1SDimitry Andric IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr, ShadowAlign);
2307fe6060f1SDimitry Andric Value *Load1 =
2308fe6060f1SDimitry Andric IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr1, ShadowAlign);
2309fe6060f1SDimitry Andric return {combineShadows(Load, Load1, Pos), Origin};
23100b57cec5SDimitry Andric }
23110b57cec5SDimitry Andric }
2312fe6060f1SDimitry Andric bool HasSizeForFastPath = DFS.hasLoadSizeForFastPath(Size);
2313e8d8bef9SDimitry Andric
2314fe6060f1SDimitry Andric if (HasSizeForFastPath)
2315fe6060f1SDimitry Andric return loadShadowFast(ShadowAddr, OriginAddr, Size, ShadowAlign,
2316fe6060f1SDimitry Andric OriginAlign, Origin, Pos);
23170b57cec5SDimitry Andric
23180b57cec5SDimitry Andric IRBuilder<> IRB(Pos);
23190b57cec5SDimitry Andric CallInst *FallbackCall = IRB.CreateCall(
2320fe6060f1SDimitry Andric DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
2321349cc55cSDimitry Andric FallbackCall->addRetAttr(Attribute::ZExt);
2322fe6060f1SDimitry Andric return {FallbackCall, Origin};
2323fe6060f1SDimitry Andric }
2324fe6060f1SDimitry Andric
loadShadowOrigin(Value * Addr,uint64_t Size,Align InstAlignment,Instruction * Pos)2325fe6060f1SDimitry Andric std::pair<Value *, Value *> DFSanFunction::loadShadowOrigin(Value *Addr,
2326fe6060f1SDimitry Andric uint64_t Size,
2327fe6060f1SDimitry Andric Align InstAlignment,
2328fe6060f1SDimitry Andric Instruction *Pos) {
2329fe6060f1SDimitry Andric Value *PrimitiveShadow, *Origin;
2330fe6060f1SDimitry Andric std::tie(PrimitiveShadow, Origin) =
2331fe6060f1SDimitry Andric loadShadowOriginSansLoadTracking(Addr, Size, InstAlignment, Pos);
2332fe6060f1SDimitry Andric if (DFS.shouldTrackOrigins()) {
2333fe6060f1SDimitry Andric if (ClTrackOrigins == 2) {
2334fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
2335fe6060f1SDimitry Andric auto *ConstantShadow = dyn_cast<Constant>(PrimitiveShadow);
2336fe6060f1SDimitry Andric if (!ConstantShadow || !ConstantShadow->isZeroValue())
2337fe6060f1SDimitry Andric Origin = updateOriginIfTainted(PrimitiveShadow, Origin, IRB);
2338fe6060f1SDimitry Andric }
2339fe6060f1SDimitry Andric }
2340fe6060f1SDimitry Andric return {PrimitiveShadow, Origin};
2341fe6060f1SDimitry Andric }
2342fe6060f1SDimitry Andric
addAcquireOrdering(AtomicOrdering AO)2343fe6060f1SDimitry Andric static AtomicOrdering addAcquireOrdering(AtomicOrdering AO) {
2344fe6060f1SDimitry Andric switch (AO) {
2345fe6060f1SDimitry Andric case AtomicOrdering::NotAtomic:
2346fe6060f1SDimitry Andric return AtomicOrdering::NotAtomic;
2347fe6060f1SDimitry Andric case AtomicOrdering::Unordered:
2348fe6060f1SDimitry Andric case AtomicOrdering::Monotonic:
2349fe6060f1SDimitry Andric case AtomicOrdering::Acquire:
2350fe6060f1SDimitry Andric return AtomicOrdering::Acquire;
2351fe6060f1SDimitry Andric case AtomicOrdering::Release:
2352fe6060f1SDimitry Andric case AtomicOrdering::AcquireRelease:
2353fe6060f1SDimitry Andric return AtomicOrdering::AcquireRelease;
2354fe6060f1SDimitry Andric case AtomicOrdering::SequentiallyConsistent:
2355fe6060f1SDimitry Andric return AtomicOrdering::SequentiallyConsistent;
2356fe6060f1SDimitry Andric }
2357fe6060f1SDimitry Andric llvm_unreachable("Unknown ordering");
23580b57cec5SDimitry Andric }
23590b57cec5SDimitry Andric
StripPointerGEPsAndCasts(Value * V)236081ad6265SDimitry Andric Value *StripPointerGEPsAndCasts(Value *V) {
236181ad6265SDimitry Andric if (!V->getType()->isPointerTy())
236281ad6265SDimitry Andric return V;
236381ad6265SDimitry Andric
236481ad6265SDimitry Andric // DFSan pass should be running on valid IR, but we'll
236581ad6265SDimitry Andric // keep a seen set to ensure there are no issues.
236681ad6265SDimitry Andric SmallPtrSet<const Value *, 4> Visited;
236781ad6265SDimitry Andric Visited.insert(V);
236881ad6265SDimitry Andric do {
236981ad6265SDimitry Andric if (auto *GEP = dyn_cast<GEPOperator>(V)) {
237081ad6265SDimitry Andric V = GEP->getPointerOperand();
237181ad6265SDimitry Andric } else if (Operator::getOpcode(V) == Instruction::BitCast) {
237281ad6265SDimitry Andric V = cast<Operator>(V)->getOperand(0);
237381ad6265SDimitry Andric if (!V->getType()->isPointerTy())
237481ad6265SDimitry Andric return V;
237581ad6265SDimitry Andric } else if (isa<GlobalAlias>(V)) {
237681ad6265SDimitry Andric V = cast<GlobalAlias>(V)->getAliasee();
237781ad6265SDimitry Andric }
237881ad6265SDimitry Andric } while (Visited.insert(V).second);
237981ad6265SDimitry Andric
238081ad6265SDimitry Andric return V;
238181ad6265SDimitry Andric }
238281ad6265SDimitry Andric
visitLoadInst(LoadInst & LI)23830b57cec5SDimitry Andric void DFSanVisitor::visitLoadInst(LoadInst &LI) {
23840b57cec5SDimitry Andric auto &DL = LI.getModule()->getDataLayout();
23850b57cec5SDimitry Andric uint64_t Size = DL.getTypeStoreSize(LI.getType());
23860b57cec5SDimitry Andric if (Size == 0) {
2387e8d8bef9SDimitry Andric DFSF.setShadow(&LI, DFSF.DFS.getZeroShadow(&LI));
2388fe6060f1SDimitry Andric DFSF.setOrigin(&LI, DFSF.DFS.ZeroOrigin);
23890b57cec5SDimitry Andric return;
23900b57cec5SDimitry Andric }
23910b57cec5SDimitry Andric
2392fe6060f1SDimitry Andric // When an application load is atomic, increase atomic ordering between
2393fe6060f1SDimitry Andric // atomic application loads and stores to ensure happen-before order; load
2394fe6060f1SDimitry Andric // shadow data after application data; store zero shadow data before
2395fe6060f1SDimitry Andric // application data. This ensure shadow loads return either labels of the
2396fe6060f1SDimitry Andric // initial application data or zeros.
2397fe6060f1SDimitry Andric if (LI.isAtomic())
2398fe6060f1SDimitry Andric LI.setOrdering(addAcquireOrdering(LI.getOrdering()));
2399fe6060f1SDimitry Andric
2400bdd1243dSDimitry Andric Instruction *AfterLi = LI.getNextNode();
2401fe6060f1SDimitry Andric Instruction *Pos = LI.isAtomic() ? LI.getNextNode() : &LI;
2402fe6060f1SDimitry Andric std::vector<Value *> Shadows;
2403fe6060f1SDimitry Andric std::vector<Value *> Origins;
2404fe6060f1SDimitry Andric Value *PrimitiveShadow, *Origin;
2405fe6060f1SDimitry Andric std::tie(PrimitiveShadow, Origin) =
2406fe6060f1SDimitry Andric DFSF.loadShadowOrigin(LI.getPointerOperand(), Size, LI.getAlign(), Pos);
2407fe6060f1SDimitry Andric const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
2408fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2409fe6060f1SDimitry Andric Shadows.push_back(PrimitiveShadow);
2410fe6060f1SDimitry Andric Origins.push_back(Origin);
2411fe6060f1SDimitry Andric }
241281ad6265SDimitry Andric if (ClCombinePointerLabelsOnLoad ||
241381ad6265SDimitry Andric DFSF.isLookupTableConstant(
241481ad6265SDimitry Andric StripPointerGEPsAndCasts(LI.getPointerOperand()))) {
24150b57cec5SDimitry Andric Value *PtrShadow = DFSF.getShadow(LI.getPointerOperand());
2416fe6060f1SDimitry Andric PrimitiveShadow = DFSF.combineShadows(PrimitiveShadow, PtrShadow, Pos);
2417fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2418fe6060f1SDimitry Andric Shadows.push_back(PtrShadow);
2419fe6060f1SDimitry Andric Origins.push_back(DFSF.getOrigin(LI.getPointerOperand()));
2420fe6060f1SDimitry Andric }
24210b57cec5SDimitry Andric }
2422e8d8bef9SDimitry Andric if (!DFSF.DFS.isZeroShadow(PrimitiveShadow))
2423e8d8bef9SDimitry Andric DFSF.NonZeroChecks.push_back(PrimitiveShadow);
24240b57cec5SDimitry Andric
2425e8d8bef9SDimitry Andric Value *Shadow =
2426fe6060f1SDimitry Andric DFSF.expandFromPrimitiveShadow(LI.getType(), PrimitiveShadow, Pos);
24270b57cec5SDimitry Andric DFSF.setShadow(&LI, Shadow);
2428fe6060f1SDimitry Andric
2429fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2430fe6060f1SDimitry Andric DFSF.setOrigin(&LI, DFSF.combineOrigins(Shadows, Origins, Pos));
2431fe6060f1SDimitry Andric }
2432fe6060f1SDimitry Andric
24335ffd83dbSDimitry Andric if (ClEventCallbacks) {
2434fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
24355f757f3fSDimitry Andric Value *Addr = LI.getPointerOperand();
2436bdd1243dSDimitry Andric CallInst *CI =
24375f757f3fSDimitry Andric IRB.CreateCall(DFSF.DFS.DFSanLoadCallbackFn, {PrimitiveShadow, Addr});
2438bdd1243dSDimitry Andric CI->addParamAttr(0, Attribute::ZExt);
24395ffd83dbSDimitry Andric }
2440bdd1243dSDimitry Andric
2441bdd1243dSDimitry Andric IRBuilder<> IRB(AfterLi);
2442bdd1243dSDimitry Andric DFSF.addReachesFunctionCallbacksIfEnabled(IRB, LI, &LI);
24430b57cec5SDimitry Andric }
24440b57cec5SDimitry Andric
updateOriginIfTainted(Value * Shadow,Value * Origin,IRBuilder<> & IRB)2445fe6060f1SDimitry Andric Value *DFSanFunction::updateOriginIfTainted(Value *Shadow, Value *Origin,
2446fe6060f1SDimitry Andric IRBuilder<> &IRB) {
2447fe6060f1SDimitry Andric assert(DFS.shouldTrackOrigins());
2448fe6060f1SDimitry Andric return IRB.CreateCall(DFS.DFSanChainOriginIfTaintedFn, {Shadow, Origin});
2449fe6060f1SDimitry Andric }
2450fe6060f1SDimitry Andric
updateOrigin(Value * V,IRBuilder<> & IRB)2451fe6060f1SDimitry Andric Value *DFSanFunction::updateOrigin(Value *V, IRBuilder<> &IRB) {
2452fe6060f1SDimitry Andric if (!DFS.shouldTrackOrigins())
2453fe6060f1SDimitry Andric return V;
2454fe6060f1SDimitry Andric return IRB.CreateCall(DFS.DFSanChainOriginFn, V);
2455fe6060f1SDimitry Andric }
2456fe6060f1SDimitry Andric
originToIntptr(IRBuilder<> & IRB,Value * Origin)2457fe6060f1SDimitry Andric Value *DFSanFunction::originToIntptr(IRBuilder<> &IRB, Value *Origin) {
2458fe6060f1SDimitry Andric const unsigned OriginSize = DataFlowSanitizer::OriginWidthBytes;
2459fe6060f1SDimitry Andric const DataLayout &DL = F->getParent()->getDataLayout();
2460fe6060f1SDimitry Andric unsigned IntptrSize = DL.getTypeStoreSize(DFS.IntptrTy);
2461fe6060f1SDimitry Andric if (IntptrSize == OriginSize)
2462fe6060f1SDimitry Andric return Origin;
2463fe6060f1SDimitry Andric assert(IntptrSize == OriginSize * 2);
2464fe6060f1SDimitry Andric Origin = IRB.CreateIntCast(Origin, DFS.IntptrTy, /* isSigned */ false);
2465fe6060f1SDimitry Andric return IRB.CreateOr(Origin, IRB.CreateShl(Origin, OriginSize * 8));
2466fe6060f1SDimitry Andric }
2467fe6060f1SDimitry Andric
paintOrigin(IRBuilder<> & IRB,Value * Origin,Value * StoreOriginAddr,uint64_t StoreOriginSize,Align Alignment)2468fe6060f1SDimitry Andric void DFSanFunction::paintOrigin(IRBuilder<> &IRB, Value *Origin,
2469fe6060f1SDimitry Andric Value *StoreOriginAddr,
2470fe6060f1SDimitry Andric uint64_t StoreOriginSize, Align Alignment) {
2471fe6060f1SDimitry Andric const unsigned OriginSize = DataFlowSanitizer::OriginWidthBytes;
2472fe6060f1SDimitry Andric const DataLayout &DL = F->getParent()->getDataLayout();
2473fe6060f1SDimitry Andric const Align IntptrAlignment = DL.getABITypeAlign(DFS.IntptrTy);
2474fe6060f1SDimitry Andric unsigned IntptrSize = DL.getTypeStoreSize(DFS.IntptrTy);
2475fe6060f1SDimitry Andric assert(IntptrAlignment >= MinOriginAlignment);
2476fe6060f1SDimitry Andric assert(IntptrSize >= OriginSize);
2477fe6060f1SDimitry Andric
2478fe6060f1SDimitry Andric unsigned Ofs = 0;
2479fe6060f1SDimitry Andric Align CurrentAlignment = Alignment;
2480fe6060f1SDimitry Andric if (Alignment >= IntptrAlignment && IntptrSize > OriginSize) {
2481fe6060f1SDimitry Andric Value *IntptrOrigin = originToIntptr(IRB, Origin);
2482fe6060f1SDimitry Andric Value *IntptrStoreOriginPtr = IRB.CreatePointerCast(
2483fe6060f1SDimitry Andric StoreOriginAddr, PointerType::get(DFS.IntptrTy, 0));
2484fe6060f1SDimitry Andric for (unsigned I = 0; I < StoreOriginSize / IntptrSize; ++I) {
2485fe6060f1SDimitry Andric Value *Ptr =
2486fe6060f1SDimitry Andric I ? IRB.CreateConstGEP1_32(DFS.IntptrTy, IntptrStoreOriginPtr, I)
2487fe6060f1SDimitry Andric : IntptrStoreOriginPtr;
2488fe6060f1SDimitry Andric IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
2489fe6060f1SDimitry Andric Ofs += IntptrSize / OriginSize;
2490fe6060f1SDimitry Andric CurrentAlignment = IntptrAlignment;
24910b57cec5SDimitry Andric }
24920b57cec5SDimitry Andric }
24930b57cec5SDimitry Andric
2494fe6060f1SDimitry Andric for (unsigned I = Ofs; I < (StoreOriginSize + OriginSize - 1) / OriginSize;
2495fe6060f1SDimitry Andric ++I) {
2496fe6060f1SDimitry Andric Value *GEP = I ? IRB.CreateConstGEP1_32(DFS.OriginTy, StoreOriginAddr, I)
2497fe6060f1SDimitry Andric : StoreOriginAddr;
2498fe6060f1SDimitry Andric IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
2499fe6060f1SDimitry Andric CurrentAlignment = MinOriginAlignment;
2500fe6060f1SDimitry Andric }
2501fe6060f1SDimitry Andric }
2502fe6060f1SDimitry Andric
convertToBool(Value * V,IRBuilder<> & IRB,const Twine & Name)2503fe6060f1SDimitry Andric Value *DFSanFunction::convertToBool(Value *V, IRBuilder<> &IRB,
2504fe6060f1SDimitry Andric const Twine &Name) {
2505fe6060f1SDimitry Andric Type *VTy = V->getType();
2506fe6060f1SDimitry Andric assert(VTy->isIntegerTy());
2507fe6060f1SDimitry Andric if (VTy->getIntegerBitWidth() == 1)
2508fe6060f1SDimitry Andric // Just converting a bool to a bool, so do nothing.
2509fe6060f1SDimitry Andric return V;
2510fe6060f1SDimitry Andric return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), Name);
2511fe6060f1SDimitry Andric }
2512fe6060f1SDimitry Andric
storeOrigin(Instruction * Pos,Value * Addr,uint64_t Size,Value * Shadow,Value * Origin,Value * StoreOriginAddr,Align InstAlignment)2513fe6060f1SDimitry Andric void DFSanFunction::storeOrigin(Instruction *Pos, Value *Addr, uint64_t Size,
2514fe6060f1SDimitry Andric Value *Shadow, Value *Origin,
2515fe6060f1SDimitry Andric Value *StoreOriginAddr, Align InstAlignment) {
2516fe6060f1SDimitry Andric // Do not write origins for zero shadows because we do not trace origins for
2517fe6060f1SDimitry Andric // untainted sinks.
2518fe6060f1SDimitry Andric const Align OriginAlignment = getOriginAlign(InstAlignment);
2519fe6060f1SDimitry Andric Value *CollapsedShadow = collapseToPrimitiveShadow(Shadow, Pos);
25200b57cec5SDimitry Andric IRBuilder<> IRB(Pos);
2521fe6060f1SDimitry Andric if (auto *ConstantShadow = dyn_cast<Constant>(CollapsedShadow)) {
2522fe6060f1SDimitry Andric if (!ConstantShadow->isZeroValue())
2523fe6060f1SDimitry Andric paintOrigin(IRB, updateOrigin(Origin, IRB), StoreOriginAddr, Size,
2524fe6060f1SDimitry Andric OriginAlignment);
2525fe6060f1SDimitry Andric return;
2526fe6060f1SDimitry Andric }
2527fe6060f1SDimitry Andric
2528fe6060f1SDimitry Andric if (shouldInstrumentWithCall()) {
25295f757f3fSDimitry Andric IRB.CreateCall(
25305f757f3fSDimitry Andric DFS.DFSanMaybeStoreOriginFn,
25315f757f3fSDimitry Andric {CollapsedShadow, Addr, ConstantInt::get(DFS.IntptrTy, Size), Origin});
2532fe6060f1SDimitry Andric } else {
2533fe6060f1SDimitry Andric Value *Cmp = convertToBool(CollapsedShadow, IRB, "_dfscmp");
253406c3fb27SDimitry Andric DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
2535fe6060f1SDimitry Andric Instruction *CheckTerm = SplitBlockAndInsertIfThen(
253606c3fb27SDimitry Andric Cmp, &*IRB.GetInsertPoint(), false, DFS.OriginStoreWeights, &DTU);
2537fe6060f1SDimitry Andric IRBuilder<> IRBNew(CheckTerm);
2538fe6060f1SDimitry Andric paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), StoreOriginAddr, Size,
2539fe6060f1SDimitry Andric OriginAlignment);
2540fe6060f1SDimitry Andric ++NumOriginStores;
2541fe6060f1SDimitry Andric }
2542fe6060f1SDimitry Andric }
2543fe6060f1SDimitry Andric
storeZeroPrimitiveShadow(Value * Addr,uint64_t Size,Align ShadowAlign,Instruction * Pos)2544fe6060f1SDimitry Andric void DFSanFunction::storeZeroPrimitiveShadow(Value *Addr, uint64_t Size,
2545fe6060f1SDimitry Andric Align ShadowAlign,
2546fe6060f1SDimitry Andric Instruction *Pos) {
2547fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
25485ffd83dbSDimitry Andric IntegerType *ShadowTy =
25495ffd83dbSDimitry Andric IntegerType::get(*DFS.Ctx, Size * DFS.ShadowWidthBits);
25500b57cec5SDimitry Andric Value *ExtZeroShadow = ConstantInt::get(ShadowTy, 0);
2551fe6060f1SDimitry Andric Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos);
25525f757f3fSDimitry Andric IRB.CreateAlignedStore(ExtZeroShadow, ShadowAddr, ShadowAlign);
2553fe6060f1SDimitry Andric // Do not write origins for 0 shadows because we do not trace origins for
2554fe6060f1SDimitry Andric // untainted sinks.
2555fe6060f1SDimitry Andric }
2556fe6060f1SDimitry Andric
storePrimitiveShadowOrigin(Value * Addr,uint64_t Size,Align InstAlignment,Value * PrimitiveShadow,Value * Origin,Instruction * Pos)2557fe6060f1SDimitry Andric void DFSanFunction::storePrimitiveShadowOrigin(Value *Addr, uint64_t Size,
2558fe6060f1SDimitry Andric Align InstAlignment,
2559fe6060f1SDimitry Andric Value *PrimitiveShadow,
2560fe6060f1SDimitry Andric Value *Origin,
2561fe6060f1SDimitry Andric Instruction *Pos) {
2562fe6060f1SDimitry Andric const bool ShouldTrackOrigins = DFS.shouldTrackOrigins() && Origin;
2563fe6060f1SDimitry Andric
2564fe6060f1SDimitry Andric if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
2565fe6060f1SDimitry Andric const auto SI = AllocaShadowMap.find(AI);
2566fe6060f1SDimitry Andric if (SI != AllocaShadowMap.end()) {
2567fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
2568fe6060f1SDimitry Andric IRB.CreateStore(PrimitiveShadow, SI->second);
2569fe6060f1SDimitry Andric
2570fe6060f1SDimitry Andric // Do not write origins for 0 shadows because we do not trace origins for
2571fe6060f1SDimitry Andric // untainted sinks.
2572fe6060f1SDimitry Andric if (ShouldTrackOrigins && !DFS.isZeroShadow(PrimitiveShadow)) {
2573fe6060f1SDimitry Andric const auto OI = AllocaOriginMap.find(AI);
2574fe6060f1SDimitry Andric assert(OI != AllocaOriginMap.end() && Origin);
2575fe6060f1SDimitry Andric IRB.CreateStore(Origin, OI->second);
2576fe6060f1SDimitry Andric }
2577fe6060f1SDimitry Andric return;
2578fe6060f1SDimitry Andric }
2579fe6060f1SDimitry Andric }
2580fe6060f1SDimitry Andric
2581fe6060f1SDimitry Andric const Align ShadowAlign = getShadowAlign(InstAlignment);
2582fe6060f1SDimitry Andric if (DFS.isZeroShadow(PrimitiveShadow)) {
2583fe6060f1SDimitry Andric storeZeroPrimitiveShadow(Addr, Size, ShadowAlign, Pos);
25840b57cec5SDimitry Andric return;
25850b57cec5SDimitry Andric }
25860b57cec5SDimitry Andric
2587fe6060f1SDimitry Andric IRBuilder<> IRB(Pos);
2588fe6060f1SDimitry Andric Value *ShadowAddr, *OriginAddr;
2589fe6060f1SDimitry Andric std::tie(ShadowAddr, OriginAddr) =
2590fe6060f1SDimitry Andric DFS.getShadowOriginAddress(Addr, InstAlignment, Pos);
2591fe6060f1SDimitry Andric
2592fe6060f1SDimitry Andric const unsigned ShadowVecSize = 8;
2593fe6060f1SDimitry Andric assert(ShadowVecSize * DFS.ShadowWidthBits <= 128 &&
2594fe6060f1SDimitry Andric "Shadow vector is too large!");
2595fe6060f1SDimitry Andric
25960b57cec5SDimitry Andric uint64_t Offset = 0;
2597fe6060f1SDimitry Andric uint64_t LeftSize = Size;
2598fe6060f1SDimitry Andric if (LeftSize >= ShadowVecSize) {
2599e8d8bef9SDimitry Andric auto *ShadowVecTy =
2600e8d8bef9SDimitry Andric FixedVectorType::get(DFS.PrimitiveShadowTy, ShadowVecSize);
2601bdd1243dSDimitry Andric Value *ShadowVec = PoisonValue::get(ShadowVecTy);
2602fe6060f1SDimitry Andric for (unsigned I = 0; I != ShadowVecSize; ++I) {
26030b57cec5SDimitry Andric ShadowVec = IRB.CreateInsertElement(
2604e8d8bef9SDimitry Andric ShadowVec, PrimitiveShadow,
2605fe6060f1SDimitry Andric ConstantInt::get(Type::getInt32Ty(*DFS.Ctx), I));
26060b57cec5SDimitry Andric }
26070b57cec5SDimitry Andric do {
26080b57cec5SDimitry Andric Value *CurShadowVecAddr =
26095f757f3fSDimitry Andric IRB.CreateConstGEP1_32(ShadowVecTy, ShadowAddr, Offset);
26100b57cec5SDimitry Andric IRB.CreateAlignedStore(ShadowVec, CurShadowVecAddr, ShadowAlign);
2611fe6060f1SDimitry Andric LeftSize -= ShadowVecSize;
26120b57cec5SDimitry Andric ++Offset;
2613fe6060f1SDimitry Andric } while (LeftSize >= ShadowVecSize);
26140b57cec5SDimitry Andric Offset *= ShadowVecSize;
26150b57cec5SDimitry Andric }
2616fe6060f1SDimitry Andric while (LeftSize > 0) {
26170b57cec5SDimitry Andric Value *CurShadowAddr =
2618e8d8bef9SDimitry Andric IRB.CreateConstGEP1_32(DFS.PrimitiveShadowTy, ShadowAddr, Offset);
2619e8d8bef9SDimitry Andric IRB.CreateAlignedStore(PrimitiveShadow, CurShadowAddr, ShadowAlign);
2620fe6060f1SDimitry Andric --LeftSize;
26210b57cec5SDimitry Andric ++Offset;
26220b57cec5SDimitry Andric }
2623fe6060f1SDimitry Andric
2624fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2625fe6060f1SDimitry Andric storeOrigin(Pos, Addr, Size, PrimitiveShadow, Origin, OriginAddr,
2626fe6060f1SDimitry Andric InstAlignment);
2627fe6060f1SDimitry Andric }
2628fe6060f1SDimitry Andric }
2629fe6060f1SDimitry Andric
addReleaseOrdering(AtomicOrdering AO)2630fe6060f1SDimitry Andric static AtomicOrdering addReleaseOrdering(AtomicOrdering AO) {
2631fe6060f1SDimitry Andric switch (AO) {
2632fe6060f1SDimitry Andric case AtomicOrdering::NotAtomic:
2633fe6060f1SDimitry Andric return AtomicOrdering::NotAtomic;
2634fe6060f1SDimitry Andric case AtomicOrdering::Unordered:
2635fe6060f1SDimitry Andric case AtomicOrdering::Monotonic:
2636fe6060f1SDimitry Andric case AtomicOrdering::Release:
2637fe6060f1SDimitry Andric return AtomicOrdering::Release;
2638fe6060f1SDimitry Andric case AtomicOrdering::Acquire:
2639fe6060f1SDimitry Andric case AtomicOrdering::AcquireRelease:
2640fe6060f1SDimitry Andric return AtomicOrdering::AcquireRelease;
2641fe6060f1SDimitry Andric case AtomicOrdering::SequentiallyConsistent:
2642fe6060f1SDimitry Andric return AtomicOrdering::SequentiallyConsistent;
2643fe6060f1SDimitry Andric }
2644fe6060f1SDimitry Andric llvm_unreachable("Unknown ordering");
26450b57cec5SDimitry Andric }
26460b57cec5SDimitry Andric
visitStoreInst(StoreInst & SI)26470b57cec5SDimitry Andric void DFSanVisitor::visitStoreInst(StoreInst &SI) {
26480b57cec5SDimitry Andric auto &DL = SI.getModule()->getDataLayout();
2649fe6060f1SDimitry Andric Value *Val = SI.getValueOperand();
2650fe6060f1SDimitry Andric uint64_t Size = DL.getTypeStoreSize(Val->getType());
26510b57cec5SDimitry Andric if (Size == 0)
26520b57cec5SDimitry Andric return;
26530b57cec5SDimitry Andric
2654fe6060f1SDimitry Andric // When an application store is atomic, increase atomic ordering between
2655fe6060f1SDimitry Andric // atomic application loads and stores to ensure happen-before order; load
2656fe6060f1SDimitry Andric // shadow data after application data; store zero shadow data before
2657fe6060f1SDimitry Andric // application data. This ensure shadow loads return either labels of the
2658fe6060f1SDimitry Andric // initial application data or zeros.
2659fe6060f1SDimitry Andric if (SI.isAtomic())
2660fe6060f1SDimitry Andric SI.setOrdering(addReleaseOrdering(SI.getOrdering()));
26610b57cec5SDimitry Andric
2662fe6060f1SDimitry Andric const bool ShouldTrackOrigins =
2663fe6060f1SDimitry Andric DFSF.DFS.shouldTrackOrigins() && !SI.isAtomic();
2664fe6060f1SDimitry Andric std::vector<Value *> Shadows;
2665fe6060f1SDimitry Andric std::vector<Value *> Origins;
2666fe6060f1SDimitry Andric
2667fe6060f1SDimitry Andric Value *Shadow =
2668fe6060f1SDimitry Andric SI.isAtomic() ? DFSF.DFS.getZeroShadow(Val) : DFSF.getShadow(Val);
2669fe6060f1SDimitry Andric
2670fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2671fe6060f1SDimitry Andric Shadows.push_back(Shadow);
2672fe6060f1SDimitry Andric Origins.push_back(DFSF.getOrigin(Val));
2673fe6060f1SDimitry Andric }
2674fe6060f1SDimitry Andric
2675e8d8bef9SDimitry Andric Value *PrimitiveShadow;
26760b57cec5SDimitry Andric if (ClCombinePointerLabelsOnStore) {
26770b57cec5SDimitry Andric Value *PtrShadow = DFSF.getShadow(SI.getPointerOperand());
2678fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2679fe6060f1SDimitry Andric Shadows.push_back(PtrShadow);
2680fe6060f1SDimitry Andric Origins.push_back(DFSF.getOrigin(SI.getPointerOperand()));
2681fe6060f1SDimitry Andric }
2682e8d8bef9SDimitry Andric PrimitiveShadow = DFSF.combineShadows(Shadow, PtrShadow, &SI);
2683e8d8bef9SDimitry Andric } else {
2684e8d8bef9SDimitry Andric PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow, &SI);
26850b57cec5SDimitry Andric }
2686fe6060f1SDimitry Andric Value *Origin = nullptr;
2687fe6060f1SDimitry Andric if (ShouldTrackOrigins)
2688fe6060f1SDimitry Andric Origin = DFSF.combineOrigins(Shadows, Origins, &SI);
2689fe6060f1SDimitry Andric DFSF.storePrimitiveShadowOrigin(SI.getPointerOperand(), Size, SI.getAlign(),
2690fe6060f1SDimitry Andric PrimitiveShadow, Origin, &SI);
26915ffd83dbSDimitry Andric if (ClEventCallbacks) {
26925ffd83dbSDimitry Andric IRBuilder<> IRB(&SI);
26935f757f3fSDimitry Andric Value *Addr = SI.getPointerOperand();
2694bdd1243dSDimitry Andric CallInst *CI =
26955f757f3fSDimitry Andric IRB.CreateCall(DFSF.DFS.DFSanStoreCallbackFn, {PrimitiveShadow, Addr});
2696bdd1243dSDimitry Andric CI->addParamAttr(0, Attribute::ZExt);
26975ffd83dbSDimitry Andric }
26980b57cec5SDimitry Andric }
26990b57cec5SDimitry Andric
visitCASOrRMW(Align InstAlignment,Instruction & I)2700fe6060f1SDimitry Andric void DFSanVisitor::visitCASOrRMW(Align InstAlignment, Instruction &I) {
2701fe6060f1SDimitry Andric assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
2702fe6060f1SDimitry Andric
2703fe6060f1SDimitry Andric Value *Val = I.getOperand(1);
2704fe6060f1SDimitry Andric const auto &DL = I.getModule()->getDataLayout();
2705fe6060f1SDimitry Andric uint64_t Size = DL.getTypeStoreSize(Val->getType());
2706fe6060f1SDimitry Andric if (Size == 0)
2707fe6060f1SDimitry Andric return;
2708fe6060f1SDimitry Andric
2709fe6060f1SDimitry Andric // Conservatively set data at stored addresses and return with zero shadow to
2710fe6060f1SDimitry Andric // prevent shadow data races.
2711fe6060f1SDimitry Andric IRBuilder<> IRB(&I);
2712fe6060f1SDimitry Andric Value *Addr = I.getOperand(0);
2713fe6060f1SDimitry Andric const Align ShadowAlign = DFSF.getShadowAlign(InstAlignment);
2714fe6060f1SDimitry Andric DFSF.storeZeroPrimitiveShadow(Addr, Size, ShadowAlign, &I);
2715fe6060f1SDimitry Andric DFSF.setShadow(&I, DFSF.DFS.getZeroShadow(&I));
2716fe6060f1SDimitry Andric DFSF.setOrigin(&I, DFSF.DFS.ZeroOrigin);
2717fe6060f1SDimitry Andric }
2718fe6060f1SDimitry Andric
visitAtomicRMWInst(AtomicRMWInst & I)2719fe6060f1SDimitry Andric void DFSanVisitor::visitAtomicRMWInst(AtomicRMWInst &I) {
2720fe6060f1SDimitry Andric visitCASOrRMW(I.getAlign(), I);
2721fe6060f1SDimitry Andric // TODO: The ordering change follows MSan. It is possible not to change
2722fe6060f1SDimitry Andric // ordering because we always set and use 0 shadows.
2723fe6060f1SDimitry Andric I.setOrdering(addReleaseOrdering(I.getOrdering()));
2724fe6060f1SDimitry Andric }
2725fe6060f1SDimitry Andric
visitAtomicCmpXchgInst(AtomicCmpXchgInst & I)2726fe6060f1SDimitry Andric void DFSanVisitor::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2727fe6060f1SDimitry Andric visitCASOrRMW(I.getAlign(), I);
2728fe6060f1SDimitry Andric // TODO: The ordering change follows MSan. It is possible not to change
2729fe6060f1SDimitry Andric // ordering because we always set and use 0 shadows.
2730fe6060f1SDimitry Andric I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
2731fe6060f1SDimitry Andric }
2732fe6060f1SDimitry Andric
visitUnaryOperator(UnaryOperator & UO)27330b57cec5SDimitry Andric void DFSanVisitor::visitUnaryOperator(UnaryOperator &UO) {
2734fe6060f1SDimitry Andric visitInstOperands(UO);
27350b57cec5SDimitry Andric }
27360b57cec5SDimitry Andric
visitBinaryOperator(BinaryOperator & BO)27370b57cec5SDimitry Andric void DFSanVisitor::visitBinaryOperator(BinaryOperator &BO) {
2738fe6060f1SDimitry Andric visitInstOperands(BO);
27390b57cec5SDimitry Andric }
27400b57cec5SDimitry Andric
visitBitCastInst(BitCastInst & BCI)2741fe6060f1SDimitry Andric void DFSanVisitor::visitBitCastInst(BitCastInst &BCI) {
2742fe6060f1SDimitry Andric // Special case: if this is the bitcast (there is exactly 1 allowed) between
2743fe6060f1SDimitry Andric // a musttail call and a ret, don't instrument. New instructions are not
2744fe6060f1SDimitry Andric // allowed after a musttail call.
2745fe6060f1SDimitry Andric if (auto *CI = dyn_cast<CallInst>(BCI.getOperand(0)))
2746fe6060f1SDimitry Andric if (CI->isMustTailCall())
2747fe6060f1SDimitry Andric return;
2748fe6060f1SDimitry Andric visitInstOperands(BCI);
2749fe6060f1SDimitry Andric }
2750fe6060f1SDimitry Andric
visitCastInst(CastInst & CI)2751fe6060f1SDimitry Andric void DFSanVisitor::visitCastInst(CastInst &CI) { visitInstOperands(CI); }
27520b57cec5SDimitry Andric
visitCmpInst(CmpInst & CI)27535ffd83dbSDimitry Andric void DFSanVisitor::visitCmpInst(CmpInst &CI) {
2754fe6060f1SDimitry Andric visitInstOperands(CI);
27555ffd83dbSDimitry Andric if (ClEventCallbacks) {
27565ffd83dbSDimitry Andric IRBuilder<> IRB(&CI);
2757fe6060f1SDimitry Andric Value *CombinedShadow = DFSF.getShadow(&CI);
2758bdd1243dSDimitry Andric CallInst *CallI =
27595ffd83dbSDimitry Andric IRB.CreateCall(DFSF.DFS.DFSanCmpCallbackFn, CombinedShadow);
2760bdd1243dSDimitry Andric CallI->addParamAttr(0, Attribute::ZExt);
27615ffd83dbSDimitry Andric }
27625ffd83dbSDimitry Andric }
27630b57cec5SDimitry Andric
visitLandingPadInst(LandingPadInst & LPI)2764fe6060f1SDimitry Andric void DFSanVisitor::visitLandingPadInst(LandingPadInst &LPI) {
2765fe6060f1SDimitry Andric // We do not need to track data through LandingPadInst.
2766fe6060f1SDimitry Andric //
2767fe6060f1SDimitry Andric // For the C++ exceptions, if a value is thrown, this value will be stored
2768fe6060f1SDimitry Andric // in a memory location provided by __cxa_allocate_exception(...) (on the
2769fe6060f1SDimitry Andric // throw side) or __cxa_begin_catch(...) (on the catch side).
2770fe6060f1SDimitry Andric // This memory will have a shadow, so with the loads and stores we will be
2771fe6060f1SDimitry Andric // able to propagate labels on data thrown through exceptions, without any
2772fe6060f1SDimitry Andric // special handling of the LandingPadInst.
2773fe6060f1SDimitry Andric //
2774fe6060f1SDimitry Andric // The second element in the pair result of the LandingPadInst is a
2775fe6060f1SDimitry Andric // register value, but it is for a type ID and should never be tainted.
2776fe6060f1SDimitry Andric DFSF.setShadow(&LPI, DFSF.DFS.getZeroShadow(&LPI));
2777fe6060f1SDimitry Andric DFSF.setOrigin(&LPI, DFSF.DFS.ZeroOrigin);
2778fe6060f1SDimitry Andric }
2779fe6060f1SDimitry Andric
visitGetElementPtrInst(GetElementPtrInst & GEPI)27800b57cec5SDimitry Andric void DFSanVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
278181ad6265SDimitry Andric if (ClCombineOffsetLabelsOnGEP ||
278281ad6265SDimitry Andric DFSF.isLookupTableConstant(
278381ad6265SDimitry Andric StripPointerGEPsAndCasts(GEPI.getPointerOperand()))) {
2784fe6060f1SDimitry Andric visitInstOperands(GEPI);
2785fe6060f1SDimitry Andric return;
2786fe6060f1SDimitry Andric }
2787fe6060f1SDimitry Andric
2788fe6060f1SDimitry Andric // Only propagate shadow/origin of base pointer value but ignore those of
2789fe6060f1SDimitry Andric // offset operands.
2790fe6060f1SDimitry Andric Value *BasePointer = GEPI.getPointerOperand();
2791fe6060f1SDimitry Andric DFSF.setShadow(&GEPI, DFSF.getShadow(BasePointer));
2792fe6060f1SDimitry Andric if (DFSF.DFS.shouldTrackOrigins())
2793fe6060f1SDimitry Andric DFSF.setOrigin(&GEPI, DFSF.getOrigin(BasePointer));
27940b57cec5SDimitry Andric }
27950b57cec5SDimitry Andric
visitExtractElementInst(ExtractElementInst & I)27960b57cec5SDimitry Andric void DFSanVisitor::visitExtractElementInst(ExtractElementInst &I) {
2797fe6060f1SDimitry Andric visitInstOperands(I);
27980b57cec5SDimitry Andric }
27990b57cec5SDimitry Andric
visitInsertElementInst(InsertElementInst & I)28000b57cec5SDimitry Andric void DFSanVisitor::visitInsertElementInst(InsertElementInst &I) {
2801fe6060f1SDimitry Andric visitInstOperands(I);
28020b57cec5SDimitry Andric }
28030b57cec5SDimitry Andric
visitShuffleVectorInst(ShuffleVectorInst & I)28040b57cec5SDimitry Andric void DFSanVisitor::visitShuffleVectorInst(ShuffleVectorInst &I) {
2805fe6060f1SDimitry Andric visitInstOperands(I);
28060b57cec5SDimitry Andric }
28070b57cec5SDimitry Andric
visitExtractValueInst(ExtractValueInst & I)28080b57cec5SDimitry Andric void DFSanVisitor::visitExtractValueInst(ExtractValueInst &I) {
2809e8d8bef9SDimitry Andric IRBuilder<> IRB(&I);
2810e8d8bef9SDimitry Andric Value *Agg = I.getAggregateOperand();
2811e8d8bef9SDimitry Andric Value *AggShadow = DFSF.getShadow(Agg);
2812e8d8bef9SDimitry Andric Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
2813e8d8bef9SDimitry Andric DFSF.setShadow(&I, ResShadow);
2814fe6060f1SDimitry Andric visitInstOperandOrigins(I);
28150b57cec5SDimitry Andric }
28160b57cec5SDimitry Andric
visitInsertValueInst(InsertValueInst & I)28170b57cec5SDimitry Andric void DFSanVisitor::visitInsertValueInst(InsertValueInst &I) {
2818e8d8bef9SDimitry Andric IRBuilder<> IRB(&I);
2819e8d8bef9SDimitry Andric Value *AggShadow = DFSF.getShadow(I.getAggregateOperand());
2820e8d8bef9SDimitry Andric Value *InsShadow = DFSF.getShadow(I.getInsertedValueOperand());
2821e8d8bef9SDimitry Andric Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
2822e8d8bef9SDimitry Andric DFSF.setShadow(&I, Res);
2823fe6060f1SDimitry Andric visitInstOperandOrigins(I);
28240b57cec5SDimitry Andric }
28250b57cec5SDimitry Andric
visitAllocaInst(AllocaInst & I)28260b57cec5SDimitry Andric void DFSanVisitor::visitAllocaInst(AllocaInst &I) {
28270b57cec5SDimitry Andric bool AllLoadsStores = true;
28280b57cec5SDimitry Andric for (User *U : I.users()) {
28290b57cec5SDimitry Andric if (isa<LoadInst>(U))
28300b57cec5SDimitry Andric continue;
28310b57cec5SDimitry Andric
28320b57cec5SDimitry Andric if (StoreInst *SI = dyn_cast<StoreInst>(U)) {
28330b57cec5SDimitry Andric if (SI->getPointerOperand() == &I)
28340b57cec5SDimitry Andric continue;
28350b57cec5SDimitry Andric }
28360b57cec5SDimitry Andric
28370b57cec5SDimitry Andric AllLoadsStores = false;
28380b57cec5SDimitry Andric break;
28390b57cec5SDimitry Andric }
28400b57cec5SDimitry Andric if (AllLoadsStores) {
28410b57cec5SDimitry Andric IRBuilder<> IRB(&I);
2842e8d8bef9SDimitry Andric DFSF.AllocaShadowMap[&I] = IRB.CreateAlloca(DFSF.DFS.PrimitiveShadowTy);
2843fe6060f1SDimitry Andric if (DFSF.DFS.shouldTrackOrigins()) {
2844fe6060f1SDimitry Andric DFSF.AllocaOriginMap[&I] =
2845fe6060f1SDimitry Andric IRB.CreateAlloca(DFSF.DFS.OriginTy, nullptr, "_dfsa");
2846fe6060f1SDimitry Andric }
28470b57cec5SDimitry Andric }
2848e8d8bef9SDimitry Andric DFSF.setShadow(&I, DFSF.DFS.ZeroPrimitiveShadow);
2849fe6060f1SDimitry Andric DFSF.setOrigin(&I, DFSF.DFS.ZeroOrigin);
28500b57cec5SDimitry Andric }
28510b57cec5SDimitry Andric
visitSelectInst(SelectInst & I)28520b57cec5SDimitry Andric void DFSanVisitor::visitSelectInst(SelectInst &I) {
28530b57cec5SDimitry Andric Value *CondShadow = DFSF.getShadow(I.getCondition());
28540b57cec5SDimitry Andric Value *TrueShadow = DFSF.getShadow(I.getTrueValue());
28550b57cec5SDimitry Andric Value *FalseShadow = DFSF.getShadow(I.getFalseValue());
2856e8d8bef9SDimitry Andric Value *ShadowSel = nullptr;
2857fe6060f1SDimitry Andric const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
2858fe6060f1SDimitry Andric std::vector<Value *> Shadows;
2859fe6060f1SDimitry Andric std::vector<Value *> Origins;
2860fe6060f1SDimitry Andric Value *TrueOrigin =
2861fe6060f1SDimitry Andric ShouldTrackOrigins ? DFSF.getOrigin(I.getTrueValue()) : nullptr;
2862fe6060f1SDimitry Andric Value *FalseOrigin =
2863fe6060f1SDimitry Andric ShouldTrackOrigins ? DFSF.getOrigin(I.getFalseValue()) : nullptr;
28640b57cec5SDimitry Andric
286504eeddc0SDimitry Andric DFSF.addConditionalCallbacksIfEnabled(I, I.getCondition());
286604eeddc0SDimitry Andric
28670b57cec5SDimitry Andric if (isa<VectorType>(I.getCondition()->getType())) {
2868e8d8bef9SDimitry Andric ShadowSel = DFSF.combineShadowsThenConvert(I.getType(), TrueShadow,
2869e8d8bef9SDimitry Andric FalseShadow, &I);
2870fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2871fe6060f1SDimitry Andric Shadows.push_back(TrueShadow);
2872fe6060f1SDimitry Andric Shadows.push_back(FalseShadow);
2873fe6060f1SDimitry Andric Origins.push_back(TrueOrigin);
2874fe6060f1SDimitry Andric Origins.push_back(FalseOrigin);
2875fe6060f1SDimitry Andric }
28760b57cec5SDimitry Andric } else {
28770b57cec5SDimitry Andric if (TrueShadow == FalseShadow) {
28780b57cec5SDimitry Andric ShadowSel = TrueShadow;
2879fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2880fe6060f1SDimitry Andric Shadows.push_back(TrueShadow);
2881fe6060f1SDimitry Andric Origins.push_back(TrueOrigin);
2882fe6060f1SDimitry Andric }
28830b57cec5SDimitry Andric } else {
28840b57cec5SDimitry Andric ShadowSel =
28850b57cec5SDimitry Andric SelectInst::Create(I.getCondition(), TrueShadow, FalseShadow, "", &I);
2886fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2887fe6060f1SDimitry Andric Shadows.push_back(ShadowSel);
2888fe6060f1SDimitry Andric Origins.push_back(SelectInst::Create(I.getCondition(), TrueOrigin,
2889fe6060f1SDimitry Andric FalseOrigin, "", &I));
2890fe6060f1SDimitry Andric }
28910b57cec5SDimitry Andric }
28920b57cec5SDimitry Andric }
2893e8d8bef9SDimitry Andric DFSF.setShadow(&I, ClTrackSelectControlFlow
2894e8d8bef9SDimitry Andric ? DFSF.combineShadowsThenConvert(
2895e8d8bef9SDimitry Andric I.getType(), CondShadow, ShadowSel, &I)
2896e8d8bef9SDimitry Andric : ShadowSel);
2897fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
2898fe6060f1SDimitry Andric if (ClTrackSelectControlFlow) {
2899fe6060f1SDimitry Andric Shadows.push_back(CondShadow);
2900fe6060f1SDimitry Andric Origins.push_back(DFSF.getOrigin(I.getCondition()));
2901fe6060f1SDimitry Andric }
2902fe6060f1SDimitry Andric DFSF.setOrigin(&I, DFSF.combineOrigins(Shadows, Origins, &I));
2903fe6060f1SDimitry Andric }
29040b57cec5SDimitry Andric }
29050b57cec5SDimitry Andric
visitMemSetInst(MemSetInst & I)29060b57cec5SDimitry Andric void DFSanVisitor::visitMemSetInst(MemSetInst &I) {
29070b57cec5SDimitry Andric IRBuilder<> IRB(&I);
29080b57cec5SDimitry Andric Value *ValShadow = DFSF.getShadow(I.getValue());
2909fe6060f1SDimitry Andric Value *ValOrigin = DFSF.DFS.shouldTrackOrigins()
2910fe6060f1SDimitry Andric ? DFSF.getOrigin(I.getValue())
2911fe6060f1SDimitry Andric : DFSF.DFS.ZeroOrigin;
29125f757f3fSDimitry Andric IRB.CreateCall(DFSF.DFS.DFSanSetLabelFn,
29135f757f3fSDimitry Andric {ValShadow, ValOrigin, I.getDest(),
29140b57cec5SDimitry Andric IRB.CreateZExtOrTrunc(I.getLength(), DFSF.DFS.IntptrTy)});
29150b57cec5SDimitry Andric }
29160b57cec5SDimitry Andric
visitMemTransferInst(MemTransferInst & I)29170b57cec5SDimitry Andric void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) {
29180b57cec5SDimitry Andric IRBuilder<> IRB(&I);
2919fe6060f1SDimitry Andric
2920fe6060f1SDimitry Andric // CopyOrMoveOrigin transfers origins by refering to their shadows. So we
2921fe6060f1SDimitry Andric // need to move origins before moving shadows.
2922fe6060f1SDimitry Andric if (DFSF.DFS.shouldTrackOrigins()) {
2923fe6060f1SDimitry Andric IRB.CreateCall(
2924fe6060f1SDimitry Andric DFSF.DFS.DFSanMemOriginTransferFn,
29255f757f3fSDimitry Andric {I.getArgOperand(0), I.getArgOperand(1),
2926fe6060f1SDimitry Andric IRB.CreateIntCast(I.getArgOperand(2), DFSF.DFS.IntptrTy, false)});
2927fe6060f1SDimitry Andric }
2928fe6060f1SDimitry Andric
29295f757f3fSDimitry Andric Value *DestShadow = DFSF.DFS.getShadowAddress(I.getDest(), &I);
29300b57cec5SDimitry Andric Value *SrcShadow = DFSF.DFS.getShadowAddress(I.getSource(), &I);
29315ffd83dbSDimitry Andric Value *LenShadow =
29325ffd83dbSDimitry Andric IRB.CreateMul(I.getLength(), ConstantInt::get(I.getLength()->getType(),
29335ffd83dbSDimitry Andric DFSF.DFS.ShadowWidthBytes));
29340b57cec5SDimitry Andric auto *MTI = cast<MemTransferInst>(
29355ffd83dbSDimitry Andric IRB.CreateCall(I.getFunctionType(), I.getCalledOperand(),
29360b57cec5SDimitry Andric {DestShadow, SrcShadow, LenShadow, I.getVolatileCst()}));
293781ad6265SDimitry Andric MTI->setDestAlignment(DFSF.getShadowAlign(I.getDestAlign().valueOrOne()));
293881ad6265SDimitry Andric MTI->setSourceAlignment(DFSF.getShadowAlign(I.getSourceAlign().valueOrOne()));
29395ffd83dbSDimitry Andric if (ClEventCallbacks) {
29405f757f3fSDimitry Andric IRB.CreateCall(
29415f757f3fSDimitry Andric DFSF.DFS.DFSanMemTransferCallbackFn,
29425f757f3fSDimitry Andric {DestShadow, IRB.CreateZExtOrTrunc(I.getLength(), DFSF.DFS.IntptrTy)});
29430b57cec5SDimitry Andric }
29440b57cec5SDimitry Andric }
29450b57cec5SDimitry Andric
visitBranchInst(BranchInst & BR)294604eeddc0SDimitry Andric void DFSanVisitor::visitBranchInst(BranchInst &BR) {
294704eeddc0SDimitry Andric if (!BR.isConditional())
294804eeddc0SDimitry Andric return;
294904eeddc0SDimitry Andric
295004eeddc0SDimitry Andric DFSF.addConditionalCallbacksIfEnabled(BR, BR.getCondition());
295104eeddc0SDimitry Andric }
295204eeddc0SDimitry Andric
visitSwitchInst(SwitchInst & SW)295304eeddc0SDimitry Andric void DFSanVisitor::visitSwitchInst(SwitchInst &SW) {
295404eeddc0SDimitry Andric DFSF.addConditionalCallbacksIfEnabled(SW, SW.getCondition());
295504eeddc0SDimitry Andric }
295604eeddc0SDimitry Andric
isAMustTailRetVal(Value * RetVal)2957fe6060f1SDimitry Andric static bool isAMustTailRetVal(Value *RetVal) {
2958fe6060f1SDimitry Andric // Tail call may have a bitcast between return.
2959fe6060f1SDimitry Andric if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
2960fe6060f1SDimitry Andric RetVal = I->getOperand(0);
2961fe6060f1SDimitry Andric }
2962fe6060f1SDimitry Andric if (auto *I = dyn_cast<CallInst>(RetVal)) {
2963fe6060f1SDimitry Andric return I->isMustTailCall();
2964fe6060f1SDimitry Andric }
2965fe6060f1SDimitry Andric return false;
2966fe6060f1SDimitry Andric }
2967fe6060f1SDimitry Andric
visitReturnInst(ReturnInst & RI)29680b57cec5SDimitry Andric void DFSanVisitor::visitReturnInst(ReturnInst &RI) {
29690b57cec5SDimitry Andric if (!DFSF.IsNativeABI && RI.getReturnValue()) {
2970fe6060f1SDimitry Andric // Don't emit the instrumentation for musttail call returns.
2971fe6060f1SDimitry Andric if (isAMustTailRetVal(RI.getReturnValue()))
2972fe6060f1SDimitry Andric return;
2973fe6060f1SDimitry Andric
29740b57cec5SDimitry Andric Value *S = DFSF.getShadow(RI.getReturnValue());
29750b57cec5SDimitry Andric IRBuilder<> IRB(&RI);
2976e8d8bef9SDimitry Andric Type *RT = DFSF.F->getFunctionType()->getReturnType();
2977349cc55cSDimitry Andric unsigned Size = getDataLayout().getTypeAllocSize(DFSF.DFS.getShadowTy(RT));
2978fe6060f1SDimitry Andric if (Size <= RetvalTLSSize) {
2979e8d8bef9SDimitry Andric // If the size overflows, stores nothing. At callsite, oversized return
2980e8d8bef9SDimitry Andric // shadows are set to zero.
2981349cc55cSDimitry Andric IRB.CreateAlignedStore(S, DFSF.getRetvalTLS(RT, IRB), ShadowTLSAlignment);
2982fe6060f1SDimitry Andric }
2983fe6060f1SDimitry Andric if (DFSF.DFS.shouldTrackOrigins()) {
2984fe6060f1SDimitry Andric Value *O = DFSF.getOrigin(RI.getReturnValue());
2985fe6060f1SDimitry Andric IRB.CreateStore(O, DFSF.getRetvalOriginTLS());
2986e8d8bef9SDimitry Andric }
29870b57cec5SDimitry Andric }
29880b57cec5SDimitry Andric }
29890b57cec5SDimitry Andric
addShadowArguments(Function & F,CallBase & CB,std::vector<Value * > & Args,IRBuilder<> & IRB)2990fe6060f1SDimitry Andric void DFSanVisitor::addShadowArguments(Function &F, CallBase &CB,
2991fe6060f1SDimitry Andric std::vector<Value *> &Args,
2992fe6060f1SDimitry Andric IRBuilder<> &IRB) {
2993fe6060f1SDimitry Andric FunctionType *FT = F.getFunctionType();
2994fe6060f1SDimitry Andric
2995fe6060f1SDimitry Andric auto *I = CB.arg_begin();
2996fe6060f1SDimitry Andric
2997fe6060f1SDimitry Andric // Adds non-variable argument shadows.
2998fe6060f1SDimitry Andric for (unsigned N = FT->getNumParams(); N != 0; ++I, --N)
2999fe6060f1SDimitry Andric Args.push_back(DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*I), &CB));
3000fe6060f1SDimitry Andric
3001fe6060f1SDimitry Andric // Adds variable argument shadows.
3002fe6060f1SDimitry Andric if (FT->isVarArg()) {
3003fe6060f1SDimitry Andric auto *LabelVATy = ArrayType::get(DFSF.DFS.PrimitiveShadowTy,
3004fe6060f1SDimitry Andric CB.arg_size() - FT->getNumParams());
3005fe6060f1SDimitry Andric auto *LabelVAAlloca =
3006fe6060f1SDimitry Andric new AllocaInst(LabelVATy, getDataLayout().getAllocaAddrSpace(),
3007fe6060f1SDimitry Andric "labelva", &DFSF.F->getEntryBlock().front());
3008fe6060f1SDimitry Andric
3009fe6060f1SDimitry Andric for (unsigned N = 0; I != CB.arg_end(); ++I, ++N) {
3010fe6060f1SDimitry Andric auto *LabelVAPtr = IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, N);
3011fe6060f1SDimitry Andric IRB.CreateStore(DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*I), &CB),
3012fe6060f1SDimitry Andric LabelVAPtr);
30130b57cec5SDimitry Andric }
30140b57cec5SDimitry Andric
3015fe6060f1SDimitry Andric Args.push_back(IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, 0));
3016fe6060f1SDimitry Andric }
30170b57cec5SDimitry Andric
3018fe6060f1SDimitry Andric // Adds the return value shadow.
3019fe6060f1SDimitry Andric if (!FT->getReturnType()->isVoidTy()) {
3020fe6060f1SDimitry Andric if (!DFSF.LabelReturnAlloca) {
3021fe6060f1SDimitry Andric DFSF.LabelReturnAlloca = new AllocaInst(
3022fe6060f1SDimitry Andric DFSF.DFS.PrimitiveShadowTy, getDataLayout().getAllocaAddrSpace(),
3023fe6060f1SDimitry Andric "labelreturn", &DFSF.F->getEntryBlock().front());
3024fe6060f1SDimitry Andric }
3025fe6060f1SDimitry Andric Args.push_back(DFSF.LabelReturnAlloca);
3026fe6060f1SDimitry Andric }
3027fe6060f1SDimitry Andric }
3028fe6060f1SDimitry Andric
addOriginArguments(Function & F,CallBase & CB,std::vector<Value * > & Args,IRBuilder<> & IRB)3029fe6060f1SDimitry Andric void DFSanVisitor::addOriginArguments(Function &F, CallBase &CB,
3030fe6060f1SDimitry Andric std::vector<Value *> &Args,
3031fe6060f1SDimitry Andric IRBuilder<> &IRB) {
3032fe6060f1SDimitry Andric FunctionType *FT = F.getFunctionType();
3033fe6060f1SDimitry Andric
3034fe6060f1SDimitry Andric auto *I = CB.arg_begin();
3035fe6060f1SDimitry Andric
3036fe6060f1SDimitry Andric // Add non-variable argument origins.
3037fe6060f1SDimitry Andric for (unsigned N = FT->getNumParams(); N != 0; ++I, --N)
3038fe6060f1SDimitry Andric Args.push_back(DFSF.getOrigin(*I));
3039fe6060f1SDimitry Andric
3040fe6060f1SDimitry Andric // Add variable argument origins.
3041fe6060f1SDimitry Andric if (FT->isVarArg()) {
3042fe6060f1SDimitry Andric auto *OriginVATy =
3043fe6060f1SDimitry Andric ArrayType::get(DFSF.DFS.OriginTy, CB.arg_size() - FT->getNumParams());
3044fe6060f1SDimitry Andric auto *OriginVAAlloca =
3045fe6060f1SDimitry Andric new AllocaInst(OriginVATy, getDataLayout().getAllocaAddrSpace(),
3046fe6060f1SDimitry Andric "originva", &DFSF.F->getEntryBlock().front());
3047fe6060f1SDimitry Andric
3048fe6060f1SDimitry Andric for (unsigned N = 0; I != CB.arg_end(); ++I, ++N) {
3049fe6060f1SDimitry Andric auto *OriginVAPtr = IRB.CreateStructGEP(OriginVATy, OriginVAAlloca, N);
3050fe6060f1SDimitry Andric IRB.CreateStore(DFSF.getOrigin(*I), OriginVAPtr);
3051fe6060f1SDimitry Andric }
3052fe6060f1SDimitry Andric
3053fe6060f1SDimitry Andric Args.push_back(IRB.CreateStructGEP(OriginVATy, OriginVAAlloca, 0));
3054fe6060f1SDimitry Andric }
3055fe6060f1SDimitry Andric
3056fe6060f1SDimitry Andric // Add the return value origin.
3057fe6060f1SDimitry Andric if (!FT->getReturnType()->isVoidTy()) {
3058fe6060f1SDimitry Andric if (!DFSF.OriginReturnAlloca) {
3059fe6060f1SDimitry Andric DFSF.OriginReturnAlloca = new AllocaInst(
3060fe6060f1SDimitry Andric DFSF.DFS.OriginTy, getDataLayout().getAllocaAddrSpace(),
3061fe6060f1SDimitry Andric "originreturn", &DFSF.F->getEntryBlock().front());
3062fe6060f1SDimitry Andric }
3063fe6060f1SDimitry Andric Args.push_back(DFSF.OriginReturnAlloca);
3064fe6060f1SDimitry Andric }
3065fe6060f1SDimitry Andric }
3066fe6060f1SDimitry Andric
visitWrappedCallBase(Function & F,CallBase & CB)3067fe6060f1SDimitry Andric bool DFSanVisitor::visitWrappedCallBase(Function &F, CallBase &CB) {
30685ffd83dbSDimitry Andric IRBuilder<> IRB(&CB);
3069fe6060f1SDimitry Andric switch (DFSF.DFS.getWrapperKind(&F)) {
30700b57cec5SDimitry Andric case DataFlowSanitizer::WK_Warning:
3071fe6060f1SDimitry Andric CB.setCalledFunction(&F);
30720b57cec5SDimitry Andric IRB.CreateCall(DFSF.DFS.DFSanUnimplementedFn,
3073fe6060f1SDimitry Andric IRB.CreateGlobalStringPtr(F.getName()));
307481ad6265SDimitry Andric DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &F);
3075e8d8bef9SDimitry Andric DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3076fe6060f1SDimitry Andric DFSF.setOrigin(&CB, DFSF.DFS.ZeroOrigin);
3077fe6060f1SDimitry Andric return true;
30780b57cec5SDimitry Andric case DataFlowSanitizer::WK_Discard:
3079fe6060f1SDimitry Andric CB.setCalledFunction(&F);
308081ad6265SDimitry Andric DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &F);
3081e8d8bef9SDimitry Andric DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3082fe6060f1SDimitry Andric DFSF.setOrigin(&CB, DFSF.DFS.ZeroOrigin);
3083fe6060f1SDimitry Andric return true;
30840b57cec5SDimitry Andric case DataFlowSanitizer::WK_Functional:
3085fe6060f1SDimitry Andric CB.setCalledFunction(&F);
308681ad6265SDimitry Andric DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &F);
3087fe6060f1SDimitry Andric visitInstOperands(CB);
3088fe6060f1SDimitry Andric return true;
30890b57cec5SDimitry Andric case DataFlowSanitizer::WK_Custom:
30900b57cec5SDimitry Andric // Don't try to handle invokes of custom functions, it's too complicated.
30910b57cec5SDimitry Andric // Instead, invoke the dfsw$ wrapper, which will in turn call the __dfsw_
30920b57cec5SDimitry Andric // wrapper.
3093fe6060f1SDimitry Andric CallInst *CI = dyn_cast<CallInst>(&CB);
3094fe6060f1SDimitry Andric if (!CI)
3095fe6060f1SDimitry Andric return false;
3096fe6060f1SDimitry Andric
3097fe6060f1SDimitry Andric const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
3098fe6060f1SDimitry Andric FunctionType *FT = F.getFunctionType();
30990b57cec5SDimitry Andric TransformedFunction CustomFn = DFSF.DFS.getCustomFunctionType(FT);
3100fe6060f1SDimitry Andric std::string CustomFName = ShouldTrackOrigins ? "__dfso_" : "__dfsw_";
3101fe6060f1SDimitry Andric CustomFName += F.getName();
31020b57cec5SDimitry Andric FunctionCallee CustomF = DFSF.DFS.Mod->getOrInsertFunction(
31030b57cec5SDimitry Andric CustomFName, CustomFn.TransformedType);
31040b57cec5SDimitry Andric if (Function *CustomFn = dyn_cast<Function>(CustomF.getCallee())) {
3105fe6060f1SDimitry Andric CustomFn->copyAttributesFrom(&F);
31060b57cec5SDimitry Andric
31070b57cec5SDimitry Andric // Custom functions returning non-void will write to the return label.
31080b57cec5SDimitry Andric if (!FT->getReturnType()->isVoidTy()) {
3109349cc55cSDimitry Andric CustomFn->removeFnAttrs(DFSF.DFS.ReadOnlyNoneAttrs);
31100b57cec5SDimitry Andric }
31110b57cec5SDimitry Andric }
31120b57cec5SDimitry Andric
31130b57cec5SDimitry Andric std::vector<Value *> Args;
31140b57cec5SDimitry Andric
3115fe6060f1SDimitry Andric // Adds non-variable arguments.
3116fe6060f1SDimitry Andric auto *I = CB.arg_begin();
3117fe6060f1SDimitry Andric for (unsigned N = FT->getNumParams(); N != 0; ++I, --N) {
3118fe6060f1SDimitry Andric Args.push_back(*I);
31190b57cec5SDimitry Andric }
31200b57cec5SDimitry Andric
3121fe6060f1SDimitry Andric // Adds shadow arguments.
31220b57cec5SDimitry Andric const unsigned ShadowArgStart = Args.size();
3123fe6060f1SDimitry Andric addShadowArguments(F, CB, Args, IRB);
31240b57cec5SDimitry Andric
3125fe6060f1SDimitry Andric // Adds origin arguments.
3126fe6060f1SDimitry Andric const unsigned OriginArgStart = Args.size();
3127fe6060f1SDimitry Andric if (ShouldTrackOrigins)
3128fe6060f1SDimitry Andric addOriginArguments(F, CB, Args, IRB);
31290b57cec5SDimitry Andric
3130fe6060f1SDimitry Andric // Adds variable arguments.
3131fe6060f1SDimitry Andric append_range(Args, drop_begin(CB.args(), FT->getNumParams()));
31320b57cec5SDimitry Andric
31330b57cec5SDimitry Andric CallInst *CustomCI = IRB.CreateCall(CustomF, Args);
31340b57cec5SDimitry Andric CustomCI->setCallingConv(CI->getCallingConv());
3135fe6060f1SDimitry Andric CustomCI->setAttributes(transformFunctionAttributes(
3136fe6060f1SDimitry Andric CustomFn, CI->getContext(), CI->getAttributes()));
31370b57cec5SDimitry Andric
31380b57cec5SDimitry Andric // Update the parameter attributes of the custom call instruction to
31390b57cec5SDimitry Andric // zero extend the shadow parameters. This is required for targets
3140e8d8bef9SDimitry Andric // which consider PrimitiveShadowTy an illegal type.
3141fe6060f1SDimitry Andric for (unsigned N = 0; N < FT->getNumParams(); N++) {
3142fe6060f1SDimitry Andric const unsigned ArgNo = ShadowArgStart + N;
3143e8d8bef9SDimitry Andric if (CustomCI->getArgOperand(ArgNo)->getType() ==
3144e8d8bef9SDimitry Andric DFSF.DFS.PrimitiveShadowTy)
31450b57cec5SDimitry Andric CustomCI->addParamAttr(ArgNo, Attribute::ZExt);
3146fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
3147fe6060f1SDimitry Andric const unsigned OriginArgNo = OriginArgStart + N;
3148fe6060f1SDimitry Andric if (CustomCI->getArgOperand(OriginArgNo)->getType() ==
3149fe6060f1SDimitry Andric DFSF.DFS.OriginTy)
3150fe6060f1SDimitry Andric CustomCI->addParamAttr(OriginArgNo, Attribute::ZExt);
3151fe6060f1SDimitry Andric }
31520b57cec5SDimitry Andric }
31530b57cec5SDimitry Andric
3154fe6060f1SDimitry Andric // Loads the return value shadow and origin.
31550b57cec5SDimitry Andric if (!FT->getReturnType()->isVoidTy()) {
3156fe6060f1SDimitry Andric LoadInst *LabelLoad =
3157fe6060f1SDimitry Andric IRB.CreateLoad(DFSF.DFS.PrimitiveShadowTy, DFSF.LabelReturnAlloca);
3158e8d8bef9SDimitry Andric DFSF.setShadow(CustomCI, DFSF.expandFromPrimitiveShadow(
3159e8d8bef9SDimitry Andric FT->getReturnType(), LabelLoad, &CB));
3160fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
3161fe6060f1SDimitry Andric LoadInst *OriginLoad =
3162fe6060f1SDimitry Andric IRB.CreateLoad(DFSF.DFS.OriginTy, DFSF.OriginReturnAlloca);
3163fe6060f1SDimitry Andric DFSF.setOrigin(CustomCI, OriginLoad);
3164fe6060f1SDimitry Andric }
31650b57cec5SDimitry Andric }
31660b57cec5SDimitry Andric
31670b57cec5SDimitry Andric CI->replaceAllUsesWith(CustomCI);
31680b57cec5SDimitry Andric CI->eraseFromParent();
3169fe6060f1SDimitry Andric return true;
31700b57cec5SDimitry Andric }
3171fe6060f1SDimitry Andric return false;
31720b57cec5SDimitry Andric }
31730b57cec5SDimitry Andric
makeAddAcquireOrderingTable(IRBuilder<> & IRB)3174bdd1243dSDimitry Andric Value *DFSanVisitor::makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
3175bdd1243dSDimitry Andric constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
3176bdd1243dSDimitry Andric uint32_t OrderingTable[NumOrderings] = {};
3177bdd1243dSDimitry Andric
3178bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::relaxed] =
3179bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::acquire] =
3180bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::consume] =
3181bdd1243dSDimitry Andric (int)AtomicOrderingCABI::acquire;
3182bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::release] =
3183bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
3184bdd1243dSDimitry Andric (int)AtomicOrderingCABI::acq_rel;
3185bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
3186bdd1243dSDimitry Andric (int)AtomicOrderingCABI::seq_cst;
3187bdd1243dSDimitry Andric
3188bdd1243dSDimitry Andric return ConstantDataVector::get(IRB.getContext(),
3189bdd1243dSDimitry Andric ArrayRef(OrderingTable, NumOrderings));
3190bdd1243dSDimitry Andric }
3191bdd1243dSDimitry Andric
visitLibAtomicLoad(CallBase & CB)3192bdd1243dSDimitry Andric void DFSanVisitor::visitLibAtomicLoad(CallBase &CB) {
3193bdd1243dSDimitry Andric // Since we use getNextNode here, we can't have CB terminate the BB.
3194bdd1243dSDimitry Andric assert(isa<CallInst>(CB));
3195bdd1243dSDimitry Andric
3196bdd1243dSDimitry Andric IRBuilder<> IRB(&CB);
3197bdd1243dSDimitry Andric Value *Size = CB.getArgOperand(0);
3198bdd1243dSDimitry Andric Value *SrcPtr = CB.getArgOperand(1);
3199bdd1243dSDimitry Andric Value *DstPtr = CB.getArgOperand(2);
3200bdd1243dSDimitry Andric Value *Ordering = CB.getArgOperand(3);
3201bdd1243dSDimitry Andric // Convert the call to have at least Acquire ordering to make sure
3202bdd1243dSDimitry Andric // the shadow operations aren't reordered before it.
3203bdd1243dSDimitry Andric Value *NewOrdering =
3204bdd1243dSDimitry Andric IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
3205bdd1243dSDimitry Andric CB.setArgOperand(3, NewOrdering);
3206bdd1243dSDimitry Andric
3207bdd1243dSDimitry Andric IRBuilder<> NextIRB(CB.getNextNode());
3208bdd1243dSDimitry Andric NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
3209bdd1243dSDimitry Andric
3210bdd1243dSDimitry Andric // TODO: Support ClCombinePointerLabelsOnLoad
3211bdd1243dSDimitry Andric // TODO: Support ClEventCallbacks
3212bdd1243dSDimitry Andric
32135f757f3fSDimitry Andric NextIRB.CreateCall(
32145f757f3fSDimitry Andric DFSF.DFS.DFSanMemShadowOriginTransferFn,
32155f757f3fSDimitry Andric {DstPtr, SrcPtr, NextIRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3216bdd1243dSDimitry Andric }
3217bdd1243dSDimitry Andric
makeAddReleaseOrderingTable(IRBuilder<> & IRB)3218bdd1243dSDimitry Andric Value *DFSanVisitor::makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
3219bdd1243dSDimitry Andric constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
3220bdd1243dSDimitry Andric uint32_t OrderingTable[NumOrderings] = {};
3221bdd1243dSDimitry Andric
3222bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::relaxed] =
3223bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::release] =
3224bdd1243dSDimitry Andric (int)AtomicOrderingCABI::release;
3225bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::consume] =
3226bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::acquire] =
3227bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
3228bdd1243dSDimitry Andric (int)AtomicOrderingCABI::acq_rel;
3229bdd1243dSDimitry Andric OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
3230bdd1243dSDimitry Andric (int)AtomicOrderingCABI::seq_cst;
3231bdd1243dSDimitry Andric
3232bdd1243dSDimitry Andric return ConstantDataVector::get(IRB.getContext(),
3233bdd1243dSDimitry Andric ArrayRef(OrderingTable, NumOrderings));
3234bdd1243dSDimitry Andric }
3235bdd1243dSDimitry Andric
visitLibAtomicStore(CallBase & CB)3236bdd1243dSDimitry Andric void DFSanVisitor::visitLibAtomicStore(CallBase &CB) {
3237bdd1243dSDimitry Andric IRBuilder<> IRB(&CB);
3238bdd1243dSDimitry Andric Value *Size = CB.getArgOperand(0);
3239bdd1243dSDimitry Andric Value *SrcPtr = CB.getArgOperand(1);
3240bdd1243dSDimitry Andric Value *DstPtr = CB.getArgOperand(2);
3241bdd1243dSDimitry Andric Value *Ordering = CB.getArgOperand(3);
3242bdd1243dSDimitry Andric // Convert the call to have at least Release ordering to make sure
3243bdd1243dSDimitry Andric // the shadow operations aren't reordered after it.
3244bdd1243dSDimitry Andric Value *NewOrdering =
3245bdd1243dSDimitry Andric IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
3246bdd1243dSDimitry Andric CB.setArgOperand(3, NewOrdering);
3247bdd1243dSDimitry Andric
3248bdd1243dSDimitry Andric // TODO: Support ClCombinePointerLabelsOnStore
3249bdd1243dSDimitry Andric // TODO: Support ClEventCallbacks
3250bdd1243dSDimitry Andric
32515f757f3fSDimitry Andric IRB.CreateCall(
32525f757f3fSDimitry Andric DFSF.DFS.DFSanMemShadowOriginTransferFn,
32535f757f3fSDimitry Andric {DstPtr, SrcPtr, IRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3254bdd1243dSDimitry Andric }
3255bdd1243dSDimitry Andric
visitLibAtomicExchange(CallBase & CB)3256bdd1243dSDimitry Andric void DFSanVisitor::visitLibAtomicExchange(CallBase &CB) {
3257bdd1243dSDimitry Andric // void __atomic_exchange(size_t size, void *ptr, void *val, void *ret, int
3258bdd1243dSDimitry Andric // ordering)
3259bdd1243dSDimitry Andric IRBuilder<> IRB(&CB);
3260bdd1243dSDimitry Andric Value *Size = CB.getArgOperand(0);
3261bdd1243dSDimitry Andric Value *TargetPtr = CB.getArgOperand(1);
3262bdd1243dSDimitry Andric Value *SrcPtr = CB.getArgOperand(2);
3263bdd1243dSDimitry Andric Value *DstPtr = CB.getArgOperand(3);
3264bdd1243dSDimitry Andric
3265bdd1243dSDimitry Andric // This operation is not atomic for the shadow and origin memory.
3266bdd1243dSDimitry Andric // This could result in DFSan false positives or false negatives.
3267bdd1243dSDimitry Andric // For now we will assume these operations are rare, and
3268bdd1243dSDimitry Andric // the additional complexity to address this is not warrented.
3269bdd1243dSDimitry Andric
3270bdd1243dSDimitry Andric // Current Target to Dest
32715f757f3fSDimitry Andric IRB.CreateCall(
32725f757f3fSDimitry Andric DFSF.DFS.DFSanMemShadowOriginTransferFn,
32735f757f3fSDimitry Andric {DstPtr, TargetPtr, IRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3274bdd1243dSDimitry Andric
3275bdd1243dSDimitry Andric // Current Src to Target (overriding)
32765f757f3fSDimitry Andric IRB.CreateCall(
32775f757f3fSDimitry Andric DFSF.DFS.DFSanMemShadowOriginTransferFn,
32785f757f3fSDimitry Andric {TargetPtr, SrcPtr, IRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3279bdd1243dSDimitry Andric }
3280bdd1243dSDimitry Andric
visitLibAtomicCompareExchange(CallBase & CB)3281bdd1243dSDimitry Andric void DFSanVisitor::visitLibAtomicCompareExchange(CallBase &CB) {
3282bdd1243dSDimitry Andric // bool __atomic_compare_exchange(size_t size, void *ptr, void *expected, void
3283bdd1243dSDimitry Andric // *desired, int success_order, int failure_order)
3284bdd1243dSDimitry Andric Value *Size = CB.getArgOperand(0);
3285bdd1243dSDimitry Andric Value *TargetPtr = CB.getArgOperand(1);
3286bdd1243dSDimitry Andric Value *ExpectedPtr = CB.getArgOperand(2);
3287bdd1243dSDimitry Andric Value *DesiredPtr = CB.getArgOperand(3);
3288bdd1243dSDimitry Andric
3289bdd1243dSDimitry Andric // This operation is not atomic for the shadow and origin memory.
3290bdd1243dSDimitry Andric // This could result in DFSan false positives or false negatives.
3291bdd1243dSDimitry Andric // For now we will assume these operations are rare, and
3292bdd1243dSDimitry Andric // the additional complexity to address this is not warrented.
3293bdd1243dSDimitry Andric
3294bdd1243dSDimitry Andric IRBuilder<> NextIRB(CB.getNextNode());
3295bdd1243dSDimitry Andric NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
3296bdd1243dSDimitry Andric
3297bdd1243dSDimitry Andric DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3298bdd1243dSDimitry Andric
3299bdd1243dSDimitry Andric // If original call returned true, copy Desired to Target.
3300bdd1243dSDimitry Andric // If original call returned false, copy Target to Expected.
33015f757f3fSDimitry Andric NextIRB.CreateCall(DFSF.DFS.DFSanMemShadowOriginConditionalExchangeFn,
3302bdd1243dSDimitry Andric {NextIRB.CreateIntCast(&CB, NextIRB.getInt8Ty(), false),
33035f757f3fSDimitry Andric TargetPtr, ExpectedPtr, DesiredPtr,
3304bdd1243dSDimitry Andric NextIRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3305bdd1243dSDimitry Andric }
3306bdd1243dSDimitry Andric
visitCallBase(CallBase & CB)3307fe6060f1SDimitry Andric void DFSanVisitor::visitCallBase(CallBase &CB) {
3308fe6060f1SDimitry Andric Function *F = CB.getCalledFunction();
3309fe6060f1SDimitry Andric if ((F && F->isIntrinsic()) || CB.isInlineAsm()) {
3310fe6060f1SDimitry Andric visitInstOperands(CB);
3311fe6060f1SDimitry Andric return;
3312fe6060f1SDimitry Andric }
3313fe6060f1SDimitry Andric
3314fe6060f1SDimitry Andric // Calls to this function are synthesized in wrappers, and we shouldn't
3315fe6060f1SDimitry Andric // instrument them.
3316fe6060f1SDimitry Andric if (F == DFSF.DFS.DFSanVarargWrapperFn.getCallee()->stripPointerCasts())
3317fe6060f1SDimitry Andric return;
3318fe6060f1SDimitry Andric
3319bdd1243dSDimitry Andric LibFunc LF;
3320bdd1243dSDimitry Andric if (DFSF.TLI.getLibFunc(CB, LF)) {
3321bdd1243dSDimitry Andric // libatomic.a functions need to have special handling because there isn't
3322bdd1243dSDimitry Andric // a good way to intercept them or compile the library with
3323bdd1243dSDimitry Andric // instrumentation.
3324bdd1243dSDimitry Andric switch (LF) {
3325bdd1243dSDimitry Andric case LibFunc_atomic_load:
3326bdd1243dSDimitry Andric if (!isa<CallInst>(CB)) {
3327bdd1243dSDimitry Andric llvm::errs() << "DFSAN -- cannot instrument invoke of libatomic load. "
3328bdd1243dSDimitry Andric "Ignoring!\n";
3329bdd1243dSDimitry Andric break;
3330bdd1243dSDimitry Andric }
3331bdd1243dSDimitry Andric visitLibAtomicLoad(CB);
3332bdd1243dSDimitry Andric return;
3333bdd1243dSDimitry Andric case LibFunc_atomic_store:
3334bdd1243dSDimitry Andric visitLibAtomicStore(CB);
3335bdd1243dSDimitry Andric return;
3336bdd1243dSDimitry Andric default:
3337bdd1243dSDimitry Andric break;
3338bdd1243dSDimitry Andric }
3339bdd1243dSDimitry Andric }
3340bdd1243dSDimitry Andric
3341bdd1243dSDimitry Andric // TODO: These are not supported by TLI? They are not in the enum.
3342bdd1243dSDimitry Andric if (F && F->hasName() && !F->isVarArg()) {
3343bdd1243dSDimitry Andric if (F->getName() == "__atomic_exchange") {
3344bdd1243dSDimitry Andric visitLibAtomicExchange(CB);
3345bdd1243dSDimitry Andric return;
3346bdd1243dSDimitry Andric }
3347bdd1243dSDimitry Andric if (F->getName() == "__atomic_compare_exchange") {
3348bdd1243dSDimitry Andric visitLibAtomicCompareExchange(CB);
3349bdd1243dSDimitry Andric return;
3350bdd1243dSDimitry Andric }
3351bdd1243dSDimitry Andric }
3352bdd1243dSDimitry Andric
3353fe6060f1SDimitry Andric DenseMap<Value *, Function *>::iterator UnwrappedFnIt =
3354fe6060f1SDimitry Andric DFSF.DFS.UnwrappedFnMap.find(CB.getCalledOperand());
3355fe6060f1SDimitry Andric if (UnwrappedFnIt != DFSF.DFS.UnwrappedFnMap.end())
3356fe6060f1SDimitry Andric if (visitWrappedCallBase(*UnwrappedFnIt->second, CB))
3357fe6060f1SDimitry Andric return;
3358fe6060f1SDimitry Andric
3359fe6060f1SDimitry Andric IRBuilder<> IRB(&CB);
3360fe6060f1SDimitry Andric
3361fe6060f1SDimitry Andric const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
33625ffd83dbSDimitry Andric FunctionType *FT = CB.getFunctionType();
3363349cc55cSDimitry Andric const DataLayout &DL = getDataLayout();
3364349cc55cSDimitry Andric
3365fe6060f1SDimitry Andric // Stores argument shadows.
3366e8d8bef9SDimitry Andric unsigned ArgOffset = 0;
3367e8d8bef9SDimitry Andric for (unsigned I = 0, N = FT->getNumParams(); I != N; ++I) {
3368fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
3369fe6060f1SDimitry Andric // Ignore overflowed origins
3370fe6060f1SDimitry Andric Value *ArgShadow = DFSF.getShadow(CB.getArgOperand(I));
3371fe6060f1SDimitry Andric if (I < DFSF.DFS.NumOfElementsInArgOrgTLS &&
3372fe6060f1SDimitry Andric !DFSF.DFS.isZeroShadow(ArgShadow))
3373fe6060f1SDimitry Andric IRB.CreateStore(DFSF.getOrigin(CB.getArgOperand(I)),
3374fe6060f1SDimitry Andric DFSF.getArgOriginTLS(I, IRB));
3375fe6060f1SDimitry Andric }
3376fe6060f1SDimitry Andric
3377e8d8bef9SDimitry Andric unsigned Size =
3378e8d8bef9SDimitry Andric DL.getTypeAllocSize(DFSF.DFS.getShadowTy(FT->getParamType(I)));
3379e8d8bef9SDimitry Andric // Stop storing if arguments' size overflows. Inside a function, arguments
3380e8d8bef9SDimitry Andric // after overflow have zero shadow values.
3381fe6060f1SDimitry Andric if (ArgOffset + Size > ArgTLSSize)
3382e8d8bef9SDimitry Andric break;
3383349cc55cSDimitry Andric IRB.CreateAlignedStore(DFSF.getShadow(CB.getArgOperand(I)),
3384e8d8bef9SDimitry Andric DFSF.getArgTLS(FT->getParamType(I), ArgOffset, IRB),
3385fe6060f1SDimitry Andric ShadowTLSAlignment);
3386fe6060f1SDimitry Andric ArgOffset += alignTo(Size, ShadowTLSAlignment);
33870b57cec5SDimitry Andric }
33880b57cec5SDimitry Andric
33890b57cec5SDimitry Andric Instruction *Next = nullptr;
33905ffd83dbSDimitry Andric if (!CB.getType()->isVoidTy()) {
33915ffd83dbSDimitry Andric if (InvokeInst *II = dyn_cast<InvokeInst>(&CB)) {
33920b57cec5SDimitry Andric if (II->getNormalDest()->getSinglePredecessor()) {
33930b57cec5SDimitry Andric Next = &II->getNormalDest()->front();
33940b57cec5SDimitry Andric } else {
33950b57cec5SDimitry Andric BasicBlock *NewBB =
33960b57cec5SDimitry Andric SplitEdge(II->getParent(), II->getNormalDest(), &DFSF.DT);
33970b57cec5SDimitry Andric Next = &NewBB->front();
33980b57cec5SDimitry Andric }
33990b57cec5SDimitry Andric } else {
34005ffd83dbSDimitry Andric assert(CB.getIterator() != CB.getParent()->end());
34015ffd83dbSDimitry Andric Next = CB.getNextNode();
34020b57cec5SDimitry Andric }
34030b57cec5SDimitry Andric
3404fe6060f1SDimitry Andric // Don't emit the epilogue for musttail call returns.
3405fe6060f1SDimitry Andric if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
3406fe6060f1SDimitry Andric return;
3407fe6060f1SDimitry Andric
3408fe6060f1SDimitry Andric // Loads the return value shadow.
34090b57cec5SDimitry Andric IRBuilder<> NextIRB(Next);
3410e8d8bef9SDimitry Andric unsigned Size = DL.getTypeAllocSize(DFSF.DFS.getShadowTy(&CB));
3411fe6060f1SDimitry Andric if (Size > RetvalTLSSize) {
3412e8d8bef9SDimitry Andric // Set overflowed return shadow to be zero.
3413e8d8bef9SDimitry Andric DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3414e8d8bef9SDimitry Andric } else {
3415e8d8bef9SDimitry Andric LoadInst *LI = NextIRB.CreateAlignedLoad(
3416e8d8bef9SDimitry Andric DFSF.DFS.getShadowTy(&CB), DFSF.getRetvalTLS(CB.getType(), NextIRB),
3417fe6060f1SDimitry Andric ShadowTLSAlignment, "_dfsret");
34180b57cec5SDimitry Andric DFSF.SkipInsts.insert(LI);
34195ffd83dbSDimitry Andric DFSF.setShadow(&CB, LI);
34200b57cec5SDimitry Andric DFSF.NonZeroChecks.push_back(LI);
34210b57cec5SDimitry Andric }
3422fe6060f1SDimitry Andric
3423fe6060f1SDimitry Andric if (ShouldTrackOrigins) {
3424349cc55cSDimitry Andric LoadInst *LI = NextIRB.CreateLoad(DFSF.DFS.OriginTy,
3425349cc55cSDimitry Andric DFSF.getRetvalOriginTLS(), "_dfsret_o");
3426fe6060f1SDimitry Andric DFSF.SkipInsts.insert(LI);
3427fe6060f1SDimitry Andric DFSF.setOrigin(&CB, LI);
3428fe6060f1SDimitry Andric }
3429bdd1243dSDimitry Andric
3430bdd1243dSDimitry Andric DFSF.addReachesFunctionCallbacksIfEnabled(NextIRB, CB, &CB);
34310b57cec5SDimitry Andric }
3432e8d8bef9SDimitry Andric }
34330b57cec5SDimitry Andric
visitPHINode(PHINode & PN)34340b57cec5SDimitry Andric void DFSanVisitor::visitPHINode(PHINode &PN) {
3435e8d8bef9SDimitry Andric Type *ShadowTy = DFSF.DFS.getShadowTy(&PN);
34360b57cec5SDimitry Andric PHINode *ShadowPN =
3437e8d8bef9SDimitry Andric PHINode::Create(ShadowTy, PN.getNumIncomingValues(), "", &PN);
34380b57cec5SDimitry Andric
34390b57cec5SDimitry Andric // Give the shadow phi node valid predecessors to fool SplitEdge into working.
3440e8d8bef9SDimitry Andric Value *UndefShadow = UndefValue::get(ShadowTy);
3441fe6060f1SDimitry Andric for (BasicBlock *BB : PN.blocks())
3442fe6060f1SDimitry Andric ShadowPN->addIncoming(UndefShadow, BB);
3443fe6060f1SDimitry Andric
3444fe6060f1SDimitry Andric DFSF.setShadow(&PN, ShadowPN);
3445fe6060f1SDimitry Andric
3446fe6060f1SDimitry Andric PHINode *OriginPN = nullptr;
3447fe6060f1SDimitry Andric if (DFSF.DFS.shouldTrackOrigins()) {
3448fe6060f1SDimitry Andric OriginPN =
3449fe6060f1SDimitry Andric PHINode::Create(DFSF.DFS.OriginTy, PN.getNumIncomingValues(), "", &PN);
3450fe6060f1SDimitry Andric Value *UndefOrigin = UndefValue::get(DFSF.DFS.OriginTy);
3451fe6060f1SDimitry Andric for (BasicBlock *BB : PN.blocks())
3452fe6060f1SDimitry Andric OriginPN->addIncoming(UndefOrigin, BB);
3453fe6060f1SDimitry Andric DFSF.setOrigin(&PN, OriginPN);
34540b57cec5SDimitry Andric }
34550b57cec5SDimitry Andric
3456fe6060f1SDimitry Andric DFSF.PHIFixups.push_back({&PN, ShadowPN, OriginPN});
34570b57cec5SDimitry Andric }
3458e8d8bef9SDimitry Andric
run(Module & M,ModuleAnalysisManager & AM)3459e8d8bef9SDimitry Andric PreservedAnalyses DataFlowSanitizerPass::run(Module &M,
3460e8d8bef9SDimitry Andric ModuleAnalysisManager &AM) {
3461bdd1243dSDimitry Andric auto GetTLI = [&](Function &F) -> TargetLibraryInfo & {
3462bdd1243dSDimitry Andric auto &FAM =
3463bdd1243dSDimitry Andric AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
3464bdd1243dSDimitry Andric return FAM.getResult<TargetLibraryAnalysis>(F);
3465bdd1243dSDimitry Andric };
3466bdd1243dSDimitry Andric if (!DataFlowSanitizer(ABIListFiles).runImpl(M, GetTLI))
3467e8d8bef9SDimitry Andric return PreservedAnalyses::all();
3468bdd1243dSDimitry Andric
3469bdd1243dSDimitry Andric PreservedAnalyses PA = PreservedAnalyses::none();
3470bdd1243dSDimitry Andric // GlobalsAA is considered stateless and does not get invalidated unless
3471bdd1243dSDimitry Andric // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
3472bdd1243dSDimitry Andric // make changes that require GlobalsAA to be invalidated.
3473bdd1243dSDimitry Andric PA.abandon<GlobalsAA>();
3474bdd1243dSDimitry Andric return PA;
3475e8d8bef9SDimitry Andric }
3476