1 //===-- memprof_rtl.cpp --------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of MemProfiler, a memory profiler.
10 //
11 // Main file of the MemProf run-time library.
12 //===----------------------------------------------------------------------===//
13 
14 #include "memprof_allocator.h"
15 #include "memprof_interceptors.h"
16 #include "memprof_interface_internal.h"
17 #include "memprof_internal.h"
18 #include "memprof_mapping.h"
19 #include "memprof_stack.h"
20 #include "memprof_stats.h"
21 #include "memprof_thread.h"
22 #include "sanitizer_common/sanitizer_atomic.h"
23 #include "sanitizer_common/sanitizer_flags.h"
24 #include "sanitizer_common/sanitizer_libc.h"
25 #include "sanitizer_common/sanitizer_symbolizer.h"
26 
27 #include <time.h>
28 
29 uptr __memprof_shadow_memory_dynamic_address; // Global interface symbol.
30 
31 // Allow the user to specify a profile output file via the binary.
32 SANITIZER_WEAK_ATTRIBUTE char __memprof_profile_filename[1];
33 
34 namespace __memprof {
35 
36 static void MemprofDie() {
37   static atomic_uint32_t num_calls;
38   if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) {
39     // Don't die twice - run a busy loop.
40     while (1) {
41     }
42   }
43   if (common_flags()->print_module_map >= 1)
44     DumpProcessMap();
45   if (flags()->unmap_shadow_on_exit) {
46     if (kHighShadowEnd)
47       UnmapOrDie((void *)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
48   }
49 }
50 
51 static void CheckUnwind() {
52   GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_check);
53   stack.Print();
54 }
55 
56 // -------------------------- Globals --------------------- {{{1
57 int memprof_inited;
58 int memprof_init_done;
59 bool memprof_init_is_running;
60 int memprof_timestamp_inited;
61 long memprof_init_timestamp_s;
62 
63 uptr kHighMemEnd;
64 
65 // -------------------------- Run-time entry ------------------- {{{1
66 // exported functions
67 
68 #define MEMPROF_MEMORY_ACCESS_CALLBACK_BODY() __memprof::RecordAccess(addr);
69 
70 #define MEMPROF_MEMORY_ACCESS_CALLBACK(type)                                   \
71   extern "C" NOINLINE INTERFACE_ATTRIBUTE void __memprof_##type(uptr addr) {   \
72     MEMPROF_MEMORY_ACCESS_CALLBACK_BODY()                                      \
73   }
74 
75 MEMPROF_MEMORY_ACCESS_CALLBACK(load)
76 MEMPROF_MEMORY_ACCESS_CALLBACK(store)
77 
78 // Force the linker to keep the symbols for various MemProf interface
79 // functions. We want to keep those in the executable in order to let the
80 // instrumented dynamic libraries access the symbol even if it is not used by
81 // the executable itself. This should help if the build system is removing dead
82 // code at link time.
83 static NOINLINE void force_interface_symbols() {
84   volatile int fake_condition = 0; // prevent dead condition elimination.
85   // clang-format off
86   switch (fake_condition) {
87     case 1: __memprof_record_access(nullptr); break;
88     case 2: __memprof_record_access_range(nullptr, 0); break;
89   }
90   // clang-format on
91 }
92 
93 static void memprof_atexit() {
94   Printf("MemProfiler exit stats:\n");
95   __memprof_print_accumulated_stats();
96 }
97 
98 static void InitializeHighMemEnd() {
99   kHighMemEnd = GetMaxUserVirtualAddress();
100   // Increase kHighMemEnd to make sure it's properly
101   // aligned together with kHighMemBeg:
102   kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1;
103 }
104 
105 void PrintAddressSpaceLayout() {
106   if (kHighMemBeg) {
107     Printf("|| `[%p, %p]` || HighMem    ||\n", (void *)kHighMemBeg,
108            (void *)kHighMemEnd);
109     Printf("|| `[%p, %p]` || HighShadow ||\n", (void *)kHighShadowBeg,
110            (void *)kHighShadowEnd);
111   }
112   Printf("|| `[%p, %p]` || ShadowGap  ||\n", (void *)kShadowGapBeg,
113          (void *)kShadowGapEnd);
114   if (kLowShadowBeg) {
115     Printf("|| `[%p, %p]` || LowShadow  ||\n", (void *)kLowShadowBeg,
116            (void *)kLowShadowEnd);
117     Printf("|| `[%p, %p]` || LowMem     ||\n", (void *)kLowMemBeg,
118            (void *)kLowMemEnd);
119   }
120   Printf("MemToShadow(shadow): %p %p", (void *)MEM_TO_SHADOW(kLowShadowBeg),
121          (void *)MEM_TO_SHADOW(kLowShadowEnd));
122   if (kHighMemBeg) {
123     Printf(" %p %p", (void *)MEM_TO_SHADOW(kHighShadowBeg),
124            (void *)MEM_TO_SHADOW(kHighShadowEnd));
125   }
126   Printf("\n");
127   Printf("malloc_context_size=%zu\n",
128          (uptr)common_flags()->malloc_context_size);
129 
130   Printf("SHADOW_SCALE: %d\n", (int)SHADOW_SCALE);
131   Printf("SHADOW_GRANULARITY: %d\n", (int)SHADOW_GRANULARITY);
132   Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)SHADOW_OFFSET);
133   CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
134 }
135 
136 static void MemprofInitInternal() {
137   if (LIKELY(memprof_inited))
138     return;
139   SanitizerToolName = "MemProfiler";
140   CHECK(!memprof_init_is_running && "MemProf init calls itself!");
141   memprof_init_is_running = true;
142 
143   CacheBinaryName();
144 
145   // Initialize flags. This must be done early, because most of the
146   // initialization steps look at flags().
147   InitializeFlags();
148 
149   AvoidCVE_2016_2143();
150 
151   SetMallocContextSize(common_flags()->malloc_context_size);
152 
153   InitializeHighMemEnd();
154 
155   // Make sure we are not statically linked.
156   MemprofDoesNotSupportStaticLinkage();
157 
158   // Install tool-specific callbacks in sanitizer_common.
159   AddDieCallback(MemprofDie);
160   SetCheckUnwindCallback(CheckUnwind);
161 
162   // Use profile name specified via the binary itself if it exists, and hasn't
163   // been overrriden by a flag at runtime.
164   if (__memprof_profile_filename[0] != 0 && !common_flags()->log_path)
165     __sanitizer_set_report_path(__memprof_profile_filename);
166   else
167     __sanitizer_set_report_path(common_flags()->log_path);
168 
169   __sanitizer::InitializePlatformEarly();
170 
171   // Re-exec ourselves if we need to set additional env or command line args.
172   MaybeReexec();
173 
174   // Setup internal allocator callback.
175   SetLowLevelAllocateMinAlignment(SHADOW_GRANULARITY);
176 
177   InitializeMemprofInterceptors();
178   CheckASLR();
179 
180   ReplaceSystemMalloc();
181 
182   DisableCoreDumperIfNecessary();
183 
184   InitializeShadowMemory();
185 
186   TSDInit(PlatformTSDDtor);
187 
188   InitializeAllocator();
189 
190   // On Linux MemprofThread::ThreadStart() calls malloc() that's why
191   // memprof_inited should be set to 1 prior to initializing the threads.
192   memprof_inited = 1;
193   memprof_init_is_running = false;
194 
195   if (flags()->atexit)
196     Atexit(memprof_atexit);
197 
198   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
199 
200   // interceptors
201   InitTlsSize();
202 
203   // Create main thread.
204   MemprofThread *main_thread = CreateMainThread();
205   CHECK_EQ(0, main_thread->tid());
206   force_interface_symbols(); // no-op.
207   SanitizerInitializeUnwinder();
208 
209   Symbolizer::LateInitialize();
210 
211   VReport(1, "MemProfiler Init done\n");
212 
213   memprof_init_done = 1;
214 }
215 
216 void MemprofInitTime() {
217   if (LIKELY(memprof_timestamp_inited))
218     return;
219   timespec ts;
220   clock_gettime(CLOCK_REALTIME, &ts);
221   memprof_init_timestamp_s = ts.tv_sec;
222   memprof_timestamp_inited = 1;
223 }
224 
225 // Initialize as requested from some part of MemProf runtime library
226 // (interceptors, allocator, etc).
227 void MemprofInitFromRtl() { MemprofInitInternal(); }
228 
229 #if MEMPROF_DYNAMIC
230 // Initialize runtime in case it's LD_PRELOAD-ed into uninstrumented executable
231 // (and thus normal initializers from .preinit_array or modules haven't run).
232 
233 class MemprofInitializer {
234 public:
235   MemprofInitializer() { MemprofInitFromRtl(); }
236 };
237 
238 static MemprofInitializer memprof_initializer;
239 #endif // MEMPROF_DYNAMIC
240 
241 } // namespace __memprof
242 
243 // ---------------------- Interface ---------------- {{{1
244 using namespace __memprof;
245 
246 // Initialize as requested from instrumented application code.
247 void __memprof_init() {
248   MemprofInitTime();
249   MemprofInitInternal();
250 }
251 
252 void __memprof_preinit() { MemprofInitInternal(); }
253 
254 void __memprof_version_mismatch_check_v1() {}
255 
256 void __memprof_record_access(void const volatile *addr) {
257   __memprof::RecordAccess((uptr)addr);
258 }
259 
260 void __memprof_record_access_range(void const volatile *addr, uptr size) {
261   for (uptr a = (uptr)addr; a < (uptr)addr + size; a += kWordSize)
262     __memprof::RecordAccess(a);
263 }
264 
265 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u16
266 __sanitizer_unaligned_load16(const uu16 *p) {
267   __memprof_record_access(p);
268   return *p;
269 }
270 
271 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u32
272 __sanitizer_unaligned_load32(const uu32 *p) {
273   __memprof_record_access(p);
274   return *p;
275 }
276 
277 extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64
278 __sanitizer_unaligned_load64(const uu64 *p) {
279   __memprof_record_access(p);
280   return *p;
281 }
282 
283 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
284 __sanitizer_unaligned_store16(uu16 *p, u16 x) {
285   __memprof_record_access(p);
286   *p = x;
287 }
288 
289 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
290 __sanitizer_unaligned_store32(uu32 *p, u32 x) {
291   __memprof_record_access(p);
292   *p = x;
293 }
294 
295 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
296 __sanitizer_unaligned_store64(uu64 *p, u64 x) {
297   __memprof_record_access(p);
298   *p = x;
299 }
300