1 /**
2 * Copyright (C) Mellanox Technologies Ltd. 2001-2015. ALL RIGHTS RESERVED.
3 * Copyright (C) ARM Ltd. 2016. ALL RIGHTS RESERVED.
4 *
5 * See file LICENSE for terms.
6 */
7
8 #ifdef HAVE_CONFIG_H
9 # include "config.h"
10 #endif
11
12 #include "mmap.h"
13
14 #include <ucm/api/ucm.h>
15 #include <ucm/event/event.h>
16 #include <ucm/util/log.h>
17 #include <ucm/util/reloc.h>
18 #include <ucm/util/sys.h>
19 #include <ucm/bistro/bistro.h>
20 #include <ucs/arch/atomic.h>
21 #include <ucs/sys/sys.h>
22 #include <ucs/sys/math.h>
23 #include <ucs/sys/checker.h>
24 #include <ucs/sys/preprocessor.h>
25 #include <ucs/arch/bitops.h>
26 #include <ucs/debug/assert.h>
27
28 #include <sys/mman.h>
29 #include <sys/shm.h>
30 #include <unistd.h>
31 #include <pthread.h>
32
33 #define UCM_IS_HOOK_ENABLED(_entry) \
34 ((_entry)->hook_type & UCS_BIT(ucm_mmap_hook_mode()))
35
36 #define UCM_HOOK_STR \
37 ((ucm_mmap_hook_mode() == UCM_MMAP_HOOK_RELOC) ? "reloc" : "bistro")
38
39 #define UCM_FIRE_EVENT(_event, _mask, _data, _call) \
40 do { \
41 int exp_events = (_event) & (_mask); \
42 (_data)->fired_events = 0; \
43 _call; \
44 ucm_trace("after %s: got 0x%x/0x%x", UCS_PP_MAKE_STRING(_call), \
45 (_data)->fired_events, exp_events); \
46 /* in case if any event is missed - set correcponding bit to 0 */ \
47 /* same as equation: */ \
48 /* (_data)->out_events &= ~(exp_events ^ */ \
49 /* ((_data)->fired_events & exp_events)); */ \
50 (_data)->out_events &= ~exp_events | (_data)->fired_events; \
51 } while(0)
52
53 #define UCM_MMAP_EVENT_NAME_ENTRY(_event) \
54 [ucs_ilog2(UCM_EVENT_##_event)] = #_event
55
56 #define UCM_MMAP_MAX_EVENT_NAME_LEN sizeof("VM_UNMAPPED")
57
58 #define UCM_MMAP_REPORT_BUF_LEN \
59 ((UCM_MMAP_MAX_EVENT_NAME_LEN + 2) * \
60 ucs_array_size(ucm_mmap_event_name))
61
62 extern const char *ucm_mmap_hook_modes[];
63
64 typedef enum ucm_mmap_hook_type {
65 UCM_HOOK_RELOC = UCS_BIT(UCM_MMAP_HOOK_RELOC),
66 UCM_HOOK_BISTRO = UCS_BIT(UCM_MMAP_HOOK_BISTRO),
67 UCM_HOOK_BOTH = UCM_HOOK_RELOC | UCM_HOOK_BISTRO
68 } ucm_mmap_hook_type_t;
69
70 typedef struct ucm_mmap_func {
71 ucm_reloc_patch_t patch;
72 ucm_event_type_t event_type;
73 ucm_event_type_t deps;
74 ucm_mmap_hook_type_t hook_type;
75 } ucm_mmap_func_t;
76
77 typedef struct ucm_mmap_test_events_data {
78 uint32_t fired_events;
79 int out_events;
80 pid_t tid;
81 } ucm_mmap_test_events_data_t;
82
83 static ucm_mmap_func_t ucm_mmap_funcs[] = {
84 { {"mmap", ucm_override_mmap}, UCM_EVENT_MMAP, UCM_EVENT_NONE, UCM_HOOK_BOTH},
85 { {"munmap", ucm_override_munmap}, UCM_EVENT_MUNMAP, UCM_EVENT_NONE, UCM_HOOK_BOTH},
86 #if HAVE_MREMAP
87 { {"mremap", ucm_override_mremap}, UCM_EVENT_MREMAP, UCM_EVENT_NONE, UCM_HOOK_BOTH},
88 #endif
89 { {"shmat", ucm_override_shmat}, UCM_EVENT_SHMAT, UCM_EVENT_NONE, UCM_HOOK_BOTH},
90 { {"shmdt", ucm_override_shmdt}, UCM_EVENT_SHMDT, UCM_EVENT_SHMAT, UCM_HOOK_BOTH},
91 { {"sbrk", ucm_override_sbrk}, UCM_EVENT_SBRK, UCM_EVENT_NONE, UCM_HOOK_RELOC},
92 #if UCM_BISTRO_HOOKS
93 { {"brk", ucm_override_brk}, UCM_EVENT_SBRK, UCM_EVENT_NONE, UCM_HOOK_BISTRO},
94 #endif
95 { {"madvise", ucm_override_madvise}, UCM_EVENT_MADVISE, UCM_EVENT_NONE, UCM_HOOK_BOTH},
96 { {NULL, NULL}, UCM_EVENT_NONE}
97 };
98
99 static pthread_mutex_t ucm_mmap_install_mutex = PTHREAD_MUTEX_INITIALIZER;
100 static int ucm_mmap_installed_events = 0; /* events that were reported as installed */
101
102 static const char *ucm_mmap_event_name[] = {
103 /* Native events */
104 UCM_MMAP_EVENT_NAME_ENTRY(MMAP),
105 UCM_MMAP_EVENT_NAME_ENTRY(MUNMAP),
106 UCM_MMAP_EVENT_NAME_ENTRY(MREMAP),
107 UCM_MMAP_EVENT_NAME_ENTRY(SHMAT),
108 UCM_MMAP_EVENT_NAME_ENTRY(SHMDT),
109 UCM_MMAP_EVENT_NAME_ENTRY(SBRK),
110 UCM_MMAP_EVENT_NAME_ENTRY(MADVISE),
111
112 /* Aggregate events */
113 UCM_MMAP_EVENT_NAME_ENTRY(VM_MAPPED),
114 UCM_MMAP_EVENT_NAME_ENTRY(VM_UNMAPPED),
115 };
116
ucm_mmap_event_test_callback(ucm_event_type_t event_type,ucm_event_t * event,void * arg)117 static void ucm_mmap_event_test_callback(ucm_event_type_t event_type,
118 ucm_event_t *event, void *arg)
119 {
120 ucm_mmap_test_events_data_t *data = arg;
121
122 /* This callback may be called from multiple threads, which are just calling
123 * memory allocations/release, and not testing mmap hooks at the moment.
124 * So ignore calls from other threads to ensure the only requested events
125 * are proceeded.
126 */
127 if (data->tid == ucs_get_tid()) {
128 data->fired_events |= event_type;
129 }
130 }
131
132 /* Fire events with pre/post action. The problem is in call sequence: we
133 * can't just fire single event - most of the system calls require set of
134 * calls to eliminate resource leaks or data corruption, such sequence
135 * produces additional events which may affect to event handling. To
136 * exclude additional events from processing used pre/post actions where
137 * set of handled events is cleared and evaluated for every system call */
138 static void
ucm_fire_mmap_events_internal(int events,ucm_mmap_test_events_data_t * data)139 ucm_fire_mmap_events_internal(int events, ucm_mmap_test_events_data_t *data)
140 {
141 size_t sbrk_size;
142 int sbrk_mask;
143 int shmid;
144 void *p;
145
146 if (events & (UCM_EVENT_MMAP|UCM_EVENT_MUNMAP|UCM_EVENT_MREMAP|
147 UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED)) {
148 UCM_FIRE_EVENT(events, UCM_EVENT_MMAP|UCM_EVENT_VM_MAPPED,
149 data, p = mmap(NULL, ucm_get_page_size(), PROT_READ | PROT_WRITE,
150 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
151 #ifdef HAVE_MREMAP
152 /* generate MAP event */
153 UCM_FIRE_EVENT(events, UCM_EVENT_MREMAP|UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED,
154 data, p = mremap(p, ucm_get_page_size(),
155 ucm_get_page_size() * 2, MREMAP_MAYMOVE));
156 /* generate UNMAP event */
157 UCM_FIRE_EVENT(events, UCM_EVENT_MREMAP|UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED,
158 data, p = mremap(p, ucm_get_page_size() * 2, ucm_get_page_size(), 0));
159 #endif
160 /* generate UNMAP event */
161 UCM_FIRE_EVENT(events, UCM_EVENT_MMAP|UCM_EVENT_VM_MAPPED,
162 data, p = mmap(p, ucm_get_page_size(), PROT_READ | PROT_WRITE,
163 MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
164 UCM_FIRE_EVENT(events, UCM_EVENT_MUNMAP|UCM_EVENT_VM_UNMAPPED,
165 data, munmap(p, ucm_get_page_size()));
166 }
167
168 if (events & (UCM_EVENT_SHMAT|UCM_EVENT_SHMDT|UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED)) {
169 shmid = shmget(IPC_PRIVATE, ucm_get_page_size(), IPC_CREAT | SHM_R | SHM_W);
170 if (shmid == -1) {
171 ucm_debug("shmget failed: %m");
172 return;
173 }
174
175 UCM_FIRE_EVENT(events, UCM_EVENT_SHMAT|UCM_EVENT_VM_MAPPED,
176 data, p = shmat(shmid, NULL, 0));
177 UCM_FIRE_EVENT(events, UCM_EVENT_SHMAT|UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED,
178 data, p = shmat(shmid, p, SHM_REMAP));
179 shmctl(shmid, IPC_RMID, NULL);
180 UCM_FIRE_EVENT(events, UCM_EVENT_SHMDT|UCM_EVENT_VM_UNMAPPED,
181 data, shmdt(p));
182 }
183
184 if (events & (UCM_EVENT_SBRK|UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED)) {
185 if (RUNNING_ON_VALGRIND) {
186 /* on valgrind, doing a non-trivial sbrk() causes heap corruption */
187 sbrk_size = 0;
188 sbrk_mask = UCM_EVENT_SBRK;
189 } else {
190 sbrk_size = ucm_get_page_size();
191 sbrk_mask = UCM_EVENT_SBRK|UCM_EVENT_VM_MAPPED|UCM_EVENT_VM_UNMAPPED;
192 }
193 UCM_FIRE_EVENT(events, (UCM_EVENT_SBRK|UCM_EVENT_VM_MAPPED) & sbrk_mask,
194 data, (void)sbrk(sbrk_size));
195 UCM_FIRE_EVENT(events, (UCM_EVENT_SBRK|UCM_EVENT_VM_UNMAPPED) & sbrk_mask,
196 data, (void)sbrk(-sbrk_size));
197 }
198
199 if (events & (UCM_EVENT_MADVISE|UCM_EVENT_VM_UNMAPPED)) {
200 UCM_FIRE_EVENT(events, UCM_EVENT_MMAP|UCM_EVENT_VM_MAPPED, data,
201 p = mmap(NULL, ucm_get_page_size(), PROT_READ|PROT_WRITE,
202 MAP_PRIVATE|MAP_ANON, -1, 0));
203 if (p != MAP_FAILED) {
204 UCM_FIRE_EVENT(events, UCM_EVENT_MADVISE|UCM_EVENT_VM_UNMAPPED, data,
205 madvise(p, ucm_get_page_size(), MADV_DONTNEED));
206 UCM_FIRE_EVENT(events, UCM_EVENT_MUNMAP|UCM_EVENT_VM_UNMAPPED, data,
207 munmap(p, ucm_get_page_size()));
208 } else {
209 ucm_debug("mmap failed: %m");
210 }
211 }
212 }
213
ucm_fire_mmap_events(int events)214 void ucm_fire_mmap_events(int events)
215 {
216 ucm_mmap_test_events_data_t data;
217
218 ucm_fire_mmap_events_internal(events, &data);
219 }
220
ucm_mmap_event_report_missing(int expected,int actual,const char * event_type)221 static void ucm_mmap_event_report_missing(int expected, int actual,
222 const char *event_type)
223 {
224 int events_count = 0;
225 int missing_events;
226 int idx;
227 char *buf;
228 char *buf_p;
229 char *end_p;
230
231 UCS_STATIC_ASSERT(UCM_MMAP_REPORT_BUF_LEN <= UCS_ALLOCA_MAX_SIZE)
232
233 buf = buf_p = ucs_alloca(UCM_MMAP_REPORT_BUF_LEN);
234 end_p = buf_p + UCM_MMAP_REPORT_BUF_LEN;
235 missing_events = expected & ~actual &
236 UCS_MASK(ucs_array_size(ucm_mmap_event_name));
237
238 ucs_for_each_bit(idx, missing_events) {
239 /* coverity[overrun-local] */
240 snprintf(buf_p, end_p - buf_p, "%s%s", ((events_count > 0) ? ", " : ""),
241 ucm_mmap_event_name[idx]);
242 events_count++;
243 buf_p += strlen(buf_p);
244 }
245
246 if (events_count) {
247 ucm_diag("missing %s memory events: %s", event_type, buf);
248 }
249 }
250
251 /* Called with lock held */
252 static ucs_status_t
ucm_mmap_test_events_nolock(int events,const char * event_type)253 ucm_mmap_test_events_nolock(int events, const char *event_type)
254 {
255 ucm_event_handler_t handler;
256 ucm_mmap_test_events_data_t data;
257
258 handler.events = events;
259 handler.priority = -1;
260 handler.cb = ucm_mmap_event_test_callback;
261 handler.arg = &data;
262 data.out_events = events;
263 data.tid = ucs_get_tid();
264
265 ucm_event_handler_add(&handler);
266 ucm_fire_mmap_events_internal(events, &data);
267 ucm_event_handler_remove(&handler);
268
269 ucm_debug("mmap test: got 0x%x out of 0x%x", data.out_events, events);
270
271 /* Return success if we caught all wanted events */
272 if (!ucs_test_all_flags(data.out_events, events)) {
273 ucm_mmap_event_report_missing(events, data.out_events, event_type);
274 return UCS_ERR_UNSUPPORTED;
275 }
276
277 return UCS_OK;
278 }
279
ucm_mmap_test_events(int events,const char * event_type)280 ucs_status_t ucm_mmap_test_events(int events, const char *event_type)
281 {
282 ucs_status_t status;
283
284 /*
285 * return UCS_OK iff all events are actually working
286 */
287 pthread_mutex_lock(&ucm_mmap_install_mutex);
288 status = ucm_mmap_test_events_nolock(events, event_type);
289 pthread_mutex_unlock(&ucm_mmap_install_mutex);
290
291 return status;
292 }
293
ucm_mmap_test_installed_events(int events)294 ucs_status_t ucm_mmap_test_installed_events(int events)
295 {
296 /*
297 * return UCS_OK iff all installed events are actually working
298 * we don't check the status of events which were not successfully installed
299 */
300 return ucm_mmap_test_events(events & ucm_mmap_installed_events, "internal");
301 }
302
303 /* Called with lock held */
ucs_mmap_install_reloc(int events)304 static ucs_status_t ucs_mmap_install_reloc(int events)
305 {
306 static int installed_events = 0;
307 ucm_mmap_func_t *entry;
308 ucs_status_t status;
309
310 if (ucm_mmap_hook_mode() == UCM_MMAP_HOOK_NONE) {
311 ucm_debug("installing mmap hooks is disabled by configuration");
312 return UCS_ERR_UNSUPPORTED;
313 }
314
315 for (entry = ucm_mmap_funcs; entry->patch.symbol != NULL; ++entry) {
316 if (!((entry->event_type|entry->deps) & events)) {
317 /* Not required */
318 continue;
319 }
320
321 if (entry->event_type & installed_events) {
322 /* Already installed */
323 continue;
324 }
325
326 if (UCM_IS_HOOK_ENABLED(entry)) {
327 ucm_debug("mmap: installing %s hook for %s = %p for event 0x%x", UCM_HOOK_STR,
328 entry->patch.symbol, entry->patch.value, entry->event_type);
329
330 if (ucm_mmap_hook_mode() == UCM_MMAP_HOOK_RELOC) {
331 status = ucm_reloc_modify(&entry->patch);
332 } else {
333 ucs_assert(ucm_mmap_hook_mode() == UCM_MMAP_HOOK_BISTRO);
334 status = ucm_bistro_patch(entry->patch.symbol, entry->patch.value, NULL);
335 }
336 if (status != UCS_OK) {
337 ucm_warn("failed to install %s hook for '%s'",
338 UCM_HOOK_STR, entry->patch.symbol);
339 return status;
340 }
341
342 installed_events |= entry->event_type;
343 }
344 }
345
346 return UCS_OK;
347 }
348
ucm_mmap_events_to_native_events(int events)349 static int ucm_mmap_events_to_native_events(int events)
350 {
351 int native_events;
352
353 native_events = events & ~(UCM_EVENT_MEM_TYPE_ALLOC |
354 UCM_EVENT_MEM_TYPE_FREE);
355
356 if (events & UCM_EVENT_VM_MAPPED) {
357 native_events |= UCM_NATIVE_EVENT_VM_MAPPED;
358 }
359 if (events & UCM_EVENT_VM_UNMAPPED) {
360 native_events |= UCM_NATIVE_EVENT_VM_UNMAPPED;
361 }
362
363 return native_events;
364 }
365
ucm_mmap_install(int events)366 ucs_status_t ucm_mmap_install(int events)
367 {
368 ucs_status_t status;
369 int native_events;
370
371 pthread_mutex_lock(&ucm_mmap_install_mutex);
372
373 /* Replace aggregate events with the native events which make them */
374 native_events = ucm_mmap_events_to_native_events(events);
375 if (ucs_test_all_flags(ucm_mmap_installed_events, native_events)) {
376 /* if we already installed these events, check that they are still
377 * working, and if not - reinstall them.
378 */
379 status = ucm_mmap_test_events_nolock(native_events, 0);
380 if (status == UCS_OK) {
381 goto out_unlock;
382 }
383 }
384
385 status = ucs_mmap_install_reloc(native_events);
386 if (status != UCS_OK) {
387 ucm_debug("failed to install relocations for mmap");
388 goto out_unlock;
389 }
390
391 status = ucm_mmap_test_events_nolock(native_events, 0);
392 if (status != UCS_OK) {
393 ucm_debug("failed to install mmap events");
394 goto out_unlock;
395 }
396
397 /* status == UCS_OK */
398 ucm_mmap_installed_events |= native_events;
399 ucm_debug("mmap installed events = 0x%x", ucm_mmap_installed_events);
400
401 out_unlock:
402 pthread_mutex_unlock(&ucm_mmap_install_mutex);
403 return status;
404 }
405