1 /**
2 * Copyright (C) Mellanox Technologies Ltd. 2001-2016. ALL RIGHTS RESERVED.
3 *
4 * See file LICENSE for terms.
5 */
6
7 #ifndef _GNU_SOURCE
8 # define _GNU_SOURCE /* for dladdr */
9 #endif
10
11 #include "sys.h"
12
13 #ifdef HAVE_CONFIG_H
14 # include "config.h"
15 #endif
16
17 #include <ucm/api/ucm.h>
18 #include <ucm/util/log.h>
19 #include <ucm/event/event.h>
20 #include <ucm/mmap/mmap.h>
21 #include <ucs/sys/math.h>
22 #include <ucs/sys/topo.h>
23 #include <linux/mman.h>
24 #include <sys/mman.h>
25 #include <pthread.h>
26 #include <string.h>
27 #include <unistd.h>
28 #include <fcntl.h>
29 #include <errno.h>
30 #include <dlfcn.h>
31
32
33 #define UCM_PROC_SELF_MAPS "/proc/self/maps"
34
35 ucm_global_config_t ucm_global_opts = {
36 .log_level = UCS_LOG_LEVEL_WARN,
37 .enable_events = 1,
38 .mmap_hook_mode = UCM_DEFAULT_HOOK_MODE,
39 .enable_malloc_hooks = 1,
40 .enable_malloc_reloc = 0,
41 .enable_cuda_reloc = 1,
42 .enable_dynamic_mmap_thresh = 1,
43 .alloc_alignment = 16,
44 .dlopen_process_rpath = 1
45 };
46
ucm_get_page_size()47 size_t ucm_get_page_size()
48 {
49 static long page_size = -1;
50 long value;
51
52 if (page_size == -1) {
53 value = sysconf(_SC_PAGESIZE);
54 if (value < 0) {
55 page_size = 4096;
56 } else {
57 page_size = value;
58 }
59 }
60 return page_size;
61 }
62
ucm_sys_complete_alloc(void * ptr,size_t size)63 static void *ucm_sys_complete_alloc(void *ptr, size_t size)
64 {
65 *(size_t*)ptr = size;
66 return UCS_PTR_BYTE_OFFSET(ptr, sizeof(size_t));
67 }
68
ucm_sys_malloc(size_t size)69 void *ucm_sys_malloc(size_t size)
70 {
71 size_t sys_size;
72 void *ptr;
73
74 sys_size = ucs_align_up_pow2(size + sizeof(size_t), ucm_get_page_size());
75 ptr = ucm_orig_mmap(NULL, sys_size, PROT_READ|PROT_WRITE,
76 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
77 if (ptr == MAP_FAILED) {
78 ucm_error("mmap(size=%zu) failed: %m", sys_size);
79 return NULL;
80 }
81
82 return ucm_sys_complete_alloc(ptr, sys_size);
83 }
84
ucm_sys_calloc(size_t nmemb,size_t size)85 void *ucm_sys_calloc(size_t nmemb, size_t size)
86 {
87 size_t total_size = size * nmemb;
88 void *ptr;
89
90 ptr = ucm_sys_malloc(total_size);
91 if (ptr == NULL) {
92 return NULL;
93 }
94
95 memset(ptr, 0, total_size);
96 return ptr;
97 }
98
ucm_sys_free(void * ptr)99 void ucm_sys_free(void *ptr)
100 {
101 size_t size;
102
103 if (ptr == NULL) {
104 return;
105 }
106
107 /* Do not use UCS_PTR_BYTE_OFFSET macro here due to coverity
108 * false positive.
109 * TODO: check for false positive on newer coverity. */
110 ptr = (char*)ptr - sizeof(size_t);
111 size = *(size_t*)ptr;
112 munmap(ptr, size);
113 }
114
ucm_sys_realloc(void * ptr,size_t size)115 void *ucm_sys_realloc(void *ptr, size_t size)
116 {
117 size_t oldsize, sys_size;
118 void *oldptr, *newptr;
119
120 if (ptr == NULL) {
121 return ucm_sys_malloc(size);
122 }
123
124 oldptr = UCS_PTR_BYTE_OFFSET(ptr, -sizeof(size_t));
125 oldsize = *(size_t*)oldptr;
126 sys_size = ucs_align_up_pow2(size + sizeof(size_t), ucm_get_page_size());
127
128 if (sys_size == oldsize) {
129 return ptr;
130 }
131
132 newptr = ucm_orig_mremap(oldptr, oldsize, sys_size, MREMAP_MAYMOVE);
133 if (newptr == MAP_FAILED) {
134 ucm_error("mremap(oldptr=%p oldsize=%zu, newsize=%zu) failed: %m",
135 oldptr, oldsize, sys_size);
136 return NULL;
137 }
138
139 return ucm_sys_complete_alloc(newptr, sys_size);
140 }
141
ucm_parse_proc_self_maps(ucm_proc_maps_cb_t cb,void * arg)142 void ucm_parse_proc_self_maps(ucm_proc_maps_cb_t cb, void *arg)
143 {
144 static char *buffer = MAP_FAILED;
145 static size_t buffer_size = 32768;
146 static pthread_rwlock_t lock = PTHREAD_RWLOCK_INITIALIZER;
147 ssize_t read_size, offset;
148 unsigned long start, end;
149 char prot_c[4];
150 int line_num;
151 int prot;
152 char *ptr, *newline;
153 int maps_fd;
154 int ret;
155 int n;
156
157 maps_fd = open(UCM_PROC_SELF_MAPS, O_RDONLY);
158 if (maps_fd < 0) {
159 ucm_fatal("cannot open %s for reading: %m", UCM_PROC_SELF_MAPS);
160 }
161
162 /* read /proc/self/maps fully into the buffer */
163 pthread_rwlock_wrlock(&lock);
164
165 if (buffer == MAP_FAILED) {
166 buffer = ucm_orig_mmap(NULL, buffer_size, PROT_READ|PROT_WRITE,
167 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
168 if (buffer == MAP_FAILED) {
169 ucm_fatal("failed to allocate maps buffer(size=%zu): %m", buffer_size);
170 }
171 }
172
173 offset = 0;
174 for (;;) {
175 read_size = read(maps_fd, buffer + offset, buffer_size - offset);
176 if (read_size < 0) {
177 /* error */
178 if (errno != EINTR) {
179 ucm_fatal("failed to read from %s: %m", UCM_PROC_SELF_MAPS);
180 }
181 } else if (read_size == buffer_size - offset) {
182 /* enlarge buffer */
183 buffer = ucm_orig_mremap(buffer, buffer_size, buffer_size * 2,
184 MREMAP_MAYMOVE);
185 if (buffer == MAP_FAILED) {
186 ucm_fatal("failed to allocate maps buffer(size=%zu)", buffer_size);
187 }
188 buffer_size *= 2;
189
190 /* read again from the beginning of the file */
191 ret = lseek(maps_fd, 0, SEEK_SET);
192 if (ret < 0) {
193 ucm_fatal("failed to lseek(0): %m");
194 }
195 offset = 0;
196 } else if (read_size == 0) {
197 /* finished reading */
198 buffer[offset] = '\0';
199 break;
200 } else {
201 /* more data could be available even if the buffer is not full */
202 offset += read_size;
203 }
204 }
205 pthread_rwlock_unlock(&lock);
206
207 close(maps_fd);
208
209 pthread_rwlock_rdlock(&lock);
210
211 ptr = buffer;
212 line_num = 1;
213 while ( (newline = strchr(ptr, '\n')) != NULL ) {
214 /* address perms offset dev inode pathname
215 * 00400000-0040b000 r-xp 00001a00 0a:0b 12345 /dev/mydev
216 */
217 *newline = '\0';
218 ret = sscanf(ptr, "%lx-%lx %4c %*x %*x:%*x %*d %n",
219 &start, &end, prot_c,
220 /* ignore offset, dev, inode */
221 &n /* save number of chars before path begins */);
222 if (ret < 3) {
223 ucm_warn("failed to parse %s line %d: '%s'",
224 UCM_PROC_SELF_MAPS, line_num, ptr);
225 } else {
226 prot = 0;
227 if (prot_c[0] == 'r') {
228 prot |= PROT_READ;
229 }
230 if (prot_c[1] == 'w') {
231 prot |= PROT_WRITE;
232 }
233 if (prot_c[2] == 'x') {
234 prot |= PROT_EXEC;
235 }
236
237 if (cb(arg, (void*)start, end - start, prot, ptr + n)) {
238 goto out;
239 }
240 }
241
242 ptr = newline + 1;
243 ++line_num;
244 }
245
246 out:
247 pthread_rwlock_unlock(&lock);
248 }
249
250 typedef struct {
251 const void *shmaddr;
252 size_t seg_size;
253 } ucm_get_shm_seg_size_ctx_t;
254
ucm_get_shm_seg_size_cb(void * arg,void * addr,size_t length,int prot,const char * path)255 static int ucm_get_shm_seg_size_cb(void *arg, void *addr, size_t length,
256 int prot, const char *path)
257 {
258 ucm_get_shm_seg_size_ctx_t *ctx = arg;
259 if (addr == ctx->shmaddr) {
260 ctx->seg_size = length;
261 return 1;
262 }
263 return 0;
264 }
265
ucm_get_shm_seg_size(const void * shmaddr)266 size_t ucm_get_shm_seg_size(const void *shmaddr)
267 {
268 ucm_get_shm_seg_size_ctx_t ctx = { shmaddr, 0 };
269 ucm_parse_proc_self_maps(ucm_get_shm_seg_size_cb, &ctx);
270 return ctx.seg_size;
271 }
272
ucm_strerror(int eno,char * buf,size_t max)273 void ucm_strerror(int eno, char *buf, size_t max)
274 {
275 #if STRERROR_R_CHAR_P
276 char *ret = strerror_r(eno, buf, max);
277 if (ret != buf) {
278 strncpy(buf, ret, max);
279 }
280 #else
281 (void)strerror_r(eno, buf, max);
282 #endif
283 }
284
ucm_prevent_dl_unload()285 void ucm_prevent_dl_unload()
286 {
287 Dl_info info;
288 void *dl;
289 int ret;
290
291 /* Get the path to current library by current function pointer */
292 (void)dlerror();
293 ret = dladdr(ucm_prevent_dl_unload, &info);
294 if (ret == 0) {
295 ucm_warn("could not find address of current library: %s", dlerror());
296 return;
297 }
298
299 /* Load the current library with NODELETE flag, to prevent it from being
300 * unloaded. This will create extra reference to the library, but also add
301 * NODELETE flag to the dynamic link map.
302 */
303 (void)dlerror();
304 dl = dlopen(info.dli_fname, RTLD_LOCAL|RTLD_LAZY|RTLD_NODELETE);
305 if (dl == NULL) {
306 ucm_warn("failed to load '%s': %s", info.dli_fname, dlerror());
307 return;
308 }
309
310 ucm_debug("reloaded '%s' at %p with NODELETE flag", info.dli_fname, dl);
311
312 /* Now we drop our reference to the lib, and it won't be unloaded anymore */
313 dlclose(dl);
314 }
315
ucm_concat_path(char * buffer,size_t max,const char * dir,const char * file)316 char *ucm_concat_path(char *buffer, size_t max, const char *dir, const char *file)
317 {
318 size_t len;
319
320 len = strlen(dir);
321 while (len && (dir[len - 1] == '/')) {
322 len--; /* trim closing '/' */
323 }
324
325 len = ucs_min(len, max);
326 memcpy(buffer, dir, len);
327 max -= len;
328 if (max < 2) { /* buffer is shorter than dir - copy dir only */
329 buffer[len - 1] = '\0';
330 return buffer;
331 }
332
333 buffer[len] = '/';
334 max--;
335
336 while (file[0] == '/') {
337 file++; /* trim beginning '/' */
338 }
339
340 strncpy(buffer + len + 1, file, max);
341 buffer[max + len] = '\0'; /* force close string */
342
343 return buffer;
344 }
345
ucm_get_mem_type_current_device_info(ucs_memory_type_t memtype,ucs_sys_bus_id_t * bus_id)346 ucs_status_t ucm_get_mem_type_current_device_info(ucs_memory_type_t memtype, ucs_sys_bus_id_t *bus_id)
347 {
348 ucs_status_t status = UCS_ERR_UNSUPPORTED;
349 ucm_event_installer_t *event_installer;
350
351 ucs_list_for_each(event_installer, &ucm_event_installer_list, list) {
352 if (NULL == event_installer->get_mem_type_current_device_info) {
353 continue;
354 }
355
356 status = event_installer->get_mem_type_current_device_info(bus_id, memtype);
357 if (UCS_OK == status) {
358 break;
359 }
360 }
361
362 return status;
363 }
364