1 /** 2 * Copyright (C) Mellanox Technologies Ltd. 2001-2015. ALL RIGHTS RESERVED. 3 * Copyright (C) Advanced Micro Devices, Inc. 2019. ALL RIGHTS RESERVED. 4 * 5 * See file LICENSE for terms. 6 */ 7 8 9 #ifndef UCM_H_ 10 #define UCM_H_ 11 12 #include <ucs/sys/compiler_def.h> 13 14 BEGIN_C_DECLS 15 16 #include <ucs/config/types.h> 17 #include <ucs/memory/memory_type.h> 18 #include <ucs/type/status.h> 19 20 #include <sys/types.h> 21 #include <stddef.h> 22 #include <stdio.h> 23 #include <stdint.h> 24 25 26 /** 27 * @brief Memory event types 28 */ 29 typedef enum ucm_event_type { 30 /* Default initialization value */ 31 UCM_EVENT_NONE = 0, 32 /* Native events */ 33 UCM_EVENT_MMAP = UCS_BIT(0), 34 UCM_EVENT_MUNMAP = UCS_BIT(1), 35 UCM_EVENT_MREMAP = UCS_BIT(2), 36 UCM_EVENT_SHMAT = UCS_BIT(3), 37 UCM_EVENT_SHMDT = UCS_BIT(4), 38 UCM_EVENT_SBRK = UCS_BIT(5), 39 UCM_EVENT_MADVISE = UCS_BIT(6), 40 41 /* Aggregate events */ 42 UCM_EVENT_VM_MAPPED = UCS_BIT(16), 43 UCM_EVENT_VM_UNMAPPED = UCS_BIT(17), 44 45 /* Non-accessible memory alloc/free events */ 46 UCM_EVENT_MEM_TYPE_ALLOC = UCS_BIT(20), 47 UCM_EVENT_MEM_TYPE_FREE = UCS_BIT(21), 48 49 /* Add event handler, but don't install new hooks */ 50 UCM_EVENT_FLAG_NO_INSTALL = UCS_BIT(24), 51 52 /* When the event handler is added, generate approximated events for 53 * existing memory allocations. 54 * Currently implemented only for @ref UCM_EVENT_MEM_TYPE_ALLOC. 55 */ 56 UCM_EVENT_FLAG_EXISTING_ALLOC = UCS_BIT(25) 57 58 } ucm_event_type_t; 59 60 61 /** 62 * @brief MMAP hook modes 63 */ 64 typedef enum ucm_mmap_hook_mode { 65 UCM_MMAP_HOOK_NONE, 66 UCM_MMAP_HOOK_RELOC, 67 UCM_MMAP_HOOK_BISTRO, 68 UCM_MMAP_HOOK_LAST 69 } ucm_mmap_hook_mode_t; 70 71 /** 72 * @brief Memory event parameters and result. 73 */ 74 typedef union ucm_event { 75 /* 76 * UCM_EVENT_MMAP 77 * mmap() is called. 78 * callbacks: pre, post 79 */ 80 struct { 81 void *result; 82 void *address; 83 size_t size; 84 int prot; 85 int flags; 86 int fd; 87 off_t offset; 88 } mmap; 89 90 /* 91 * UCM_EVENT_MUNMAP 92 * munmap() is called. 93 */ 94 struct { 95 int result; 96 void *address; 97 size_t size; 98 } munmap; 99 100 /* 101 * UCM_EVENT_MREMAP 102 * mremap() is called. 103 */ 104 struct { 105 void *result; 106 void *address; 107 size_t old_size; 108 size_t new_size; 109 int flags; 110 } mremap; 111 112 /* 113 * UCM_EVENT_SHMAT 114 * shmat() is called. 115 */ 116 struct { 117 void *result; 118 int shmid; 119 const void *shmaddr; 120 int shmflg; 121 } shmat; 122 123 /* 124 * UCM_EVENT_SHMDT 125 * shmdt() is called. 126 */ 127 struct { 128 int result; 129 const void *shmaddr; 130 } shmdt; 131 132 /* 133 * UCM_EVENT_SBRK 134 * sbrk() is called. 135 */ 136 struct { 137 void *result; 138 intptr_t increment; 139 } sbrk; 140 141 /* 142 * UCM_EVENT_MADVISE 143 * madvise() is called. 144 */ 145 struct { 146 int result; 147 void *addr; 148 size_t length; 149 int advice; 150 } madvise; 151 152 /* 153 * UCM_EVENT_VM_MAPPED, UCM_EVENT_VM_UNMAPPED 154 * 155 * This is a "read-only" event which is called whenever memory is mapped 156 * or unmapped from process address space, in addition to the other events. 157 * It can return only UCM_EVENT_STATUS_NEXT. 158 * 159 * For UCM_EVENT_VM_MAPPED, callbacks are post 160 * For UCM_EVENT_VM_UNMAPPED, callbacks are pre 161 */ 162 struct { 163 void *address; 164 size_t size; 165 } vm_mapped, vm_unmapped; 166 167 /* 168 * UCM_EVENT_MEM_TYPE_ALLOC, UCM_EVENT_MEM_TYPE_FREE 169 * 170 * Memory type allocation and deallocation event. 171 * If mem_type is @ref UCS_MEMORY_TYPE_LAST, the memory type is unknown, and 172 * further memory type detection is required. 173 */ 174 struct { 175 void *address; 176 size_t size; 177 ucs_memory_type_t mem_type; 178 } mem_type; 179 180 } ucm_event_t; 181 182 183 /** 184 * @brief Global UCM configuration. 185 * 186 * Can be safely modified before using UCM functions. 187 */ 188 typedef struct ucm_global_config { 189 ucs_log_level_t log_level; /* Logging level */ 190 int enable_events; /* Enable memory events */ 191 ucm_mmap_hook_mode_t mmap_hook_mode; /* MMAP hook mode */ 192 int enable_malloc_hooks; /* Enable installing malloc hooks */ 193 int enable_malloc_reloc; /* Enable installing malloc relocations */ 194 int enable_cuda_reloc; /* Enable installing CUDA relocations */ 195 int enable_dynamic_mmap_thresh; /* Enable adaptive mmap threshold */ 196 size_t alloc_alignment; /* Alignment for memory allocations */ 197 int dlopen_process_rpath; /* Process RPATH section in dlopen hook */ 198 } ucm_global_config_t; 199 200 201 /* Global UCM configuration */ 202 extern ucm_global_config_t ucm_global_opts; 203 204 205 /** 206 * @brief Memory event callback. 207 * 208 * This type describes a callback which handles memory events in the current process. 209 * 210 * @param [in] event_type Type of the event being fired. see @ref ucm_event_type_t. 211 * @param [inout] event Event information. This structure can be updated by 212 * this callback, as described below. 213 * @param [in] arg User-defined argument as passed to @ref ucm_set_event_handler. 214 * 215 * 216 * Events are dispatched in order of callback priority (low to high). 217 * 218 * The fields of the relevant part of the union are initialized as follows: 219 * - "result" - to an invalid erroneous return value (depends on the specific event). 220 * - the rest - to the input parameters of the event. 221 * 222 * The callback is allowed to modify the fields, and those modifications will 223 * be passed to the next callback. Also, the callback is allowed to modify the 224 * result, but **only if it's currently invalid**. A valid result indicates that 225 * a previous callback already performed the requested memory operation, so a 226 * callback should **refrain from actions with side-effects** in this case. 227 * 228 * If the result is still invalid after all callbacks are called, the parameters, 229 * possibly modified by the callbacks, will be passed to the original handler. 230 * 231 * 232 * Important Note: The callback must not call any memory allocation routines, or 233 * anything which may trigger or wait for memory allocation, because it 234 * may lead to deadlock or infinite recursion. 235 * 236 * @todo describe use cases 237 * 238 */ 239 typedef void (*ucm_event_callback_t)(ucm_event_type_t event_type, 240 ucm_event_t *event, void *arg); 241 242 243 /** 244 * @brief Install a handler for memory events. 245 * 246 * @param [in] events Bit-mask of events to handle. 247 * @param [in] priority Priority value which defines the order in which event 248 * callbacks are called. 249 * < 0 - called before the original implementation, 250 * >= 0 - called after the original implementation. 251 * @param [in] cb Event-handling callback. 252 * @param [in] arg User-defined argument for the callback. 253 * 254 * @note If UCM_EVENT_FLAG_NO_INSTALL flag is passed in @a events argument, 255 * only @cb handler will be registered for @a events. No memory 256 * events/hooks will be installed. 257 * 258 * @return Status code. 259 */ 260 ucs_status_t ucm_set_event_handler(int events, int priority, 261 ucm_event_callback_t cb, void *arg); 262 263 264 /** 265 * @brief Remove a handler for memory events. 266 * 267 * @param [in] events Which events to remove. The handler is removed 268 * completely when all its events are removed. 269 * @param [in] cb Event-handling callback. 270 * @param [in] arg User-defined argument for the callback. 271 */ 272 void ucm_unset_event_handler(int events, ucm_event_callback_t cb, void *arg); 273 274 275 /** 276 * @brief Add memory events to the external events list. 277 * 278 * When the event is set to be external, it means that user is responsible for 279 * handling it. So, setting a handler for external event will not trigger 280 * installing of UCM memory hooks (if they were not installed before). In this 281 * case the corresponding UCM function needs to be invoked to trigger event 282 * handlers. 283 * Usage example is when the user disables UCM memory hooks (he may have its 284 * own hooks, like Open MPI), but it wants to use some UCM based functionality, 285 * e.g. IB registration cache. IB registration cache needs to be notified about 286 * UCM_EVENT_VM_UNMAPPED events, therefore it adds specific handler for it. 287 * In this case user needs to declare UCM_EVENT_VM_UNMAPPED event as external 288 * and explicitly call ucm_vm_munmap() when some memory release operation 289 * occurs. 290 * 291 * @param [in] events Bit-mask of events which are supposed to be handled 292 * externally. 293 * 294 * @note To take an effect, the event should be set external prior to adding 295 * event handlers for it. 296 */ 297 void ucm_set_external_event(int events); 298 299 300 /** 301 * @brief Remove memory events from the external events list. 302 * 303 * When the event is removed from the external events list, any subsequent call 304 * to ucm_set_event_handler() for that event will trigger installing of UCM 305 * memory hooks (if they are enabled and were not installed before). 306 * 307 * @param [in] events Which events to remove from the external events list. 308 */ 309 void ucm_unset_external_event(int events); 310 311 312 /** 313 * @brief Test event handlers 314 * 315 * This routine checks if event handlers are called when corresponding system API 316 * is invoked. 317 * 318 * @param [in] events Bit-mask of events which are supposed to be handled 319 * externally. 320 * 321 * @return Status code. 322 */ 323 ucs_status_t ucm_test_events(int events); 324 325 326 /** 327 * @brief Test event external handlers 328 * 329 * This routine checks if external events, as set by @ref ucm_set_external_event, 330 * are actually being reported (by calling APIs such as @ref ucm_vm_munmap). 331 * 332 * @param [in] events Bit-mask of events which are supposed to be handled 333 * externally. 334 * 335 * @return Status code. 336 */ 337 ucs_status_t ucm_test_external_events(int events); 338 339 340 /** 341 * @brief Call the original implementation of @ref mmap without triggering events. 342 */ 343 void *ucm_orig_mmap(void *addr, size_t length, int prot, int flags, int fd, 344 off_t offset); 345 346 347 /** 348 * @brief Call the original implementation of @ref munmap without triggering events. 349 */ 350 int ucm_orig_munmap(void *addr, size_t length); 351 352 353 /** 354 * @brief Call the original implementation of @ref mremap without triggering events. 355 */ 356 void *ucm_orig_mremap(void *old_address, size_t old_size, size_t new_size, 357 int flags); 358 359 360 /** 361 * @brief Call the original implementation of @ref shmat without triggering events. 362 */ 363 void *ucm_orig_shmat(int shmid, const void *shmaddr, int shmflg); 364 365 366 /** 367 * @brief Call the original implementation of @ref shmdt without triggering events. 368 */ 369 int ucm_orig_shmdt(const void *shmaddr); 370 371 372 /** 373 * @brief Call the original implementation of @ref sbrk without triggering events. 374 */ 375 void *ucm_orig_sbrk(intptr_t increment); 376 377 378 /** 379 * @brief Call the original implementation of @ref brk without triggering events. 380 */ 381 int ucm_orig_brk(void *addr); 382 383 384 /** 385 * @brief Call the original implementation of @ref madvise without triggering events. 386 */ 387 int ucm_orig_madvise(void *addr, size_t length, int advice); 388 389 390 /** 391 * @brief Call the original implementation of @ref mmap and all handlers 392 * associated with it. 393 */ 394 void *ucm_mmap(void *addr, size_t length, int prot, int flags, int fd, 395 off_t offset); 396 397 398 /** 399 * @brief Call the original implementation of @ref munmap and all handlers 400 * associated with it. 401 */ 402 int ucm_munmap(void *addr, size_t length); 403 404 405 /** 406 * @brief Call the handlers registered for aggregated VM_MMAP event. 407 */ 408 void ucm_vm_mmap(void *addr, size_t length); 409 410 411 /** 412 * @brief Call the handlers registered for aggregated VM_MUNMAP event. 413 */ 414 void ucm_vm_munmap(void *addr, size_t length); 415 416 417 /** 418 * @brief Call the original implementation of @ref mremap and all handlers 419 * associated with it. 420 */ 421 void *ucm_mremap(void *old_address, size_t old_size, size_t new_size, int flags); 422 423 424 /** 425 * @brief Call the original implementation of @ref shmat and all handlers 426 * associated with it. 427 */ 428 void *ucm_shmat(int shmid, const void *shmaddr, int shmflg); 429 430 431 /** 432 * @brief Call the original implementation of @ref shmdt and all handlers 433 * associated with it. 434 */ 435 int ucm_shmdt(const void *shmaddr); 436 437 438 /** 439 * @brief Call the original implementation of @ref sbrk and all handlers 440 * associated with it. 441 */ 442 void *ucm_sbrk(intptr_t increment); 443 444 445 /** 446 * @brief Call the original implementation of @ref brk and all handlers 447 * associated with it. 448 */ 449 int ucm_brk(void *addr); 450 451 452 /** 453 * @brief Call the original implementation of @ref madvise and all handlers 454 * associated with it. 455 */ 456 int ucm_madvise(void *addr, size_t length, int advice); 457 458 459 /** 460 * @brief Call the original implementation of @ref dlopen and all handlers 461 * associated with it. 462 */ 463 void *ucm_dlopen(const char *filename, int flag); 464 465 466 END_C_DECLS 467 468 #endif 469