1 /******************************************************************************* 2 Copyright (c) 2016-2021 NVIDIA Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 22 *******************************************************************************/ 23 24 #ifndef __UVM_PERF_EVENTS_H__ 25 #define __UVM_PERF_EVENTS_H__ 26 27 #include "uvm_linux.h" 28 #include "uvm_forward_decl.h" 29 #include "uvm_processors.h" 30 #include "uvm_hal_types.h" 31 #include "uvm_lock.h" 32 #include "uvm_va_block_types.h" 33 34 // uvm_perf_events is an event notification dispatcher that broadcasts events 35 // to clients. Clients register functions to be called under specific events. 36 // The callback lists are stored per va_space and, therefore, different 37 // callbacks can be registered per client. This will be useful to use 38 // different performance heuristic implementations depending on the GPU 39 // resources used by each process. For example, on a system with Pascal + 40 // Maxwell GPUs, VA spaces which have Maxwell GPU VA spaces will be restrited 41 // to the UVM-Lite feature set, while a VA space which only uses the Pascal 42 // GPU will not be downgraded. Registering/unregistering callbacks requires 43 // holding the VA space events lock in write mode. The exact locking 44 // guarantees under which callbacks are executed depend on the specific event, 45 // but the VA space events lock is held in read mode for all of them. The 46 // additional locking guarantees are defined in each event definition. 47 48 // Performance-related events that can be notified 49 typedef enum 50 { 51 // Locking: uvm_va_space: at least in read mode, uvm_va_block: exclusive / nobody is referencing the block anymore 52 UVM_PERF_EVENT_BLOCK_DESTROY = 0, 53 54 // Locking: uvm_va_space: write 55 UVM_PERF_EVENT_BLOCK_SHRINK, 56 57 // Locking: HMM uvm_va_block lock 58 UVM_PERF_EVENT_BLOCK_MUNMAP, 59 60 // Locking: uvm_va_space: write 61 UVM_PERF_EVENT_RANGE_DESTROY, 62 63 // Locking: uvm_va_space: write 64 UVM_PERF_EVENT_RANGE_SHRINK, 65 66 // Locking: uvm_va_space: write 67 UVM_PERF_EVENT_MODULE_UNLOAD, 68 69 // Locking: uvm_va_space: at least in read mode, uvm_va_block: exclusive (if uvm_va_block is not NULL) 70 UVM_PERF_EVENT_FAULT, 71 72 // Locking: uvm_va_block: exclusive. Notably the uvm_va_space lock may not be held on eviction. 73 UVM_PERF_EVENT_MIGRATION, 74 75 // Locking: uvm_va_space: at least in read mode, uvm_va_block: exclusive 76 UVM_PERF_EVENT_REVOCATION, 77 78 UVM_PERF_EVENT_COUNT, 79 } uvm_perf_event_t; 80 81 // Format of the data passed to callbacks. Scope must be filled with the appropriate values by the code which notifies 82 // the event 83 typedef union 84 { 85 struct 86 { 87 uvm_va_block_t *block; 88 } block_destroy; 89 90 struct 91 { 92 uvm_va_block_t *block; 93 } block_shrink; 94 95 struct 96 { 97 uvm_va_block_t *block; 98 uvm_va_block_region_t region; 99 } block_munmap; 100 101 struct 102 { 103 uvm_va_range_t *range; 104 } range_destroy; 105 106 struct 107 { 108 uvm_va_range_t *range; 109 } range_shrink; 110 111 struct 112 { 113 uvm_perf_module_t *module; 114 115 // Only one of these two can be set. The other one must be NULL 116 uvm_va_block_t *block; 117 uvm_va_range_t *range; 118 } module_unload; 119 120 struct 121 { 122 // This field contains the VA space where this fault was reported. 123 // If block is not NULL, this field must match 124 // uvm_va_block_get_va_space(block). 125 uvm_va_space_t *space; 126 127 // VA block for the page where the fault was triggered if it exists, 128 // NULL otherwise (this can happen if the fault is fatal or the 129 // VA block could not be created). 130 uvm_va_block_t *block; 131 132 // ID of the faulting processor 133 uvm_processor_id_t proc_id; 134 135 // ID of the preferred location processor 136 uvm_processor_id_t preferred_location; 137 138 // Fault descriptor 139 union 140 { 141 struct 142 { 143 uvm_fault_buffer_entry_t *buffer_entry; 144 145 NvU32 batch_id; 146 147 bool is_duplicate; 148 } gpu; 149 150 struct 151 { 152 NvU64 fault_va; 153 154 bool is_write; 155 156 NvU64 pc; 157 } cpu; 158 }; 159 } fault; 160 161 // This event is emitted during migration and the residency bits may be 162 // stale. Do not rely on them in the callbacks. 163 struct 164 { 165 uvm_push_t *push; 166 uvm_va_block_t *block; 167 168 // ID of the destination processor of the migration 169 uvm_processor_id_t dst; 170 171 // ID of the source processor of the migration 172 uvm_processor_id_t src; 173 174 // Start address of the memory range being migrated 175 NvU64 address; 176 177 // Number of bytes being migrated 178 NvU64 bytes; 179 180 // Whether the page has been copied or moved 181 uvm_va_block_transfer_mode_t transfer_mode; 182 183 // Event that performed the call to make_resident 184 uvm_make_resident_cause_t cause; 185 186 // Pointer to the make_resident context from the va_block_context 187 // struct used by the operation that triggered the make_resident call. 188 uvm_make_resident_context_t *make_resident_context; 189 } migration; 190 191 struct 192 { 193 uvm_va_block_t *block; 194 195 // ID of the processor whose access permissions have been revoked 196 uvm_processor_id_t proc_id; 197 198 // Start address of the memory range being revoked 199 NvU64 address; 200 201 // Number of bytes of the memory range being revoked 202 NvU64 bytes; 203 204 // Old access permission 205 uvm_prot_t old_prot; 206 207 // New access permission 208 uvm_prot_t new_prot; 209 } revocation; 210 } uvm_perf_event_data_t; 211 212 // Type of the function that can be registered as a callback 213 // 214 // event_id: is the event being notified. Passing it to the callback enables using the same function to handle 215 // different events. 216 // event_data: extra event data that is passed to the callback function. The format of data passed for each event type 217 // is declared in the uvm_perf_event_data_t union 218 typedef void (*uvm_perf_event_callback_t)(uvm_perf_event_t event_id, uvm_perf_event_data_t *event_data); 219 220 typedef struct 221 { 222 // Lock protecting the events 223 // 224 // Held for write during registration/unregistration of callbacks and for 225 // read during notification of events. 226 // 227 // Also used by tools to protect their state and registration of perf event callbacks. 228 uvm_rw_semaphore_t lock; 229 230 // Array of callbacks for event notification 231 struct list_head event_callbacks[UVM_PERF_EVENT_COUNT]; 232 233 uvm_va_space_t *va_space; 234 } uvm_perf_va_space_events_t; 235 236 // Initialize event notifiction for a va_space. This must be called from va_space construction. No locking required 237 NV_STATUS uvm_perf_init_va_space_events(uvm_va_space_t *va_space, uvm_perf_va_space_events_t *va_space_events); 238 239 // Finalize event notifiction for a va_space. Caller must hold va_space lock in write mode 240 void uvm_perf_destroy_va_space_events(uvm_perf_va_space_events_t *va_space_events); 241 242 // Register a callback to be executed under the given event. The given callback cannot have been already registered for 243 // the same event, although the same callback can be registered for different events. 244 NV_STATUS uvm_perf_register_event_callback(uvm_perf_va_space_events_t *va_space_events, 245 uvm_perf_event_t event_id, uvm_perf_event_callback_t callback); 246 247 // Same as uvm_perf_register_event_callback(), but the caller must hold 248 // va_space_events lock in write mode. 249 NV_STATUS uvm_perf_register_event_callback_locked(uvm_perf_va_space_events_t *va_space_events, 250 uvm_perf_event_t event_id, uvm_perf_event_callback_t callback); 251 252 // Removes a callback for the given event. It's safe to call with a callback that hasn't been registered. 253 void uvm_perf_unregister_event_callback(uvm_perf_va_space_events_t *va_space_events, uvm_perf_event_t event_id, 254 uvm_perf_event_callback_t callback); 255 256 // Same as uvm_perf_unregister_event_callback(), but the caller must hold 257 // va_space_events lock in write mode. 258 void uvm_perf_unregister_event_callback_locked(uvm_perf_va_space_events_t *va_space_events, uvm_perf_event_t event_id, 259 uvm_perf_event_callback_t callback); 260 261 // Invoke the callbacks registered for the given event. Callbacks cannot fail. 262 // Acquires the va_space_events lock internally 263 void uvm_perf_event_notify(uvm_perf_va_space_events_t *va_space_events, uvm_perf_event_t event_id, 264 uvm_perf_event_data_t *event_data); 265 266 // Checks if the given callback is already registered for the event. 267 // va_space_events.lock must be held in either mode by the caller. 268 bool uvm_perf_is_event_callback_registered(uvm_perf_va_space_events_t *va_space_events, 269 uvm_perf_event_t event_id, 270 uvm_perf_event_callback_t callback); 271 272 // Initialization/cleanup functions 273 NV_STATUS uvm_perf_events_init(void); 274 void uvm_perf_events_exit(void); 275 276 // Helper to notify migration events 277 static inline void uvm_perf_event_notify_migration(uvm_perf_va_space_events_t *va_space_events, 278 uvm_push_t *push, 279 uvm_va_block_t *va_block, 280 uvm_processor_id_t dst, 281 uvm_processor_id_t src, 282 NvU64 address, 283 NvU64 bytes, 284 uvm_va_block_transfer_mode_t transfer_mode, 285 uvm_make_resident_cause_t cause, 286 uvm_make_resident_context_t *make_resident_context) 287 { 288 uvm_perf_event_data_t event_data = 289 { 290 .migration = 291 { 292 .push = push, 293 .block = va_block, 294 .dst = dst, 295 .src = src, 296 .address = address, 297 .bytes = bytes, 298 .transfer_mode = transfer_mode, 299 .cause = cause, 300 .make_resident_context = make_resident_context, 301 } 302 }; 303 304 uvm_perf_event_notify(va_space_events, UVM_PERF_EVENT_MIGRATION, &event_data); 305 } 306 307 // Helper to notify gpu fault events 308 static inline void uvm_perf_event_notify_gpu_fault(uvm_perf_va_space_events_t *va_space_events, 309 uvm_va_block_t *va_block, 310 uvm_gpu_id_t gpu_id, 311 uvm_processor_id_t preferred_location, 312 uvm_fault_buffer_entry_t *buffer_entry, 313 NvU32 batch_id, 314 bool is_duplicate) 315 { 316 uvm_perf_event_data_t event_data = 317 { 318 .fault = 319 { 320 .space = va_space_events->va_space, 321 .block = va_block, 322 .proc_id = gpu_id, 323 .preferred_location = preferred_location, 324 }, 325 }; 326 327 event_data.fault.gpu.buffer_entry = buffer_entry; 328 event_data.fault.gpu.batch_id = batch_id; 329 event_data.fault.gpu.is_duplicate = is_duplicate; 330 331 uvm_perf_event_notify(va_space_events, UVM_PERF_EVENT_FAULT, &event_data); 332 } 333 334 // Helper to notify cpu fault events 335 static inline void uvm_perf_event_notify_cpu_fault(uvm_perf_va_space_events_t *va_space_events, 336 uvm_va_block_t *va_block, 337 uvm_processor_id_t preferred_location, 338 NvU64 fault_va, 339 bool is_write, 340 NvU64 pc) 341 { 342 uvm_perf_event_data_t event_data = 343 { 344 .fault = 345 { 346 .space = va_space_events->va_space, 347 .block = va_block, 348 .proc_id = UVM_ID_CPU, 349 .preferred_location = preferred_location, 350 } 351 }; 352 353 event_data.fault.cpu.fault_va = fault_va, 354 event_data.fault.cpu.is_write = is_write, 355 event_data.fault.cpu.pc = pc, 356 357 uvm_perf_event_notify(va_space_events, UVM_PERF_EVENT_FAULT, &event_data); 358 } 359 360 // Helper to notify permission revocation 361 static inline void uvm_perf_event_notify_revocation(uvm_perf_va_space_events_t *va_space_events, 362 uvm_va_block_t *va_block, 363 uvm_processor_id_t id, 364 NvU64 addr, 365 NvU64 bytes, 366 uvm_prot_t old_prot, 367 uvm_prot_t new_prot) 368 { 369 uvm_perf_event_data_t event_data = 370 { 371 .revocation = 372 { 373 .block = va_block, 374 .proc_id = id, 375 .address = addr, 376 .bytes = bytes, 377 .old_prot = old_prot, 378 .new_prot = new_prot, 379 } 380 }; 381 382 uvm_perf_event_notify(va_space_events, UVM_PERF_EVENT_REVOCATION, &event_data); 383 } 384 385 #endif 386