1 /******************************************************************************* 2 Copyright (c) 2015-2022 NVidia Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 *******************************************************************************/ 22 23 #ifndef __UVM_TEST_IOCTL_H__ 24 #define __UVM_TEST_IOCTL_H__ 25 26 27 #include "uvm_types.h" 28 #include "uvm_ioctl.h" 29 #include "nv_uvm_types.h" 30 31 #ifdef __cplusplus 32 extern "C" { 33 #endif 34 35 // Offset the test ioctl to leave space for the api ones 36 #define UVM_TEST_IOCTL_BASE(i) UVM_IOCTL_BASE(200 + i) 37 38 #define UVM_TEST_GET_GPU_REF_COUNT UVM_TEST_IOCTL_BASE(0) 39 typedef struct 40 { 41 // In params 42 NvProcessorUuid gpu_uuid; 43 NvU32 swizz_id; 44 // Out params 45 NvU64 ref_count NV_ALIGN_BYTES(8); 46 NV_STATUS rmStatus; 47 } UVM_TEST_GET_GPU_REF_COUNT_PARAMS; 48 49 #define UVM_TEST_RNG_SANITY UVM_TEST_IOCTL_BASE(1) 50 typedef struct 51 { 52 NV_STATUS rmStatus; 53 } UVM_TEST_RNG_SANITY_PARAMS; 54 55 #define UVM_TEST_RANGE_TREE_DIRECTED UVM_TEST_IOCTL_BASE(2) 56 typedef struct 57 { 58 NV_STATUS rmStatus; 59 } UVM_TEST_RANGE_TREE_DIRECTED_PARAMS; 60 61 #define UVM_TEST_RANGE_TREE_RANDOM UVM_TEST_IOCTL_BASE(3) 62 typedef struct 63 { 64 NvU32 seed; // In 65 NvU64 main_iterations NV_ALIGN_BYTES(8); // In 66 NvU32 verbose; // In 67 68 // Probability (0-100) 69 // 70 // When the test starts up, it adds and splits ranges with high_probability. 71 // Eventually when adds and splits fail too often, they'll invert their 72 // probability to 100 - high_probability. They'll switch back when the tree 73 // becomes too empty. 74 // 75 // This can be < 50, but the test will not be very interesting. 76 NvU32 high_probability; // In 77 78 // Probability (0-100) 79 // 80 // Every main iteration a group of operations is selected with this 81 // probability. The group consists of either "add/remove" or "split/merge." 82 // This is the chance that the "add/remove" group is selected each 83 // iteration. 84 NvU32 add_remove_shrink_group_probability; 85 86 // Probability (0-100) 87 // 88 // Probability of picking the shrink operation instead of add/remove if the 89 // add/remove/shrink group of operations is selected. 90 NvU32 shrink_probability; 91 92 // The number of collision verification checks to make each main iteration 93 NvU32 collision_checks; // In 94 95 // The number of tree iterator verification checks to make each main 96 // iteration. 97 NvU32 iterator_checks; // In 98 99 // Highest range value to use 100 NvU64 max_end NV_ALIGN_BYTES(8); // In 101 102 // Maximum number of range nodes to put in the tree 103 NvU64 max_ranges NV_ALIGN_BYTES(8); // In 104 105 // Maximum number of range nodes to add or remove at one time 106 NvU64 max_batch_count NV_ALIGN_BYTES(8); // In 107 108 // add, split, and merge operations all operate on randomly-selected ranges 109 // or nodes. It's possible, sometimes even likely, that the operation cannot 110 // be performed on the selected range or node. 111 // 112 // For example, when a range node is added its range is selected at random 113 // without regard to range nodes already in the tree. If a collision occurs 114 // when the test attempts to add that node to the tree, a new, smaller 115 // random range is selected and the attempt is made again. 116 // 117 // max_attempts is the maximum number of times to keep picking new ranges or 118 // nodes before giving up on the operation. 119 NvU32 max_attempts; // In 120 121 struct 122 { 123 NvU64 total_adds NV_ALIGN_BYTES(8); 124 NvU64 failed_adds NV_ALIGN_BYTES(8); 125 NvU64 max_attempts_add NV_ALIGN_BYTES(8); 126 NvU64 total_removes NV_ALIGN_BYTES(8); 127 NvU64 total_splits NV_ALIGN_BYTES(8); 128 NvU64 failed_splits NV_ALIGN_BYTES(8); 129 NvU64 max_attempts_split NV_ALIGN_BYTES(8); 130 NvU64 total_merges NV_ALIGN_BYTES(8); 131 NvU64 failed_merges NV_ALIGN_BYTES(8); 132 NvU64 max_attempts_merge NV_ALIGN_BYTES(8); 133 NvU64 total_shrinks NV_ALIGN_BYTES(8); 134 NvU64 failed_shrinks NV_ALIGN_BYTES(8); 135 } stats; // Out 136 137 NV_STATUS rmStatus; // Out 138 } UVM_TEST_RANGE_TREE_RANDOM_PARAMS; 139 140 // Keep this in sync with uvm_va_range_type_t in uvm_va_range.h 141 typedef enum 142 { 143 UVM_TEST_VA_RANGE_TYPE_INVALID = 0, 144 UVM_TEST_VA_RANGE_TYPE_MANAGED, 145 UVM_TEST_VA_RANGE_TYPE_EXTERNAL, 146 UVM_TEST_VA_RANGE_TYPE_CHANNEL, 147 UVM_TEST_VA_RANGE_TYPE_SKED_REFLECTED, 148 UVM_TEST_VA_RANGE_TYPE_SEMAPHORE_POOL, 149 UVM_TEST_VA_RANGE_TYPE_MAX 150 } UVM_TEST_VA_RANGE_TYPE; 151 152 typedef enum 153 { 154 UVM_TEST_RANGE_SUBTYPE_INVALID = 0, 155 UVM_TEST_RANGE_SUBTYPE_UVM, 156 UVM_TEST_RANGE_SUBTYPE_HMM, 157 UVM_TEST_RANGE_SUBTYPE_MAX 158 } UVM_TEST_RANGE_SUBTYPE; 159 160 // Keep this in sync with uvm_read_duplication_t in uvm_va_range.h 161 typedef enum 162 { 163 UVM_TEST_READ_DUPLICATION_UNSET = 0, 164 UVM_TEST_READ_DUPLICATION_ENABLED, 165 UVM_TEST_READ_DUPLICATION_DISABLED, 166 UVM_TEST_READ_DUPLICATION_MAX 167 } UVM_TEST_READ_DUPLICATION_POLICY; 168 169 typedef struct 170 { 171 // Note: if this is a zombie or not owned by the calling process, the vma info 172 // will not be filled out and is invalid. 173 NvU64 vma_start NV_ALIGN_BYTES(8); // Out 174 NvU64 vma_end NV_ALIGN_BYTES(8); // Out, inclusive 175 NvBool is_zombie; // Out 176 // Note: if this is a zombie, this field is meaningless. 177 NvBool owned_by_calling_process; // Out 178 NvU32 subtype; // Out (UVM_TEST_RANGE_SUBTYPE) 179 } UVM_TEST_VA_RANGE_INFO_MANAGED; 180 181 #define UVM_TEST_VA_RANGE_INFO UVM_TEST_IOCTL_BASE(4) 182 typedef struct 183 { 184 NvU64 lookup_address NV_ALIGN_BYTES(8); // In 185 186 // For HMM ranges va_range_start/end will contain the lookup address but not 187 // neccessarily the maximal range over which the returned policy applies. 188 // For example there could be adjacent ranges with the same policy, implying 189 // the returned range could be as small as a page in the worst case for HMM. 190 NvU64 va_range_start NV_ALIGN_BYTES(8); // Out 191 NvU64 va_range_end NV_ALIGN_BYTES(8); // Out, inclusive 192 NvU32 read_duplication; // Out (UVM_TEST_READ_DUPLICATION_POLICY) 193 NvProcessorUuid preferred_location; // Out 194 NvProcessorUuid accessed_by[UVM_MAX_PROCESSORS]; // Out 195 NvU32 accessed_by_count; // Out 196 NvU32 type; // Out (UVM_TEST_VA_RANGE_TYPE) 197 union 198 { 199 UVM_TEST_VA_RANGE_INFO_MANAGED managed NV_ALIGN_BYTES(8); // Out 200 // More here eventually 201 }; 202 203 // NV_ERR_INVALID_ADDRESS lookup_address doesn't match a UVM range 204 NV_STATUS rmStatus; // Out 205 } UVM_TEST_VA_RANGE_INFO_PARAMS; 206 207 #define UVM_TEST_RM_MEM_SANITY UVM_TEST_IOCTL_BASE(5) 208 typedef struct 209 { 210 // Out params 211 NV_STATUS rmStatus; 212 } UVM_TEST_RM_MEM_SANITY_PARAMS; 213 214 #define UVM_TEST_GPU_SEMAPHORE_SANITY UVM_TEST_IOCTL_BASE(6) 215 typedef struct 216 { 217 // Out params 218 NV_STATUS rmStatus; 219 } UVM_TEST_GPU_SEMAPHORE_SANITY_PARAMS; 220 221 #define UVM_TEST_PEER_REF_COUNT UVM_TEST_IOCTL_BASE(7) 222 typedef struct 223 { 224 // In params 225 NvProcessorUuid gpu_uuid_1; 226 NvProcessorUuid gpu_uuid_2; 227 228 // Out params 229 NV_STATUS rmStatus; 230 NvU64 ref_count NV_ALIGN_BYTES(8); 231 } UVM_TEST_PEER_REF_COUNT_PARAMS; 232 233 // Force an existing UVM range to split. split_address will be the new end of 234 // the existing range. A new range will be created covering 235 // [split_address+1, original end]. 236 // 237 // Error returns: 238 // NV_ERR_INVALID_ADDRESS 239 // - split_address+1 isn't page-aligned 240 // - split_address doesn't match a splittable UVM range 241 // - The range cannot be split at split_address because split_address is 242 // already the end of the range. 243 #define UVM_TEST_VA_RANGE_SPLIT UVM_TEST_IOCTL_BASE(8) 244 typedef struct 245 { 246 NvU64 split_address NV_ALIGN_BYTES(8); // In 247 NV_STATUS rmStatus; // Out 248 } UVM_TEST_VA_RANGE_SPLIT_PARAMS; 249 250 // Forces the next range split on the range covering lookup_address to fail with 251 // an out-of-memory error. Only the next split will fail. Subsequent ones will 252 // succeed. The split can come from any source, such as vma splitting or 253 // UVM_TEST_VA_RANGE_SPLIT. 254 // 255 // Error returns: 256 // NV_ERR_INVALID_ADDRESS 257 // - lookup_address doesn't match a UVM range 258 #define UVM_TEST_VA_RANGE_INJECT_SPLIT_ERROR UVM_TEST_IOCTL_BASE(9) 259 typedef struct 260 { 261 NvU64 lookup_address NV_ALIGN_BYTES(8); // In 262 NV_STATUS rmStatus; // Out 263 } UVM_TEST_VA_RANGE_INJECT_SPLIT_ERROR_PARAMS; 264 265 #define UVM_TEST_PAGE_TREE UVM_TEST_IOCTL_BASE(10) 266 typedef struct 267 { 268 NV_STATUS rmStatus; // Out 269 } UVM_TEST_PAGE_TREE_PARAMS; 270 271 // Given a VA and a target processor, forcibly set that processor's mapping to 272 // the VA to the given permissions. This may require changing other processors' 273 // mappings. For example, setting an atomic mapping for a given GPU might make 274 // other GPUs' mappings read-only. 275 // 276 // If the mapping changes from invalid to anything else, this call always 277 // attempts to create direct mappings from the given processor to the current 278 // physical memory backing the target address. If a direct mapping cannot be 279 // created, or no physical memory currently backs the VA, 280 // NV_ERR_INVALID_OPERATION is returned. 281 // 282 // uuid is allowed to be NV_PROCESSOR_UUID_CPU_DEFAULT. 283 // 284 // Error returns: 285 // NV_ERR_INVALID_DEVICE 286 // - uuid is an unknown value 287 // - uuid is a GPU that hasn't been registered with this process 288 // 289 // NV_ERR_INVALID_ADDRESS 290 // - VA is unknown to the kernel 291 // - VA isn't aligned to the system page size 292 // 293 // NV_ERR_INVALID_STATE 294 // - A mapping for va can't be accessed because it belongs to another process 295 // 296 // NV_ERR_INVALID_ARGUMENT 297 // - mapping is not a valid enum value 298 // 299 // NV_ERR_INVALID_ACCESS_TYPE 300 // - The mapping permissions aren't logically allowed. For example, 301 // UVM_TEST_PTE_MAPPING_READ_WRITE can't be set on a read-only mapping. 302 // 303 // NV_ERR_INVALID_OPERATION 304 // - mapping is not UVM_TEST_PTE_MAPPING_INVALID, and a direct mapping from the 305 // given processor to the physical memory currently backing VA cannot be 306 // created. 307 #define UVM_TEST_CHANGE_PTE_MAPPING UVM_TEST_IOCTL_BASE(11) 308 309 typedef enum 310 { 311 UVM_TEST_PTE_MAPPING_INVALID = 0, 312 UVM_TEST_PTE_MAPPING_READ_ONLY, 313 UVM_TEST_PTE_MAPPING_READ_WRITE, 314 UVM_TEST_PTE_MAPPING_READ_WRITE_ATOMIC, 315 UVM_TEST_PTE_MAPPING_MAX 316 } UVM_TEST_PTE_MAPPING; 317 318 typedef struct 319 { 320 NvProcessorUuid uuid NV_ALIGN_BYTES(8); // In 321 NvU64 va NV_ALIGN_BYTES(8); // In 322 NvU32 mapping; // In (UVM_TEST_PTE_MAPPING) 323 NV_STATUS rmStatus; // Out 324 } UVM_TEST_CHANGE_PTE_MAPPING_PARAMS; 325 326 #define UVM_TEST_TRACKER_SANITY UVM_TEST_IOCTL_BASE(12) 327 typedef struct 328 { 329 NV_STATUS rmStatus; // Out 330 } UVM_TEST_TRACKER_SANITY_PARAMS; 331 332 #define UVM_TEST_PUSH_SANITY UVM_TEST_IOCTL_BASE(13) 333 typedef struct 334 { 335 NvBool skipTimestampTest; // In 336 NV_STATUS rmStatus; // Out 337 } UVM_TEST_PUSH_SANITY_PARAMS; 338 339 #define UVM_TEST_CHANNEL_SANITY UVM_TEST_IOCTL_BASE(14) 340 typedef struct 341 { 342 NV_STATUS rmStatus; // Out 343 } UVM_TEST_CHANNEL_SANITY_PARAMS; 344 345 typedef enum 346 { 347 UVM_TEST_CHANNEL_STRESS_MODE_NOOP_PUSH = 0, 348 UVM_TEST_CHANNEL_STRESS_MODE_UPDATE_CHANNELS, 349 UVM_TEST_CHANNEL_STRESS_MODE_STREAM, 350 } UVM_TEST_CHANNEL_STRESS_MODE; 351 352 #define UVM_TEST_CHANNEL_STRESS UVM_TEST_IOCTL_BASE(15) 353 typedef struct 354 { 355 NvU32 mode; // In 356 357 // Number of iterations: 358 // mode == NOOP_PUSH: number of noop pushes 359 // mode == UPDATE_CHANNELS: number of updates 360 // mode == STREAM: number of iterations per stream 361 NvU32 iterations; 362 363 NvU32 num_streams; // In, used only for mode == UVM_TEST_CHANNEL_STRESS_MODE_STREAM 364 NvU32 seed; // In 365 NvU32 verbose; // In 366 NV_STATUS rmStatus; // Out 367 } UVM_TEST_CHANNEL_STRESS_PARAMS; 368 369 #define UVM_TEST_CE_SANITY UVM_TEST_IOCTL_BASE(16) 370 typedef struct 371 { 372 NvBool skipTimestampTest; // In 373 NV_STATUS rmStatus; // Out 374 } UVM_TEST_CE_SANITY_PARAMS; 375 376 #define UVM_TEST_VA_BLOCK_INFO UVM_TEST_IOCTL_BASE(17) 377 378 // See UVM_VA_BLOCK_SIZE in uvm_va_block.h for an explanation of this number 379 #define UVM_TEST_VA_BLOCK_SIZE (2ull*1024*1024) 380 381 typedef struct 382 { 383 NvU64 lookup_address NV_ALIGN_BYTES(8); // In 384 385 386 NvU64 va_block_start NV_ALIGN_BYTES(8); // Out 387 NvU64 va_block_end NV_ALIGN_BYTES(8); // Out, inclusive 388 389 // NV_ERR_INVALID_ADDRESS lookup_address doesn't match a UVM range 390 // 391 // NV_ERR_OBJECT_NOT_FOUND lookup_address matched a UVM range on this file 392 // but the corresponding block has not yet been 393 // populated 394 NV_STATUS rmStatus; // Out 395 } UVM_TEST_VA_BLOCK_INFO_PARAMS; 396 397 #define UVM_TEST_LOCK_SANITY UVM_TEST_IOCTL_BASE(18) 398 typedef struct 399 { 400 NV_STATUS rmStatus; // Out 401 } UVM_TEST_LOCK_SANITY_PARAMS; 402 403 #define UVM_TEST_PERF_UTILS_SANITY UVM_TEST_IOCTL_BASE(19) 404 typedef struct 405 { 406 NV_STATUS rmStatus; // Out 407 } UVM_TEST_PERF_UTILS_SANITY_PARAMS; 408 409 #define UVM_TEST_KVMALLOC UVM_TEST_IOCTL_BASE(20) 410 typedef struct 411 { 412 NV_STATUS rmStatus; // Out 413 } UVM_TEST_KVMALLOC_PARAMS; 414 415 #define UVM_TEST_PMM_QUERY UVM_TEST_IOCTL_BASE(21) 416 typedef enum 417 { 418 // Get the value of valid user allocations as key 419 UVM_TEST_CHUNK_SIZE_GET_USER_SIZE 420 } uvm_test_pmm_query_key_t; 421 422 typedef struct 423 { 424 // In params 425 NvProcessorUuid gpu_uuid; 426 NvU64 key; 427 // Out params 428 NvU64 value; 429 NV_STATUS rmStatus; 430 } UVM_TEST_PMM_QUERY_PARAMS; 431 432 #define UVM_TEST_PMM_CHECK_LEAK UVM_TEST_IOCTL_BASE(22) 433 434 typedef struct 435 { 436 NvProcessorUuid gpu_uuid; // In 437 NvU64 chunk_size; // In 438 NvS64 alloc_limit; // In. Number of chunks to allocate. -1 means unlimited 439 NvU64 allocated; // Out. Number of chunks actually allocated 440 NV_STATUS rmStatus; // Out 441 } UVM_TEST_PMM_CHECK_LEAK_PARAMS; 442 443 #define UVM_TEST_PERF_EVENTS_SANITY UVM_TEST_IOCTL_BASE(23) 444 typedef struct 445 { 446 // Out params 447 NV_STATUS rmStatus; 448 } UVM_TEST_PERF_EVENTS_SANITY_PARAMS; 449 450 #define UVM_TEST_PERF_MODULE_SANITY UVM_TEST_IOCTL_BASE(24) 451 typedef struct 452 { 453 // In params 454 NvU64 range_address NV_ALIGN_BYTES(8); 455 NvU32 range_size; 456 // Out params 457 NV_STATUS rmStatus; 458 } UVM_TEST_PERF_MODULE_SANITY_PARAMS; 459 460 #define UVM_TEST_RANGE_ALLOCATOR_SANITY UVM_TEST_IOCTL_BASE(25) 461 typedef struct 462 { 463 // In params 464 NvU32 verbose; 465 NvU32 seed; 466 NvU32 iters; 467 468 // Out params 469 NV_STATUS rmStatus; 470 } UVM_TEST_RANGE_ALLOCATOR_SANITY_PARAMS; 471 472 #define UVM_TEST_GET_RM_PTES UVM_TEST_IOCTL_BASE(26) 473 typedef enum 474 { 475 UVM_TEST_GET_RM_PTES_SINGLE_GPU = 0, 476 UVM_TEST_GET_RM_PTES_MULTI_GPU_SUPPORTED, 477 UVM_TEST_GET_RM_PTES_MULTI_GPU_SLI_SUPPORTED, 478 UVM_TEST_GET_RM_PTES_MULTI_GPU_NOT_SUPPORTED, 479 UVM_TEST_GET_RM_PTES_MAX 480 } UVM_TEST_PTE_RM_PTES_TEST_MODE; 481 482 typedef struct 483 { 484 // In 485 NvS32 rmCtrlFd; // For future use. (security check) 486 NvHandle hClient; 487 NvHandle hMemory; 488 NvU32 test_mode; // (UVM_TEST_PTE_RM_PTES_TEST_MODE) 489 NvU64 size NV_ALIGN_BYTES(8); 490 NvProcessorUuid gpu_uuid; 491 492 // Out 493 NV_STATUS rmStatus; 494 } UVM_TEST_GET_RM_PTES_PARAMS; 495 496 #define UVM_TEST_FAULT_BUFFER_FLUSH UVM_TEST_IOCTL_BASE(27) 497 typedef struct 498 { 499 NvU64 iterations; // In 500 NV_STATUS rmStatus; // Out 501 } UVM_TEST_FAULT_BUFFER_FLUSH_PARAMS; 502 503 #define UVM_TEST_INJECT_TOOLS_EVENT UVM_TEST_IOCTL_BASE(28) 504 typedef struct 505 { 506 // In params 507 UvmEventEntry entry; // contains only NvUxx types 508 NvU32 count; 509 510 // Out param 511 NV_STATUS rmStatus; 512 } UVM_TEST_INJECT_TOOLS_EVENT_PARAMS; 513 514 #define UVM_TEST_INCREMENT_TOOLS_COUNTER UVM_TEST_IOCTL_BASE(29) 515 typedef struct 516 { 517 // In params 518 NvU64 amount NV_ALIGN_BYTES(8); // amount to increment 519 NvU32 counter; // name of counter 520 NvProcessorUuid processor; 521 NvU32 count; // number of times to increment 522 523 // Out param 524 NV_STATUS rmStatus; 525 } UVM_TEST_INCREMENT_TOOLS_COUNTER_PARAMS; 526 527 #define UVM_TEST_MEM_SANITY UVM_TEST_IOCTL_BASE(30) 528 typedef struct 529 { 530 // Out params 531 NV_STATUS rmStatus; 532 } UVM_TEST_MEM_SANITY_PARAMS; 533 534 #define UVM_TEST_MAKE_CHANNEL_STOPS_IMMEDIATE UVM_TEST_IOCTL_BASE(32) 535 typedef struct 536 { 537 // Out params 538 NV_STATUS rmStatus; 539 } UVM_TEST_MAKE_CHANNEL_STOPS_IMMEDIATE_PARAMS; 540 541 // Inject an error into the VA block covering the lookup_address 542 // 543 // If page_table_allocation_retry_force_count is non-0 then the next count 544 // page table allocations under the VA block will be forced to do 545 // allocation-retry. 546 // 547 // If user_pages_allocation_retry_force_count is non-0 then the next count user 548 // memory allocations under the VA block will be forced to do allocation-retry. 549 // 550 // If cpu_pages_allocation_error_count is not zero, the subsequent operations 551 // that need to allocate CPU pages will fail with NV_ERR_NO_MEMORY for 552 // cpu_pages_allocation_error_count times. If cpu_pages_allocation_error_count 553 // is equal to ~0U, the count is infinite. 554 // 555 // If eviction_failure is NV_TRUE, the next eviction attempt from the VA block 556 // will fail with NV_ERR_NO_MEMORY. 557 // 558 // If populate_failure is NV_TRUE, a retry error will be injected after the next 559 // successful user memory allocation under the VA block but before that 560 // allocation is used by the block. This is similar to 561 // user_pages_allocation_retry_force_count, but the injection point simulates 562 // driver metadata allocation failure. 563 // 564 // cpu_chunk_allocation_target_id and cpu_chunk_allocation_actual_id are used 565 // to control the NUMA node IDs for CPU chunk allocations, specifically for 566 // testing overlapping CPU chunk allocations. 567 // 568 // Currently, uvm_api_migrate() does not pass the preferred CPU NUMA node to for 569 // managed memory so it is not possible to request a specific node. 570 // cpu_chunk_allocation_target_id is used to request the allocation be made on 571 // specific node. On the other hand, cpu_chunk_allocation_actual_id is the node 572 // on which the allocation will actually be made. 573 // 574 // The two parameters can be used to force a CPU chunk allocation to overlap a 575 // previously allocated chunk. 576 // 577 // Please note that even when specifying cpu_cpu_allocation_actual_id, the 578 // kernel may end up allocating on a different node. 579 // 580 // Error returns: 581 // NV_ERR_INVALID_ADDRESS 582 // - lookup_address doesn't match a UVM range 583 #define UVM_TEST_VA_BLOCK_INJECT_ERROR UVM_TEST_IOCTL_BASE(33) 584 typedef struct 585 { 586 NvU64 lookup_address NV_ALIGN_BYTES(8); // In 587 NvU32 page_table_allocation_retry_force_count; // In 588 NvU32 user_pages_allocation_retry_force_count; // In 589 NvU32 cpu_chunk_allocation_size_mask; // In 590 NvS32 cpu_chunk_allocation_target_id; // In 591 NvS32 cpu_chunk_allocation_actual_id; // In 592 NvU32 cpu_pages_allocation_error_count; // In 593 NvBool eviction_error; // In 594 NvBool populate_error; // In 595 NV_STATUS rmStatus; // Out 596 } UVM_TEST_VA_BLOCK_INJECT_ERROR_PARAMS; 597 598 #define UVM_TEST_PEER_IDENTITY_MAPPINGS UVM_TEST_IOCTL_BASE(34) 599 typedef struct 600 { 601 // In params 602 NvProcessorUuid gpuA; 603 NvProcessorUuid gpuB; 604 // Out param 605 NV_STATUS rmStatus; 606 } UVM_TEST_PEER_IDENTITY_MAPPINGS_PARAMS; 607 608 #define UVM_TEST_VA_RESIDENCY_INFO UVM_TEST_IOCTL_BASE(35) 609 typedef struct 610 { 611 NvU64 lookup_address NV_ALIGN_BYTES(8); // In 612 613 // Whether to wait on the block tracker before returning. Fields like 614 // resident_on and mapped_on represent state which will be valid when the 615 // block tracker is complete. If is_async is true, then those fields will 616 // still be filled out as if the tracker is done, but the actual residency 617 // or mapping changes may not have been performed yet. 618 NvBool is_async; // In 619 620 // Array of processors which have a resident copy of the page containing 621 // lookup_address. 622 NvProcessorUuid resident_on[UVM_MAX_PROCESSORS]; // Out 623 NvU32 resident_on_count; // Out 624 625 // If the memory is resident on the CPU, the NUMA node on which the page 626 // is resident. Otherwise, -1. 627 NvS32 resident_nid; // Out 628 629 // The size of the physical allocation backing lookup_address. Only the 630 // system-page-sized portion of this allocation which contains 631 // lookup_address is guaranteed to be resident on the corresponding 632 // processor. 633 NvU32 resident_physical_size[UVM_MAX_PROCESSORS]; // Out 634 635 // The physical address of the physical allocation backing lookup_address. 636 NvU64 resident_physical_address[UVM_MAX_PROCESSORS] NV_ALIGN_BYTES(8); // Out 637 638 // Array of processors which have a virtual mapping covering lookup_address. 639 NvProcessorUuid mapped_on[UVM_MAX_PROCESSORS]; // Out 640 NvU32 mapping_type[UVM_MAX_PROCESSORS]; // Out 641 NvU64 mapping_physical_address[UVM_MAX_PROCESSORS] NV_ALIGN_BYTES(8); // Out 642 NvU32 mapped_on_count; // Out 643 644 // The size of the virtual mapping covering lookup_address on each 645 // mapped_on processor. 646 NvU32 page_size[UVM_MAX_PROCESSORS]; // Out 647 648 // Array of processors which have physical memory populated that would back 649 // lookup_address if it was resident. 650 NvProcessorUuid populated_on[UVM_MAX_PROCESSORS]; // Out 651 NvU32 populated_on_count; // Out 652 653 NV_STATUS rmStatus; // Out 654 } UVM_TEST_VA_RESIDENCY_INFO_PARAMS; 655 656 #define UVM_TEST_PMM_ASYNC_ALLOC UVM_TEST_IOCTL_BASE(36) 657 typedef struct 658 { 659 NvProcessorUuid gpu_uuid; // In 660 NvU32 num_chunks; // In 661 NvU32 num_work_iterations; // In 662 NV_STATUS rmStatus; // Out 663 } UVM_TEST_PMM_ASYNC_ALLOC_PARAMS; 664 665 typedef enum 666 { 667 UVM_TEST_PREFETCH_FILTERING_MODE_FILTER_ALL, // Disable all prefetch faults 668 UVM_TEST_PREFETCH_FILTERING_MODE_FILTER_NONE, // Enable all prefetch faults 669 } UvmTestPrefetchFilteringMode; 670 671 #define UVM_TEST_SET_PREFETCH_FILTERING UVM_TEST_IOCTL_BASE(37) 672 typedef struct 673 { 674 NvProcessorUuid gpu_uuid; // In 675 NvU32 filtering_mode; // In (UvmTestPrefetchFilteringMode) 676 NV_STATUS rmStatus; // Out 677 } UVM_TEST_SET_PREFETCH_FILTERING_PARAMS; 678 679 typedef enum 680 { 681 UvmTestPmmSanityModeFull = 1, 682 UvmTestPmmSanityModeBasic = 2, 683 } UvmTestPmmSanityMode; 684 685 #define UVM_TEST_PMM_SANITY UVM_TEST_IOCTL_BASE(40) 686 typedef struct 687 { 688 // Test mode of type UvmTestPmmSanityMode 689 NvU32 mode; // In 690 NV_STATUS rmStatus; // Out 691 } UVM_TEST_PMM_SANITY_PARAMS; 692 693 typedef enum 694 { 695 UvmInvalidateTlbMemBarNone = 1, 696 UvmInvalidateTlbMemBarSys = 2, 697 UvmInvalidateTlbMemBarLocal = 3, 698 } UvmInvalidateTlbMembarType; 699 700 typedef enum 701 { 702 UvmInvalidatePageTableLevelAll = 1, 703 UvmInvalidatePageTableLevelPte = 2, 704 UvmInvalidatePageTableLevelPde0 = 3, 705 UvmInvalidatePageTableLevelPde1 = 4, 706 UvmInvalidatePageTableLevelPde2 = 5, 707 UvmInvalidatePageTableLevelPde3 = 6, 708 UvmInvalidatePageTableLevelPde4 = 7, 709 } UvmInvalidatePageTableLevel; 710 711 typedef enum 712 { 713 UvmTargetVaModeAll = 1, 714 UvmTargetVaModeTargeted = 2, 715 } UvmTargetVaMode; 716 717 #define UVM_TEST_INVALIDATE_TLB UVM_TEST_IOCTL_BASE(41) 718 typedef struct 719 { 720 // In params 721 NvProcessorUuid gpu_uuid; 722 NvU64 va NV_ALIGN_BYTES(8); 723 NvU32 target_va_mode; // UvmTargetVaMode 724 NvU32 page_table_level; // UvmInvalidatePageTableLevel 725 NvU32 membar; // UvmInvalidateTlbMembarType 726 NvBool disable_gpc_invalidate; 727 728 // Out params 729 NV_STATUS rmStatus; 730 } UVM_TEST_INVALIDATE_TLB_PARAMS; 731 732 #define UVM_TEST_VA_BLOCK UVM_TEST_IOCTL_BASE(42) 733 typedef struct 734 { 735 NV_STATUS rmStatus; // Out 736 } UVM_TEST_VA_BLOCK_PARAMS; 737 738 typedef enum 739 { 740 // Default policy based eviction 741 // 742 // Evicts a chunk that the default eviction path would pick. 743 UvmTestEvictModeDefault = 1, 744 745 // Virtual address based eviction 746 // 747 // Evicts the root chunk that the chunk backing the provided virtual address 748 // belongs to. 749 UvmTestEvictModeVirtual, 750 751 // Physical address based eviction 752 // 753 // Evicts the root chunk covering the provided physical address. 754 UvmTestEvictModePhysical, 755 } UvmTestEvictMode; 756 757 // Evict a chunk chosen according to one the test eviction modes specified 758 // above. Eviction may not always be possible, but as long as the arguments are 759 // valid NV_OK will be returned. To check whether eviction happened, the 760 // chunk_was_evicted flag needs to be inspected. 761 #define UVM_TEST_EVICT_CHUNK UVM_TEST_IOCTL_BASE(43) 762 typedef struct 763 { 764 // The GPU to evict from, has to be registered in the VA space. 765 NvProcessorUuid gpu_uuid; // In 766 767 // UvmTestEvictMode 768 NvU32 eviction_mode; // In 769 770 // Virtual or physical address if evictionMode is UvmTestEvictModeVirtual or 771 // UvmTestEvictModePhysical. 772 NvU64 address NV_ALIGN_BYTES(8); // In 773 774 // Flag indicating whether the eviction was performed. 775 NvBool chunk_was_evicted; // Out 776 777 // Physical address of the evicted root chunk. Notably 0 is a valid physical address. 778 NvU64 evicted_physical_address NV_ALIGN_BYTES(8); // Out 779 780 // For the virtual eviction mode, returns the size of the chunk that was 781 // backing the virtual address before being evicted. 0 otherwise. 782 NvU64 chunk_size_backing_virtual NV_ALIGN_BYTES(8); // Out 783 784 NV_STATUS rmStatus; // Out 785 } UVM_TEST_EVICT_CHUNK_PARAMS; 786 787 typedef enum 788 { 789 // Flush deferred accessed by mappings 790 UvmTestDeferredWorkTypeAcessedByMappings = 1, 791 } UvmTestDeferredWorkType; 792 793 #define UVM_TEST_FLUSH_DEFERRED_WORK UVM_TEST_IOCTL_BASE(44) 794 typedef struct 795 { 796 // UvmTestDeferredWorkType 797 NvU32 work_type; // In 798 799 NV_STATUS rmStatus; // Out 800 } UVM_TEST_FLUSH_DEFERRED_WORK_PARAMS; 801 802 #define UVM_TEST_NV_KTHREAD_Q UVM_TEST_IOCTL_BASE(45) 803 typedef struct 804 { 805 NV_STATUS rmStatus; // Out 806 } UVM_TEST_NV_KTHREAD_Q_PARAMS; 807 808 typedef enum 809 { 810 UVM_TEST_PAGE_PREFETCH_POLICY_ENABLE = 0, 811 UVM_TEST_PAGE_PREFETCH_POLICY_DISABLE, 812 UVM_TEST_PAGE_PREFETCH_POLICY_MAX 813 } UVM_TEST_PAGE_PREFETCH_POLICY; 814 815 #define UVM_TEST_SET_PAGE_PREFETCH_POLICY UVM_TEST_IOCTL_BASE(46) 816 typedef struct 817 { 818 NvU32 policy; // In (UVM_TEST_PAGE_PREFETCH_POLICY) 819 NV_STATUS rmStatus; // Out 820 } UVM_TEST_SET_PAGE_PREFETCH_POLICY_PARAMS; 821 822 #define UVM_TEST_RANGE_GROUP_TREE UVM_TEST_IOCTL_BASE(47) 823 typedef struct 824 { 825 NvU64 rangeGroupIds[4] NV_ALIGN_BYTES(8); // In 826 NV_STATUS rmStatus; // Out 827 } UVM_TEST_RANGE_GROUP_TREE_PARAMS; 828 829 #define UVM_TEST_RANGE_GROUP_RANGE_INFO UVM_TEST_IOCTL_BASE(48) 830 typedef struct 831 { 832 NvU64 lookup_address NV_ALIGN_BYTES(8); // In 833 834 NvU64 range_group_range_start NV_ALIGN_BYTES(8); // Out 835 NvU64 range_group_range_end NV_ALIGN_BYTES(8); // Out, inclusive 836 NvU64 range_group_id NV_ALIGN_BYTES(8); // Out 837 NvU32 range_group_present; // Out 838 NV_STATUS rmStatus; // Out 839 } UVM_TEST_RANGE_GROUP_RANGE_INFO_PARAMS; 840 841 #define UVM_TEST_RANGE_GROUP_RANGE_COUNT UVM_TEST_IOCTL_BASE(49) 842 typedef struct 843 { 844 NvU64 rangeGroupId NV_ALIGN_BYTES(8); // In 845 NvU64 count NV_ALIGN_BYTES(8); // Out 846 NV_STATUS rmStatus; // Out 847 } UVM_TEST_RANGE_GROUP_RANGE_COUNT_PARAMS; 848 849 #define UVM_TEST_GET_PREFETCH_FAULTS_REENABLE_LAPSE UVM_TEST_IOCTL_BASE(50) 850 typedef struct 851 { 852 NvU32 reenable_lapse; // Out: Lapse in miliseconds 853 NV_STATUS rmStatus; // Out 854 } UVM_TEST_GET_PREFETCH_FAULTS_REENABLE_LAPSE_PARAMS; 855 856 #define UVM_TEST_SET_PREFETCH_FAULTS_REENABLE_LAPSE UVM_TEST_IOCTL_BASE(51) 857 typedef struct 858 { 859 NvU32 reenable_lapse; // In: Lapse in miliseconds 860 NV_STATUS rmStatus; // Out 861 } UVM_TEST_SET_PREFETCH_FAULTS_REENABLE_LAPSE_PARAMS; 862 863 #define UVM_TEST_GET_KERNEL_VIRTUAL_ADDRESS UVM_TEST_IOCTL_BASE(52) 864 typedef struct 865 { 866 NvU64 addr NV_ALIGN_BYTES(8); // Out 867 NV_STATUS rmStatus; // Out 868 } UVM_TEST_GET_KERNEL_VIRTUAL_ADDRESS_PARAMS; 869 870 // Allocate and free memory directly from PMA with eviction enabled. This allows 871 // to simulate RM-like allocations, but without the RM API lock serializing 872 // everything. 873 #define UVM_TEST_PMA_ALLOC_FREE UVM_TEST_IOCTL_BASE(53) 874 typedef struct 875 { 876 NvProcessorUuid gpu_uuid; // In 877 NvU32 page_size; 878 NvBool contiguous; 879 NvU64 num_pages NV_ALIGN_BYTES(8); // In 880 NvU64 phys_begin NV_ALIGN_BYTES(8); // In 881 NvU64 phys_end NV_ALIGN_BYTES(8); // In 882 NvU32 nap_us_before_free; // In 883 NV_STATUS rmStatus; // Out 884 } UVM_TEST_PMA_ALLOC_FREE_PARAMS; 885 886 // Allocate and free user memory directly from PMM with eviction enabled. 887 // 888 // Provides a direct way of exercising PMM allocs, eviction and frees of user 889 // memory type. 890 #define UVM_TEST_PMM_ALLOC_FREE_ROOT UVM_TEST_IOCTL_BASE(54) 891 typedef struct 892 { 893 NvProcessorUuid gpu_uuid; // In 894 NvU32 nap_us_before_free; // In 895 NV_STATUS rmStatus; // Out 896 } UVM_TEST_PMM_ALLOC_FREE_ROOT_PARAMS; 897 898 // Inject a PMA eviction error after the specified number of chunks are 899 // evicted. 900 #define UVM_TEST_PMM_INJECT_PMA_EVICT_ERROR UVM_TEST_IOCTL_BASE(55) 901 typedef struct 902 { 903 NvProcessorUuid gpu_uuid; // In 904 NvU32 error_after_num_chunks; // In 905 NV_STATUS rmStatus; // Out 906 } UVM_TEST_PMM_INJECT_PMA_EVICT_ERROR_PARAMS; 907 908 // Change configuration of access counters. This call will disable access 909 // counters and reenable them using the new configuration. All previous 910 // notifications will be lost 911 // 912 // The reconfiguration affects all VA spaces that rely on the access 913 // counters information for the same GPU. To avoid conflicting configurations, 914 // only one VA space is allowed to reconfigure the GPU at a time. 915 // 916 // Error returns: 917 // NV_ERR_INVALID_STATE 918 // - The GPU has already been reconfigured in a different VA space 919 #define UVM_TEST_RECONFIGURE_ACCESS_COUNTERS UVM_TEST_IOCTL_BASE(56) 920 typedef struct 921 { 922 NvProcessorUuid gpu_uuid; // In 923 924 // Type UVM_ACCESS_COUNTER_GRANULARITY from nv_uvm_types.h 925 NvU32 mimc_granularity; // In 926 NvU32 momc_granularity; // In 927 928 // Type UVM_ACCESS_COUNTER_USE_LIMIT from nv_uvm_types.h 929 NvU32 mimc_use_limit; // In 930 NvU32 momc_use_limit; // In 931 932 NvU32 threshold; // In 933 NvBool enable_mimc_migrations; // In 934 NvBool enable_momc_migrations; // In 935 936 NV_STATUS rmStatus; // Out 937 } UVM_TEST_RECONFIGURE_ACCESS_COUNTERS_PARAMS; 938 939 typedef enum 940 { 941 UVM_TEST_ACCESS_COUNTER_RESET_MODE_ALL = 0, 942 UVM_TEST_ACCESS_COUNTER_RESET_MODE_TARGETED, 943 UVM_TEST_ACCESS_COUNTER_RESET_MODE_MAX 944 } UVM_TEST_ACCESS_COUNTER_RESET_MODE; 945 946 typedef enum 947 { 948 UVM_TEST_ACCESS_COUNTER_TYPE_MIMC = 0, 949 UVM_TEST_ACCESS_COUNTER_TYPE_MOMC, 950 UVM_TEST_ACCESS_COUNTER_TYPE_MAX 951 } UVM_TEST_ACCESS_COUNTER_TYPE; 952 953 // Clear the contents of the access counters. This call supports different 954 // modes for targeted/global resets. 955 #define UVM_TEST_RESET_ACCESS_COUNTERS UVM_TEST_IOCTL_BASE(57) 956 typedef struct 957 { 958 NvProcessorUuid gpu_uuid; // In 959 960 // Type UVM_TEST_ACCESS_COUNTER_RESET_MODE 961 NvU32 mode; // In 962 963 // Type UVM_TEST_ACCESS_COUNTER_TYPE 964 NvU32 counter_type; // In 965 966 NvU32 bank; // In 967 NvU32 tag; // In 968 NV_STATUS rmStatus; // Out 969 } UVM_TEST_RESET_ACCESS_COUNTERS_PARAMS; 970 971 // Do not handle access counter notifications when they arrive. This call is 972 // used to force an overflow of the access counter notification buffer 973 #define UVM_TEST_SET_IGNORE_ACCESS_COUNTERS UVM_TEST_IOCTL_BASE(58) 974 typedef struct 975 { 976 NvProcessorUuid gpu_uuid; // In 977 NvBool ignore; // In 978 NV_STATUS rmStatus; // Out 979 } UVM_TEST_SET_IGNORE_ACCESS_COUNTERS_PARAMS; 980 981 // Verifies that the given channel is registered under the UVM VA space of 982 // vaSpaceFd. Returns NV_OK if so, NV_ERR_INVALID_CHANNEL if not. 983 #define UVM_TEST_CHECK_CHANNEL_VA_SPACE UVM_TEST_IOCTL_BASE(59) 984 typedef struct 985 { 986 NvProcessorUuid gpu_uuid; // In 987 NvS32 rm_ctrl_fd; // In 988 NvHandle client; // In 989 NvHandle channel; // In 990 NvU32 ve_id; // In 991 NvS32 va_space_fd; // In 992 NV_STATUS rmStatus; // Out 993 } UVM_TEST_CHECK_CHANNEL_VA_SPACE_PARAMS; 994 995 // 996 // UvmTestEnableNvlinkPeerAccess 997 // 998 #define UVM_TEST_ENABLE_NVLINK_PEER_ACCESS UVM_TEST_IOCTL_BASE(60) 999 typedef struct 1000 { 1001 NvProcessorUuid gpuUuidA; // IN 1002 NvProcessorUuid gpuUuidB; // IN 1003 NV_STATUS rmStatus; // OUT 1004 } UVM_TEST_ENABLE_NVLINK_PEER_ACCESS_PARAMS; 1005 1006 // 1007 // UvmTestDisableNvlinkPeerAccess 1008 // 1009 #define UVM_TEST_DISABLE_NVLINK_PEER_ACCESS UVM_TEST_IOCTL_BASE(61) 1010 typedef struct 1011 { 1012 NvProcessorUuid gpuUuidA; // IN 1013 NvProcessorUuid gpuUuidB; // IN 1014 NV_STATUS rmStatus; // OUT 1015 } UVM_TEST_DISABLE_NVLINK_PEER_ACCESS_PARAMS; 1016 1017 typedef enum 1018 { 1019 UVM_TEST_PAGE_THRASHING_POLICY_ENABLE = 0, 1020 UVM_TEST_PAGE_THRASHING_POLICY_DISABLE, 1021 UVM_TEST_PAGE_THRASHING_POLICY_MAX 1022 } UVM_TEST_PAGE_THRASHING_POLICY; 1023 1024 // This ioctl returns the thrashing mitigation parameters on the current VA 1025 // space. Note that these values may change after a simulated/emulated GPU is 1026 // registered on the VA space. 1027 #define UVM_TEST_GET_PAGE_THRASHING_POLICY UVM_TEST_IOCTL_BASE(62) 1028 typedef struct 1029 { 1030 NvU32 policy; // Out (UVM_TEST_PAGE_THRASHING_POLICY) 1031 NvU64 nap_ns NV_ALIGN_BYTES(8); // Out 1032 NvU64 pin_ns NV_ALIGN_BYTES(8); // Out 1033 NvBool map_remote_on_native_atomics_fault; // Out 1034 NV_STATUS rmStatus; // Out 1035 } UVM_TEST_GET_PAGE_THRASHING_POLICY_PARAMS; 1036 1037 #define UVM_TEST_SET_PAGE_THRASHING_POLICY UVM_TEST_IOCTL_BASE(63) 1038 typedef struct 1039 { 1040 NvU32 policy; // In (UVM_TEST_PAGE_THRASHING_POLICY) 1041 NvU64 pin_ns NV_ALIGN_BYTES(8); // In 1042 NV_STATUS rmStatus; // Out 1043 } UVM_TEST_SET_PAGE_THRASHING_POLICY_PARAMS; 1044 1045 #define UVM_TEST_PMM_SYSMEM UVM_TEST_IOCTL_BASE(64) 1046 typedef struct 1047 { 1048 NvU64 range_address1 NV_ALIGN_BYTES(8); // In 1049 NvU64 range_address2 NV_ALIGN_BYTES(8); // In 1050 NV_STATUS rmStatus; // Out 1051 } UVM_TEST_PMM_SYSMEM_PARAMS; 1052 1053 #define UVM_TEST_PMM_REVERSE_MAP UVM_TEST_IOCTL_BASE(65) 1054 typedef struct 1055 { 1056 NvProcessorUuid gpu_uuid; // In 1057 NvU64 range_address1 NV_ALIGN_BYTES(8); // In 1058 NvU64 range_address2 NV_ALIGN_BYTES(8); // In 1059 NvU64 range_size2 NV_ALIGN_BYTES(8); // In 1060 NV_STATUS rmStatus; // Out 1061 } UVM_TEST_PMM_REVERSE_MAP_PARAMS; 1062 1063 #define UVM_TEST_PMM_INDIRECT_PEERS UVM_TEST_IOCTL_BASE(66) 1064 typedef struct 1065 { 1066 NV_STATUS rmStatus; // Out 1067 } UVM_TEST_PMM_INDIRECT_PEERS_PARAMS; 1068 1069 // Calls uvm_va_space_mm_retain on a VA space, operates on the mm, optionally 1070 // sleeps for a while, then releases the va_space_mm and returns. The idea is to 1071 // simulate retaining a va_space_mm from a thread like the GPU fault handler 1072 // which operates outside of the normal context of the VA space. 1073 #define UVM_TEST_VA_SPACE_MM_RETAIN UVM_TEST_IOCTL_BASE(67) 1074 typedef struct 1075 { 1076 // The kernel virtual address of the uvm_va_space on which to attempt 1077 // retain. This can be obtained via UVM_TEST_GET_KERNEL_VIRTUAL_ADDRESS. 1078 // 1079 // The reason to use this instead of looking it up from an fd as normal is 1080 // to allow testing of calling threads which race with UVM VA space destroy 1081 // (file close). We wouldn't be able to test that path if this was an fd. 1082 NvU64 va_space_ptr NV_ALIGN_BYTES(8); // In 1083 1084 // User virtual address within the va_space_mm. If the va_space_mm is 1085 // successfully retained, this address is read once before sleeping and once 1086 // after (if sleep_us > 0). 1087 NvU64 addr NV_ALIGN_BYTES(8); // In 1088 1089 // On success, this contains the value of addr read prior to the sleep. 1090 NvU64 val_before NV_ALIGN_BYTES(8); // In 1091 1092 // On success, and if sleep_us > 0, this contains the value of addr read 1093 // after the sleep. This is invalid if sleep_us == 0. 1094 NvU64 val_after NV_ALIGN_BYTES(8); // In 1095 1096 // Approximate duration for which to sleep with the va_space_mm retained. 1097 NvU64 sleep_us NV_ALIGN_BYTES(8); // In 1098 1099 // NV_ERR_MISSING_TABLE_ENTRY va_space_ptr is not a valid VA space 1100 // NV_ERR_PAGE_TABLE_NOT_AVAIL Could not retain va_space_mm 1101 // (uvm_va_space_mm_retain returned NULL) 1102 // NV_ERR_INVALID_ADDRESS addr is invalid in va_space_mm 1103 NV_STATUS rmStatus; // Out 1104 } UVM_TEST_VA_SPACE_MM_RETAIN_PARAMS; 1105 1106 #define UVM_TEST_PMM_CHUNK_WITH_ELEVATED_PAGE UVM_TEST_IOCTL_BASE(69) 1107 typedef struct 1108 { 1109 NV_STATUS rmStatus; // Out 1110 } UVM_TEST_PMM_CHUNK_WITH_ELEVATED_PAGE_PARAMS; 1111 1112 #define UVM_TEST_GET_GPU_TIME UVM_TEST_IOCTL_BASE(70) 1113 typedef struct 1114 { 1115 // GPU to query time from. GPU must have been previously registered 1116 NvProcessorUuid gpu_uuid; // In 1117 1118 NvU64 timestamp_ns NV_ALIGN_BYTES(8); // Out 1119 NV_STATUS rmStatus; // Out 1120 } UVM_TEST_GET_GPU_TIME_PARAMS; 1121 1122 // Check if access counters are enabled upon registration of the given GPU 1123 #define UVM_TEST_ACCESS_COUNTERS_ENABLED_BY_DEFAULT UVM_TEST_IOCTL_BASE(71) 1124 typedef struct 1125 { 1126 NvProcessorUuid gpu_uuid; // In 1127 NvBool enabled; // Out 1128 1129 NV_STATUS rmStatus; // Out 1130 } UVM_TEST_ACCESS_COUNTERS_ENABLED_BY_DEFAULT_PARAMS; 1131 1132 // Inject an error into the VA space 1133 // 1134 // If migrate_vma_allocation_fail_nth is greater than 0, the nth page 1135 // allocation within migrate_vma will fail. 1136 // 1137 // If va_block_allocation_fail_nth is greater than 0, the nth call to 1138 // uvm_va_block_find_create() will fail with NV_ERR_NO_MEMORY. 1139 #define UVM_TEST_VA_SPACE_INJECT_ERROR UVM_TEST_IOCTL_BASE(72) 1140 typedef struct 1141 { 1142 NvU32 migrate_vma_allocation_fail_nth; // In 1143 NvU32 va_block_allocation_fail_nth; // In 1144 1145 NV_STATUS rmStatus; // Out 1146 } UVM_TEST_VA_SPACE_INJECT_ERROR_PARAMS; 1147 1148 // Release to PMA all free root chunks 1149 #define UVM_TEST_PMM_RELEASE_FREE_ROOT_CHUNKS UVM_TEST_IOCTL_BASE(73) 1150 typedef struct 1151 { 1152 NvProcessorUuid gpu_uuid; // In 1153 1154 NV_STATUS rmStatus; // Out 1155 } UVM_TEST_PMM_RELEASE_FREE_ROOT_CHUNKS_PARAMS; 1156 1157 // Wait until all pending replayable faults have been processed. If there are 1158 // still pending packets when timeout_ns is reached, the ioctl returns 1159 // NV_ERR_TIMEOUT. 1160 // 1161 // This function should be called after the kernel producing the faults has been 1162 // synchronized. This should ensure that PUT != GET and faults will not be 1163 // missed even if the driver has not started to process them, yet. 1164 #define UVM_TEST_DRAIN_REPLAYABLE_FAULTS UVM_TEST_IOCTL_BASE(74) 1165 typedef struct 1166 { 1167 NvProcessorUuid gpu_uuid; // In 1168 NvU64 timeout_ns; // In 1169 1170 NV_STATUS rmStatus; // Out 1171 } UVM_TEST_DRAIN_REPLAYABLE_FAULTS_PARAMS; 1172 1173 // Get module config PMA batch size in bytes 1174 #define UVM_TEST_PMA_GET_BATCH_SIZE UVM_TEST_IOCTL_BASE(75) 1175 typedef struct 1176 { 1177 NvProcessorUuid gpu_uuid; // In 1178 NvU64 pma_batch_size; NV_ALIGN_BYTES(8) // Out 1179 1180 NV_STATUS rmStatus; // Out 1181 } UVM_TEST_PMA_GET_BATCH_SIZE_PARAMS; 1182 1183 // Request PMA's global statistics 1184 #define UVM_TEST_PMM_QUERY_PMA_STATS UVM_TEST_IOCTL_BASE(76) 1185 typedef struct 1186 { 1187 NvProcessorUuid gpu_uuid; // In 1188 UvmPmaStatistics pma_stats; // Out 1189 1190 NV_STATUS rmStatus; // Out 1191 } UVM_TEST_PMM_QUERY_PMA_STATS_PARAMS; 1192 1193 // Test whether the bottom halves have run on the correct CPUs based on the 1194 // NUMA node locality of the GPU. 1195 // 1196 // Failure is reported if: 1197 // 1. The GPU has serviced faults but the mask tracking which CPUs the 1198 // bottom half ran on was empty, or 1199 // 2. The set of CPUs where the bottom half ran is not a subset of the CPUs 1200 // attached to the NUMA node. 1201 // 1202 // This IOCTL returns NV_OK on success, NV_ERR_INVALID_STATE on failure, or 1203 // NV_ERR_NOT_SUPPORTED if UVM thread affinity is not supported. 1204 #define UVM_TEST_NUMA_CHECK_AFFINITY UVM_TEST_IOCTL_BASE(78) 1205 typedef struct 1206 { 1207 NvProcessorUuid gpu_uuid; // In 1208 NvHandle client; // In 1209 NvHandle smc_part_ref; // In 1210 1211 NV_STATUS rmStatus; // Out 1212 } UVM_TEST_NUMA_CHECK_AFFINITY_PARAMS; 1213 1214 #define UVM_TEST_VA_SPACE_ADD_DUMMY_THREAD_CONTEXTS UVM_TEST_IOCTL_BASE(79) 1215 typedef struct 1216 { 1217 // Number of thread contexts to add per thread context table entry 1218 NvU32 num_dummy_thread_contexts; // In 1219 1220 NV_STATUS rmStatus; // Out 1221 } UVM_TEST_VA_SPACE_ADD_DUMMY_THREAD_CONTEXTS_PARAMS; 1222 1223 #define UVM_TEST_VA_SPACE_REMOVE_DUMMY_THREAD_CONTEXTS UVM_TEST_IOCTL_BASE(80) 1224 typedef struct 1225 { 1226 NV_STATUS rmStatus; // Out 1227 } UVM_TEST_VA_SPACE_REMOVE_DUMMY_THREAD_CONTEXTS_PARAMS; 1228 1229 #define UVM_TEST_THREAD_CONTEXT_SANITY UVM_TEST_IOCTL_BASE(81) 1230 typedef struct 1231 { 1232 // Iterations to run. 1233 NvU32 iterations; // In 1234 1235 NV_STATUS rmStatus; // Out 1236 } UVM_TEST_THREAD_CONTEXT_SANITY_PARAMS; 1237 1238 #define UVM_TEST_THREAD_CONTEXT_PERF UVM_TEST_IOCTL_BASE(82) 1239 typedef struct 1240 { 1241 // Iterations to run. 1242 NvU32 iterations; // In 1243 1244 // Delay, in microseconds, between thread context addition and removal 1245 NvU32 delay_us; // In 1246 1247 // Median time, in nanoseconds, spent in adding and then deleting a thread 1248 // context. 1249 NvU64 ns NV_ALIGN_BYTES(8); // Out 1250 1251 NV_STATUS rmStatus; // Out 1252 } UVM_TEST_THREAD_CONTEXT_PERF_PARAMS; 1253 1254 typedef enum 1255 { 1256 UVM_TEST_PAGEABLE_MEM_ACCESS_TYPE_NONE = 0, 1257 1258 // Pageable memory cannot be accessed, but there is an association between 1259 // this VA space and its owning process. For example, this enables the GPU 1260 // fault handler to establish CPU mappings. 1261 UVM_TEST_PAGEABLE_MEM_ACCESS_TYPE_MMU_NOTIFIER, 1262 1263 UVM_TEST_PAGEABLE_MEM_ACCESS_TYPE_HMM, 1264 UVM_TEST_PAGEABLE_MEM_ACCESS_TYPE_ATS_KERNEL, 1265 UVM_TEST_PAGEABLE_MEM_ACCESS_TYPE_ATS_DRIVER, 1266 UVM_TEST_PAGEABLE_MEM_ACCESS_TYPE_COUNT 1267 } UVM_TEST_PAGEABLE_MEM_ACCESS_TYPE; 1268 1269 #define UVM_TEST_GET_PAGEABLE_MEM_ACCESS_TYPE UVM_TEST_IOCTL_BASE(83) 1270 typedef struct 1271 { 1272 // UVM_TEST_PAGEABLE_MEM_ACCESS_TYPE 1273 NvU32 type; // Out 1274 1275 NV_STATUS rmStatus; // Out 1276 } UVM_TEST_GET_PAGEABLE_MEM_ACCESS_TYPE_PARAMS; 1277 1278 // Some events, like fault replays, may not immediately show up in the events 1279 // queue despite calling UVM_TOOLS_FLUSH_EVENTS since that will only flush 1280 // completed events but not pending events. Successful completion of this IOCTL 1281 // guarantees that any replays issued on the given GPU prior to the call will 1282 // have its event enqueued in all the tools sessions which have replay events 1283 // enabled. Also, this IOCTL includes an implicit UVM_TOOLS_FLUSH_EVENTS call. 1284 // Hence, this IOCTL is a superset of UVM_TOOLS_FLUSH_EVENTS. Since this call is 1285 // more expensive than UVM_TOOLS_FLUSH_EVENTS, callers who don't need the above 1286 // mentioned guarantee should consider calling UVM_TOOLS_FLUSH_EVENTS instead. 1287 #define UVM_TEST_TOOLS_FLUSH_REPLAY_EVENTS UVM_TEST_IOCTL_BASE(84) 1288 typedef struct 1289 { 1290 NvProcessorUuid gpuUuid; // In 1291 1292 NV_STATUS rmStatus; // Out 1293 } UVM_TEST_TOOLS_FLUSH_REPLAY_EVENTS_PARAMS; 1294 1295 // Many checks are performed when the driver is unloaded. In the event of an 1296 // error, a warning message may be printed to the kernel log. In automated 1297 // testing, a systematic way to check the state of the driver after it is 1298 // unloaded is required for additional test coverage. One userland process may 1299 // register to receive the driver state after its unload, since we cannot use 1300 // /proc or /sys to retrieve driver-specific information for an unloaded driver. 1301 // Any userland process registers the given address (unload_state_buf) with the 1302 // UVM driver. On module unload, if an address has been registered, debugging 1303 // state is written to that address. The data in the address is valid once 1304 // module unload completes. 1305 // Error returns: 1306 // NV_ERR_IN_USE 1307 // - The unload state buffer has already been registered. 1308 // NV_ERR_INVALID_ADDRESS 1309 // - unload_state_buf is invalid. 1310 // - unload_state_buf is not 8-byte aligned. 1311 1312 #define UVM_TEST_REGISTER_UNLOAD_STATE_BUFFER UVM_TEST_IOCTL_BASE(85) 1313 1314 // Unload debugging states: 1315 #define UVM_TEST_UNLOAD_STATE_MEMORY_LEAK ((NvU64)0x1) 1316 1317 typedef struct 1318 { 1319 // unload_state_buf points to a 8-byte buf and must be aligned to 8 bytes. 1320 NvU64 unload_state_buf; // In 1321 1322 NV_STATUS rmStatus; // Out 1323 } UVM_TEST_REGISTER_UNLOAD_STATE_BUFFER_PARAMS; 1324 1325 #define UVM_TEST_RB_TREE_DIRECTED UVM_TEST_IOCTL_BASE(86) 1326 1327 typedef struct 1328 { 1329 NV_STATUS rmStatus; // Out 1330 } UVM_TEST_RB_TREE_DIRECTED_PARAMS; 1331 1332 #define UVM_TEST_RB_TREE_RANDOM UVM_TEST_IOCTL_BASE(87) 1333 1334 typedef struct 1335 { 1336 NvU64 iterations NV_ALIGN_BYTES(8); // In 1337 1338 // Upper key range bound. Randomly generated node keys will not exceed this 1339 // value. 1340 NvU64 range_max; // In 1341 1342 // This parameter is used to control the size of the tree. 1343 // The number of nodes in the tree will bounce between 0 and this limit. 1344 // See uvm_rb_tree_test.c:rbtt_test_get_random_op() for full description. 1345 NvU32 node_limit; // In 1346 NvU32 seed; // In 1347 1348 NV_STATUS rmStatus; // Out 1349 } UVM_TEST_RB_TREE_RANDOM_PARAMS; 1350 1351 #define UVM_TEST_HOST_SANITY UVM_TEST_IOCTL_BASE(88) 1352 typedef struct 1353 { 1354 NV_STATUS rmStatus; // Out 1355 } UVM_TEST_HOST_SANITY_PARAMS; 1356 1357 // Calls uvm_va_space_mm_or_current_retain() on a VA space, 1358 // then releases the va_space_mm and returns. 1359 #define UVM_TEST_VA_SPACE_MM_OR_CURRENT_RETAIN UVM_TEST_IOCTL_BASE(89) 1360 typedef struct 1361 { 1362 // User address of a flag to act as a semaphore. If non-NULL, the address 1363 // is set to 1 after successful retain but before the sleep. 1364 NvU64 retain_done_ptr NV_ALIGN_BYTES(8); // In 1365 1366 // Approximate duration for which to sleep with the va_space_mm retained. 1367 NvU64 sleep_us NV_ALIGN_BYTES(8); // In 1368 1369 // NV_ERR_PAGE_TABLE_NOT_AVAIL Could not retain va_space_mm 1370 // (uvm_va_space_mm_or_current_retain returned 1371 // NULL) 1372 NV_STATUS rmStatus; // Out 1373 } UVM_TEST_VA_SPACE_MM_OR_CURRENT_RETAIN_PARAMS; 1374 1375 #define UVM_TEST_GET_USER_SPACE_END_ADDRESS UVM_TEST_IOCTL_BASE(90) 1376 typedef struct 1377 { 1378 NvU64 user_space_end_address; // Out 1379 NV_STATUS rmStatus; // Out 1380 } UVM_TEST_GET_USER_SPACE_END_ADDRESS_PARAMS; 1381 1382 #define UVM_TEST_GET_CPU_CHUNK_ALLOC_SIZES UVM_TEST_IOCTL_BASE(91) 1383 typedef struct 1384 { 1385 NvU32 alloc_size_mask; // Out 1386 NvU32 rmStatus; // Out 1387 } UVM_TEST_GET_CPU_CHUNK_ALLOC_SIZES_PARAMS; 1388 1389 // Forces the next range covering the lookup_address to fail in 1390 // uvm_va_range_add_gpu_va_space() with an out-of-memory error. Only the next 1391 // uvm_va_range_add_gpu_va_space() will fail. Subsequent ones will succeed. 1392 // 1393 // Error returns: 1394 // NV_ERR_INVALID_ADDRESS 1395 // - lookup_address doesn't match a UVM range 1396 #define UVM_TEST_VA_RANGE_INJECT_ADD_GPU_VA_SPACE_ERROR UVM_TEST_IOCTL_BASE(93) 1397 typedef struct 1398 { 1399 NvU64 lookup_address NV_ALIGN_BYTES(8); // In 1400 NV_STATUS rmStatus; // Out 1401 } UVM_TEST_VA_RANGE_INJECT_ADD_GPU_VA_SPACE_ERROR_PARAMS; 1402 1403 // Forces destroy_gpu_va_space() to delay execution. This provides a high 1404 // probability of exercising the race condition between concurrent 1405 // UvmRegisterGpuVaSpace() calls on the same {va_space, gpu} pair in the 1406 // ATS_KERNEL case. 1407 #define UVM_TEST_DESTROY_GPU_VA_SPACE_DELAY UVM_TEST_IOCTL_BASE(94) 1408 typedef struct 1409 { 1410 NvU64 delay_us; // In 1411 NV_STATUS rmStatus; // Out 1412 } UVM_TEST_DESTROY_GPU_VA_SPACE_DELAY_PARAMS; 1413 1414 #define UVM_TEST_SEC2_SANITY UVM_TEST_IOCTL_BASE(95) 1415 typedef struct 1416 { 1417 NV_STATUS rmStatus; // Out 1418 } UVM_TEST_SEC2_SANITY_PARAMS; 1419 1420 #define UVM_TEST_CGROUP_ACCOUNTING_SUPPORTED UVM_TEST_IOCTL_BASE(96) 1421 typedef struct 1422 { 1423 NV_STATUS rmStatus; // Out 1424 } UVM_TEST_CGROUP_ACCOUNTING_SUPPORTED_PARAMS; 1425 1426 #define UVM_TEST_SPLIT_INVALIDATE_DELAY UVM_TEST_IOCTL_BASE(98) 1427 typedef struct 1428 { 1429 NvU64 delay_us; // In 1430 NV_STATUS rmStatus; // Out 1431 } UVM_TEST_SPLIT_INVALIDATE_DELAY_PARAMS; 1432 1433 // Tests the CSL/SEC2 encryption/decryption methods by doing a secure transfer 1434 // of memory from CPU->GPU and a subsequent GPU->CPU transfer. 1435 #define UVM_TEST_SEC2_CPU_GPU_ROUNDTRIP UVM_TEST_IOCTL_BASE(99) 1436 typedef struct 1437 { 1438 NV_STATUS rmStatus; // Out 1439 } UVM_TEST_SEC2_CPU_GPU_ROUNDTRIP_PARAMS; 1440 1441 #define UVM_TEST_CPU_CHUNK_API UVM_TEST_IOCTL_BASE(100) 1442 typedef struct 1443 { 1444 NV_STATUS rmStatus; // Out 1445 } UVM_TEST_CPU_CHUNK_API_PARAMS; 1446 #ifdef __cplusplus 1447 } 1448 #endif 1449 1450 #endif // __UVM_TEST_IOCTL_H__ 1451