1 /******************************************************************************* 2 Copyright (c) 2018 NVIDIA Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 *******************************************************************************/ 22 23 #include "uvm_api.h" 24 #include "uvm_tools.h" 25 #include "uvm_va_range.h" 26 #include "uvm_ats.h" 27 #include "uvm_ats_faults.h" 28 #include "uvm_migrate_pageable.h" 29 #include <linux/nodemask.h> 30 #include <linux/mempolicy.h> 31 #include <linux/mmu_notifier.h> 32 33 #if UVM_ATS_PREFETCH_SUPPORTED() 34 #include <linux/hmm.h> 35 #endif 36 37 static NV_STATUS service_ats_faults(uvm_gpu_va_space_t *gpu_va_space, 38 struct vm_area_struct *vma, 39 NvU64 start, 40 size_t length, 41 uvm_fault_access_type_t access_type, 42 uvm_ats_fault_context_t *ats_context) 43 { 44 uvm_va_space_t *va_space = gpu_va_space->va_space; 45 struct mm_struct *mm = va_space->va_space_mm.mm; 46 bool write = (access_type >= UVM_FAULT_ACCESS_TYPE_WRITE); 47 NV_STATUS status; 48 NvU64 user_space_start; 49 NvU64 user_space_length; 50 51 // Request uvm_migrate_pageable() to touch the corresponding page after 52 // population. 53 // Under virtualization ATS provides two translations: 54 // 1) guest virtual -> guest physical 55 // 2) guest physical -> host physical 56 // 57 // The overall ATS translation will fault if either of those translations is 58 // invalid. The pin_user_pages() call within uvm_migrate_pageable() call 59 // below handles translation #1, but not #2. We don't know if we're running 60 // as a guest, but in case we are we can force that translation to be valid 61 // by touching the guest physical address from the CPU. If the translation 62 // is not valid then the access will cause a hypervisor fault. Note that 63 // dma_map_page() can't establish mappings used by GPU ATS SVA translations. 64 // GPU accesses to host physical addresses obtained as a result of the 65 // address translation request uses the CPU address space instead of the 66 // IOMMU address space since the translated host physical address isn't 67 // necessarily an IOMMU address. The only way to establish guest physical to 68 // host physical mapping in the CPU address space is to touch the page from 69 // the CPU. 70 // 71 // We assume that the hypervisor mappings are all VM_PFNMAP, VM_SHARED, and 72 // VM_WRITE, meaning that the mappings are all granted write access on any 73 // fault and that the kernel will never revoke them. 74 // drivers/vfio/pci/vfio_pci_nvlink2.c enforces this. Thus we can assume 75 // that a read fault is always sufficient to also enable write access on the 76 // guest translation. 77 78 uvm_migrate_args_t uvm_migrate_args = 79 { 80 .va_space = va_space, 81 .mm = mm, 82 .dst_id = ats_context->residency_id, 83 .dst_node_id = ats_context->residency_node, 84 .start = start, 85 .length = length, 86 .populate_permissions = write ? UVM_POPULATE_PERMISSIONS_WRITE : UVM_POPULATE_PERMISSIONS_ANY, 87 .touch = true, 88 .skip_mapped = true, 89 .populate_on_cpu_alloc_failures = true, 90 .user_space_start = &user_space_start, 91 .user_space_length = &user_space_length, 92 }; 93 94 UVM_ASSERT(uvm_ats_can_service_faults(gpu_va_space, mm)); 95 96 // We are trying to use migrate_vma API in the kernel (if it exists) to 97 // populate and map the faulting region on the GPU. We want to do this only 98 // on the first touch. That is, pages which are not already mapped. So, we 99 // set skip_mapped to true. For pages already mapped, this will only handle 100 // PTE upgrades if needed. 101 status = uvm_migrate_pageable(&uvm_migrate_args); 102 if (status == NV_WARN_NOTHING_TO_DO) 103 status = NV_OK; 104 105 UVM_ASSERT(status != NV_ERR_MORE_PROCESSING_REQUIRED); 106 107 return status; 108 } 109 110 static void flush_tlb_write_faults(uvm_gpu_va_space_t *gpu_va_space, 111 NvU64 addr, 112 size_t size, 113 uvm_fault_client_type_t client_type) 114 { 115 uvm_ats_fault_invalidate_t *ats_invalidate; 116 117 if (client_type == UVM_FAULT_CLIENT_TYPE_GPC) 118 ats_invalidate = &gpu_va_space->gpu->parent->fault_buffer_info.replayable.ats_invalidate; 119 else 120 ats_invalidate = &gpu_va_space->gpu->parent->fault_buffer_info.non_replayable.ats_invalidate; 121 122 if (!ats_invalidate->write_faults_in_batch) { 123 uvm_tlb_batch_begin(&gpu_va_space->page_tables, &ats_invalidate->write_faults_tlb_batch); 124 ats_invalidate->write_faults_in_batch = true; 125 } 126 127 uvm_tlb_batch_invalidate(&ats_invalidate->write_faults_tlb_batch, addr, size, PAGE_SIZE, UVM_MEMBAR_NONE); 128 } 129 130 static void ats_batch_select_residency(uvm_gpu_va_space_t *gpu_va_space, 131 struct vm_area_struct *vma, 132 uvm_ats_fault_context_t *ats_context) 133 { 134 uvm_gpu_t *gpu = gpu_va_space->gpu; 135 int residency = uvm_gpu_numa_node(gpu); 136 137 #if defined(NV_MEMPOLICY_HAS_UNIFIED_NODES) 138 struct mempolicy *vma_policy = vma_policy(vma); 139 unsigned short mode; 140 141 ats_context->prefetch_state.has_preferred_location = false; 142 143 // It's safe to read vma_policy since the mmap_lock is held in at least read 144 // mode in this path. 145 uvm_assert_mmap_lock_locked(vma->vm_mm); 146 147 if (!vma_policy) 148 goto done; 149 150 mode = vma_policy->mode; 151 152 if ((mode == MPOL_BIND) || (mode == MPOL_PREFERRED_MANY) || (mode == MPOL_PREFERRED)) { 153 int home_node = NUMA_NO_NODE; 154 155 #if defined(NV_MEMPOLICY_HAS_HOME_NODE) 156 if ((mode != MPOL_PREFERRED) && (vma_policy->home_node != NUMA_NO_NODE)) 157 home_node = vma_policy->home_node; 158 #endif 159 160 // Prefer home_node if set. Otherwise, prefer the faulting GPU if it's 161 // in the list of preferred nodes, else prefer the closest_cpu_numa_node 162 // to the GPU if closest_cpu_numa_node is in the list of preferred 163 // nodes. Fallback to the faulting GPU if all else fails. 164 if (home_node != NUMA_NO_NODE) { 165 residency = home_node; 166 } 167 else if (!node_isset(residency, vma_policy->nodes)) { 168 int closest_cpu_numa_node = gpu->parent->closest_cpu_numa_node; 169 170 if ((closest_cpu_numa_node != NUMA_NO_NODE) && node_isset(closest_cpu_numa_node, vma_policy->nodes)) 171 residency = gpu->parent->closest_cpu_numa_node; 172 else 173 residency = first_node(vma_policy->nodes); 174 } 175 176 if (!nodes_empty(vma_policy->nodes)) 177 ats_context->prefetch_state.has_preferred_location = true; 178 } 179 180 // Update gpu if residency is not the faulting gpu. 181 if (residency != uvm_gpu_numa_node(gpu)) 182 gpu = uvm_va_space_find_gpu_with_memory_node_id(gpu_va_space->va_space, residency); 183 184 done: 185 #else 186 ats_context->prefetch_state.has_preferred_location = false; 187 #endif 188 189 ats_context->residency_id = gpu ? gpu->parent->id : UVM_ID_CPU; 190 ats_context->residency_node = residency; 191 } 192 193 static void get_range_in_vma(struct vm_area_struct *vma, NvU64 base, NvU64 *start, NvU64 *end) 194 { 195 *start = max(vma->vm_start, (unsigned long) base); 196 *end = min(vma->vm_end, (unsigned long) (base + UVM_VA_BLOCK_SIZE)); 197 } 198 199 static uvm_page_index_t uvm_ats_cpu_page_index(NvU64 base, NvU64 addr) 200 { 201 UVM_ASSERT(addr >= base); 202 UVM_ASSERT(addr <= (base + UVM_VA_BLOCK_SIZE)); 203 204 return (addr - base) / PAGE_SIZE; 205 } 206 207 // start and end must be aligned to PAGE_SIZE and must fall within 208 // [base, base + UVM_VA_BLOCK_SIZE] 209 static uvm_va_block_region_t uvm_ats_region_from_start_end(NvU64 start, NvU64 end) 210 { 211 // base can be greater than, less than or equal to the start of a VMA. 212 NvU64 base = UVM_VA_BLOCK_ALIGN_DOWN(start); 213 214 UVM_ASSERT(start < end); 215 UVM_ASSERT(PAGE_ALIGNED(start)); 216 UVM_ASSERT(PAGE_ALIGNED(end)); 217 UVM_ASSERT(IS_ALIGNED(base, UVM_VA_BLOCK_SIZE)); 218 219 return uvm_va_block_region(uvm_ats_cpu_page_index(base, start), uvm_ats_cpu_page_index(base, end)); 220 } 221 222 static uvm_va_block_region_t uvm_ats_region_from_vma(struct vm_area_struct *vma, NvU64 base) 223 { 224 NvU64 start; 225 NvU64 end; 226 227 get_range_in_vma(vma, base, &start, &end); 228 229 return uvm_ats_region_from_start_end(start, end); 230 } 231 232 #if UVM_ATS_PREFETCH_SUPPORTED() 233 234 static bool uvm_ats_invalidate_notifier(struct mmu_interval_notifier *mni, unsigned long cur_seq) 235 { 236 uvm_ats_fault_context_t *ats_context = container_of(mni, uvm_ats_fault_context_t, prefetch_state.notifier); 237 uvm_va_space_t *va_space = ats_context->prefetch_state.va_space; 238 239 // The following write lock protects against concurrent invalidates while 240 // hmm_range_fault() is being called in ats_compute_residency_mask(). 241 uvm_down_write(&va_space->ats.lock); 242 243 mmu_interval_set_seq(mni, cur_seq); 244 245 uvm_up_write(&va_space->ats.lock); 246 247 return true; 248 } 249 250 static bool uvm_ats_invalidate_notifier_entry(struct mmu_interval_notifier *mni, 251 const struct mmu_notifier_range *range, 252 unsigned long cur_seq) 253 { 254 UVM_ENTRY_RET(uvm_ats_invalidate_notifier(mni, cur_seq)); 255 } 256 257 static const struct mmu_interval_notifier_ops uvm_ats_notifier_ops = 258 { 259 .invalidate = uvm_ats_invalidate_notifier_entry, 260 }; 261 262 #endif 263 264 static NV_STATUS ats_compute_residency_mask(uvm_gpu_va_space_t *gpu_va_space, 265 struct vm_area_struct *vma, 266 NvU64 base, 267 uvm_ats_fault_context_t *ats_context) 268 { 269 NV_STATUS status = NV_OK; 270 271 #if UVM_ATS_PREFETCH_SUPPORTED() 272 int ret; 273 NvU64 start; 274 NvU64 end; 275 uvm_page_mask_t *residency_mask = &ats_context->prefetch_state.residency_mask; 276 struct hmm_range range; 277 uvm_page_index_t page_index; 278 uvm_va_block_region_t vma_region; 279 uvm_va_space_t *va_space = gpu_va_space->va_space; 280 struct mm_struct *mm = va_space->va_space_mm.mm; 281 282 uvm_assert_rwsem_locked_read(&va_space->lock); 283 284 ats_context->prefetch_state.first_touch = true; 285 286 uvm_page_mask_zero(residency_mask); 287 288 get_range_in_vma(vma, base, &start, &end); 289 290 vma_region = uvm_ats_region_from_start_end(start, end); 291 292 range.notifier = &ats_context->prefetch_state.notifier; 293 range.start = start; 294 range.end = end; 295 range.hmm_pfns = ats_context->prefetch_state.pfns; 296 range.default_flags = 0; 297 range.pfn_flags_mask = 0; 298 range.dev_private_owner = NULL; 299 300 ats_context->prefetch_state.va_space = va_space; 301 302 // mmu_interval_notifier_insert() will try to acquire mmap_lock for write 303 // and will deadlock since mmap_lock is already held for read in this path. 304 // This is prevented by calling __mmu_notifier_register() during va_space 305 // creation. See the comment in uvm_mmu_notifier_register() for more 306 // details. 307 ret = mmu_interval_notifier_insert(range.notifier, mm, start, end, &uvm_ats_notifier_ops); 308 if (ret) 309 return errno_to_nv_status(ret); 310 311 while (true) { 312 range.notifier_seq = mmu_interval_read_begin(range.notifier); 313 ret = hmm_range_fault(&range); 314 if (ret == -EBUSY) 315 continue; 316 if (ret) { 317 status = errno_to_nv_status(ret); 318 UVM_ASSERT(status != NV_OK); 319 break; 320 } 321 322 uvm_down_read(&va_space->ats.lock); 323 324 // Pages may have been freed or re-allocated after hmm_range_fault() is 325 // called. So the PTE might point to a different page or nothing. In the 326 // memory hot-unplug case it is not safe to call page_to_nid() on the 327 // page as the struct page itself may have been freed. To protect 328 // against these cases, uvm_ats_invalidate_entry() blocks on va_space 329 // ATS write lock for concurrent invalidates since va_space ATS lock is 330 // held for read in this path. 331 if (!mmu_interval_read_retry(range.notifier, range.notifier_seq)) 332 break; 333 334 uvm_up_read(&va_space->ats.lock); 335 } 336 337 if (status == NV_OK) { 338 for_each_va_block_page_in_region(page_index, vma_region) { 339 unsigned long pfn = ats_context->prefetch_state.pfns[page_index - vma_region.first]; 340 341 if (pfn & HMM_PFN_VALID) { 342 struct page *page = hmm_pfn_to_page(pfn); 343 344 if (page_to_nid(page) == ats_context->residency_node) 345 uvm_page_mask_set(residency_mask, page_index); 346 347 ats_context->prefetch_state.first_touch = false; 348 } 349 } 350 351 uvm_up_read(&va_space->ats.lock); 352 } 353 354 mmu_interval_notifier_remove(range.notifier); 355 356 #endif 357 358 return status; 359 } 360 361 static void ats_expand_fault_region(uvm_gpu_va_space_t *gpu_va_space, 362 struct vm_area_struct *vma, 363 uvm_ats_fault_context_t *ats_context, 364 uvm_va_block_region_t max_prefetch_region, 365 uvm_page_mask_t *faulted_mask) 366 { 367 uvm_page_mask_t *read_fault_mask = &ats_context->read_fault_mask; 368 uvm_page_mask_t *write_fault_mask = &ats_context->write_fault_mask; 369 uvm_page_mask_t *residency_mask = &ats_context->prefetch_state.residency_mask; 370 uvm_page_mask_t *prefetch_mask = &ats_context->prefetch_state.prefetch_pages_mask; 371 uvm_perf_prefetch_bitmap_tree_t *bitmap_tree = &ats_context->prefetch_state.bitmap_tree; 372 373 if (uvm_page_mask_empty(faulted_mask)) 374 return; 375 376 uvm_perf_prefetch_compute_ats(gpu_va_space->va_space, 377 faulted_mask, 378 uvm_va_block_region_from_mask(NULL, faulted_mask), 379 max_prefetch_region, 380 residency_mask, 381 bitmap_tree, 382 prefetch_mask); 383 384 uvm_page_mask_or(read_fault_mask, read_fault_mask, prefetch_mask); 385 386 if (vma->vm_flags & VM_WRITE) 387 uvm_page_mask_or(write_fault_mask, write_fault_mask, prefetch_mask); 388 } 389 390 static NV_STATUS ats_fault_prefetch(uvm_gpu_va_space_t *gpu_va_space, 391 struct vm_area_struct *vma, 392 NvU64 base, 393 uvm_ats_fault_context_t *ats_context) 394 { 395 NV_STATUS status = NV_OK; 396 uvm_page_mask_t *read_fault_mask = &ats_context->read_fault_mask; 397 uvm_page_mask_t *write_fault_mask = &ats_context->write_fault_mask; 398 uvm_page_mask_t *faulted_mask = &ats_context->faulted_mask; 399 uvm_page_mask_t *prefetch_mask = &ats_context->prefetch_state.prefetch_pages_mask; 400 uvm_va_block_region_t max_prefetch_region = uvm_ats_region_from_vma(vma, base); 401 402 if (!uvm_perf_prefetch_enabled(gpu_va_space->va_space)) 403 return status; 404 405 if (uvm_page_mask_empty(faulted_mask)) 406 return status; 407 408 status = ats_compute_residency_mask(gpu_va_space, vma, base, ats_context); 409 if (status != NV_OK) 410 return status; 411 412 // Prefetch the entire region if none of the pages are resident on any node 413 // and if preferred_location is the faulting GPU. 414 if (ats_context->prefetch_state.has_preferred_location && 415 ats_context->prefetch_state.first_touch && 416 uvm_id_equal(ats_context->residency_id, gpu_va_space->gpu->parent->id)) { 417 418 uvm_page_mask_init_from_region(prefetch_mask, max_prefetch_region, NULL); 419 uvm_page_mask_or(read_fault_mask, read_fault_mask, prefetch_mask); 420 421 if (vma->vm_flags & VM_WRITE) 422 uvm_page_mask_or(write_fault_mask, write_fault_mask, prefetch_mask); 423 424 return status; 425 } 426 427 ats_expand_fault_region(gpu_va_space, vma, ats_context, max_prefetch_region, faulted_mask); 428 429 return status; 430 } 431 432 NV_STATUS uvm_ats_service_faults(uvm_gpu_va_space_t *gpu_va_space, 433 struct vm_area_struct *vma, 434 NvU64 base, 435 uvm_ats_fault_context_t *ats_context) 436 { 437 NV_STATUS status = NV_OK; 438 uvm_va_block_region_t subregion; 439 uvm_va_block_region_t region = uvm_va_block_region(0, PAGES_PER_UVM_VA_BLOCK); 440 uvm_page_mask_t *read_fault_mask = &ats_context->read_fault_mask; 441 uvm_page_mask_t *write_fault_mask = &ats_context->write_fault_mask; 442 uvm_page_mask_t *faults_serviced_mask = &ats_context->faults_serviced_mask; 443 uvm_page_mask_t *reads_serviced_mask = &ats_context->reads_serviced_mask; 444 uvm_fault_client_type_t client_type = ats_context->client_type; 445 446 UVM_ASSERT(vma); 447 UVM_ASSERT(IS_ALIGNED(base, UVM_VA_BLOCK_SIZE)); 448 UVM_ASSERT(g_uvm_global.ats.enabled); 449 UVM_ASSERT(gpu_va_space); 450 UVM_ASSERT(gpu_va_space->ats.enabled); 451 UVM_ASSERT(uvm_gpu_va_space_state(gpu_va_space) == UVM_GPU_VA_SPACE_STATE_ACTIVE); 452 453 uvm_page_mask_zero(faults_serviced_mask); 454 uvm_page_mask_zero(reads_serviced_mask); 455 456 if (!(vma->vm_flags & VM_READ)) 457 return status; 458 459 if (!(vma->vm_flags & VM_WRITE)) { 460 // If VMA doesn't have write permissions, all write faults are fatal. 461 // Try servicing such faults for read iff they are also present in 462 // read_fault_mask. This is because for replayable faults, if there are 463 // pending read accesses on the same page, we have to service them 464 // before we can cancel the write/atomic faults. So we try with read 465 // fault access type even though these write faults are fatal. 466 if (ats_context->client_type == UVM_FAULT_CLIENT_TYPE_GPC) 467 uvm_page_mask_and(write_fault_mask, write_fault_mask, read_fault_mask); 468 else 469 uvm_page_mask_zero(write_fault_mask); 470 } 471 472 ats_batch_select_residency(gpu_va_space, vma, ats_context); 473 474 ats_fault_prefetch(gpu_va_space, vma, base, ats_context); 475 476 for_each_va_block_subregion_in_mask(subregion, write_fault_mask, region) { 477 NvU64 start = base + (subregion.first * PAGE_SIZE); 478 size_t length = uvm_va_block_region_num_pages(subregion) * PAGE_SIZE; 479 uvm_fault_access_type_t access_type = (vma->vm_flags & VM_WRITE) ? 480 UVM_FAULT_ACCESS_TYPE_WRITE : 481 UVM_FAULT_ACCESS_TYPE_READ; 482 483 UVM_ASSERT(start >= vma->vm_start); 484 UVM_ASSERT((start + length) <= vma->vm_end); 485 486 status = service_ats_faults(gpu_va_space, vma, start, length, access_type, ats_context); 487 if (status != NV_OK) 488 return status; 489 490 if (vma->vm_flags & VM_WRITE) { 491 uvm_page_mask_region_fill(faults_serviced_mask, subregion); 492 493 // The Linux kernel never invalidates TLB entries on mapping 494 // permission upgrade. This is a problem if the GPU has cached 495 // entries with the old permission. The GPU will re-fetch the entry 496 // if the PTE is invalid and page size is not 4K (this is the case 497 // on P9). However, if a page gets upgraded from R/O to R/W and GPU 498 // has the PTEs cached with R/O permissions we will enter an 499 // infinite loop because we just forward the fault to the Linux 500 // kernel and it will see that the permissions in the page table are 501 // correct. Therefore, we flush TLB entries on ATS write faults. 502 flush_tlb_write_faults(gpu_va_space, start, length, client_type); 503 } 504 else { 505 uvm_page_mask_region_fill(reads_serviced_mask, subregion); 506 } 507 } 508 509 // Remove write faults from read_fault_mask 510 uvm_page_mask_andnot(read_fault_mask, read_fault_mask, write_fault_mask); 511 512 for_each_va_block_subregion_in_mask(subregion, read_fault_mask, region) { 513 NvU64 start = base + (subregion.first * PAGE_SIZE); 514 size_t length = uvm_va_block_region_num_pages(subregion) * PAGE_SIZE; 515 uvm_fault_access_type_t access_type = UVM_FAULT_ACCESS_TYPE_READ; 516 517 UVM_ASSERT(start >= vma->vm_start); 518 UVM_ASSERT((start + length) <= vma->vm_end); 519 520 status = service_ats_faults(gpu_va_space, vma, start, length, access_type, ats_context); 521 if (status != NV_OK) 522 return status; 523 524 uvm_page_mask_region_fill(faults_serviced_mask, subregion); 525 } 526 527 return status; 528 } 529 530 bool uvm_ats_check_in_gmmu_region(uvm_va_space_t *va_space, NvU64 address, uvm_va_range_t *next) 531 { 532 uvm_va_range_t *prev; 533 NvU64 gmmu_region_base = UVM_ALIGN_DOWN(address, UVM_GMMU_ATS_GRANULARITY); 534 535 UVM_ASSERT(va_space); 536 537 if (next) { 538 if (next->node.start <= gmmu_region_base + UVM_GMMU_ATS_GRANULARITY - 1) 539 return true; 540 541 prev = uvm_va_range_container(uvm_range_tree_prev(&va_space->va_range_tree, &next->node)); 542 } 543 else { 544 // No VA range exists after address, so check the last VA range in the 545 // tree. 546 prev = uvm_va_range_container(uvm_range_tree_last(&va_space->va_range_tree)); 547 } 548 549 return prev && (prev->node.end >= gmmu_region_base); 550 } 551 552 NV_STATUS uvm_ats_invalidate_tlbs(uvm_gpu_va_space_t *gpu_va_space, 553 uvm_ats_fault_invalidate_t *ats_invalidate, 554 uvm_tracker_t *out_tracker) 555 { 556 NV_STATUS status; 557 uvm_push_t push; 558 559 if (!ats_invalidate->write_faults_in_batch) 560 return NV_OK; 561 562 UVM_ASSERT(gpu_va_space); 563 UVM_ASSERT(gpu_va_space->ats.enabled); 564 565 status = uvm_push_begin(gpu_va_space->gpu->channel_manager, 566 UVM_CHANNEL_TYPE_MEMOPS, 567 &push, 568 "Invalidate ATS entries"); 569 570 if (status == NV_OK) { 571 uvm_tlb_batch_end(&ats_invalidate->write_faults_tlb_batch, &push, UVM_MEMBAR_NONE); 572 uvm_push_end(&push); 573 574 // Add this push to the GPU's tracker so that fault replays/clears can 575 // wait on it 576 status = uvm_tracker_add_push_safe(out_tracker, &push); 577 } 578 579 ats_invalidate->write_faults_in_batch = false; 580 581 return status; 582 } 583 584