1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /************************************************************************** 3 * 4 * Copyright 2019 VMware, Inc., Palo Alto, CA., USA 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 #include "vmwgfx_drv.h" 28 29 /* 30 * Different methods for tracking dirty: 31 * VMW_BO_DIRTY_PAGETABLE - Scan the pagetable for hardware dirty bits 32 * VMW_BO_DIRTY_MKWRITE - Write-protect page table entries and record write- 33 * accesses in the VM mkwrite() callback 34 */ 35 enum vmw_bo_dirty_method { 36 VMW_BO_DIRTY_PAGETABLE, 37 VMW_BO_DIRTY_MKWRITE, 38 }; 39 40 /* 41 * No dirtied pages at scan trigger a transition to the _MKWRITE method, 42 * similarly a certain percentage of dirty pages trigger a transition to 43 * the _PAGETABLE method. How many triggers should we wait for before 44 * changing method? 45 */ 46 #define VMW_DIRTY_NUM_CHANGE_TRIGGERS 2 47 48 /* Percentage to trigger a transition to the _PAGETABLE method */ 49 #define VMW_DIRTY_PERCENTAGE 10 50 51 /** 52 * struct vmw_bo_dirty - Dirty information for buffer objects 53 * @start: First currently dirty bit 54 * @end: Last currently dirty bit + 1 55 * @method: The currently used dirty method 56 * @change_count: Number of consecutive method change triggers 57 * @ref_count: Reference count for this structure 58 * @bitmap_size: The size of the bitmap in bits. Typically equal to the 59 * nuber of pages in the bo. 60 * @size: The accounting size for this struct. 61 * @bitmap: A bitmap where each bit represents a page. A set bit means a 62 * dirty page. 63 */ 64 struct vmw_bo_dirty { 65 unsigned long start; 66 unsigned long end; 67 enum vmw_bo_dirty_method method; 68 unsigned int change_count; 69 unsigned int ref_count; 70 unsigned long bitmap_size; 71 size_t size; 72 unsigned long bitmap[0]; 73 }; 74 75 /** 76 * vmw_bo_dirty_scan_pagetable - Perform a pagetable scan for dirty bits 77 * @vbo: The buffer object to scan 78 * 79 * Scans the pagetable for dirty bits. Clear those bits and modify the 80 * dirty structure with the results. This function may change the 81 * dirty-tracking method. 82 */ 83 static void vmw_bo_dirty_scan_pagetable(struct vmw_buffer_object *vbo) 84 { 85 struct vmw_bo_dirty *dirty = vbo->dirty; 86 pgoff_t offset = drm_vma_node_start(&vbo->base.base.vma_node); 87 struct address_space *mapping = vbo->base.bdev->dev_mapping; 88 pgoff_t num_marked; 89 90 num_marked = clean_record_shared_mapping_range 91 (mapping, 92 offset, dirty->bitmap_size, 93 offset, &dirty->bitmap[0], 94 &dirty->start, &dirty->end); 95 if (num_marked == 0) 96 dirty->change_count++; 97 else 98 dirty->change_count = 0; 99 100 if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) { 101 dirty->change_count = 0; 102 dirty->method = VMW_BO_DIRTY_MKWRITE; 103 wp_shared_mapping_range(mapping, 104 offset, dirty->bitmap_size); 105 clean_record_shared_mapping_range(mapping, 106 offset, dirty->bitmap_size, 107 offset, &dirty->bitmap[0], 108 &dirty->start, &dirty->end); 109 } 110 } 111 112 /** 113 * vmw_bo_dirty_scan_mkwrite - Reset the mkwrite dirty-tracking method 114 * @vbo: The buffer object to scan 115 * 116 * Write-protect pages written to so that consecutive write accesses will 117 * trigger a call to mkwrite. 118 * 119 * This function may change the dirty-tracking method. 120 */ 121 static void vmw_bo_dirty_scan_mkwrite(struct vmw_buffer_object *vbo) 122 { 123 struct vmw_bo_dirty *dirty = vbo->dirty; 124 unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node); 125 struct address_space *mapping = vbo->base.bdev->dev_mapping; 126 pgoff_t num_marked; 127 128 if (dirty->end <= dirty->start) 129 return; 130 131 num_marked = wp_shared_mapping_range(vbo->base.bdev->dev_mapping, 132 dirty->start + offset, 133 dirty->end - dirty->start); 134 135 if (100UL * num_marked / dirty->bitmap_size > 136 VMW_DIRTY_PERCENTAGE) { 137 dirty->change_count++; 138 } else { 139 dirty->change_count = 0; 140 } 141 142 if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) { 143 pgoff_t start = 0; 144 pgoff_t end = dirty->bitmap_size; 145 146 dirty->method = VMW_BO_DIRTY_PAGETABLE; 147 clean_record_shared_mapping_range(mapping, offset, end, offset, 148 &dirty->bitmap[0], 149 &start, &end); 150 bitmap_clear(&dirty->bitmap[0], 0, dirty->bitmap_size); 151 if (dirty->start < dirty->end) 152 bitmap_set(&dirty->bitmap[0], dirty->start, 153 dirty->end - dirty->start); 154 dirty->change_count = 0; 155 } 156 } 157 158 /** 159 * vmw_bo_dirty_scan - Scan for dirty pages and add them to the dirty 160 * tracking structure 161 * @vbo: The buffer object to scan 162 * 163 * This function may change the dirty tracking method. 164 */ 165 void vmw_bo_dirty_scan(struct vmw_buffer_object *vbo) 166 { 167 struct vmw_bo_dirty *dirty = vbo->dirty; 168 169 if (dirty->method == VMW_BO_DIRTY_PAGETABLE) 170 vmw_bo_dirty_scan_pagetable(vbo); 171 else 172 vmw_bo_dirty_scan_mkwrite(vbo); 173 } 174 175 /** 176 * vmw_bo_dirty_pre_unmap - write-protect and pick up dirty pages before 177 * an unmap_mapping_range operation. 178 * @vbo: The buffer object, 179 * @start: First page of the range within the buffer object. 180 * @end: Last page of the range within the buffer object + 1. 181 * 182 * If we're using the _PAGETABLE scan method, we may leak dirty pages 183 * when calling unmap_mapping_range(). This function makes sure we pick 184 * up all dirty pages. 185 */ 186 static void vmw_bo_dirty_pre_unmap(struct vmw_buffer_object *vbo, 187 pgoff_t start, pgoff_t end) 188 { 189 struct vmw_bo_dirty *dirty = vbo->dirty; 190 unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node); 191 struct address_space *mapping = vbo->base.bdev->dev_mapping; 192 193 if (dirty->method != VMW_BO_DIRTY_PAGETABLE || start >= end) 194 return; 195 196 wp_shared_mapping_range(mapping, start + offset, end - start); 197 clean_record_shared_mapping_range(mapping, start + offset, 198 end - start, offset, 199 &dirty->bitmap[0], &dirty->start, 200 &dirty->end); 201 } 202 203 /** 204 * vmw_bo_dirty_unmap - Clear all ptes pointing to a range within a bo 205 * @vbo: The buffer object, 206 * @start: First page of the range within the buffer object. 207 * @end: Last page of the range within the buffer object + 1. 208 * 209 * This is similar to ttm_bo_unmap_virtual_locked() except it takes a subrange. 210 */ 211 void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo, 212 pgoff_t start, pgoff_t end) 213 { 214 unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node); 215 struct address_space *mapping = vbo->base.bdev->dev_mapping; 216 217 vmw_bo_dirty_pre_unmap(vbo, start, end); 218 unmap_shared_mapping_range(mapping, (offset + start) << PAGE_SHIFT, 219 (loff_t) (end - start) << PAGE_SHIFT); 220 } 221 222 /** 223 * vmw_bo_dirty_add - Add a dirty-tracking user to a buffer object 224 * @vbo: The buffer object 225 * 226 * This function registers a dirty-tracking user to a buffer object. 227 * A user can be for example a resource or a vma in a special user-space 228 * mapping. 229 * 230 * Return: Zero on success, -ENOMEM on memory allocation failure. 231 */ 232 int vmw_bo_dirty_add(struct vmw_buffer_object *vbo) 233 { 234 struct vmw_bo_dirty *dirty = vbo->dirty; 235 pgoff_t num_pages = vbo->base.num_pages; 236 size_t size, acc_size; 237 int ret; 238 static struct ttm_operation_ctx ctx = { 239 .interruptible = false, 240 .no_wait_gpu = false 241 }; 242 243 if (dirty) { 244 dirty->ref_count++; 245 return 0; 246 } 247 248 size = sizeof(*dirty) + BITS_TO_LONGS(num_pages) * sizeof(long); 249 acc_size = ttm_round_pot(size); 250 ret = ttm_mem_global_alloc(&ttm_mem_glob, acc_size, &ctx); 251 if (ret) { 252 VMW_DEBUG_USER("Out of graphics memory for buffer object " 253 "dirty tracker.\n"); 254 return ret; 255 } 256 dirty = kvzalloc(size, GFP_KERNEL); 257 if (!dirty) { 258 ret = -ENOMEM; 259 goto out_no_dirty; 260 } 261 262 dirty->size = acc_size; 263 dirty->bitmap_size = num_pages; 264 dirty->start = dirty->bitmap_size; 265 dirty->end = 0; 266 dirty->ref_count = 1; 267 if (num_pages < PAGE_SIZE / sizeof(pte_t)) { 268 dirty->method = VMW_BO_DIRTY_PAGETABLE; 269 } else { 270 struct address_space *mapping = vbo->base.bdev->dev_mapping; 271 pgoff_t offset = drm_vma_node_start(&vbo->base.base.vma_node); 272 273 dirty->method = VMW_BO_DIRTY_MKWRITE; 274 275 /* Write-protect and then pick up already dirty bits */ 276 wp_shared_mapping_range(mapping, offset, num_pages); 277 clean_record_shared_mapping_range(mapping, offset, num_pages, 278 offset, 279 &dirty->bitmap[0], 280 &dirty->start, &dirty->end); 281 } 282 283 vbo->dirty = dirty; 284 285 return 0; 286 287 out_no_dirty: 288 ttm_mem_global_free(&ttm_mem_glob, acc_size); 289 return ret; 290 } 291 292 /** 293 * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object 294 * @vbo: The buffer object 295 * 296 * This function releases a dirty-tracking user from a buffer object. 297 * If the reference count reaches zero, then the dirty-tracking object is 298 * freed and the pointer to it cleared. 299 * 300 * Return: Zero on success, -ENOMEM on memory allocation failure. 301 */ 302 void vmw_bo_dirty_release(struct vmw_buffer_object *vbo) 303 { 304 struct vmw_bo_dirty *dirty = vbo->dirty; 305 306 if (dirty && --dirty->ref_count == 0) { 307 size_t acc_size = dirty->size; 308 309 kvfree(dirty); 310 ttm_mem_global_free(&ttm_mem_glob, acc_size); 311 vbo->dirty = NULL; 312 } 313 } 314 315 /** 316 * vmw_bo_dirty_transfer_to_res - Pick up a resource's dirty region from 317 * its backing mob. 318 * @res: The resource 319 * 320 * This function will pick up all dirty ranges affecting the resource from 321 * it's backup mob, and call vmw_resource_dirty_update() once for each 322 * range. The transferred ranges will be cleared from the backing mob's 323 * dirty tracking. 324 */ 325 void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res) 326 { 327 struct vmw_buffer_object *vbo = res->backup; 328 struct vmw_bo_dirty *dirty = vbo->dirty; 329 pgoff_t start, cur, end; 330 unsigned long res_start = res->backup_offset; 331 unsigned long res_end = res->backup_offset + res->backup_size; 332 333 WARN_ON_ONCE(res_start & ~PAGE_MASK); 334 res_start >>= PAGE_SHIFT; 335 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE); 336 337 if (res_start >= dirty->end || res_end <= dirty->start) 338 return; 339 340 cur = max(res_start, dirty->start); 341 res_end = max(res_end, dirty->end); 342 while (cur < res_end) { 343 unsigned long num; 344 345 start = find_next_bit(&dirty->bitmap[0], res_end, cur); 346 if (start >= res_end) 347 break; 348 349 end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1); 350 cur = end + 1; 351 num = end - start; 352 bitmap_clear(&dirty->bitmap[0], start, num); 353 vmw_resource_dirty_update(res, start, end); 354 } 355 356 if (res_start <= dirty->start && res_end > dirty->start) 357 dirty->start = res_end; 358 if (res_start < dirty->end && res_end >= dirty->end) 359 dirty->end = res_start; 360 } 361 362 /** 363 * vmw_bo_dirty_clear_res - Clear a resource's dirty region from 364 * its backing mob. 365 * @res: The resource 366 * 367 * This function will clear all dirty ranges affecting the resource from 368 * it's backup mob's dirty tracking. 369 */ 370 void vmw_bo_dirty_clear_res(struct vmw_resource *res) 371 { 372 unsigned long res_start = res->backup_offset; 373 unsigned long res_end = res->backup_offset + res->backup_size; 374 struct vmw_buffer_object *vbo = res->backup; 375 struct vmw_bo_dirty *dirty = vbo->dirty; 376 377 res_start >>= PAGE_SHIFT; 378 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE); 379 380 if (res_start >= dirty->end || res_end <= dirty->start) 381 return; 382 383 res_start = max(res_start, dirty->start); 384 res_end = min(res_end, dirty->end); 385 bitmap_clear(&dirty->bitmap[0], res_start, res_end - res_start); 386 387 if (res_start <= dirty->start && res_end > dirty->start) 388 dirty->start = res_end; 389 if (res_start < dirty->end && res_end >= dirty->end) 390 dirty->end = res_start; 391 } 392 393 vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf) 394 { 395 struct vm_area_struct *vma = vmf->vma; 396 struct ttm_buffer_object *bo = (struct ttm_buffer_object *) 397 vma->vm_private_data; 398 vm_fault_t ret; 399 unsigned long page_offset; 400 unsigned int save_flags; 401 struct vmw_buffer_object *vbo = 402 container_of(bo, typeof(*vbo), base); 403 404 /* 405 * mkwrite() doesn't handle the VM_FAULT_RETRY return value correctly. 406 * So make sure the TTM helpers are aware. 407 */ 408 save_flags = vmf->flags; 409 vmf->flags &= ~FAULT_FLAG_ALLOW_RETRY; 410 ret = ttm_bo_vm_reserve(bo, vmf); 411 vmf->flags = save_flags; 412 if (ret) 413 return ret; 414 415 page_offset = vmf->pgoff - drm_vma_node_start(&bo->base.vma_node); 416 if (unlikely(page_offset >= bo->num_pages)) { 417 ret = VM_FAULT_SIGBUS; 418 goto out_unlock; 419 } 420 421 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE && 422 !test_bit(page_offset, &vbo->dirty->bitmap[0])) { 423 struct vmw_bo_dirty *dirty = vbo->dirty; 424 425 __set_bit(page_offset, &dirty->bitmap[0]); 426 dirty->start = min(dirty->start, page_offset); 427 dirty->end = max(dirty->end, page_offset + 1); 428 } 429 430 out_unlock: 431 dma_resv_unlock(bo->base.resv); 432 return ret; 433 } 434 435 vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf) 436 { 437 struct vm_area_struct *vma = vmf->vma; 438 struct ttm_buffer_object *bo = (struct ttm_buffer_object *) 439 vma->vm_private_data; 440 struct vmw_buffer_object *vbo = 441 container_of(bo, struct vmw_buffer_object, base); 442 pgoff_t num_prefault; 443 pgprot_t prot; 444 vm_fault_t ret; 445 446 ret = ttm_bo_vm_reserve(bo, vmf); 447 if (ret) 448 return ret; 449 450 num_prefault = (vma->vm_flags & VM_RAND_READ) ? 1 : 451 TTM_BO_VM_NUM_PREFAULT; 452 453 if (vbo->dirty) { 454 pgoff_t allowed_prefault; 455 unsigned long page_offset; 456 457 page_offset = vmf->pgoff - 458 drm_vma_node_start(&bo->base.vma_node); 459 if (page_offset >= bo->num_pages || 460 vmw_resources_clean(vbo, page_offset, 461 page_offset + PAGE_SIZE, 462 &allowed_prefault)) { 463 ret = VM_FAULT_SIGBUS; 464 goto out_unlock; 465 } 466 467 num_prefault = min(num_prefault, allowed_prefault); 468 } 469 470 /* 471 * If we don't track dirty using the MKWRITE method, make sure 472 * sure the page protection is write-enabled so we don't get 473 * a lot of unnecessary write faults. 474 */ 475 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE) 476 prot = vma->vm_page_prot; 477 else 478 prot = vm_get_page_prot(vma->vm_flags); 479 480 ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault); 481 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) 482 return ret; 483 484 out_unlock: 485 dma_resv_unlock(bo->base.resv); 486 487 return ret; 488 } 489