1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/list_sort.h> 26 #include <linux/prime_numbers.h> 27 28 #include "gem/i915_gem_context.h" 29 #include "gem/i915_gem_internal.h" 30 #include "gem/i915_gem_region.h" 31 #include "gem/selftests/mock_context.h" 32 #include "gt/intel_context.h" 33 #include "gt/intel_gpu_commands.h" 34 #include "gt/intel_gtt.h" 35 36 #include "i915_random.h" 37 #include "i915_selftest.h" 38 #include "i915_vma_resource.h" 39 40 #include "mock_drm.h" 41 #include "mock_gem_device.h" 42 #include "mock_gtt.h" 43 #include "igt_flush_test.h" 44 45 static void cleanup_freed_objects(struct drm_i915_private *i915) 46 { 47 i915_gem_drain_freed_objects(i915); 48 } 49 50 static void fake_free_pages(struct drm_i915_gem_object *obj, 51 struct sg_table *pages) 52 { 53 sg_free_table(pages); 54 kfree(pages); 55 } 56 57 static int fake_get_pages(struct drm_i915_gem_object *obj) 58 { 59 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 60 #define PFN_BIAS 0x1000 61 struct sg_table *pages; 62 struct scatterlist *sg; 63 unsigned int sg_page_sizes; 64 typeof(obj->base.size) rem; 65 66 pages = kmalloc(sizeof(*pages), GFP); 67 if (!pages) 68 return -ENOMEM; 69 70 rem = round_up(obj->base.size, BIT(31)) >> 31; 71 if (sg_alloc_table(pages, rem, GFP)) { 72 kfree(pages); 73 return -ENOMEM; 74 } 75 76 sg_page_sizes = 0; 77 rem = obj->base.size; 78 for (sg = pages->sgl; sg; sg = sg_next(sg)) { 79 unsigned long len = min_t(typeof(rem), rem, BIT(31)); 80 81 GEM_BUG_ON(!len); 82 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0); 83 sg_dma_address(sg) = page_to_phys(sg_page(sg)); 84 sg_dma_len(sg) = len; 85 sg_page_sizes |= len; 86 87 rem -= len; 88 } 89 GEM_BUG_ON(rem); 90 91 __i915_gem_object_set_pages(obj, pages, sg_page_sizes); 92 93 return 0; 94 #undef GFP 95 } 96 97 static void fake_put_pages(struct drm_i915_gem_object *obj, 98 struct sg_table *pages) 99 { 100 fake_free_pages(obj, pages); 101 obj->mm.dirty = false; 102 } 103 104 static const struct drm_i915_gem_object_ops fake_ops = { 105 .name = "fake-gem", 106 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 107 .get_pages = fake_get_pages, 108 .put_pages = fake_put_pages, 109 }; 110 111 static struct drm_i915_gem_object * 112 fake_dma_object(struct drm_i915_private *i915, u64 size) 113 { 114 static struct lock_class_key lock_class; 115 struct drm_i915_gem_object *obj; 116 117 GEM_BUG_ON(!size); 118 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 119 120 if (overflows_type(size, obj->base.size)) 121 return ERR_PTR(-E2BIG); 122 123 obj = i915_gem_object_alloc(); 124 if (!obj) 125 goto err; 126 127 drm_gem_private_object_init(&i915->drm, &obj->base, size); 128 i915_gem_object_init(obj, &fake_ops, &lock_class, 0); 129 130 i915_gem_object_set_volatile(obj); 131 132 obj->write_domain = I915_GEM_DOMAIN_CPU; 133 obj->read_domains = I915_GEM_DOMAIN_CPU; 134 obj->cache_level = I915_CACHE_NONE; 135 136 /* Preallocate the "backing storage" */ 137 if (i915_gem_object_pin_pages_unlocked(obj)) 138 goto err_obj; 139 140 i915_gem_object_unpin_pages(obj); 141 return obj; 142 143 err_obj: 144 i915_gem_object_put(obj); 145 err: 146 return ERR_PTR(-ENOMEM); 147 } 148 149 static int igt_ppgtt_alloc(void *arg) 150 { 151 struct drm_i915_private *dev_priv = arg; 152 struct i915_ppgtt *ppgtt; 153 struct i915_gem_ww_ctx ww; 154 u64 size, last, limit; 155 int err = 0; 156 157 /* Allocate a ppggt and try to fill the entire range */ 158 159 if (!HAS_PPGTT(dev_priv)) 160 return 0; 161 162 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0); 163 if (IS_ERR(ppgtt)) 164 return PTR_ERR(ppgtt); 165 166 if (!ppgtt->vm.allocate_va_range) 167 goto err_ppgtt_cleanup; 168 169 /* 170 * While we only allocate the page tables here and so we could 171 * address a much larger GTT than we could actually fit into 172 * RAM, a practical limit is the amount of physical pages in the system. 173 * This should ensure that we do not run into the oomkiller during 174 * the test and take down the machine wilfully. 175 */ 176 limit = totalram_pages() << PAGE_SHIFT; 177 limit = min(ppgtt->vm.total, limit); 178 179 i915_gem_ww_ctx_init(&ww, false); 180 retry: 181 err = i915_vm_lock_objects(&ppgtt->vm, &ww); 182 if (err) 183 goto err_ppgtt_cleanup; 184 185 /* Check we can allocate the entire range */ 186 for (size = 4096; size <= limit; size <<= 2) { 187 struct i915_vm_pt_stash stash = {}; 188 189 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size); 190 if (err) 191 goto err_ppgtt_cleanup; 192 193 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 194 if (err) { 195 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 196 goto err_ppgtt_cleanup; 197 } 198 199 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size); 200 cond_resched(); 201 202 ppgtt->vm.clear_range(&ppgtt->vm, 0, size); 203 204 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 205 } 206 207 /* Check we can incrementally allocate the entire range */ 208 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) { 209 struct i915_vm_pt_stash stash = {}; 210 211 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last); 212 if (err) 213 goto err_ppgtt_cleanup; 214 215 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 216 if (err) { 217 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 218 goto err_ppgtt_cleanup; 219 } 220 221 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 222 last, size - last); 223 cond_resched(); 224 225 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 226 } 227 228 err_ppgtt_cleanup: 229 if (err == -EDEADLK) { 230 err = i915_gem_ww_ctx_backoff(&ww); 231 if (!err) 232 goto retry; 233 } 234 i915_gem_ww_ctx_fini(&ww); 235 236 i915_vm_put(&ppgtt->vm); 237 return err; 238 } 239 240 static int lowlevel_hole(struct i915_address_space *vm, 241 u64 hole_start, u64 hole_end, 242 unsigned long end_time) 243 { 244 const unsigned int min_alignment = 245 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 246 I915_RND_STATE(seed_prng); 247 struct i915_vma_resource *mock_vma_res; 248 unsigned int size; 249 250 mock_vma_res = kzalloc(sizeof(*mock_vma_res), GFP_KERNEL); 251 if (!mock_vma_res) 252 return -ENOMEM; 253 254 /* Keep creating larger objects until one cannot fit into the hole */ 255 for (size = 12; (hole_end - hole_start) >> size; size++) { 256 I915_RND_SUBSTATE(prng, seed_prng); 257 struct drm_i915_gem_object *obj; 258 unsigned int *order, count, n; 259 u64 hole_size, aligned_size; 260 261 aligned_size = max_t(u32, ilog2(min_alignment), size); 262 hole_size = (hole_end - hole_start) >> aligned_size; 263 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 264 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 265 count = hole_size >> 1; 266 if (!count) { 267 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 268 __func__, hole_start, hole_end, size, hole_size); 269 break; 270 } 271 272 do { 273 order = i915_random_order(count, &prng); 274 if (order) 275 break; 276 } while (count >>= 1); 277 if (!count) { 278 kfree(mock_vma_res); 279 return -ENOMEM; 280 } 281 GEM_BUG_ON(!order); 282 283 GEM_BUG_ON(count * BIT_ULL(aligned_size) > vm->total); 284 GEM_BUG_ON(hole_start + count * BIT_ULL(aligned_size) > hole_end); 285 286 /* Ignore allocation failures (i.e. don't report them as 287 * a test failure) as we are purposefully allocating very 288 * large objects without checking that we have sufficient 289 * memory. We expect to hit -ENOMEM. 290 */ 291 292 obj = fake_dma_object(vm->i915, BIT_ULL(size)); 293 if (IS_ERR(obj)) { 294 kfree(order); 295 break; 296 } 297 298 GEM_BUG_ON(obj->base.size != BIT_ULL(size)); 299 300 if (i915_gem_object_pin_pages_unlocked(obj)) { 301 i915_gem_object_put(obj); 302 kfree(order); 303 break; 304 } 305 306 for (n = 0; n < count; n++) { 307 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 308 intel_wakeref_t wakeref; 309 310 GEM_BUG_ON(addr + BIT_ULL(aligned_size) > vm->total); 311 312 if (igt_timeout(end_time, 313 "%s timed out before %d/%d\n", 314 __func__, n, count)) { 315 hole_end = hole_start; /* quit */ 316 break; 317 } 318 319 if (vm->allocate_va_range) { 320 struct i915_vm_pt_stash stash = {}; 321 struct i915_gem_ww_ctx ww; 322 int err; 323 324 i915_gem_ww_ctx_init(&ww, false); 325 retry: 326 err = i915_vm_lock_objects(vm, &ww); 327 if (err) 328 goto alloc_vm_end; 329 330 err = -ENOMEM; 331 if (i915_vm_alloc_pt_stash(vm, &stash, 332 BIT_ULL(size))) 333 goto alloc_vm_end; 334 335 err = i915_vm_map_pt_stash(vm, &stash); 336 if (!err) 337 vm->allocate_va_range(vm, &stash, 338 addr, BIT_ULL(size)); 339 i915_vm_free_pt_stash(vm, &stash); 340 alloc_vm_end: 341 if (err == -EDEADLK) { 342 err = i915_gem_ww_ctx_backoff(&ww); 343 if (!err) 344 goto retry; 345 } 346 i915_gem_ww_ctx_fini(&ww); 347 348 if (err) 349 break; 350 } 351 352 mock_vma_res->bi.pages = obj->mm.pages; 353 mock_vma_res->node_size = BIT_ULL(aligned_size); 354 mock_vma_res->start = addr; 355 356 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) 357 vm->insert_entries(vm, mock_vma_res, 358 I915_CACHE_NONE, 0); 359 } 360 count = n; 361 362 i915_random_reorder(order, count, &prng); 363 for (n = 0; n < count; n++) { 364 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 365 intel_wakeref_t wakeref; 366 367 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 368 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) 369 vm->clear_range(vm, addr, BIT_ULL(size)); 370 } 371 372 i915_gem_object_unpin_pages(obj); 373 i915_gem_object_put(obj); 374 375 kfree(order); 376 377 cleanup_freed_objects(vm->i915); 378 } 379 380 kfree(mock_vma_res); 381 return 0; 382 } 383 384 static void close_object_list(struct list_head *objects, 385 struct i915_address_space *vm) 386 { 387 struct drm_i915_gem_object *obj, *on; 388 int ignored; 389 390 list_for_each_entry_safe(obj, on, objects, st_link) { 391 struct i915_vma *vma; 392 393 vma = i915_vma_instance(obj, vm, NULL); 394 if (!IS_ERR(vma)) 395 ignored = i915_vma_unbind_unlocked(vma); 396 397 list_del(&obj->st_link); 398 i915_gem_object_put(obj); 399 } 400 } 401 402 static int fill_hole(struct i915_address_space *vm, 403 u64 hole_start, u64 hole_end, 404 unsigned long end_time) 405 { 406 const u64 hole_size = hole_end - hole_start; 407 struct drm_i915_gem_object *obj; 408 const unsigned int min_alignment = 409 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 410 const unsigned long max_pages = 411 min_t(u64, ULONG_MAX - 1, (hole_size / 2) >> ilog2(min_alignment)); 412 const unsigned long max_step = max(int_sqrt(max_pages), 2UL); 413 unsigned long npages, prime, flags; 414 struct i915_vma *vma; 415 LIST_HEAD(objects); 416 int err; 417 418 /* Try binding many VMA working inwards from either edge */ 419 420 flags = PIN_OFFSET_FIXED | PIN_USER; 421 if (i915_is_ggtt(vm)) 422 flags |= PIN_GLOBAL; 423 424 for_each_prime_number_from(prime, 2, max_step) { 425 for (npages = 1; npages <= max_pages; npages *= prime) { 426 const u64 full_size = npages << PAGE_SHIFT; 427 const struct { 428 const char *name; 429 u64 offset; 430 int step; 431 } phases[] = { 432 { "top-down", hole_end, -1, }, 433 { "bottom-up", hole_start, 1, }, 434 { } 435 }, *p; 436 437 obj = fake_dma_object(vm->i915, full_size); 438 if (IS_ERR(obj)) 439 break; 440 441 list_add(&obj->st_link, &objects); 442 443 /* Align differing sized objects against the edges, and 444 * check we don't walk off into the void when binding 445 * them into the GTT. 446 */ 447 for (p = phases; p->name; p++) { 448 u64 offset; 449 450 offset = p->offset; 451 list_for_each_entry(obj, &objects, st_link) { 452 u64 aligned_size = round_up(obj->base.size, 453 min_alignment); 454 455 vma = i915_vma_instance(obj, vm, NULL); 456 if (IS_ERR(vma)) 457 continue; 458 459 if (p->step < 0) { 460 if (offset < hole_start + aligned_size) 461 break; 462 offset -= aligned_size; 463 } 464 465 err = i915_vma_pin(vma, 0, 0, offset | flags); 466 if (err) { 467 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 468 __func__, p->name, err, npages, prime, offset); 469 goto err; 470 } 471 472 if (!drm_mm_node_allocated(&vma->node) || 473 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 474 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 475 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 476 offset); 477 err = -EINVAL; 478 goto err; 479 } 480 481 i915_vma_unpin(vma); 482 483 if (p->step > 0) { 484 if (offset + aligned_size > hole_end) 485 break; 486 offset += aligned_size; 487 } 488 } 489 490 offset = p->offset; 491 list_for_each_entry(obj, &objects, st_link) { 492 u64 aligned_size = round_up(obj->base.size, 493 min_alignment); 494 495 vma = i915_vma_instance(obj, vm, NULL); 496 if (IS_ERR(vma)) 497 continue; 498 499 if (p->step < 0) { 500 if (offset < hole_start + aligned_size) 501 break; 502 offset -= aligned_size; 503 } 504 505 if (!drm_mm_node_allocated(&vma->node) || 506 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 507 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n", 508 __func__, p->name, vma->node.start, vma->node.size, 509 offset); 510 err = -EINVAL; 511 goto err; 512 } 513 514 err = i915_vma_unbind_unlocked(vma); 515 if (err) { 516 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n", 517 __func__, p->name, vma->node.start, vma->node.size, 518 err); 519 goto err; 520 } 521 522 if (p->step > 0) { 523 if (offset + aligned_size > hole_end) 524 break; 525 offset += aligned_size; 526 } 527 } 528 529 offset = p->offset; 530 list_for_each_entry_reverse(obj, &objects, st_link) { 531 u64 aligned_size = round_up(obj->base.size, 532 min_alignment); 533 534 vma = i915_vma_instance(obj, vm, NULL); 535 if (IS_ERR(vma)) 536 continue; 537 538 if (p->step < 0) { 539 if (offset < hole_start + aligned_size) 540 break; 541 offset -= aligned_size; 542 } 543 544 err = i915_vma_pin(vma, 0, 0, offset | flags); 545 if (err) { 546 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 547 __func__, p->name, err, npages, prime, offset); 548 goto err; 549 } 550 551 if (!drm_mm_node_allocated(&vma->node) || 552 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 553 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 554 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 555 offset); 556 err = -EINVAL; 557 goto err; 558 } 559 560 i915_vma_unpin(vma); 561 562 if (p->step > 0) { 563 if (offset + aligned_size > hole_end) 564 break; 565 offset += aligned_size; 566 } 567 } 568 569 offset = p->offset; 570 list_for_each_entry_reverse(obj, &objects, st_link) { 571 u64 aligned_size = round_up(obj->base.size, 572 min_alignment); 573 574 vma = i915_vma_instance(obj, vm, NULL); 575 if (IS_ERR(vma)) 576 continue; 577 578 if (p->step < 0) { 579 if (offset < hole_start + aligned_size) 580 break; 581 offset -= aligned_size; 582 } 583 584 if (!drm_mm_node_allocated(&vma->node) || 585 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 586 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 587 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 588 offset); 589 err = -EINVAL; 590 goto err; 591 } 592 593 err = i915_vma_unbind_unlocked(vma); 594 if (err) { 595 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n", 596 __func__, p->name, vma->node.start, vma->node.size, 597 err); 598 goto err; 599 } 600 601 if (p->step > 0) { 602 if (offset + aligned_size > hole_end) 603 break; 604 offset += aligned_size; 605 } 606 } 607 } 608 609 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n", 610 __func__, npages, prime)) { 611 err = -EINTR; 612 goto err; 613 } 614 } 615 616 close_object_list(&objects, vm); 617 cleanup_freed_objects(vm->i915); 618 } 619 620 return 0; 621 622 err: 623 close_object_list(&objects, vm); 624 return err; 625 } 626 627 static int walk_hole(struct i915_address_space *vm, 628 u64 hole_start, u64 hole_end, 629 unsigned long end_time) 630 { 631 const u64 hole_size = hole_end - hole_start; 632 const unsigned long max_pages = 633 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT); 634 unsigned long min_alignment; 635 unsigned long flags; 636 u64 size; 637 638 /* Try binding a single VMA in different positions within the hole */ 639 640 flags = PIN_OFFSET_FIXED | PIN_USER; 641 if (i915_is_ggtt(vm)) 642 flags |= PIN_GLOBAL; 643 644 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 645 646 for_each_prime_number_from(size, 1, max_pages) { 647 struct drm_i915_gem_object *obj; 648 struct i915_vma *vma; 649 u64 addr; 650 int err = 0; 651 652 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT); 653 if (IS_ERR(obj)) 654 break; 655 656 vma = i915_vma_instance(obj, vm, NULL); 657 if (IS_ERR(vma)) { 658 err = PTR_ERR(vma); 659 goto err_put; 660 } 661 662 for (addr = hole_start; 663 addr + obj->base.size < hole_end; 664 addr += round_up(obj->base.size, min_alignment)) { 665 err = i915_vma_pin(vma, 0, 0, addr | flags); 666 if (err) { 667 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", 668 __func__, addr, vma->size, 669 hole_start, hole_end, err); 670 goto err_put; 671 } 672 i915_vma_unpin(vma); 673 674 if (!drm_mm_node_allocated(&vma->node) || 675 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 676 pr_err("%s incorrect at %llx + %llx\n", 677 __func__, addr, vma->size); 678 err = -EINVAL; 679 goto err_put; 680 } 681 682 err = i915_vma_unbind_unlocked(vma); 683 if (err) { 684 pr_err("%s unbind failed at %llx + %llx with err=%d\n", 685 __func__, addr, vma->size, err); 686 goto err_put; 687 } 688 689 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 690 691 if (igt_timeout(end_time, 692 "%s timed out at %llx\n", 693 __func__, addr)) { 694 err = -EINTR; 695 goto err_put; 696 } 697 } 698 699 err_put: 700 i915_gem_object_put(obj); 701 if (err) 702 return err; 703 704 cleanup_freed_objects(vm->i915); 705 } 706 707 return 0; 708 } 709 710 static int pot_hole(struct i915_address_space *vm, 711 u64 hole_start, u64 hole_end, 712 unsigned long end_time) 713 { 714 struct drm_i915_gem_object *obj; 715 struct i915_vma *vma; 716 unsigned int min_alignment; 717 unsigned long flags; 718 unsigned int pot; 719 int err = 0; 720 721 flags = PIN_OFFSET_FIXED | PIN_USER; 722 if (i915_is_ggtt(vm)) 723 flags |= PIN_GLOBAL; 724 725 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 726 727 obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE); 728 if (IS_ERR(obj)) 729 return PTR_ERR(obj); 730 731 vma = i915_vma_instance(obj, vm, NULL); 732 if (IS_ERR(vma)) { 733 err = PTR_ERR(vma); 734 goto err_obj; 735 } 736 737 /* Insert a pair of pages across every pot boundary within the hole */ 738 for (pot = fls64(hole_end - 1) - 1; 739 pot > ilog2(2 * min_alignment); 740 pot--) { 741 u64 step = BIT_ULL(pot); 742 u64 addr; 743 744 for (addr = round_up(hole_start + min_alignment, step) - min_alignment; 745 hole_end > addr && hole_end - addr >= 2 * min_alignment; 746 addr += step) { 747 err = i915_vma_pin(vma, 0, 0, addr | flags); 748 if (err) { 749 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n", 750 __func__, 751 addr, 752 hole_start, hole_end, 753 err); 754 goto err_obj; 755 } 756 757 if (!drm_mm_node_allocated(&vma->node) || 758 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 759 pr_err("%s incorrect at %llx + %llx\n", 760 __func__, addr, vma->size); 761 i915_vma_unpin(vma); 762 err = i915_vma_unbind_unlocked(vma); 763 err = -EINVAL; 764 goto err_obj; 765 } 766 767 i915_vma_unpin(vma); 768 err = i915_vma_unbind_unlocked(vma); 769 GEM_BUG_ON(err); 770 } 771 772 if (igt_timeout(end_time, 773 "%s timed out after %d/%d\n", 774 __func__, pot, fls64(hole_end - 1) - 1)) { 775 err = -EINTR; 776 goto err_obj; 777 } 778 } 779 780 err_obj: 781 i915_gem_object_put(obj); 782 return err; 783 } 784 785 static int drunk_hole(struct i915_address_space *vm, 786 u64 hole_start, u64 hole_end, 787 unsigned long end_time) 788 { 789 I915_RND_STATE(prng); 790 unsigned int min_alignment; 791 unsigned int size; 792 unsigned long flags; 793 794 flags = PIN_OFFSET_FIXED | PIN_USER; 795 if (i915_is_ggtt(vm)) 796 flags |= PIN_GLOBAL; 797 798 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 799 800 /* Keep creating larger objects until one cannot fit into the hole */ 801 for (size = 12; (hole_end - hole_start) >> size; size++) { 802 struct drm_i915_gem_object *obj; 803 unsigned int *order, count, n; 804 struct i915_vma *vma; 805 u64 hole_size, aligned_size; 806 int err = -ENODEV; 807 808 aligned_size = max_t(u32, ilog2(min_alignment), size); 809 hole_size = (hole_end - hole_start) >> aligned_size; 810 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 811 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 812 count = hole_size >> 1; 813 if (!count) { 814 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 815 __func__, hole_start, hole_end, size, hole_size); 816 break; 817 } 818 819 do { 820 order = i915_random_order(count, &prng); 821 if (order) 822 break; 823 } while (count >>= 1); 824 if (!count) 825 return -ENOMEM; 826 GEM_BUG_ON(!order); 827 828 /* Ignore allocation failures (i.e. don't report them as 829 * a test failure) as we are purposefully allocating very 830 * large objects without checking that we have sufficient 831 * memory. We expect to hit -ENOMEM. 832 */ 833 834 obj = fake_dma_object(vm->i915, BIT_ULL(size)); 835 if (IS_ERR(obj)) { 836 kfree(order); 837 break; 838 } 839 840 vma = i915_vma_instance(obj, vm, NULL); 841 if (IS_ERR(vma)) { 842 err = PTR_ERR(vma); 843 goto err_obj; 844 } 845 846 GEM_BUG_ON(vma->size != BIT_ULL(size)); 847 848 for (n = 0; n < count; n++) { 849 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 850 851 err = i915_vma_pin(vma, 0, 0, addr | flags); 852 if (err) { 853 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 854 __func__, 855 addr, BIT_ULL(size), 856 hole_start, hole_end, 857 err); 858 goto err_obj; 859 } 860 861 if (!drm_mm_node_allocated(&vma->node) || 862 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 863 pr_err("%s incorrect at %llx + %llx\n", 864 __func__, addr, BIT_ULL(size)); 865 i915_vma_unpin(vma); 866 err = i915_vma_unbind_unlocked(vma); 867 err = -EINVAL; 868 goto err_obj; 869 } 870 871 i915_vma_unpin(vma); 872 err = i915_vma_unbind_unlocked(vma); 873 GEM_BUG_ON(err); 874 875 if (igt_timeout(end_time, 876 "%s timed out after %d/%d\n", 877 __func__, n, count)) { 878 err = -EINTR; 879 goto err_obj; 880 } 881 } 882 883 err_obj: 884 i915_gem_object_put(obj); 885 kfree(order); 886 if (err) 887 return err; 888 889 cleanup_freed_objects(vm->i915); 890 } 891 892 return 0; 893 } 894 895 static int __shrink_hole(struct i915_address_space *vm, 896 u64 hole_start, u64 hole_end, 897 unsigned long end_time) 898 { 899 struct drm_i915_gem_object *obj; 900 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 901 unsigned int min_alignment; 902 unsigned int order = 12; 903 LIST_HEAD(objects); 904 int err = 0; 905 u64 addr; 906 907 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 908 909 /* Keep creating larger objects until one cannot fit into the hole */ 910 for (addr = hole_start; addr < hole_end; ) { 911 struct i915_vma *vma; 912 u64 size = BIT_ULL(order++); 913 914 size = min(size, hole_end - addr); 915 obj = fake_dma_object(vm->i915, size); 916 if (IS_ERR(obj)) { 917 err = PTR_ERR(obj); 918 break; 919 } 920 921 list_add(&obj->st_link, &objects); 922 923 vma = i915_vma_instance(obj, vm, NULL); 924 if (IS_ERR(vma)) { 925 err = PTR_ERR(vma); 926 break; 927 } 928 929 GEM_BUG_ON(vma->size != size); 930 931 err = i915_vma_pin(vma, 0, 0, addr | flags); 932 if (err) { 933 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 934 __func__, addr, size, hole_start, hole_end, err); 935 break; 936 } 937 938 if (!drm_mm_node_allocated(&vma->node) || 939 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 940 pr_err("%s incorrect at %llx + %llx\n", 941 __func__, addr, size); 942 i915_vma_unpin(vma); 943 err = i915_vma_unbind_unlocked(vma); 944 err = -EINVAL; 945 break; 946 } 947 948 i915_vma_unpin(vma); 949 addr += round_up(size, min_alignment); 950 951 /* 952 * Since we are injecting allocation faults at random intervals, 953 * wait for this allocation to complete before we change the 954 * faultinjection. 955 */ 956 err = i915_vma_sync(vma); 957 if (err) 958 break; 959 960 if (igt_timeout(end_time, 961 "%s timed out at ofset %llx [%llx - %llx]\n", 962 __func__, addr, hole_start, hole_end)) { 963 err = -EINTR; 964 break; 965 } 966 } 967 968 close_object_list(&objects, vm); 969 cleanup_freed_objects(vm->i915); 970 return err; 971 } 972 973 static int shrink_hole(struct i915_address_space *vm, 974 u64 hole_start, u64 hole_end, 975 unsigned long end_time) 976 { 977 unsigned long prime; 978 int err; 979 980 vm->fault_attr.probability = 999; 981 atomic_set(&vm->fault_attr.times, -1); 982 983 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) { 984 vm->fault_attr.interval = prime; 985 err = __shrink_hole(vm, hole_start, hole_end, end_time); 986 if (err) 987 break; 988 } 989 990 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 991 992 return err; 993 } 994 995 static int shrink_boom(struct i915_address_space *vm, 996 u64 hole_start, u64 hole_end, 997 unsigned long end_time) 998 { 999 unsigned int sizes[] = { SZ_2M, SZ_1G }; 1000 struct drm_i915_gem_object *purge; 1001 struct drm_i915_gem_object *explode; 1002 int err; 1003 int i; 1004 1005 /* 1006 * Catch the case which shrink_hole seems to miss. The setup here 1007 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while 1008 * ensuring that all vma assiocated with the respective pd/pdp are 1009 * unpinned at the time. 1010 */ 1011 1012 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1013 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1014 unsigned int size = sizes[i]; 1015 struct i915_vma *vma; 1016 1017 purge = fake_dma_object(vm->i915, size); 1018 if (IS_ERR(purge)) 1019 return PTR_ERR(purge); 1020 1021 vma = i915_vma_instance(purge, vm, NULL); 1022 if (IS_ERR(vma)) { 1023 err = PTR_ERR(vma); 1024 goto err_purge; 1025 } 1026 1027 err = i915_vma_pin(vma, 0, 0, flags); 1028 if (err) 1029 goto err_purge; 1030 1031 /* Should now be ripe for purging */ 1032 i915_vma_unpin(vma); 1033 1034 explode = fake_dma_object(vm->i915, size); 1035 if (IS_ERR(explode)) { 1036 err = PTR_ERR(explode); 1037 goto err_purge; 1038 } 1039 1040 vm->fault_attr.probability = 100; 1041 vm->fault_attr.interval = 1; 1042 atomic_set(&vm->fault_attr.times, -1); 1043 1044 vma = i915_vma_instance(explode, vm, NULL); 1045 if (IS_ERR(vma)) { 1046 err = PTR_ERR(vma); 1047 goto err_explode; 1048 } 1049 1050 err = i915_vma_pin(vma, 0, 0, flags | size); 1051 if (err) 1052 goto err_explode; 1053 1054 i915_vma_unpin(vma); 1055 1056 i915_gem_object_put(purge); 1057 i915_gem_object_put(explode); 1058 1059 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 1060 cleanup_freed_objects(vm->i915); 1061 } 1062 1063 return 0; 1064 1065 err_explode: 1066 i915_gem_object_put(explode); 1067 err_purge: 1068 i915_gem_object_put(purge); 1069 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 1070 return err; 1071 } 1072 1073 static int misaligned_case(struct i915_address_space *vm, struct intel_memory_region *mr, 1074 u64 addr, u64 size, unsigned long flags) 1075 { 1076 struct drm_i915_gem_object *obj; 1077 struct i915_vma *vma; 1078 int err = 0; 1079 u64 expected_vma_size, expected_node_size; 1080 bool is_stolen = mr->type == INTEL_MEMORY_STOLEN_SYSTEM || 1081 mr->type == INTEL_MEMORY_STOLEN_LOCAL; 1082 1083 obj = i915_gem_object_create_region(mr, size, 0, I915_BO_ALLOC_GPU_ONLY); 1084 if (IS_ERR(obj)) { 1085 /* if iGVT-g or DMAR is active, stolen mem will be uninitialized */ 1086 if (PTR_ERR(obj) == -ENODEV && is_stolen) 1087 return 0; 1088 return PTR_ERR(obj); 1089 } 1090 1091 vma = i915_vma_instance(obj, vm, NULL); 1092 if (IS_ERR(vma)) { 1093 err = PTR_ERR(vma); 1094 goto err_put; 1095 } 1096 1097 err = i915_vma_pin(vma, 0, 0, addr | flags); 1098 if (err) 1099 goto err_put; 1100 i915_vma_unpin(vma); 1101 1102 if (!drm_mm_node_allocated(&vma->node)) { 1103 err = -EINVAL; 1104 goto err_put; 1105 } 1106 1107 if (i915_vma_misplaced(vma, 0, 0, addr | flags)) { 1108 err = -EINVAL; 1109 goto err_put; 1110 } 1111 1112 expected_vma_size = round_up(size, 1 << (ffs(vma->resource->page_sizes_gtt) - 1)); 1113 expected_node_size = expected_vma_size; 1114 1115 if (HAS_64K_PAGES(vm->i915) && i915_gem_object_is_lmem(obj)) { 1116 /* 1117 * The compact-pt should expand lmem node to 2MB for the ppGTT, 1118 * for all other cases we should only expect 64K. 1119 */ 1120 expected_vma_size = round_up(size, I915_GTT_PAGE_SIZE_64K); 1121 if (NEEDS_COMPACT_PT(vm->i915) && !i915_is_ggtt(vm)) 1122 expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_2M); 1123 else 1124 expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_64K); 1125 } 1126 1127 if (vma->size != expected_vma_size || vma->node.size != expected_node_size) { 1128 err = i915_vma_unbind_unlocked(vma); 1129 err = -EBADSLT; 1130 goto err_put; 1131 } 1132 1133 err = i915_vma_unbind_unlocked(vma); 1134 if (err) 1135 goto err_put; 1136 1137 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 1138 1139 err_put: 1140 i915_gem_object_put(obj); 1141 cleanup_freed_objects(vm->i915); 1142 return err; 1143 } 1144 1145 static int misaligned_pin(struct i915_address_space *vm, 1146 u64 hole_start, u64 hole_end, 1147 unsigned long end_time) 1148 { 1149 struct intel_memory_region *mr; 1150 enum intel_region_id id; 1151 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 1152 int err = 0; 1153 u64 hole_size = hole_end - hole_start; 1154 1155 if (i915_is_ggtt(vm)) 1156 flags |= PIN_GLOBAL; 1157 1158 for_each_memory_region(mr, vm->i915, id) { 1159 u64 min_alignment = i915_vm_min_alignment(vm, mr->type); 1160 u64 size = min_alignment; 1161 u64 addr = round_down(hole_start + (hole_size / 2), min_alignment); 1162 1163 /* avoid -ENOSPC on very small hole setups */ 1164 if (hole_size < 3 * min_alignment) 1165 continue; 1166 1167 /* we can't test < 4k alignment due to flags being encoded in lower bits */ 1168 if (min_alignment != I915_GTT_PAGE_SIZE_4K) { 1169 err = misaligned_case(vm, mr, addr + (min_alignment / 2), size, flags); 1170 /* misaligned should error with -EINVAL*/ 1171 if (!err) 1172 err = -EBADSLT; 1173 if (err != -EINVAL) 1174 return err; 1175 } 1176 1177 /* test for vma->size expansion to min page size */ 1178 err = misaligned_case(vm, mr, addr, PAGE_SIZE, flags); 1179 if (err) 1180 return err; 1181 1182 /* test for intermediate size not expanding vma->size for large alignments */ 1183 err = misaligned_case(vm, mr, addr, size / 2, flags); 1184 if (err) 1185 return err; 1186 } 1187 1188 return 0; 1189 } 1190 1191 static int exercise_ppgtt(struct drm_i915_private *dev_priv, 1192 int (*func)(struct i915_address_space *vm, 1193 u64 hole_start, u64 hole_end, 1194 unsigned long end_time)) 1195 { 1196 struct i915_ppgtt *ppgtt; 1197 IGT_TIMEOUT(end_time); 1198 struct file *file; 1199 int err; 1200 1201 if (!HAS_FULL_PPGTT(dev_priv)) 1202 return 0; 1203 1204 file = mock_file(dev_priv); 1205 if (IS_ERR(file)) 1206 return PTR_ERR(file); 1207 1208 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0); 1209 if (IS_ERR(ppgtt)) { 1210 err = PTR_ERR(ppgtt); 1211 goto out_free; 1212 } 1213 GEM_BUG_ON(offset_in_page(ppgtt->vm.total)); 1214 assert_vm_alive(&ppgtt->vm); 1215 1216 err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time); 1217 1218 i915_vm_put(&ppgtt->vm); 1219 1220 out_free: 1221 fput(file); 1222 return err; 1223 } 1224 1225 static int igt_ppgtt_fill(void *arg) 1226 { 1227 return exercise_ppgtt(arg, fill_hole); 1228 } 1229 1230 static int igt_ppgtt_walk(void *arg) 1231 { 1232 return exercise_ppgtt(arg, walk_hole); 1233 } 1234 1235 static int igt_ppgtt_pot(void *arg) 1236 { 1237 return exercise_ppgtt(arg, pot_hole); 1238 } 1239 1240 static int igt_ppgtt_drunk(void *arg) 1241 { 1242 return exercise_ppgtt(arg, drunk_hole); 1243 } 1244 1245 static int igt_ppgtt_lowlevel(void *arg) 1246 { 1247 return exercise_ppgtt(arg, lowlevel_hole); 1248 } 1249 1250 static int igt_ppgtt_shrink(void *arg) 1251 { 1252 return exercise_ppgtt(arg, shrink_hole); 1253 } 1254 1255 static int igt_ppgtt_shrink_boom(void *arg) 1256 { 1257 return exercise_ppgtt(arg, shrink_boom); 1258 } 1259 1260 static int igt_ppgtt_misaligned_pin(void *arg) 1261 { 1262 return exercise_ppgtt(arg, misaligned_pin); 1263 } 1264 1265 static int sort_holes(void *priv, const struct list_head *A, 1266 const struct list_head *B) 1267 { 1268 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack); 1269 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack); 1270 1271 if (a->start < b->start) 1272 return -1; 1273 else 1274 return 1; 1275 } 1276 1277 static int exercise_ggtt(struct drm_i915_private *i915, 1278 int (*func)(struct i915_address_space *vm, 1279 u64 hole_start, u64 hole_end, 1280 unsigned long end_time)) 1281 { 1282 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 1283 u64 hole_start, hole_end, last = 0; 1284 struct drm_mm_node *node; 1285 IGT_TIMEOUT(end_time); 1286 int err = 0; 1287 1288 restart: 1289 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes); 1290 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) { 1291 if (hole_start < last) 1292 continue; 1293 1294 if (ggtt->vm.mm.color_adjust) 1295 ggtt->vm.mm.color_adjust(node, 0, 1296 &hole_start, &hole_end); 1297 if (hole_start >= hole_end) 1298 continue; 1299 1300 err = func(&ggtt->vm, hole_start, hole_end, end_time); 1301 if (err) 1302 break; 1303 1304 /* As we have manipulated the drm_mm, the list may be corrupt */ 1305 last = hole_end; 1306 goto restart; 1307 } 1308 1309 return err; 1310 } 1311 1312 static int igt_ggtt_fill(void *arg) 1313 { 1314 return exercise_ggtt(arg, fill_hole); 1315 } 1316 1317 static int igt_ggtt_walk(void *arg) 1318 { 1319 return exercise_ggtt(arg, walk_hole); 1320 } 1321 1322 static int igt_ggtt_pot(void *arg) 1323 { 1324 return exercise_ggtt(arg, pot_hole); 1325 } 1326 1327 static int igt_ggtt_drunk(void *arg) 1328 { 1329 return exercise_ggtt(arg, drunk_hole); 1330 } 1331 1332 static int igt_ggtt_lowlevel(void *arg) 1333 { 1334 return exercise_ggtt(arg, lowlevel_hole); 1335 } 1336 1337 static int igt_ggtt_misaligned_pin(void *arg) 1338 { 1339 return exercise_ggtt(arg, misaligned_pin); 1340 } 1341 1342 static int igt_ggtt_page(void *arg) 1343 { 1344 const unsigned int count = PAGE_SIZE/sizeof(u32); 1345 I915_RND_STATE(prng); 1346 struct drm_i915_private *i915 = arg; 1347 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 1348 struct drm_i915_gem_object *obj; 1349 intel_wakeref_t wakeref; 1350 struct drm_mm_node tmp; 1351 unsigned int *order, n; 1352 int err; 1353 1354 if (!i915_ggtt_has_aperture(ggtt)) 1355 return 0; 1356 1357 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 1358 if (IS_ERR(obj)) 1359 return PTR_ERR(obj); 1360 1361 err = i915_gem_object_pin_pages_unlocked(obj); 1362 if (err) 1363 goto out_free; 1364 1365 memset(&tmp, 0, sizeof(tmp)); 1366 mutex_lock(&ggtt->vm.mutex); 1367 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp, 1368 count * PAGE_SIZE, 0, 1369 I915_COLOR_UNEVICTABLE, 1370 0, ggtt->mappable_end, 1371 DRM_MM_INSERT_LOW); 1372 mutex_unlock(&ggtt->vm.mutex); 1373 if (err) 1374 goto out_unpin; 1375 1376 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1377 1378 for (n = 0; n < count; n++) { 1379 u64 offset = tmp.start + n * PAGE_SIZE; 1380 1381 ggtt->vm.insert_page(&ggtt->vm, 1382 i915_gem_object_get_dma_address(obj, 0), 1383 offset, I915_CACHE_NONE, 0); 1384 } 1385 1386 order = i915_random_order(count, &prng); 1387 if (!order) { 1388 err = -ENOMEM; 1389 goto out_remove; 1390 } 1391 1392 for (n = 0; n < count; n++) { 1393 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1394 u32 __iomem *vaddr; 1395 1396 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1397 iowrite32(n, vaddr + n); 1398 io_mapping_unmap_atomic(vaddr); 1399 } 1400 intel_gt_flush_ggtt_writes(ggtt->vm.gt); 1401 1402 i915_random_reorder(order, count, &prng); 1403 for (n = 0; n < count; n++) { 1404 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1405 u32 __iomem *vaddr; 1406 u32 val; 1407 1408 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1409 val = ioread32(vaddr + n); 1410 io_mapping_unmap_atomic(vaddr); 1411 1412 if (val != n) { 1413 pr_err("insert page failed: found %d, expected %d\n", 1414 val, n); 1415 err = -EINVAL; 1416 break; 1417 } 1418 } 1419 1420 kfree(order); 1421 out_remove: 1422 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size); 1423 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1424 mutex_lock(&ggtt->vm.mutex); 1425 drm_mm_remove_node(&tmp); 1426 mutex_unlock(&ggtt->vm.mutex); 1427 out_unpin: 1428 i915_gem_object_unpin_pages(obj); 1429 out_free: 1430 i915_gem_object_put(obj); 1431 return err; 1432 } 1433 1434 static void track_vma_bind(struct i915_vma *vma) 1435 { 1436 struct drm_i915_gem_object *obj = vma->obj; 1437 1438 __i915_gem_object_pin_pages(obj); 1439 1440 GEM_BUG_ON(atomic_read(&vma->pages_count)); 1441 atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE); 1442 __i915_gem_object_pin_pages(obj); 1443 vma->pages = obj->mm.pages; 1444 vma->resource->bi.pages = vma->pages; 1445 1446 mutex_lock(&vma->vm->mutex); 1447 list_move_tail(&vma->vm_link, &vma->vm->bound_list); 1448 mutex_unlock(&vma->vm->mutex); 1449 } 1450 1451 static int exercise_mock(struct drm_i915_private *i915, 1452 int (*func)(struct i915_address_space *vm, 1453 u64 hole_start, u64 hole_end, 1454 unsigned long end_time)) 1455 { 1456 const u64 limit = totalram_pages() << PAGE_SHIFT; 1457 struct i915_address_space *vm; 1458 struct i915_gem_context *ctx; 1459 IGT_TIMEOUT(end_time); 1460 int err; 1461 1462 ctx = mock_context(i915, "mock"); 1463 if (!ctx) 1464 return -ENOMEM; 1465 1466 vm = i915_gem_context_get_eb_vm(ctx); 1467 err = func(vm, 0, min(vm->total, limit), end_time); 1468 i915_vm_put(vm); 1469 1470 mock_context_close(ctx); 1471 return err; 1472 } 1473 1474 static int igt_mock_fill(void *arg) 1475 { 1476 struct i915_ggtt *ggtt = arg; 1477 1478 return exercise_mock(ggtt->vm.i915, fill_hole); 1479 } 1480 1481 static int igt_mock_walk(void *arg) 1482 { 1483 struct i915_ggtt *ggtt = arg; 1484 1485 return exercise_mock(ggtt->vm.i915, walk_hole); 1486 } 1487 1488 static int igt_mock_pot(void *arg) 1489 { 1490 struct i915_ggtt *ggtt = arg; 1491 1492 return exercise_mock(ggtt->vm.i915, pot_hole); 1493 } 1494 1495 static int igt_mock_drunk(void *arg) 1496 { 1497 struct i915_ggtt *ggtt = arg; 1498 1499 return exercise_mock(ggtt->vm.i915, drunk_hole); 1500 } 1501 1502 static int reserve_gtt_with_resource(struct i915_vma *vma, u64 offset) 1503 { 1504 struct i915_address_space *vm = vma->vm; 1505 struct i915_vma_resource *vma_res; 1506 struct drm_i915_gem_object *obj = vma->obj; 1507 int err; 1508 1509 vma_res = i915_vma_resource_alloc(); 1510 if (IS_ERR(vma_res)) 1511 return PTR_ERR(vma_res); 1512 1513 mutex_lock(&vm->mutex); 1514 err = i915_gem_gtt_reserve(vm, NULL, &vma->node, obj->base.size, 1515 offset, 1516 obj->cache_level, 1517 0); 1518 if (!err) { 1519 i915_vma_resource_init_from_vma(vma_res, vma); 1520 vma->resource = vma_res; 1521 } else { 1522 kfree(vma_res); 1523 } 1524 mutex_unlock(&vm->mutex); 1525 1526 return err; 1527 } 1528 1529 static int igt_gtt_reserve(void *arg) 1530 { 1531 struct i915_ggtt *ggtt = arg; 1532 struct drm_i915_gem_object *obj, *on; 1533 I915_RND_STATE(prng); 1534 LIST_HEAD(objects); 1535 u64 total; 1536 int err = -ENODEV; 1537 1538 /* i915_gem_gtt_reserve() tries to reserve the precise range 1539 * for the node, and evicts if it has to. So our test checks that 1540 * it can give us the requsted space and prevent overlaps. 1541 */ 1542 1543 /* Start by filling the GGTT */ 1544 for (total = 0; 1545 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1546 total += 2 * I915_GTT_PAGE_SIZE) { 1547 struct i915_vma *vma; 1548 1549 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1550 2 * PAGE_SIZE); 1551 if (IS_ERR(obj)) { 1552 err = PTR_ERR(obj); 1553 goto out; 1554 } 1555 1556 err = i915_gem_object_pin_pages_unlocked(obj); 1557 if (err) { 1558 i915_gem_object_put(obj); 1559 goto out; 1560 } 1561 1562 list_add(&obj->st_link, &objects); 1563 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1564 if (IS_ERR(vma)) { 1565 err = PTR_ERR(vma); 1566 goto out; 1567 } 1568 1569 err = reserve_gtt_with_resource(vma, total); 1570 if (err) { 1571 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n", 1572 total, ggtt->vm.total, err); 1573 goto out; 1574 } 1575 track_vma_bind(vma); 1576 1577 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1578 if (vma->node.start != total || 1579 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1580 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1581 vma->node.start, vma->node.size, 1582 total, 2*I915_GTT_PAGE_SIZE); 1583 err = -EINVAL; 1584 goto out; 1585 } 1586 } 1587 1588 /* Now we start forcing evictions */ 1589 for (total = I915_GTT_PAGE_SIZE; 1590 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1591 total += 2 * I915_GTT_PAGE_SIZE) { 1592 struct i915_vma *vma; 1593 1594 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1595 2 * PAGE_SIZE); 1596 if (IS_ERR(obj)) { 1597 err = PTR_ERR(obj); 1598 goto out; 1599 } 1600 1601 err = i915_gem_object_pin_pages_unlocked(obj); 1602 if (err) { 1603 i915_gem_object_put(obj); 1604 goto out; 1605 } 1606 1607 list_add(&obj->st_link, &objects); 1608 1609 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1610 if (IS_ERR(vma)) { 1611 err = PTR_ERR(vma); 1612 goto out; 1613 } 1614 1615 err = reserve_gtt_with_resource(vma, total); 1616 if (err) { 1617 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n", 1618 total, ggtt->vm.total, err); 1619 goto out; 1620 } 1621 track_vma_bind(vma); 1622 1623 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1624 if (vma->node.start != total || 1625 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1626 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1627 vma->node.start, vma->node.size, 1628 total, 2*I915_GTT_PAGE_SIZE); 1629 err = -EINVAL; 1630 goto out; 1631 } 1632 } 1633 1634 /* And then try at random */ 1635 list_for_each_entry_safe(obj, on, &objects, st_link) { 1636 struct i915_vma *vma; 1637 u64 offset; 1638 1639 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1640 if (IS_ERR(vma)) { 1641 err = PTR_ERR(vma); 1642 goto out; 1643 } 1644 1645 err = i915_vma_unbind_unlocked(vma); 1646 if (err) { 1647 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1648 goto out; 1649 } 1650 1651 offset = igt_random_offset(&prng, 1652 0, ggtt->vm.total, 1653 2 * I915_GTT_PAGE_SIZE, 1654 I915_GTT_MIN_ALIGNMENT); 1655 1656 err = reserve_gtt_with_resource(vma, offset); 1657 if (err) { 1658 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n", 1659 total, ggtt->vm.total, err); 1660 goto out; 1661 } 1662 track_vma_bind(vma); 1663 1664 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1665 if (vma->node.start != offset || 1666 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1667 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1668 vma->node.start, vma->node.size, 1669 offset, 2*I915_GTT_PAGE_SIZE); 1670 err = -EINVAL; 1671 goto out; 1672 } 1673 } 1674 1675 out: 1676 list_for_each_entry_safe(obj, on, &objects, st_link) { 1677 i915_gem_object_unpin_pages(obj); 1678 i915_gem_object_put(obj); 1679 } 1680 return err; 1681 } 1682 1683 static int insert_gtt_with_resource(struct i915_vma *vma) 1684 { 1685 struct i915_address_space *vm = vma->vm; 1686 struct i915_vma_resource *vma_res; 1687 struct drm_i915_gem_object *obj = vma->obj; 1688 int err; 1689 1690 vma_res = i915_vma_resource_alloc(); 1691 if (IS_ERR(vma_res)) 1692 return PTR_ERR(vma_res); 1693 1694 mutex_lock(&vm->mutex); 1695 err = i915_gem_gtt_insert(vm, NULL, &vma->node, obj->base.size, 0, 1696 obj->cache_level, 0, vm->total, 0); 1697 if (!err) { 1698 i915_vma_resource_init_from_vma(vma_res, vma); 1699 vma->resource = vma_res; 1700 } else { 1701 kfree(vma_res); 1702 } 1703 mutex_unlock(&vm->mutex); 1704 1705 return err; 1706 } 1707 1708 static int igt_gtt_insert(void *arg) 1709 { 1710 struct i915_ggtt *ggtt = arg; 1711 struct drm_i915_gem_object *obj, *on; 1712 struct drm_mm_node tmp = {}; 1713 const struct invalid_insert { 1714 u64 size; 1715 u64 alignment; 1716 u64 start, end; 1717 } invalid_insert[] = { 1718 { 1719 ggtt->vm.total + I915_GTT_PAGE_SIZE, 0, 1720 0, ggtt->vm.total, 1721 }, 1722 { 1723 2*I915_GTT_PAGE_SIZE, 0, 1724 0, I915_GTT_PAGE_SIZE, 1725 }, 1726 { 1727 -(u64)I915_GTT_PAGE_SIZE, 0, 1728 0, 4*I915_GTT_PAGE_SIZE, 1729 }, 1730 { 1731 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE, 1732 0, 4*I915_GTT_PAGE_SIZE, 1733 }, 1734 { 1735 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1, 1736 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1, 1737 }, 1738 {} 1739 }, *ii; 1740 LIST_HEAD(objects); 1741 u64 total; 1742 int err = -ENODEV; 1743 1744 /* i915_gem_gtt_insert() tries to allocate some free space in the GTT 1745 * to the node, evicting if required. 1746 */ 1747 1748 /* Check a couple of obviously invalid requests */ 1749 for (ii = invalid_insert; ii->size; ii++) { 1750 mutex_lock(&ggtt->vm.mutex); 1751 err = i915_gem_gtt_insert(&ggtt->vm, NULL, &tmp, 1752 ii->size, ii->alignment, 1753 I915_COLOR_UNEVICTABLE, 1754 ii->start, ii->end, 1755 0); 1756 mutex_unlock(&ggtt->vm.mutex); 1757 if (err != -ENOSPC) { 1758 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n", 1759 ii->size, ii->alignment, ii->start, ii->end, 1760 err); 1761 return -EINVAL; 1762 } 1763 } 1764 1765 /* Start by filling the GGTT */ 1766 for (total = 0; 1767 total + I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1768 total += I915_GTT_PAGE_SIZE) { 1769 struct i915_vma *vma; 1770 1771 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1772 I915_GTT_PAGE_SIZE); 1773 if (IS_ERR(obj)) { 1774 err = PTR_ERR(obj); 1775 goto out; 1776 } 1777 1778 err = i915_gem_object_pin_pages_unlocked(obj); 1779 if (err) { 1780 i915_gem_object_put(obj); 1781 goto out; 1782 } 1783 1784 list_add(&obj->st_link, &objects); 1785 1786 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1787 if (IS_ERR(vma)) { 1788 err = PTR_ERR(vma); 1789 goto out; 1790 } 1791 1792 err = insert_gtt_with_resource(vma); 1793 if (err == -ENOSPC) { 1794 /* maxed out the GGTT space */ 1795 i915_gem_object_put(obj); 1796 break; 1797 } 1798 if (err) { 1799 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n", 1800 total, ggtt->vm.total, err); 1801 goto out; 1802 } 1803 track_vma_bind(vma); 1804 __i915_vma_pin(vma); 1805 1806 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1807 } 1808 1809 list_for_each_entry(obj, &objects, st_link) { 1810 struct i915_vma *vma; 1811 1812 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1813 if (IS_ERR(vma)) { 1814 err = PTR_ERR(vma); 1815 goto out; 1816 } 1817 1818 if (!drm_mm_node_allocated(&vma->node)) { 1819 pr_err("VMA was unexpectedly evicted!\n"); 1820 err = -EINVAL; 1821 goto out; 1822 } 1823 1824 __i915_vma_unpin(vma); 1825 } 1826 1827 /* If we then reinsert, we should find the same hole */ 1828 list_for_each_entry_safe(obj, on, &objects, st_link) { 1829 struct i915_vma *vma; 1830 u64 offset; 1831 1832 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1833 if (IS_ERR(vma)) { 1834 err = PTR_ERR(vma); 1835 goto out; 1836 } 1837 1838 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1839 offset = vma->node.start; 1840 1841 err = i915_vma_unbind_unlocked(vma); 1842 if (err) { 1843 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1844 goto out; 1845 } 1846 1847 err = insert_gtt_with_resource(vma); 1848 if (err) { 1849 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n", 1850 total, ggtt->vm.total, err); 1851 goto out; 1852 } 1853 track_vma_bind(vma); 1854 1855 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1856 if (vma->node.start != offset) { 1857 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n", 1858 offset, vma->node.start); 1859 err = -EINVAL; 1860 goto out; 1861 } 1862 } 1863 1864 /* And then force evictions */ 1865 for (total = 0; 1866 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1867 total += 2 * I915_GTT_PAGE_SIZE) { 1868 struct i915_vma *vma; 1869 1870 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1871 2 * I915_GTT_PAGE_SIZE); 1872 if (IS_ERR(obj)) { 1873 err = PTR_ERR(obj); 1874 goto out; 1875 } 1876 1877 err = i915_gem_object_pin_pages_unlocked(obj); 1878 if (err) { 1879 i915_gem_object_put(obj); 1880 goto out; 1881 } 1882 1883 list_add(&obj->st_link, &objects); 1884 1885 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1886 if (IS_ERR(vma)) { 1887 err = PTR_ERR(vma); 1888 goto out; 1889 } 1890 1891 err = insert_gtt_with_resource(vma); 1892 if (err) { 1893 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n", 1894 total, ggtt->vm.total, err); 1895 goto out; 1896 } 1897 track_vma_bind(vma); 1898 1899 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1900 } 1901 1902 out: 1903 list_for_each_entry_safe(obj, on, &objects, st_link) { 1904 i915_gem_object_unpin_pages(obj); 1905 i915_gem_object_put(obj); 1906 } 1907 return err; 1908 } 1909 1910 int i915_gem_gtt_mock_selftests(void) 1911 { 1912 static const struct i915_subtest tests[] = { 1913 SUBTEST(igt_mock_drunk), 1914 SUBTEST(igt_mock_walk), 1915 SUBTEST(igt_mock_pot), 1916 SUBTEST(igt_mock_fill), 1917 SUBTEST(igt_gtt_reserve), 1918 SUBTEST(igt_gtt_insert), 1919 }; 1920 struct drm_i915_private *i915; 1921 struct intel_gt *gt; 1922 int err; 1923 1924 i915 = mock_gem_device(); 1925 if (!i915) 1926 return -ENOMEM; 1927 1928 /* allocate the ggtt */ 1929 err = intel_gt_assign_ggtt(to_gt(i915)); 1930 if (err) 1931 goto out_put; 1932 1933 gt = to_gt(i915); 1934 1935 mock_init_ggtt(gt); 1936 1937 err = i915_subtests(tests, gt->ggtt); 1938 1939 mock_device_flush(i915); 1940 i915_gem_drain_freed_objects(i915); 1941 mock_fini_ggtt(gt->ggtt); 1942 1943 out_put: 1944 mock_destroy_device(i915); 1945 return err; 1946 } 1947 1948 static int context_sync(struct intel_context *ce) 1949 { 1950 struct i915_request *rq; 1951 long timeout; 1952 1953 rq = intel_context_create_request(ce); 1954 if (IS_ERR(rq)) 1955 return PTR_ERR(rq); 1956 1957 i915_request_get(rq); 1958 i915_request_add(rq); 1959 1960 timeout = i915_request_wait(rq, 0, HZ / 5); 1961 i915_request_put(rq); 1962 1963 return timeout < 0 ? -EIO : 0; 1964 } 1965 1966 static struct i915_request * 1967 submit_batch(struct intel_context *ce, u64 addr) 1968 { 1969 struct i915_request *rq; 1970 int err; 1971 1972 rq = intel_context_create_request(ce); 1973 if (IS_ERR(rq)) 1974 return rq; 1975 1976 err = 0; 1977 if (rq->engine->emit_init_breadcrumb) /* detect a hang */ 1978 err = rq->engine->emit_init_breadcrumb(rq); 1979 if (err == 0) 1980 err = rq->engine->emit_bb_start(rq, addr, 0, 0); 1981 1982 if (err == 0) 1983 i915_request_get(rq); 1984 i915_request_add(rq); 1985 1986 return err ? ERR_PTR(err) : rq; 1987 } 1988 1989 static u32 *spinner(u32 *batch, int i) 1990 { 1991 return batch + i * 64 / sizeof(*batch) + 4; 1992 } 1993 1994 static void end_spin(u32 *batch, int i) 1995 { 1996 *spinner(batch, i) = MI_BATCH_BUFFER_END; 1997 wmb(); 1998 } 1999 2000 static int igt_cs_tlb(void *arg) 2001 { 2002 const unsigned int count = PAGE_SIZE / 64; 2003 const unsigned int chunk_size = count * PAGE_SIZE; 2004 struct drm_i915_private *i915 = arg; 2005 struct drm_i915_gem_object *bbe, *act, *out; 2006 struct i915_gem_engines_iter it; 2007 struct i915_address_space *vm; 2008 struct i915_gem_context *ctx; 2009 struct intel_context *ce; 2010 struct i915_vma *vma; 2011 I915_RND_STATE(prng); 2012 struct file *file; 2013 unsigned int i; 2014 u32 *result; 2015 u32 *batch; 2016 int err = 0; 2017 2018 /* 2019 * Our mission here is to fool the hardware to execute something 2020 * from scratch as it has not seen the batch move (due to missing 2021 * the TLB invalidate). 2022 */ 2023 2024 file = mock_file(i915); 2025 if (IS_ERR(file)) 2026 return PTR_ERR(file); 2027 2028 ctx = live_context(i915, file); 2029 if (IS_ERR(ctx)) { 2030 err = PTR_ERR(ctx); 2031 goto out_unlock; 2032 } 2033 2034 vm = i915_gem_context_get_eb_vm(ctx); 2035 if (i915_is_ggtt(vm)) 2036 goto out_vm; 2037 2038 /* Create two pages; dummy we prefill the TLB, and intended */ 2039 bbe = i915_gem_object_create_internal(i915, PAGE_SIZE); 2040 if (IS_ERR(bbe)) { 2041 err = PTR_ERR(bbe); 2042 goto out_vm; 2043 } 2044 2045 batch = i915_gem_object_pin_map_unlocked(bbe, I915_MAP_WC); 2046 if (IS_ERR(batch)) { 2047 err = PTR_ERR(batch); 2048 goto out_put_bbe; 2049 } 2050 memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32)); 2051 i915_gem_object_flush_map(bbe); 2052 i915_gem_object_unpin_map(bbe); 2053 2054 act = i915_gem_object_create_internal(i915, PAGE_SIZE); 2055 if (IS_ERR(act)) { 2056 err = PTR_ERR(act); 2057 goto out_put_bbe; 2058 } 2059 2060 /* Track the execution of each request by writing into different slot */ 2061 batch = i915_gem_object_pin_map_unlocked(act, I915_MAP_WC); 2062 if (IS_ERR(batch)) { 2063 err = PTR_ERR(batch); 2064 goto out_put_act; 2065 } 2066 for (i = 0; i < count; i++) { 2067 u32 *cs = batch + i * 64 / sizeof(*cs); 2068 u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32); 2069 2070 GEM_BUG_ON(GRAPHICS_VER(i915) < 6); 2071 cs[0] = MI_STORE_DWORD_IMM_GEN4; 2072 if (GRAPHICS_VER(i915) >= 8) { 2073 cs[1] = lower_32_bits(addr); 2074 cs[2] = upper_32_bits(addr); 2075 cs[3] = i; 2076 cs[4] = MI_NOOP; 2077 cs[5] = MI_BATCH_BUFFER_START_GEN8; 2078 } else { 2079 cs[1] = 0; 2080 cs[2] = lower_32_bits(addr); 2081 cs[3] = i; 2082 cs[4] = MI_NOOP; 2083 cs[5] = MI_BATCH_BUFFER_START; 2084 } 2085 } 2086 2087 out = i915_gem_object_create_internal(i915, PAGE_SIZE); 2088 if (IS_ERR(out)) { 2089 err = PTR_ERR(out); 2090 goto out_put_batch; 2091 } 2092 i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED); 2093 2094 vma = i915_vma_instance(out, vm, NULL); 2095 if (IS_ERR(vma)) { 2096 err = PTR_ERR(vma); 2097 goto out_put_out; 2098 } 2099 2100 err = i915_vma_pin(vma, 0, 0, 2101 PIN_USER | 2102 PIN_OFFSET_FIXED | 2103 (vm->total - PAGE_SIZE)); 2104 if (err) 2105 goto out_put_out; 2106 GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE); 2107 2108 result = i915_gem_object_pin_map_unlocked(out, I915_MAP_WB); 2109 if (IS_ERR(result)) { 2110 err = PTR_ERR(result); 2111 goto out_put_out; 2112 } 2113 2114 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 2115 IGT_TIMEOUT(end_time); 2116 unsigned long pass = 0; 2117 2118 if (!intel_engine_can_store_dword(ce->engine)) 2119 continue; 2120 2121 while (!__igt_timeout(end_time, NULL)) { 2122 struct i915_vm_pt_stash stash = {}; 2123 struct i915_request *rq; 2124 struct i915_gem_ww_ctx ww; 2125 struct i915_vma_resource *vma_res; 2126 u64 offset; 2127 2128 offset = igt_random_offset(&prng, 2129 0, vm->total - PAGE_SIZE, 2130 chunk_size, PAGE_SIZE); 2131 2132 memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32)); 2133 2134 vma = i915_vma_instance(bbe, vm, NULL); 2135 if (IS_ERR(vma)) { 2136 err = PTR_ERR(vma); 2137 goto end; 2138 } 2139 2140 i915_gem_object_lock(bbe, NULL); 2141 err = i915_vma_get_pages(vma); 2142 i915_gem_object_unlock(bbe); 2143 if (err) 2144 goto end; 2145 2146 vma_res = i915_vma_resource_alloc(); 2147 if (IS_ERR(vma_res)) { 2148 i915_vma_put_pages(vma); 2149 err = PTR_ERR(vma_res); 2150 goto end; 2151 } 2152 2153 i915_gem_ww_ctx_init(&ww, false); 2154 retry: 2155 err = i915_vm_lock_objects(vm, &ww); 2156 if (err) 2157 goto end_ww; 2158 2159 err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size); 2160 if (err) 2161 goto end_ww; 2162 2163 err = i915_vm_map_pt_stash(vm, &stash); 2164 if (!err) 2165 vm->allocate_va_range(vm, &stash, offset, chunk_size); 2166 i915_vm_free_pt_stash(vm, &stash); 2167 end_ww: 2168 if (err == -EDEADLK) { 2169 err = i915_gem_ww_ctx_backoff(&ww); 2170 if (!err) 2171 goto retry; 2172 } 2173 i915_gem_ww_ctx_fini(&ww); 2174 if (err) { 2175 kfree(vma_res); 2176 goto end; 2177 } 2178 2179 i915_vma_resource_init_from_vma(vma_res, vma); 2180 /* Prime the TLB with the dummy pages */ 2181 for (i = 0; i < count; i++) { 2182 vma_res->start = offset + i * PAGE_SIZE; 2183 vm->insert_entries(vm, vma_res, I915_CACHE_NONE, 2184 0); 2185 2186 rq = submit_batch(ce, vma_res->start); 2187 if (IS_ERR(rq)) { 2188 err = PTR_ERR(rq); 2189 i915_vma_resource_fini(vma_res); 2190 kfree(vma_res); 2191 goto end; 2192 } 2193 i915_request_put(rq); 2194 } 2195 i915_vma_resource_fini(vma_res); 2196 i915_vma_put_pages(vma); 2197 2198 err = context_sync(ce); 2199 if (err) { 2200 pr_err("%s: dummy setup timed out\n", 2201 ce->engine->name); 2202 kfree(vma_res); 2203 goto end; 2204 } 2205 2206 vma = i915_vma_instance(act, vm, NULL); 2207 if (IS_ERR(vma)) { 2208 kfree(vma_res); 2209 err = PTR_ERR(vma); 2210 goto end; 2211 } 2212 2213 i915_gem_object_lock(act, NULL); 2214 err = i915_vma_get_pages(vma); 2215 i915_gem_object_unlock(act); 2216 if (err) { 2217 kfree(vma_res); 2218 goto end; 2219 } 2220 2221 i915_vma_resource_init_from_vma(vma_res, vma); 2222 /* Replace the TLB with target batches */ 2223 for (i = 0; i < count; i++) { 2224 struct i915_request *rq; 2225 u32 *cs = batch + i * 64 / sizeof(*cs); 2226 u64 addr; 2227 2228 vma_res->start = offset + i * PAGE_SIZE; 2229 vm->insert_entries(vm, vma_res, I915_CACHE_NONE, 0); 2230 2231 addr = vma_res->start + i * 64; 2232 cs[4] = MI_NOOP; 2233 cs[6] = lower_32_bits(addr); 2234 cs[7] = upper_32_bits(addr); 2235 wmb(); 2236 2237 rq = submit_batch(ce, addr); 2238 if (IS_ERR(rq)) { 2239 err = PTR_ERR(rq); 2240 i915_vma_resource_fini(vma_res); 2241 kfree(vma_res); 2242 goto end; 2243 } 2244 2245 /* Wait until the context chain has started */ 2246 if (i == 0) { 2247 while (READ_ONCE(result[i]) && 2248 !i915_request_completed(rq)) 2249 cond_resched(); 2250 } else { 2251 end_spin(batch, i - 1); 2252 } 2253 2254 i915_request_put(rq); 2255 } 2256 end_spin(batch, count - 1); 2257 2258 i915_vma_resource_fini(vma_res); 2259 kfree(vma_res); 2260 i915_vma_put_pages(vma); 2261 2262 err = context_sync(ce); 2263 if (err) { 2264 pr_err("%s: writes timed out\n", 2265 ce->engine->name); 2266 goto end; 2267 } 2268 2269 for (i = 0; i < count; i++) { 2270 if (result[i] != i) { 2271 pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n", 2272 ce->engine->name, pass, 2273 offset, i, result[i], i); 2274 err = -EINVAL; 2275 goto end; 2276 } 2277 } 2278 2279 vm->clear_range(vm, offset, chunk_size); 2280 pass++; 2281 } 2282 } 2283 end: 2284 if (igt_flush_test(i915)) 2285 err = -EIO; 2286 i915_gem_context_unlock_engines(ctx); 2287 i915_gem_object_unpin_map(out); 2288 out_put_out: 2289 i915_gem_object_put(out); 2290 out_put_batch: 2291 i915_gem_object_unpin_map(act); 2292 out_put_act: 2293 i915_gem_object_put(act); 2294 out_put_bbe: 2295 i915_gem_object_put(bbe); 2296 out_vm: 2297 i915_vm_put(vm); 2298 out_unlock: 2299 fput(file); 2300 return err; 2301 } 2302 2303 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) 2304 { 2305 static const struct i915_subtest tests[] = { 2306 SUBTEST(igt_ppgtt_alloc), 2307 SUBTEST(igt_ppgtt_lowlevel), 2308 SUBTEST(igt_ppgtt_drunk), 2309 SUBTEST(igt_ppgtt_walk), 2310 SUBTEST(igt_ppgtt_pot), 2311 SUBTEST(igt_ppgtt_fill), 2312 SUBTEST(igt_ppgtt_shrink), 2313 SUBTEST(igt_ppgtt_shrink_boom), 2314 SUBTEST(igt_ppgtt_misaligned_pin), 2315 SUBTEST(igt_ggtt_lowlevel), 2316 SUBTEST(igt_ggtt_drunk), 2317 SUBTEST(igt_ggtt_walk), 2318 SUBTEST(igt_ggtt_pot), 2319 SUBTEST(igt_ggtt_fill), 2320 SUBTEST(igt_ggtt_page), 2321 SUBTEST(igt_ggtt_misaligned_pin), 2322 SUBTEST(igt_cs_tlb), 2323 }; 2324 2325 GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total)); 2326 2327 return i915_live_subtests(tests, i915); 2328 } 2329