1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/list_sort.h> 26 #include <linux/prime_numbers.h> 27 28 #include "gem/i915_gem_context.h" 29 #include "gem/i915_gem_internal.h" 30 #include "gem/i915_gem_region.h" 31 #include "gem/selftests/mock_context.h" 32 #include "gt/intel_context.h" 33 #include "gt/intel_gpu_commands.h" 34 #include "gt/intel_gtt.h" 35 36 #include "i915_random.h" 37 #include "i915_selftest.h" 38 #include "i915_vma_resource.h" 39 40 #include "mock_drm.h" 41 #include "mock_gem_device.h" 42 #include "mock_gtt.h" 43 #include "igt_flush_test.h" 44 45 static void cleanup_freed_objects(struct drm_i915_private *i915) 46 { 47 i915_gem_drain_freed_objects(i915); 48 } 49 50 static void fake_free_pages(struct drm_i915_gem_object *obj, 51 struct sg_table *pages) 52 { 53 sg_free_table(pages); 54 kfree(pages); 55 } 56 57 static int fake_get_pages(struct drm_i915_gem_object *obj) 58 { 59 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 60 #define PFN_BIAS 0x1000 61 struct sg_table *pages; 62 struct scatterlist *sg; 63 unsigned int sg_page_sizes; 64 typeof(obj->base.size) rem; 65 66 pages = kmalloc(sizeof(*pages), GFP); 67 if (!pages) 68 return -ENOMEM; 69 70 rem = round_up(obj->base.size, BIT(31)) >> 31; 71 if (sg_alloc_table(pages, rem, GFP)) { 72 kfree(pages); 73 return -ENOMEM; 74 } 75 76 sg_page_sizes = 0; 77 rem = obj->base.size; 78 for (sg = pages->sgl; sg; sg = sg_next(sg)) { 79 unsigned long len = min_t(typeof(rem), rem, BIT(31)); 80 81 GEM_BUG_ON(!len); 82 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0); 83 sg_dma_address(sg) = page_to_phys(sg_page(sg)); 84 sg_dma_len(sg) = len; 85 sg_page_sizes |= len; 86 87 rem -= len; 88 } 89 GEM_BUG_ON(rem); 90 91 __i915_gem_object_set_pages(obj, pages, sg_page_sizes); 92 93 return 0; 94 #undef GFP 95 } 96 97 static void fake_put_pages(struct drm_i915_gem_object *obj, 98 struct sg_table *pages) 99 { 100 fake_free_pages(obj, pages); 101 obj->mm.dirty = false; 102 } 103 104 static const struct drm_i915_gem_object_ops fake_ops = { 105 .name = "fake-gem", 106 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 107 .get_pages = fake_get_pages, 108 .put_pages = fake_put_pages, 109 }; 110 111 static struct drm_i915_gem_object * 112 fake_dma_object(struct drm_i915_private *i915, u64 size) 113 { 114 static struct lock_class_key lock_class; 115 struct drm_i915_gem_object *obj; 116 117 GEM_BUG_ON(!size); 118 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 119 120 if (overflows_type(size, obj->base.size)) 121 return ERR_PTR(-E2BIG); 122 123 obj = i915_gem_object_alloc(); 124 if (!obj) 125 goto err; 126 127 drm_gem_private_object_init(&i915->drm, &obj->base, size); 128 i915_gem_object_init(obj, &fake_ops, &lock_class, 0); 129 130 i915_gem_object_set_volatile(obj); 131 132 obj->write_domain = I915_GEM_DOMAIN_CPU; 133 obj->read_domains = I915_GEM_DOMAIN_CPU; 134 obj->cache_level = I915_CACHE_NONE; 135 136 /* Preallocate the "backing storage" */ 137 if (i915_gem_object_pin_pages_unlocked(obj)) 138 goto err_obj; 139 140 i915_gem_object_unpin_pages(obj); 141 return obj; 142 143 err_obj: 144 i915_gem_object_put(obj); 145 err: 146 return ERR_PTR(-ENOMEM); 147 } 148 149 static int igt_ppgtt_alloc(void *arg) 150 { 151 struct drm_i915_private *dev_priv = arg; 152 struct i915_ppgtt *ppgtt; 153 struct i915_gem_ww_ctx ww; 154 u64 size, last, limit; 155 int err = 0; 156 157 /* Allocate a ppggt and try to fill the entire range */ 158 159 if (!HAS_PPGTT(dev_priv)) 160 return 0; 161 162 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0); 163 if (IS_ERR(ppgtt)) 164 return PTR_ERR(ppgtt); 165 166 if (!ppgtt->vm.allocate_va_range) 167 goto err_ppgtt_cleanup; 168 169 /* 170 * While we only allocate the page tables here and so we could 171 * address a much larger GTT than we could actually fit into 172 * RAM, a practical limit is the amount of physical pages in the system. 173 * This should ensure that we do not run into the oomkiller during 174 * the test and take down the machine wilfully. 175 */ 176 limit = totalram_pages() << PAGE_SHIFT; 177 limit = min(ppgtt->vm.total, limit); 178 179 i915_gem_ww_ctx_init(&ww, false); 180 retry: 181 err = i915_vm_lock_objects(&ppgtt->vm, &ww); 182 if (err) 183 goto err_ppgtt_cleanup; 184 185 /* Check we can allocate the entire range */ 186 for (size = 4096; size <= limit; size <<= 2) { 187 struct i915_vm_pt_stash stash = {}; 188 189 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size); 190 if (err) 191 goto err_ppgtt_cleanup; 192 193 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 194 if (err) { 195 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 196 goto err_ppgtt_cleanup; 197 } 198 199 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size); 200 cond_resched(); 201 202 ppgtt->vm.clear_range(&ppgtt->vm, 0, size); 203 204 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 205 } 206 207 /* Check we can incrementally allocate the entire range */ 208 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) { 209 struct i915_vm_pt_stash stash = {}; 210 211 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last); 212 if (err) 213 goto err_ppgtt_cleanup; 214 215 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 216 if (err) { 217 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 218 goto err_ppgtt_cleanup; 219 } 220 221 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 222 last, size - last); 223 cond_resched(); 224 225 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 226 } 227 228 err_ppgtt_cleanup: 229 if (err == -EDEADLK) { 230 err = i915_gem_ww_ctx_backoff(&ww); 231 if (!err) 232 goto retry; 233 } 234 i915_gem_ww_ctx_fini(&ww); 235 236 i915_vm_put(&ppgtt->vm); 237 return err; 238 } 239 240 static int lowlevel_hole(struct i915_address_space *vm, 241 u64 hole_start, u64 hole_end, 242 unsigned long end_time) 243 { 244 const unsigned int min_alignment = 245 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 246 I915_RND_STATE(seed_prng); 247 struct i915_vma_resource *mock_vma_res; 248 unsigned int size; 249 250 mock_vma_res = kzalloc(sizeof(*mock_vma_res), GFP_KERNEL); 251 if (!mock_vma_res) 252 return -ENOMEM; 253 254 /* Keep creating larger objects until one cannot fit into the hole */ 255 for (size = 12; (hole_end - hole_start) >> size; size++) { 256 I915_RND_SUBSTATE(prng, seed_prng); 257 struct drm_i915_gem_object *obj; 258 unsigned int *order, count, n; 259 u64 hole_size, aligned_size; 260 261 aligned_size = max_t(u32, ilog2(min_alignment), size); 262 hole_size = (hole_end - hole_start) >> aligned_size; 263 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 264 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 265 count = hole_size >> 1; 266 if (!count) { 267 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 268 __func__, hole_start, hole_end, size, hole_size); 269 break; 270 } 271 272 do { 273 order = i915_random_order(count, &prng); 274 if (order) 275 break; 276 } while (count >>= 1); 277 if (!count) { 278 kfree(mock_vma_res); 279 return -ENOMEM; 280 } 281 GEM_BUG_ON(!order); 282 283 GEM_BUG_ON(count * BIT_ULL(aligned_size) > vm->total); 284 GEM_BUG_ON(hole_start + count * BIT_ULL(aligned_size) > hole_end); 285 286 /* Ignore allocation failures (i.e. don't report them as 287 * a test failure) as we are purposefully allocating very 288 * large objects without checking that we have sufficient 289 * memory. We expect to hit -ENOMEM. 290 */ 291 292 obj = fake_dma_object(vm->i915, BIT_ULL(size)); 293 if (IS_ERR(obj)) { 294 kfree(order); 295 break; 296 } 297 298 GEM_BUG_ON(obj->base.size != BIT_ULL(size)); 299 300 if (i915_gem_object_pin_pages_unlocked(obj)) { 301 i915_gem_object_put(obj); 302 kfree(order); 303 break; 304 } 305 306 for (n = 0; n < count; n++) { 307 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 308 intel_wakeref_t wakeref; 309 310 GEM_BUG_ON(addr + BIT_ULL(aligned_size) > vm->total); 311 312 if (igt_timeout(end_time, 313 "%s timed out before %d/%d\n", 314 __func__, n, count)) { 315 hole_end = hole_start; /* quit */ 316 break; 317 } 318 319 if (vm->allocate_va_range) { 320 struct i915_vm_pt_stash stash = {}; 321 struct i915_gem_ww_ctx ww; 322 int err; 323 324 i915_gem_ww_ctx_init(&ww, false); 325 retry: 326 err = i915_vm_lock_objects(vm, &ww); 327 if (err) 328 goto alloc_vm_end; 329 330 err = -ENOMEM; 331 if (i915_vm_alloc_pt_stash(vm, &stash, 332 BIT_ULL(size))) 333 goto alloc_vm_end; 334 335 err = i915_vm_map_pt_stash(vm, &stash); 336 if (!err) 337 vm->allocate_va_range(vm, &stash, 338 addr, BIT_ULL(size)); 339 i915_vm_free_pt_stash(vm, &stash); 340 alloc_vm_end: 341 if (err == -EDEADLK) { 342 err = i915_gem_ww_ctx_backoff(&ww); 343 if (!err) 344 goto retry; 345 } 346 i915_gem_ww_ctx_fini(&ww); 347 348 if (err) 349 break; 350 } 351 352 mock_vma_res->bi.pages = obj->mm.pages; 353 mock_vma_res->node_size = BIT_ULL(aligned_size); 354 mock_vma_res->start = addr; 355 356 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) 357 vm->insert_entries(vm, mock_vma_res, 358 I915_CACHE_NONE, 0); 359 } 360 count = n; 361 362 i915_random_reorder(order, count, &prng); 363 for (n = 0; n < count; n++) { 364 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 365 intel_wakeref_t wakeref; 366 367 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 368 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) 369 vm->clear_range(vm, addr, BIT_ULL(size)); 370 } 371 372 i915_gem_object_unpin_pages(obj); 373 i915_gem_object_put(obj); 374 375 kfree(order); 376 377 cleanup_freed_objects(vm->i915); 378 } 379 380 kfree(mock_vma_res); 381 return 0; 382 } 383 384 static void close_object_list(struct list_head *objects, 385 struct i915_address_space *vm) 386 { 387 struct drm_i915_gem_object *obj, *on; 388 int ignored; 389 390 list_for_each_entry_safe(obj, on, objects, st_link) { 391 struct i915_vma *vma; 392 393 vma = i915_vma_instance(obj, vm, NULL); 394 if (!IS_ERR(vma)) 395 ignored = i915_vma_unbind_unlocked(vma); 396 397 list_del(&obj->st_link); 398 i915_gem_object_put(obj); 399 } 400 } 401 402 static int fill_hole(struct i915_address_space *vm, 403 u64 hole_start, u64 hole_end, 404 unsigned long end_time) 405 { 406 const u64 hole_size = hole_end - hole_start; 407 struct drm_i915_gem_object *obj; 408 const unsigned int min_alignment = 409 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 410 const unsigned long max_pages = 411 min_t(u64, ULONG_MAX - 1, (hole_size / 2) >> ilog2(min_alignment)); 412 const unsigned long max_step = max(int_sqrt(max_pages), 2UL); 413 unsigned long npages, prime, flags; 414 struct i915_vma *vma; 415 LIST_HEAD(objects); 416 int err; 417 418 /* Try binding many VMA working inwards from either edge */ 419 420 flags = PIN_OFFSET_FIXED | PIN_USER; 421 if (i915_is_ggtt(vm)) 422 flags |= PIN_GLOBAL; 423 424 for_each_prime_number_from(prime, 2, max_step) { 425 for (npages = 1; npages <= max_pages; npages *= prime) { 426 const u64 full_size = npages << PAGE_SHIFT; 427 const struct { 428 const char *name; 429 u64 offset; 430 int step; 431 } phases[] = { 432 { "top-down", hole_end, -1, }, 433 { "bottom-up", hole_start, 1, }, 434 { } 435 }, *p; 436 437 obj = fake_dma_object(vm->i915, full_size); 438 if (IS_ERR(obj)) 439 break; 440 441 list_add(&obj->st_link, &objects); 442 443 /* Align differing sized objects against the edges, and 444 * check we don't walk off into the void when binding 445 * them into the GTT. 446 */ 447 for (p = phases; p->name; p++) { 448 u64 offset; 449 450 offset = p->offset; 451 list_for_each_entry(obj, &objects, st_link) { 452 u64 aligned_size = round_up(obj->base.size, 453 min_alignment); 454 455 vma = i915_vma_instance(obj, vm, NULL); 456 if (IS_ERR(vma)) 457 continue; 458 459 if (p->step < 0) { 460 if (offset < hole_start + aligned_size) 461 break; 462 offset -= aligned_size; 463 } 464 465 err = i915_vma_pin(vma, 0, 0, offset | flags); 466 if (err) { 467 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 468 __func__, p->name, err, npages, prime, offset); 469 goto err; 470 } 471 472 if (!drm_mm_node_allocated(&vma->node) || 473 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 474 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 475 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 476 offset); 477 err = -EINVAL; 478 goto err; 479 } 480 481 i915_vma_unpin(vma); 482 483 if (p->step > 0) { 484 if (offset + aligned_size > hole_end) 485 break; 486 offset += aligned_size; 487 } 488 } 489 490 offset = p->offset; 491 list_for_each_entry(obj, &objects, st_link) { 492 u64 aligned_size = round_up(obj->base.size, 493 min_alignment); 494 495 vma = i915_vma_instance(obj, vm, NULL); 496 if (IS_ERR(vma)) 497 continue; 498 499 if (p->step < 0) { 500 if (offset < hole_start + aligned_size) 501 break; 502 offset -= aligned_size; 503 } 504 505 if (!drm_mm_node_allocated(&vma->node) || 506 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 507 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n", 508 __func__, p->name, vma->node.start, vma->node.size, 509 offset); 510 err = -EINVAL; 511 goto err; 512 } 513 514 err = i915_vma_unbind_unlocked(vma); 515 if (err) { 516 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n", 517 __func__, p->name, vma->node.start, vma->node.size, 518 err); 519 goto err; 520 } 521 522 if (p->step > 0) { 523 if (offset + aligned_size > hole_end) 524 break; 525 offset += aligned_size; 526 } 527 } 528 529 offset = p->offset; 530 list_for_each_entry_reverse(obj, &objects, st_link) { 531 u64 aligned_size = round_up(obj->base.size, 532 min_alignment); 533 534 vma = i915_vma_instance(obj, vm, NULL); 535 if (IS_ERR(vma)) 536 continue; 537 538 if (p->step < 0) { 539 if (offset < hole_start + aligned_size) 540 break; 541 offset -= aligned_size; 542 } 543 544 err = i915_vma_pin(vma, 0, 0, offset | flags); 545 if (err) { 546 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 547 __func__, p->name, err, npages, prime, offset); 548 goto err; 549 } 550 551 if (!drm_mm_node_allocated(&vma->node) || 552 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 553 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 554 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 555 offset); 556 err = -EINVAL; 557 goto err; 558 } 559 560 i915_vma_unpin(vma); 561 562 if (p->step > 0) { 563 if (offset + aligned_size > hole_end) 564 break; 565 offset += aligned_size; 566 } 567 } 568 569 offset = p->offset; 570 list_for_each_entry_reverse(obj, &objects, st_link) { 571 u64 aligned_size = round_up(obj->base.size, 572 min_alignment); 573 574 vma = i915_vma_instance(obj, vm, NULL); 575 if (IS_ERR(vma)) 576 continue; 577 578 if (p->step < 0) { 579 if (offset < hole_start + aligned_size) 580 break; 581 offset -= aligned_size; 582 } 583 584 if (!drm_mm_node_allocated(&vma->node) || 585 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 586 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 587 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 588 offset); 589 err = -EINVAL; 590 goto err; 591 } 592 593 err = i915_vma_unbind_unlocked(vma); 594 if (err) { 595 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n", 596 __func__, p->name, vma->node.start, vma->node.size, 597 err); 598 goto err; 599 } 600 601 if (p->step > 0) { 602 if (offset + aligned_size > hole_end) 603 break; 604 offset += aligned_size; 605 } 606 } 607 } 608 609 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n", 610 __func__, npages, prime)) { 611 err = -EINTR; 612 goto err; 613 } 614 } 615 616 close_object_list(&objects, vm); 617 cleanup_freed_objects(vm->i915); 618 } 619 620 return 0; 621 622 err: 623 close_object_list(&objects, vm); 624 return err; 625 } 626 627 static int walk_hole(struct i915_address_space *vm, 628 u64 hole_start, u64 hole_end, 629 unsigned long end_time) 630 { 631 const u64 hole_size = hole_end - hole_start; 632 const unsigned long max_pages = 633 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT); 634 unsigned long min_alignment; 635 unsigned long flags; 636 u64 size; 637 638 /* Try binding a single VMA in different positions within the hole */ 639 640 flags = PIN_OFFSET_FIXED | PIN_USER; 641 if (i915_is_ggtt(vm)) 642 flags |= PIN_GLOBAL; 643 644 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 645 646 for_each_prime_number_from(size, 1, max_pages) { 647 struct drm_i915_gem_object *obj; 648 struct i915_vma *vma; 649 u64 addr; 650 int err = 0; 651 652 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT); 653 if (IS_ERR(obj)) 654 break; 655 656 vma = i915_vma_instance(obj, vm, NULL); 657 if (IS_ERR(vma)) { 658 err = PTR_ERR(vma); 659 goto err_put; 660 } 661 662 for (addr = hole_start; 663 addr + obj->base.size < hole_end; 664 addr += round_up(obj->base.size, min_alignment)) { 665 err = i915_vma_pin(vma, 0, 0, addr | flags); 666 if (err) { 667 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", 668 __func__, addr, vma->size, 669 hole_start, hole_end, err); 670 goto err_put; 671 } 672 i915_vma_unpin(vma); 673 674 if (!drm_mm_node_allocated(&vma->node) || 675 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 676 pr_err("%s incorrect at %llx + %llx\n", 677 __func__, addr, vma->size); 678 err = -EINVAL; 679 goto err_put; 680 } 681 682 err = i915_vma_unbind_unlocked(vma); 683 if (err) { 684 pr_err("%s unbind failed at %llx + %llx with err=%d\n", 685 __func__, addr, vma->size, err); 686 goto err_put; 687 } 688 689 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 690 691 if (igt_timeout(end_time, 692 "%s timed out at %llx\n", 693 __func__, addr)) { 694 err = -EINTR; 695 goto err_put; 696 } 697 } 698 699 err_put: 700 i915_gem_object_put(obj); 701 if (err) 702 return err; 703 704 cleanup_freed_objects(vm->i915); 705 } 706 707 return 0; 708 } 709 710 static int pot_hole(struct i915_address_space *vm, 711 u64 hole_start, u64 hole_end, 712 unsigned long end_time) 713 { 714 struct drm_i915_gem_object *obj; 715 struct i915_vma *vma; 716 unsigned int min_alignment; 717 unsigned long flags; 718 unsigned int pot; 719 int err = 0; 720 721 flags = PIN_OFFSET_FIXED | PIN_USER; 722 if (i915_is_ggtt(vm)) 723 flags |= PIN_GLOBAL; 724 725 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 726 727 obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE); 728 if (IS_ERR(obj)) 729 return PTR_ERR(obj); 730 731 vma = i915_vma_instance(obj, vm, NULL); 732 if (IS_ERR(vma)) { 733 err = PTR_ERR(vma); 734 goto err_obj; 735 } 736 737 /* Insert a pair of pages across every pot boundary within the hole */ 738 for (pot = fls64(hole_end - 1) - 1; 739 pot > ilog2(2 * min_alignment); 740 pot--) { 741 u64 step = BIT_ULL(pot); 742 u64 addr; 743 744 for (addr = round_up(hole_start + min_alignment, step) - min_alignment; 745 addr <= round_down(hole_end - (2 * min_alignment), step) - min_alignment; 746 addr += step) { 747 err = i915_vma_pin(vma, 0, 0, addr | flags); 748 if (err) { 749 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n", 750 __func__, 751 addr, 752 hole_start, hole_end, 753 err); 754 goto err_obj; 755 } 756 757 if (!drm_mm_node_allocated(&vma->node) || 758 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 759 pr_err("%s incorrect at %llx + %llx\n", 760 __func__, addr, vma->size); 761 i915_vma_unpin(vma); 762 err = i915_vma_unbind_unlocked(vma); 763 err = -EINVAL; 764 goto err_obj; 765 } 766 767 i915_vma_unpin(vma); 768 err = i915_vma_unbind_unlocked(vma); 769 GEM_BUG_ON(err); 770 } 771 772 if (igt_timeout(end_time, 773 "%s timed out after %d/%d\n", 774 __func__, pot, fls64(hole_end - 1) - 1)) { 775 err = -EINTR; 776 goto err_obj; 777 } 778 } 779 780 err_obj: 781 i915_gem_object_put(obj); 782 return err; 783 } 784 785 static int drunk_hole(struct i915_address_space *vm, 786 u64 hole_start, u64 hole_end, 787 unsigned long end_time) 788 { 789 I915_RND_STATE(prng); 790 unsigned int min_alignment; 791 unsigned int size; 792 unsigned long flags; 793 794 flags = PIN_OFFSET_FIXED | PIN_USER; 795 if (i915_is_ggtt(vm)) 796 flags |= PIN_GLOBAL; 797 798 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 799 800 /* Keep creating larger objects until one cannot fit into the hole */ 801 for (size = 12; (hole_end - hole_start) >> size; size++) { 802 struct drm_i915_gem_object *obj; 803 unsigned int *order, count, n; 804 struct i915_vma *vma; 805 u64 hole_size, aligned_size; 806 int err = -ENODEV; 807 808 aligned_size = max_t(u32, ilog2(min_alignment), size); 809 hole_size = (hole_end - hole_start) >> aligned_size; 810 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 811 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 812 count = hole_size >> 1; 813 if (!count) { 814 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 815 __func__, hole_start, hole_end, size, hole_size); 816 break; 817 } 818 819 do { 820 order = i915_random_order(count, &prng); 821 if (order) 822 break; 823 } while (count >>= 1); 824 if (!count) 825 return -ENOMEM; 826 GEM_BUG_ON(!order); 827 828 /* Ignore allocation failures (i.e. don't report them as 829 * a test failure) as we are purposefully allocating very 830 * large objects without checking that we have sufficient 831 * memory. We expect to hit -ENOMEM. 832 */ 833 834 obj = fake_dma_object(vm->i915, BIT_ULL(size)); 835 if (IS_ERR(obj)) { 836 kfree(order); 837 break; 838 } 839 840 vma = i915_vma_instance(obj, vm, NULL); 841 if (IS_ERR(vma)) { 842 err = PTR_ERR(vma); 843 goto err_obj; 844 } 845 846 GEM_BUG_ON(vma->size != BIT_ULL(size)); 847 848 for (n = 0; n < count; n++) { 849 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 850 851 err = i915_vma_pin(vma, 0, 0, addr | flags); 852 if (err) { 853 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 854 __func__, 855 addr, BIT_ULL(size), 856 hole_start, hole_end, 857 err); 858 goto err_obj; 859 } 860 861 if (!drm_mm_node_allocated(&vma->node) || 862 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 863 pr_err("%s incorrect at %llx + %llx\n", 864 __func__, addr, BIT_ULL(size)); 865 i915_vma_unpin(vma); 866 err = i915_vma_unbind_unlocked(vma); 867 err = -EINVAL; 868 goto err_obj; 869 } 870 871 i915_vma_unpin(vma); 872 err = i915_vma_unbind_unlocked(vma); 873 GEM_BUG_ON(err); 874 875 if (igt_timeout(end_time, 876 "%s timed out after %d/%d\n", 877 __func__, n, count)) { 878 err = -EINTR; 879 goto err_obj; 880 } 881 } 882 883 err_obj: 884 i915_gem_object_put(obj); 885 kfree(order); 886 if (err) 887 return err; 888 889 cleanup_freed_objects(vm->i915); 890 } 891 892 return 0; 893 } 894 895 static int __shrink_hole(struct i915_address_space *vm, 896 u64 hole_start, u64 hole_end, 897 unsigned long end_time) 898 { 899 struct drm_i915_gem_object *obj; 900 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 901 unsigned int min_alignment; 902 unsigned int order = 12; 903 LIST_HEAD(objects); 904 int err = 0; 905 u64 addr; 906 907 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 908 909 /* Keep creating larger objects until one cannot fit into the hole */ 910 for (addr = hole_start; addr < hole_end; ) { 911 struct i915_vma *vma; 912 u64 size = BIT_ULL(order++); 913 914 size = min(size, hole_end - addr); 915 obj = fake_dma_object(vm->i915, size); 916 if (IS_ERR(obj)) { 917 err = PTR_ERR(obj); 918 break; 919 } 920 921 list_add(&obj->st_link, &objects); 922 923 vma = i915_vma_instance(obj, vm, NULL); 924 if (IS_ERR(vma)) { 925 err = PTR_ERR(vma); 926 break; 927 } 928 929 GEM_BUG_ON(vma->size != size); 930 931 err = i915_vma_pin(vma, 0, 0, addr | flags); 932 if (err) { 933 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 934 __func__, addr, size, hole_start, hole_end, err); 935 break; 936 } 937 938 if (!drm_mm_node_allocated(&vma->node) || 939 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 940 pr_err("%s incorrect at %llx + %llx\n", 941 __func__, addr, size); 942 i915_vma_unpin(vma); 943 err = i915_vma_unbind_unlocked(vma); 944 err = -EINVAL; 945 break; 946 } 947 948 i915_vma_unpin(vma); 949 addr += round_up(size, min_alignment); 950 951 /* 952 * Since we are injecting allocation faults at random intervals, 953 * wait for this allocation to complete before we change the 954 * faultinjection. 955 */ 956 err = i915_vma_sync(vma); 957 if (err) 958 break; 959 960 if (igt_timeout(end_time, 961 "%s timed out at ofset %llx [%llx - %llx]\n", 962 __func__, addr, hole_start, hole_end)) { 963 err = -EINTR; 964 break; 965 } 966 } 967 968 close_object_list(&objects, vm); 969 cleanup_freed_objects(vm->i915); 970 return err; 971 } 972 973 static int shrink_hole(struct i915_address_space *vm, 974 u64 hole_start, u64 hole_end, 975 unsigned long end_time) 976 { 977 unsigned long prime; 978 int err; 979 980 vm->fault_attr.probability = 999; 981 atomic_set(&vm->fault_attr.times, -1); 982 983 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) { 984 vm->fault_attr.interval = prime; 985 err = __shrink_hole(vm, hole_start, hole_end, end_time); 986 if (err) 987 break; 988 } 989 990 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 991 992 return err; 993 } 994 995 static int shrink_boom(struct i915_address_space *vm, 996 u64 hole_start, u64 hole_end, 997 unsigned long end_time) 998 { 999 unsigned int sizes[] = { SZ_2M, SZ_1G }; 1000 struct drm_i915_gem_object *purge; 1001 struct drm_i915_gem_object *explode; 1002 int err; 1003 int i; 1004 1005 /* 1006 * Catch the case which shrink_hole seems to miss. The setup here 1007 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while 1008 * ensuring that all vma assiocated with the respective pd/pdp are 1009 * unpinned at the time. 1010 */ 1011 1012 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1013 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1014 unsigned int size = sizes[i]; 1015 struct i915_vma *vma; 1016 1017 purge = fake_dma_object(vm->i915, size); 1018 if (IS_ERR(purge)) 1019 return PTR_ERR(purge); 1020 1021 vma = i915_vma_instance(purge, vm, NULL); 1022 if (IS_ERR(vma)) { 1023 err = PTR_ERR(vma); 1024 goto err_purge; 1025 } 1026 1027 err = i915_vma_pin(vma, 0, 0, flags); 1028 if (err) 1029 goto err_purge; 1030 1031 /* Should now be ripe for purging */ 1032 i915_vma_unpin(vma); 1033 1034 explode = fake_dma_object(vm->i915, size); 1035 if (IS_ERR(explode)) { 1036 err = PTR_ERR(explode); 1037 goto err_purge; 1038 } 1039 1040 vm->fault_attr.probability = 100; 1041 vm->fault_attr.interval = 1; 1042 atomic_set(&vm->fault_attr.times, -1); 1043 1044 vma = i915_vma_instance(explode, vm, NULL); 1045 if (IS_ERR(vma)) { 1046 err = PTR_ERR(vma); 1047 goto err_explode; 1048 } 1049 1050 err = i915_vma_pin(vma, 0, 0, flags | size); 1051 if (err) 1052 goto err_explode; 1053 1054 i915_vma_unpin(vma); 1055 1056 i915_gem_object_put(purge); 1057 i915_gem_object_put(explode); 1058 1059 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 1060 cleanup_freed_objects(vm->i915); 1061 } 1062 1063 return 0; 1064 1065 err_explode: 1066 i915_gem_object_put(explode); 1067 err_purge: 1068 i915_gem_object_put(purge); 1069 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 1070 return err; 1071 } 1072 1073 static int misaligned_case(struct i915_address_space *vm, struct intel_memory_region *mr, 1074 u64 addr, u64 size, unsigned long flags) 1075 { 1076 struct drm_i915_gem_object *obj; 1077 struct i915_vma *vma; 1078 int err = 0; 1079 u64 expected_vma_size, expected_node_size; 1080 bool is_stolen = mr->type == INTEL_MEMORY_STOLEN_SYSTEM || 1081 mr->type == INTEL_MEMORY_STOLEN_LOCAL; 1082 1083 obj = i915_gem_object_create_region(mr, size, 0, 0); 1084 if (IS_ERR(obj)) { 1085 /* if iGVT-g or DMAR is active, stolen mem will be uninitialized */ 1086 if (PTR_ERR(obj) == -ENODEV && is_stolen) 1087 return 0; 1088 return PTR_ERR(obj); 1089 } 1090 1091 vma = i915_vma_instance(obj, vm, NULL); 1092 if (IS_ERR(vma)) { 1093 err = PTR_ERR(vma); 1094 goto err_put; 1095 } 1096 1097 err = i915_vma_pin(vma, 0, 0, addr | flags); 1098 if (err) 1099 goto err_put; 1100 i915_vma_unpin(vma); 1101 1102 if (!drm_mm_node_allocated(&vma->node)) { 1103 err = -EINVAL; 1104 goto err_put; 1105 } 1106 1107 if (i915_vma_misplaced(vma, 0, 0, addr | flags)) { 1108 err = -EINVAL; 1109 goto err_put; 1110 } 1111 1112 expected_vma_size = round_up(size, 1 << (ffs(vma->resource->page_sizes_gtt) - 1)); 1113 expected_node_size = expected_vma_size; 1114 1115 if (NEEDS_COMPACT_PT(vm->i915) && i915_gem_object_is_lmem(obj)) { 1116 /* compact-pt should expand lmem node to 2MB */ 1117 expected_vma_size = round_up(size, I915_GTT_PAGE_SIZE_64K); 1118 expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_2M); 1119 } 1120 1121 if (vma->size != expected_vma_size || vma->node.size != expected_node_size) { 1122 err = i915_vma_unbind_unlocked(vma); 1123 err = -EBADSLT; 1124 goto err_put; 1125 } 1126 1127 err = i915_vma_unbind_unlocked(vma); 1128 if (err) 1129 goto err_put; 1130 1131 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 1132 1133 err_put: 1134 i915_gem_object_put(obj); 1135 cleanup_freed_objects(vm->i915); 1136 return err; 1137 } 1138 1139 static int misaligned_pin(struct i915_address_space *vm, 1140 u64 hole_start, u64 hole_end, 1141 unsigned long end_time) 1142 { 1143 struct intel_memory_region *mr; 1144 enum intel_region_id id; 1145 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 1146 int err = 0; 1147 u64 hole_size = hole_end - hole_start; 1148 1149 if (i915_is_ggtt(vm)) 1150 flags |= PIN_GLOBAL; 1151 1152 for_each_memory_region(mr, vm->i915, id) { 1153 u64 min_alignment = i915_vm_min_alignment(vm, (enum intel_memory_type)id); 1154 u64 size = min_alignment; 1155 u64 addr = round_down(hole_start + (hole_size / 2), min_alignment); 1156 1157 /* avoid -ENOSPC on very small hole setups */ 1158 if (hole_size < 3 * min_alignment) 1159 continue; 1160 1161 /* we can't test < 4k alignment due to flags being encoded in lower bits */ 1162 if (min_alignment != I915_GTT_PAGE_SIZE_4K) { 1163 err = misaligned_case(vm, mr, addr + (min_alignment / 2), size, flags); 1164 /* misaligned should error with -EINVAL*/ 1165 if (!err) 1166 err = -EBADSLT; 1167 if (err != -EINVAL) 1168 return err; 1169 } 1170 1171 /* test for vma->size expansion to min page size */ 1172 err = misaligned_case(vm, mr, addr, PAGE_SIZE, flags); 1173 if (err) 1174 return err; 1175 1176 /* test for intermediate size not expanding vma->size for large alignments */ 1177 err = misaligned_case(vm, mr, addr, size / 2, flags); 1178 if (err) 1179 return err; 1180 } 1181 1182 return 0; 1183 } 1184 1185 static int exercise_ppgtt(struct drm_i915_private *dev_priv, 1186 int (*func)(struct i915_address_space *vm, 1187 u64 hole_start, u64 hole_end, 1188 unsigned long end_time)) 1189 { 1190 struct i915_ppgtt *ppgtt; 1191 IGT_TIMEOUT(end_time); 1192 struct file *file; 1193 int err; 1194 1195 if (!HAS_FULL_PPGTT(dev_priv)) 1196 return 0; 1197 1198 file = mock_file(dev_priv); 1199 if (IS_ERR(file)) 1200 return PTR_ERR(file); 1201 1202 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0); 1203 if (IS_ERR(ppgtt)) { 1204 err = PTR_ERR(ppgtt); 1205 goto out_free; 1206 } 1207 GEM_BUG_ON(offset_in_page(ppgtt->vm.total)); 1208 GEM_BUG_ON(!atomic_read(&ppgtt->vm.open)); 1209 1210 err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time); 1211 1212 i915_vm_put(&ppgtt->vm); 1213 1214 out_free: 1215 fput(file); 1216 return err; 1217 } 1218 1219 static int igt_ppgtt_fill(void *arg) 1220 { 1221 return exercise_ppgtt(arg, fill_hole); 1222 } 1223 1224 static int igt_ppgtt_walk(void *arg) 1225 { 1226 return exercise_ppgtt(arg, walk_hole); 1227 } 1228 1229 static int igt_ppgtt_pot(void *arg) 1230 { 1231 return exercise_ppgtt(arg, pot_hole); 1232 } 1233 1234 static int igt_ppgtt_drunk(void *arg) 1235 { 1236 return exercise_ppgtt(arg, drunk_hole); 1237 } 1238 1239 static int igt_ppgtt_lowlevel(void *arg) 1240 { 1241 return exercise_ppgtt(arg, lowlevel_hole); 1242 } 1243 1244 static int igt_ppgtt_shrink(void *arg) 1245 { 1246 return exercise_ppgtt(arg, shrink_hole); 1247 } 1248 1249 static int igt_ppgtt_shrink_boom(void *arg) 1250 { 1251 return exercise_ppgtt(arg, shrink_boom); 1252 } 1253 1254 static int igt_ppgtt_misaligned_pin(void *arg) 1255 { 1256 return exercise_ppgtt(arg, misaligned_pin); 1257 } 1258 1259 static int sort_holes(void *priv, const struct list_head *A, 1260 const struct list_head *B) 1261 { 1262 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack); 1263 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack); 1264 1265 if (a->start < b->start) 1266 return -1; 1267 else 1268 return 1; 1269 } 1270 1271 static int exercise_ggtt(struct drm_i915_private *i915, 1272 int (*func)(struct i915_address_space *vm, 1273 u64 hole_start, u64 hole_end, 1274 unsigned long end_time)) 1275 { 1276 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 1277 u64 hole_start, hole_end, last = 0; 1278 struct drm_mm_node *node; 1279 IGT_TIMEOUT(end_time); 1280 int err = 0; 1281 1282 restart: 1283 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes); 1284 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) { 1285 if (hole_start < last) 1286 continue; 1287 1288 if (ggtt->vm.mm.color_adjust) 1289 ggtt->vm.mm.color_adjust(node, 0, 1290 &hole_start, &hole_end); 1291 if (hole_start >= hole_end) 1292 continue; 1293 1294 err = func(&ggtt->vm, hole_start, hole_end, end_time); 1295 if (err) 1296 break; 1297 1298 /* As we have manipulated the drm_mm, the list may be corrupt */ 1299 last = hole_end; 1300 goto restart; 1301 } 1302 1303 return err; 1304 } 1305 1306 static int igt_ggtt_fill(void *arg) 1307 { 1308 return exercise_ggtt(arg, fill_hole); 1309 } 1310 1311 static int igt_ggtt_walk(void *arg) 1312 { 1313 return exercise_ggtt(arg, walk_hole); 1314 } 1315 1316 static int igt_ggtt_pot(void *arg) 1317 { 1318 return exercise_ggtt(arg, pot_hole); 1319 } 1320 1321 static int igt_ggtt_drunk(void *arg) 1322 { 1323 return exercise_ggtt(arg, drunk_hole); 1324 } 1325 1326 static int igt_ggtt_lowlevel(void *arg) 1327 { 1328 return exercise_ggtt(arg, lowlevel_hole); 1329 } 1330 1331 static int igt_ggtt_misaligned_pin(void *arg) 1332 { 1333 return exercise_ggtt(arg, misaligned_pin); 1334 } 1335 1336 static int igt_ggtt_page(void *arg) 1337 { 1338 const unsigned int count = PAGE_SIZE/sizeof(u32); 1339 I915_RND_STATE(prng); 1340 struct drm_i915_private *i915 = arg; 1341 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 1342 struct drm_i915_gem_object *obj; 1343 intel_wakeref_t wakeref; 1344 struct drm_mm_node tmp; 1345 unsigned int *order, n; 1346 int err; 1347 1348 if (!i915_ggtt_has_aperture(ggtt)) 1349 return 0; 1350 1351 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 1352 if (IS_ERR(obj)) 1353 return PTR_ERR(obj); 1354 1355 err = i915_gem_object_pin_pages_unlocked(obj); 1356 if (err) 1357 goto out_free; 1358 1359 memset(&tmp, 0, sizeof(tmp)); 1360 mutex_lock(&ggtt->vm.mutex); 1361 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp, 1362 count * PAGE_SIZE, 0, 1363 I915_COLOR_UNEVICTABLE, 1364 0, ggtt->mappable_end, 1365 DRM_MM_INSERT_LOW); 1366 mutex_unlock(&ggtt->vm.mutex); 1367 if (err) 1368 goto out_unpin; 1369 1370 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1371 1372 for (n = 0; n < count; n++) { 1373 u64 offset = tmp.start + n * PAGE_SIZE; 1374 1375 ggtt->vm.insert_page(&ggtt->vm, 1376 i915_gem_object_get_dma_address(obj, 0), 1377 offset, I915_CACHE_NONE, 0); 1378 } 1379 1380 order = i915_random_order(count, &prng); 1381 if (!order) { 1382 err = -ENOMEM; 1383 goto out_remove; 1384 } 1385 1386 for (n = 0; n < count; n++) { 1387 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1388 u32 __iomem *vaddr; 1389 1390 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1391 iowrite32(n, vaddr + n); 1392 io_mapping_unmap_atomic(vaddr); 1393 } 1394 intel_gt_flush_ggtt_writes(ggtt->vm.gt); 1395 1396 i915_random_reorder(order, count, &prng); 1397 for (n = 0; n < count; n++) { 1398 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1399 u32 __iomem *vaddr; 1400 u32 val; 1401 1402 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1403 val = ioread32(vaddr + n); 1404 io_mapping_unmap_atomic(vaddr); 1405 1406 if (val != n) { 1407 pr_err("insert page failed: found %d, expected %d\n", 1408 val, n); 1409 err = -EINVAL; 1410 break; 1411 } 1412 } 1413 1414 kfree(order); 1415 out_remove: 1416 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size); 1417 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1418 mutex_lock(&ggtt->vm.mutex); 1419 drm_mm_remove_node(&tmp); 1420 mutex_unlock(&ggtt->vm.mutex); 1421 out_unpin: 1422 i915_gem_object_unpin_pages(obj); 1423 out_free: 1424 i915_gem_object_put(obj); 1425 return err; 1426 } 1427 1428 static void track_vma_bind(struct i915_vma *vma) 1429 { 1430 struct drm_i915_gem_object *obj = vma->obj; 1431 1432 __i915_gem_object_pin_pages(obj); 1433 1434 GEM_BUG_ON(atomic_read(&vma->pages_count)); 1435 atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE); 1436 __i915_gem_object_pin_pages(obj); 1437 vma->pages = obj->mm.pages; 1438 vma->resource->bi.pages = vma->pages; 1439 1440 mutex_lock(&vma->vm->mutex); 1441 list_add_tail(&vma->vm_link, &vma->vm->bound_list); 1442 mutex_unlock(&vma->vm->mutex); 1443 } 1444 1445 static int exercise_mock(struct drm_i915_private *i915, 1446 int (*func)(struct i915_address_space *vm, 1447 u64 hole_start, u64 hole_end, 1448 unsigned long end_time)) 1449 { 1450 const u64 limit = totalram_pages() << PAGE_SHIFT; 1451 struct i915_address_space *vm; 1452 struct i915_gem_context *ctx; 1453 IGT_TIMEOUT(end_time); 1454 int err; 1455 1456 ctx = mock_context(i915, "mock"); 1457 if (!ctx) 1458 return -ENOMEM; 1459 1460 vm = i915_gem_context_get_eb_vm(ctx); 1461 err = func(vm, 0, min(vm->total, limit), end_time); 1462 i915_vm_put(vm); 1463 1464 mock_context_close(ctx); 1465 return err; 1466 } 1467 1468 static int igt_mock_fill(void *arg) 1469 { 1470 struct i915_ggtt *ggtt = arg; 1471 1472 return exercise_mock(ggtt->vm.i915, fill_hole); 1473 } 1474 1475 static int igt_mock_walk(void *arg) 1476 { 1477 struct i915_ggtt *ggtt = arg; 1478 1479 return exercise_mock(ggtt->vm.i915, walk_hole); 1480 } 1481 1482 static int igt_mock_pot(void *arg) 1483 { 1484 struct i915_ggtt *ggtt = arg; 1485 1486 return exercise_mock(ggtt->vm.i915, pot_hole); 1487 } 1488 1489 static int igt_mock_drunk(void *arg) 1490 { 1491 struct i915_ggtt *ggtt = arg; 1492 1493 return exercise_mock(ggtt->vm.i915, drunk_hole); 1494 } 1495 1496 static int reserve_gtt_with_resource(struct i915_vma *vma, u64 offset) 1497 { 1498 struct i915_address_space *vm = vma->vm; 1499 struct i915_vma_resource *vma_res; 1500 struct drm_i915_gem_object *obj = vma->obj; 1501 int err; 1502 1503 vma_res = i915_vma_resource_alloc(); 1504 if (IS_ERR(vma_res)) 1505 return PTR_ERR(vma_res); 1506 1507 mutex_lock(&vm->mutex); 1508 err = i915_gem_gtt_reserve(vm, NULL, &vma->node, obj->base.size, 1509 offset, 1510 obj->cache_level, 1511 0); 1512 if (!err) { 1513 i915_vma_resource_init_from_vma(vma_res, vma); 1514 vma->resource = vma_res; 1515 } else { 1516 kfree(vma_res); 1517 } 1518 mutex_unlock(&vm->mutex); 1519 1520 return err; 1521 } 1522 1523 static int igt_gtt_reserve(void *arg) 1524 { 1525 struct i915_ggtt *ggtt = arg; 1526 struct drm_i915_gem_object *obj, *on; 1527 I915_RND_STATE(prng); 1528 LIST_HEAD(objects); 1529 u64 total; 1530 int err = -ENODEV; 1531 1532 /* i915_gem_gtt_reserve() tries to reserve the precise range 1533 * for the node, and evicts if it has to. So our test checks that 1534 * it can give us the requsted space and prevent overlaps. 1535 */ 1536 1537 /* Start by filling the GGTT */ 1538 for (total = 0; 1539 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1540 total += 2 * I915_GTT_PAGE_SIZE) { 1541 struct i915_vma *vma; 1542 1543 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1544 2 * PAGE_SIZE); 1545 if (IS_ERR(obj)) { 1546 err = PTR_ERR(obj); 1547 goto out; 1548 } 1549 1550 err = i915_gem_object_pin_pages_unlocked(obj); 1551 if (err) { 1552 i915_gem_object_put(obj); 1553 goto out; 1554 } 1555 1556 list_add(&obj->st_link, &objects); 1557 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1558 if (IS_ERR(vma)) { 1559 err = PTR_ERR(vma); 1560 goto out; 1561 } 1562 1563 err = reserve_gtt_with_resource(vma, total); 1564 if (err) { 1565 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n", 1566 total, ggtt->vm.total, err); 1567 goto out; 1568 } 1569 track_vma_bind(vma); 1570 1571 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1572 if (vma->node.start != total || 1573 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1574 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1575 vma->node.start, vma->node.size, 1576 total, 2*I915_GTT_PAGE_SIZE); 1577 err = -EINVAL; 1578 goto out; 1579 } 1580 } 1581 1582 /* Now we start forcing evictions */ 1583 for (total = I915_GTT_PAGE_SIZE; 1584 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1585 total += 2 * I915_GTT_PAGE_SIZE) { 1586 struct i915_vma *vma; 1587 1588 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1589 2 * PAGE_SIZE); 1590 if (IS_ERR(obj)) { 1591 err = PTR_ERR(obj); 1592 goto out; 1593 } 1594 1595 err = i915_gem_object_pin_pages_unlocked(obj); 1596 if (err) { 1597 i915_gem_object_put(obj); 1598 goto out; 1599 } 1600 1601 list_add(&obj->st_link, &objects); 1602 1603 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1604 if (IS_ERR(vma)) { 1605 err = PTR_ERR(vma); 1606 goto out; 1607 } 1608 1609 err = reserve_gtt_with_resource(vma, total); 1610 if (err) { 1611 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n", 1612 total, ggtt->vm.total, err); 1613 goto out; 1614 } 1615 track_vma_bind(vma); 1616 1617 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1618 if (vma->node.start != total || 1619 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1620 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1621 vma->node.start, vma->node.size, 1622 total, 2*I915_GTT_PAGE_SIZE); 1623 err = -EINVAL; 1624 goto out; 1625 } 1626 } 1627 1628 /* And then try at random */ 1629 list_for_each_entry_safe(obj, on, &objects, st_link) { 1630 struct i915_vma *vma; 1631 u64 offset; 1632 1633 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1634 if (IS_ERR(vma)) { 1635 err = PTR_ERR(vma); 1636 goto out; 1637 } 1638 1639 err = i915_vma_unbind_unlocked(vma); 1640 if (err) { 1641 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1642 goto out; 1643 } 1644 1645 offset = igt_random_offset(&prng, 1646 0, ggtt->vm.total, 1647 2 * I915_GTT_PAGE_SIZE, 1648 I915_GTT_MIN_ALIGNMENT); 1649 1650 err = reserve_gtt_with_resource(vma, offset); 1651 if (err) { 1652 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n", 1653 total, ggtt->vm.total, err); 1654 goto out; 1655 } 1656 track_vma_bind(vma); 1657 1658 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1659 if (vma->node.start != offset || 1660 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1661 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1662 vma->node.start, vma->node.size, 1663 offset, 2*I915_GTT_PAGE_SIZE); 1664 err = -EINVAL; 1665 goto out; 1666 } 1667 } 1668 1669 out: 1670 list_for_each_entry_safe(obj, on, &objects, st_link) { 1671 i915_gem_object_unpin_pages(obj); 1672 i915_gem_object_put(obj); 1673 } 1674 return err; 1675 } 1676 1677 static int insert_gtt_with_resource(struct i915_vma *vma) 1678 { 1679 struct i915_address_space *vm = vma->vm; 1680 struct i915_vma_resource *vma_res; 1681 struct drm_i915_gem_object *obj = vma->obj; 1682 int err; 1683 1684 vma_res = i915_vma_resource_alloc(); 1685 if (IS_ERR(vma_res)) 1686 return PTR_ERR(vma_res); 1687 1688 mutex_lock(&vm->mutex); 1689 err = i915_gem_gtt_insert(vm, NULL, &vma->node, obj->base.size, 0, 1690 obj->cache_level, 0, vm->total, 0); 1691 if (!err) { 1692 i915_vma_resource_init_from_vma(vma_res, vma); 1693 vma->resource = vma_res; 1694 } else { 1695 kfree(vma_res); 1696 } 1697 mutex_unlock(&vm->mutex); 1698 1699 return err; 1700 } 1701 1702 static int igt_gtt_insert(void *arg) 1703 { 1704 struct i915_ggtt *ggtt = arg; 1705 struct drm_i915_gem_object *obj, *on; 1706 struct drm_mm_node tmp = {}; 1707 const struct invalid_insert { 1708 u64 size; 1709 u64 alignment; 1710 u64 start, end; 1711 } invalid_insert[] = { 1712 { 1713 ggtt->vm.total + I915_GTT_PAGE_SIZE, 0, 1714 0, ggtt->vm.total, 1715 }, 1716 { 1717 2*I915_GTT_PAGE_SIZE, 0, 1718 0, I915_GTT_PAGE_SIZE, 1719 }, 1720 { 1721 -(u64)I915_GTT_PAGE_SIZE, 0, 1722 0, 4*I915_GTT_PAGE_SIZE, 1723 }, 1724 { 1725 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE, 1726 0, 4*I915_GTT_PAGE_SIZE, 1727 }, 1728 { 1729 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1, 1730 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1, 1731 }, 1732 {} 1733 }, *ii; 1734 LIST_HEAD(objects); 1735 u64 total; 1736 int err = -ENODEV; 1737 1738 /* i915_gem_gtt_insert() tries to allocate some free space in the GTT 1739 * to the node, evicting if required. 1740 */ 1741 1742 /* Check a couple of obviously invalid requests */ 1743 for (ii = invalid_insert; ii->size; ii++) { 1744 mutex_lock(&ggtt->vm.mutex); 1745 err = i915_gem_gtt_insert(&ggtt->vm, NULL, &tmp, 1746 ii->size, ii->alignment, 1747 I915_COLOR_UNEVICTABLE, 1748 ii->start, ii->end, 1749 0); 1750 mutex_unlock(&ggtt->vm.mutex); 1751 if (err != -ENOSPC) { 1752 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n", 1753 ii->size, ii->alignment, ii->start, ii->end, 1754 err); 1755 return -EINVAL; 1756 } 1757 } 1758 1759 /* Start by filling the GGTT */ 1760 for (total = 0; 1761 total + I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1762 total += I915_GTT_PAGE_SIZE) { 1763 struct i915_vma *vma; 1764 1765 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1766 I915_GTT_PAGE_SIZE); 1767 if (IS_ERR(obj)) { 1768 err = PTR_ERR(obj); 1769 goto out; 1770 } 1771 1772 err = i915_gem_object_pin_pages_unlocked(obj); 1773 if (err) { 1774 i915_gem_object_put(obj); 1775 goto out; 1776 } 1777 1778 list_add(&obj->st_link, &objects); 1779 1780 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1781 if (IS_ERR(vma)) { 1782 err = PTR_ERR(vma); 1783 goto out; 1784 } 1785 1786 err = insert_gtt_with_resource(vma); 1787 if (err == -ENOSPC) { 1788 /* maxed out the GGTT space */ 1789 i915_gem_object_put(obj); 1790 break; 1791 } 1792 if (err) { 1793 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n", 1794 total, ggtt->vm.total, err); 1795 goto out; 1796 } 1797 track_vma_bind(vma); 1798 __i915_vma_pin(vma); 1799 1800 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1801 } 1802 1803 list_for_each_entry(obj, &objects, st_link) { 1804 struct i915_vma *vma; 1805 1806 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1807 if (IS_ERR(vma)) { 1808 err = PTR_ERR(vma); 1809 goto out; 1810 } 1811 1812 if (!drm_mm_node_allocated(&vma->node)) { 1813 pr_err("VMA was unexpectedly evicted!\n"); 1814 err = -EINVAL; 1815 goto out; 1816 } 1817 1818 __i915_vma_unpin(vma); 1819 } 1820 1821 /* If we then reinsert, we should find the same hole */ 1822 list_for_each_entry_safe(obj, on, &objects, st_link) { 1823 struct i915_vma *vma; 1824 u64 offset; 1825 1826 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1827 if (IS_ERR(vma)) { 1828 err = PTR_ERR(vma); 1829 goto out; 1830 } 1831 1832 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1833 offset = vma->node.start; 1834 1835 err = i915_vma_unbind_unlocked(vma); 1836 if (err) { 1837 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1838 goto out; 1839 } 1840 1841 err = insert_gtt_with_resource(vma); 1842 if (err) { 1843 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n", 1844 total, ggtt->vm.total, err); 1845 goto out; 1846 } 1847 track_vma_bind(vma); 1848 1849 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1850 if (vma->node.start != offset) { 1851 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n", 1852 offset, vma->node.start); 1853 err = -EINVAL; 1854 goto out; 1855 } 1856 } 1857 1858 /* And then force evictions */ 1859 for (total = 0; 1860 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1861 total += 2 * I915_GTT_PAGE_SIZE) { 1862 struct i915_vma *vma; 1863 1864 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1865 2 * I915_GTT_PAGE_SIZE); 1866 if (IS_ERR(obj)) { 1867 err = PTR_ERR(obj); 1868 goto out; 1869 } 1870 1871 err = i915_gem_object_pin_pages_unlocked(obj); 1872 if (err) { 1873 i915_gem_object_put(obj); 1874 goto out; 1875 } 1876 1877 list_add(&obj->st_link, &objects); 1878 1879 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1880 if (IS_ERR(vma)) { 1881 err = PTR_ERR(vma); 1882 goto out; 1883 } 1884 1885 err = insert_gtt_with_resource(vma); 1886 if (err) { 1887 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n", 1888 total, ggtt->vm.total, err); 1889 goto out; 1890 } 1891 track_vma_bind(vma); 1892 1893 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1894 } 1895 1896 out: 1897 list_for_each_entry_safe(obj, on, &objects, st_link) { 1898 i915_gem_object_unpin_pages(obj); 1899 i915_gem_object_put(obj); 1900 } 1901 return err; 1902 } 1903 1904 int i915_gem_gtt_mock_selftests(void) 1905 { 1906 static const struct i915_subtest tests[] = { 1907 SUBTEST(igt_mock_drunk), 1908 SUBTEST(igt_mock_walk), 1909 SUBTEST(igt_mock_pot), 1910 SUBTEST(igt_mock_fill), 1911 SUBTEST(igt_gtt_reserve), 1912 SUBTEST(igt_gtt_insert), 1913 }; 1914 struct drm_i915_private *i915; 1915 struct intel_gt *gt; 1916 int err; 1917 1918 i915 = mock_gem_device(); 1919 if (!i915) 1920 return -ENOMEM; 1921 1922 /* allocate the ggtt */ 1923 err = intel_gt_assign_ggtt(to_gt(i915)); 1924 if (err) 1925 goto out_put; 1926 1927 gt = to_gt(i915); 1928 1929 mock_init_ggtt(gt); 1930 1931 err = i915_subtests(tests, gt->ggtt); 1932 1933 mock_device_flush(i915); 1934 i915_gem_drain_freed_objects(i915); 1935 mock_fini_ggtt(gt->ggtt); 1936 1937 out_put: 1938 mock_destroy_device(i915); 1939 return err; 1940 } 1941 1942 static int context_sync(struct intel_context *ce) 1943 { 1944 struct i915_request *rq; 1945 long timeout; 1946 1947 rq = intel_context_create_request(ce); 1948 if (IS_ERR(rq)) 1949 return PTR_ERR(rq); 1950 1951 i915_request_get(rq); 1952 i915_request_add(rq); 1953 1954 timeout = i915_request_wait(rq, 0, HZ / 5); 1955 i915_request_put(rq); 1956 1957 return timeout < 0 ? -EIO : 0; 1958 } 1959 1960 static struct i915_request * 1961 submit_batch(struct intel_context *ce, u64 addr) 1962 { 1963 struct i915_request *rq; 1964 int err; 1965 1966 rq = intel_context_create_request(ce); 1967 if (IS_ERR(rq)) 1968 return rq; 1969 1970 err = 0; 1971 if (rq->engine->emit_init_breadcrumb) /* detect a hang */ 1972 err = rq->engine->emit_init_breadcrumb(rq); 1973 if (err == 0) 1974 err = rq->engine->emit_bb_start(rq, addr, 0, 0); 1975 1976 if (err == 0) 1977 i915_request_get(rq); 1978 i915_request_add(rq); 1979 1980 return err ? ERR_PTR(err) : rq; 1981 } 1982 1983 static u32 *spinner(u32 *batch, int i) 1984 { 1985 return batch + i * 64 / sizeof(*batch) + 4; 1986 } 1987 1988 static void end_spin(u32 *batch, int i) 1989 { 1990 *spinner(batch, i) = MI_BATCH_BUFFER_END; 1991 wmb(); 1992 } 1993 1994 static int igt_cs_tlb(void *arg) 1995 { 1996 const unsigned int count = PAGE_SIZE / 64; 1997 const unsigned int chunk_size = count * PAGE_SIZE; 1998 struct drm_i915_private *i915 = arg; 1999 struct drm_i915_gem_object *bbe, *act, *out; 2000 struct i915_gem_engines_iter it; 2001 struct i915_address_space *vm; 2002 struct i915_gem_context *ctx; 2003 struct intel_context *ce; 2004 struct i915_vma *vma; 2005 I915_RND_STATE(prng); 2006 struct file *file; 2007 unsigned int i; 2008 u32 *result; 2009 u32 *batch; 2010 int err = 0; 2011 2012 /* 2013 * Our mission here is to fool the hardware to execute something 2014 * from scratch as it has not seen the batch move (due to missing 2015 * the TLB invalidate). 2016 */ 2017 2018 file = mock_file(i915); 2019 if (IS_ERR(file)) 2020 return PTR_ERR(file); 2021 2022 ctx = live_context(i915, file); 2023 if (IS_ERR(ctx)) { 2024 err = PTR_ERR(ctx); 2025 goto out_unlock; 2026 } 2027 2028 vm = i915_gem_context_get_eb_vm(ctx); 2029 if (i915_is_ggtt(vm)) 2030 goto out_vm; 2031 2032 /* Create two pages; dummy we prefill the TLB, and intended */ 2033 bbe = i915_gem_object_create_internal(i915, PAGE_SIZE); 2034 if (IS_ERR(bbe)) { 2035 err = PTR_ERR(bbe); 2036 goto out_vm; 2037 } 2038 2039 batch = i915_gem_object_pin_map_unlocked(bbe, I915_MAP_WC); 2040 if (IS_ERR(batch)) { 2041 err = PTR_ERR(batch); 2042 goto out_put_bbe; 2043 } 2044 memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32)); 2045 i915_gem_object_flush_map(bbe); 2046 i915_gem_object_unpin_map(bbe); 2047 2048 act = i915_gem_object_create_internal(i915, PAGE_SIZE); 2049 if (IS_ERR(act)) { 2050 err = PTR_ERR(act); 2051 goto out_put_bbe; 2052 } 2053 2054 /* Track the execution of each request by writing into different slot */ 2055 batch = i915_gem_object_pin_map_unlocked(act, I915_MAP_WC); 2056 if (IS_ERR(batch)) { 2057 err = PTR_ERR(batch); 2058 goto out_put_act; 2059 } 2060 for (i = 0; i < count; i++) { 2061 u32 *cs = batch + i * 64 / sizeof(*cs); 2062 u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32); 2063 2064 GEM_BUG_ON(GRAPHICS_VER(i915) < 6); 2065 cs[0] = MI_STORE_DWORD_IMM_GEN4; 2066 if (GRAPHICS_VER(i915) >= 8) { 2067 cs[1] = lower_32_bits(addr); 2068 cs[2] = upper_32_bits(addr); 2069 cs[3] = i; 2070 cs[4] = MI_NOOP; 2071 cs[5] = MI_BATCH_BUFFER_START_GEN8; 2072 } else { 2073 cs[1] = 0; 2074 cs[2] = lower_32_bits(addr); 2075 cs[3] = i; 2076 cs[4] = MI_NOOP; 2077 cs[5] = MI_BATCH_BUFFER_START; 2078 } 2079 } 2080 2081 out = i915_gem_object_create_internal(i915, PAGE_SIZE); 2082 if (IS_ERR(out)) { 2083 err = PTR_ERR(out); 2084 goto out_put_batch; 2085 } 2086 i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED); 2087 2088 vma = i915_vma_instance(out, vm, NULL); 2089 if (IS_ERR(vma)) { 2090 err = PTR_ERR(vma); 2091 goto out_put_out; 2092 } 2093 2094 err = i915_vma_pin(vma, 0, 0, 2095 PIN_USER | 2096 PIN_OFFSET_FIXED | 2097 (vm->total - PAGE_SIZE)); 2098 if (err) 2099 goto out_put_out; 2100 GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE); 2101 2102 result = i915_gem_object_pin_map_unlocked(out, I915_MAP_WB); 2103 if (IS_ERR(result)) { 2104 err = PTR_ERR(result); 2105 goto out_put_out; 2106 } 2107 2108 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 2109 IGT_TIMEOUT(end_time); 2110 unsigned long pass = 0; 2111 2112 if (!intel_engine_can_store_dword(ce->engine)) 2113 continue; 2114 2115 while (!__igt_timeout(end_time, NULL)) { 2116 struct i915_vm_pt_stash stash = {}; 2117 struct i915_request *rq; 2118 struct i915_gem_ww_ctx ww; 2119 struct i915_vma_resource *vma_res; 2120 u64 offset; 2121 2122 offset = igt_random_offset(&prng, 2123 0, vm->total - PAGE_SIZE, 2124 chunk_size, PAGE_SIZE); 2125 2126 memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32)); 2127 2128 vma = i915_vma_instance(bbe, vm, NULL); 2129 if (IS_ERR(vma)) { 2130 err = PTR_ERR(vma); 2131 goto end; 2132 } 2133 2134 i915_gem_object_lock(bbe, NULL); 2135 err = i915_vma_get_pages(vma); 2136 i915_gem_object_unlock(bbe); 2137 if (err) 2138 goto end; 2139 2140 vma_res = i915_vma_resource_alloc(); 2141 if (IS_ERR(vma_res)) { 2142 i915_vma_put_pages(vma); 2143 err = PTR_ERR(vma_res); 2144 goto end; 2145 } 2146 2147 i915_gem_ww_ctx_init(&ww, false); 2148 retry: 2149 err = i915_vm_lock_objects(vm, &ww); 2150 if (err) 2151 goto end_ww; 2152 2153 err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size); 2154 if (err) 2155 goto end_ww; 2156 2157 err = i915_vm_map_pt_stash(vm, &stash); 2158 if (!err) 2159 vm->allocate_va_range(vm, &stash, offset, chunk_size); 2160 i915_vm_free_pt_stash(vm, &stash); 2161 end_ww: 2162 if (err == -EDEADLK) { 2163 err = i915_gem_ww_ctx_backoff(&ww); 2164 if (!err) 2165 goto retry; 2166 } 2167 i915_gem_ww_ctx_fini(&ww); 2168 if (err) { 2169 kfree(vma_res); 2170 goto end; 2171 } 2172 2173 i915_vma_resource_init_from_vma(vma_res, vma); 2174 /* Prime the TLB with the dummy pages */ 2175 for (i = 0; i < count; i++) { 2176 vma_res->start = offset + i * PAGE_SIZE; 2177 vm->insert_entries(vm, vma_res, I915_CACHE_NONE, 2178 0); 2179 2180 rq = submit_batch(ce, vma_res->start); 2181 if (IS_ERR(rq)) { 2182 err = PTR_ERR(rq); 2183 i915_vma_resource_fini(vma_res); 2184 kfree(vma_res); 2185 goto end; 2186 } 2187 i915_request_put(rq); 2188 } 2189 i915_vma_resource_fini(vma_res); 2190 i915_vma_put_pages(vma); 2191 2192 err = context_sync(ce); 2193 if (err) { 2194 pr_err("%s: dummy setup timed out\n", 2195 ce->engine->name); 2196 kfree(vma_res); 2197 goto end; 2198 } 2199 2200 vma = i915_vma_instance(act, vm, NULL); 2201 if (IS_ERR(vma)) { 2202 kfree(vma_res); 2203 err = PTR_ERR(vma); 2204 goto end; 2205 } 2206 2207 i915_gem_object_lock(act, NULL); 2208 err = i915_vma_get_pages(vma); 2209 i915_gem_object_unlock(act); 2210 if (err) { 2211 kfree(vma_res); 2212 goto end; 2213 } 2214 2215 i915_vma_resource_init_from_vma(vma_res, vma); 2216 /* Replace the TLB with target batches */ 2217 for (i = 0; i < count; i++) { 2218 struct i915_request *rq; 2219 u32 *cs = batch + i * 64 / sizeof(*cs); 2220 u64 addr; 2221 2222 vma_res->start = offset + i * PAGE_SIZE; 2223 vm->insert_entries(vm, vma_res, I915_CACHE_NONE, 0); 2224 2225 addr = vma_res->start + i * 64; 2226 cs[4] = MI_NOOP; 2227 cs[6] = lower_32_bits(addr); 2228 cs[7] = upper_32_bits(addr); 2229 wmb(); 2230 2231 rq = submit_batch(ce, addr); 2232 if (IS_ERR(rq)) { 2233 err = PTR_ERR(rq); 2234 i915_vma_resource_fini(vma_res); 2235 kfree(vma_res); 2236 goto end; 2237 } 2238 2239 /* Wait until the context chain has started */ 2240 if (i == 0) { 2241 while (READ_ONCE(result[i]) && 2242 !i915_request_completed(rq)) 2243 cond_resched(); 2244 } else { 2245 end_spin(batch, i - 1); 2246 } 2247 2248 i915_request_put(rq); 2249 } 2250 end_spin(batch, count - 1); 2251 2252 i915_vma_resource_fini(vma_res); 2253 kfree(vma_res); 2254 i915_vma_put_pages(vma); 2255 2256 err = context_sync(ce); 2257 if (err) { 2258 pr_err("%s: writes timed out\n", 2259 ce->engine->name); 2260 goto end; 2261 } 2262 2263 for (i = 0; i < count; i++) { 2264 if (result[i] != i) { 2265 pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n", 2266 ce->engine->name, pass, 2267 offset, i, result[i], i); 2268 err = -EINVAL; 2269 goto end; 2270 } 2271 } 2272 2273 vm->clear_range(vm, offset, chunk_size); 2274 pass++; 2275 } 2276 } 2277 end: 2278 if (igt_flush_test(i915)) 2279 err = -EIO; 2280 i915_gem_context_unlock_engines(ctx); 2281 i915_gem_object_unpin_map(out); 2282 out_put_out: 2283 i915_gem_object_put(out); 2284 out_put_batch: 2285 i915_gem_object_unpin_map(act); 2286 out_put_act: 2287 i915_gem_object_put(act); 2288 out_put_bbe: 2289 i915_gem_object_put(bbe); 2290 out_vm: 2291 i915_vm_put(vm); 2292 out_unlock: 2293 fput(file); 2294 return err; 2295 } 2296 2297 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) 2298 { 2299 static const struct i915_subtest tests[] = { 2300 SUBTEST(igt_ppgtt_alloc), 2301 SUBTEST(igt_ppgtt_lowlevel), 2302 SUBTEST(igt_ppgtt_drunk), 2303 SUBTEST(igt_ppgtt_walk), 2304 SUBTEST(igt_ppgtt_pot), 2305 SUBTEST(igt_ppgtt_fill), 2306 SUBTEST(igt_ppgtt_shrink), 2307 SUBTEST(igt_ppgtt_shrink_boom), 2308 SUBTEST(igt_ppgtt_misaligned_pin), 2309 SUBTEST(igt_ggtt_lowlevel), 2310 SUBTEST(igt_ggtt_drunk), 2311 SUBTEST(igt_ggtt_walk), 2312 SUBTEST(igt_ggtt_pot), 2313 SUBTEST(igt_ggtt_fill), 2314 SUBTEST(igt_ggtt_page), 2315 SUBTEST(igt_ggtt_misaligned_pin), 2316 SUBTEST(igt_cs_tlb), 2317 }; 2318 2319 GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total)); 2320 2321 return i915_subtests(tests, i915); 2322 } 2323