1 /* 2 * Copyright (c) Red Hat Inc. 3 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sub license, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the 12 * next paragraph) shall be included in all copies or substantial portions 13 * of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 * 23 * Authors: Dave Airlie <airlied@redhat.com> 24 * Jerome Glisse <jglisse@redhat.com> 25 * Pauli Nieminen <suokkos@gmail.com> 26 */ 27 /* 28 * Copyright (c) 2013 The FreeBSD Foundation 29 * All rights reserved. 30 * 31 * Portions of this software were developed by Konstantin Belousov 32 * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 33 */ 34 35 /* simple list based uncached page pool 36 * - Pool collects resently freed pages for reuse 37 * - Use page->lru to keep a free list 38 * - doesn't track currently in use pages 39 */ 40 41 #define pr_fmt(fmt) "[TTM] " fmt 42 43 #include <linux/list.h> 44 #include <linux/spinlock.h> 45 #include <linux/highmem.h> 46 #include <linux/mm_types.h> 47 #include <linux/module.h> 48 #include <linux/mm.h> 49 #include <linux/seq_file.h> /* for seq_printf */ 50 #include <linux/dma-mapping.h> 51 52 #include <linux/atomic.h> 53 54 #include <drm/ttm/ttm_bo_driver.h> 55 #include <drm/ttm/ttm_page_alloc.h> 56 #include <drm/ttm/ttm_set_memory.h> 57 58 #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) 59 #define SMALL_ALLOCATION 16 60 #define FREE_ALL_PAGES (~0U) 61 /* times are in msecs */ 62 #define PAGE_FREE_INTERVAL 1000 63 64 /** 65 * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. 66 * 67 * @lock: Protects the shared pool from concurrnet access. Must be used with 68 * irqsave/irqrestore variants because pool allocator maybe called from 69 * delayed work. 70 * @fill_lock: Prevent concurrent calls to fill. 71 * @list: Pool of free uc/wc pages for fast reuse. 72 * @gfp_flags: Flags to pass for alloc_page. 73 * @npages: Number of pages in pool. 74 */ 75 struct ttm_page_pool { 76 struct lock lock; 77 bool fill_lock; 78 struct pglist list; 79 gfp_t gfp_flags; 80 unsigned npages; 81 char *name; 82 unsigned long nfrees; 83 unsigned long nrefills; 84 unsigned int order; 85 }; 86 87 /** 88 * Limits for the pool. They are handled without locks because only place where 89 * they may change is in sysfs store. They won't have immediate effect anyway 90 * so forcing serialization to access them is pointless. 91 */ 92 93 struct ttm_pool_opts { 94 unsigned alloc_size; 95 unsigned max_size; 96 unsigned small; 97 }; 98 99 #define NUM_POOLS 4 100 101 /** 102 * struct ttm_pool_manager - Holds memory pools for fst allocation 103 * 104 * Manager is read only object for pool code so it doesn't need locking. 105 * 106 * @free_interval: minimum number of jiffies between freeing pages from pool. 107 * @page_alloc_inited: reference counting for pool allocation. 108 * @work: Work that is used to shrink the pool. Work is only run when there is 109 * some pages to free. 110 * @small_allocation: Limit in number of pages what is small allocation. 111 * 112 * @pools: All pool objects in use. 113 **/ 114 struct ttm_pool_manager { 115 struct kobject kobj; 116 struct shrinker mm_shrink; 117 eventhandler_tag lowmem_handler; 118 struct ttm_pool_opts options; 119 120 union { 121 struct ttm_page_pool pools[NUM_POOLS]; 122 struct { 123 struct ttm_page_pool wc_pool; 124 struct ttm_page_pool uc_pool; 125 struct ttm_page_pool wc_pool_dma32; 126 struct ttm_page_pool uc_pool_dma32; 127 struct ttm_page_pool wc_pool_huge; 128 struct ttm_page_pool uc_pool_huge; 129 } ; 130 }; 131 }; 132 133 static struct attribute ttm_page_pool_max = { 134 .name = "pool_max_size", 135 .mode = S_IRUGO | S_IWUSR 136 }; 137 static struct attribute ttm_page_pool_small = { 138 .name = "pool_small_allocation", 139 .mode = S_IRUGO | S_IWUSR 140 }; 141 static struct attribute ttm_page_pool_alloc_size = { 142 .name = "pool_allocation_size", 143 .mode = S_IRUGO | S_IWUSR 144 }; 145 146 static struct attribute *ttm_pool_attrs[] = { 147 &ttm_page_pool_max, 148 &ttm_page_pool_small, 149 &ttm_page_pool_alloc_size, 150 NULL 151 }; 152 153 static void ttm_pool_kobj_release(struct kobject *kobj) 154 { 155 struct ttm_pool_manager *m = 156 container_of(kobj, struct ttm_pool_manager, kobj); 157 kfree(m); 158 } 159 160 static ssize_t ttm_pool_store(struct kobject *kobj, 161 struct attribute *attr, const char *buffer, size_t size) 162 { 163 struct ttm_pool_manager *m = 164 container_of(kobj, struct ttm_pool_manager, kobj); 165 int chars; 166 unsigned val; 167 chars = ksscanf(buffer, "%u", &val); 168 if (chars == 0) 169 return size; 170 171 /* Convert kb to number of pages */ 172 val = val / (PAGE_SIZE >> 10); 173 174 if (attr == &ttm_page_pool_max) 175 m->options.max_size = val; 176 else if (attr == &ttm_page_pool_small) 177 m->options.small = val; 178 else if (attr == &ttm_page_pool_alloc_size) { 179 if (val > NUM_PAGES_TO_ALLOC*8) { 180 pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", 181 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), 182 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); 183 return size; 184 } else if (val > NUM_PAGES_TO_ALLOC) { 185 pr_warn("Setting allocation size to larger than %lu is not recommended\n", 186 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); 187 } 188 m->options.alloc_size = val; 189 } 190 191 return size; 192 } 193 194 static ssize_t ttm_pool_show(struct kobject *kobj, 195 struct attribute *attr, char *buffer) 196 { 197 struct ttm_pool_manager *m = 198 container_of(kobj, struct ttm_pool_manager, kobj); 199 unsigned val = 0; 200 201 if (attr == &ttm_page_pool_max) 202 val = m->options.max_size; 203 else if (attr == &ttm_page_pool_small) 204 val = m->options.small; 205 else if (attr == &ttm_page_pool_alloc_size) 206 val = m->options.alloc_size; 207 208 val = val * (PAGE_SIZE >> 10); 209 210 return ksnprintf(buffer, PAGE_SIZE, "%u\n", val); 211 } 212 213 static const struct sysfs_ops ttm_pool_sysfs_ops = { 214 .show = &ttm_pool_show, 215 .store = &ttm_pool_store, 216 }; 217 218 static struct kobj_type ttm_pool_kobj_type = { 219 .release = &ttm_pool_kobj_release, 220 .sysfs_ops = &ttm_pool_sysfs_ops, 221 .default_attrs = ttm_pool_attrs, 222 }; 223 224 static struct ttm_pool_manager *_manager; 225 226 /** 227 * Select the right pool or requested caching state and ttm flags. */ 228 static struct ttm_page_pool *ttm_get_pool(int flags, bool huge, 229 enum ttm_caching_state cstate) 230 { 231 int pool_index; 232 233 if (cstate == tt_cached) 234 return NULL; 235 236 if (cstate == tt_wc) 237 pool_index = 0x0; 238 else 239 pool_index = 0x1; 240 241 if (flags & TTM_PAGE_FLAG_DMA32) { 242 if (huge) 243 return NULL; 244 pool_index |= 0x2; 245 246 } else if (huge) { 247 pool_index |= 0x4; 248 } 249 250 return &_manager->pools[pool_index]; 251 } 252 253 /* set memory back to wb and free the pages. */ 254 static void ttm_pages_put(struct page *pages[], unsigned npages, 255 unsigned int order) 256 { 257 unsigned int i, pages_nr = (1 << order); 258 259 if (order == 0) { 260 if (ttm_set_pages_array_wb(pages, npages)) 261 pr_err("Failed to set %d pages to wb!\n", npages); 262 } 263 264 for (i = 0; i < npages; ++i) { 265 if (order > 0) { 266 if (ttm_set_pages_wb(pages[i], pages_nr)) 267 pr_err("Failed to set %d pages to wb!\n", pages_nr); 268 } 269 __free_pages(pages[i], order); 270 } 271 } 272 273 static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, 274 unsigned freed_pages) 275 { 276 pool->npages -= freed_pages; 277 pool->nfrees += freed_pages; 278 } 279 280 /** 281 * Free pages from pool. 282 * 283 * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC 284 * number of pages in one go. 285 * 286 * @pool: to free the pages from 287 * @free_all: If set to true will free all pages in pool 288 * @use_static: Safe to use static buffer 289 **/ 290 static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, 291 bool use_static) 292 { 293 static struct page *static_buf[NUM_PAGES_TO_ALLOC]; 294 unsigned long irq_flags; 295 struct vm_page *p, *p1; 296 struct page **pages_to_free; 297 unsigned freed_pages = 0, 298 npages_to_free = nr_free; 299 unsigned i; 300 301 if (NUM_PAGES_TO_ALLOC < nr_free) 302 npages_to_free = NUM_PAGES_TO_ALLOC; 303 304 if (use_static) 305 pages_to_free = static_buf; 306 else 307 pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), 308 M_DRM, GFP_KERNEL); 309 if (!pages_to_free) { 310 pr_err("Failed to allocate memory for pool free operation\n"); 311 return 0; 312 } 313 314 restart: 315 spin_lock_irqsave(&pool->lock, irq_flags); 316 317 TAILQ_FOREACH_REVERSE_MUTABLE(p, &pool->list, pglist, pageq, p1) { 318 if (freed_pages >= npages_to_free) 319 break; 320 321 pages_to_free[freed_pages++] = (struct page *)p; 322 /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ 323 if (freed_pages >= NUM_PAGES_TO_ALLOC) { 324 /* remove range of pages from the pool */ 325 for (i = 0; i < freed_pages; i++) 326 TAILQ_REMOVE(&pool->list, (struct vm_page *)pages_to_free[i], pageq); 327 328 ttm_pool_update_free_locked(pool, freed_pages); 329 /** 330 * Because changing page caching is costly 331 * we unlock the pool to prevent stalling. 332 */ 333 spin_unlock_irqrestore(&pool->lock, irq_flags); 334 335 ttm_pages_put(pages_to_free, freed_pages, pool->order); 336 if (likely(nr_free != FREE_ALL_PAGES)) 337 nr_free -= freed_pages; 338 339 if (NUM_PAGES_TO_ALLOC >= nr_free) 340 npages_to_free = nr_free; 341 else 342 npages_to_free = NUM_PAGES_TO_ALLOC; 343 344 freed_pages = 0; 345 346 /* free all so restart the processing */ 347 if (nr_free) 348 goto restart; 349 350 /* Not allowed to fall through or break because 351 * following context is inside spinlock while we are 352 * outside here. 353 */ 354 goto out; 355 356 } 357 } 358 359 /* remove range of pages from the pool */ 360 if (freed_pages) { 361 for (i = 0; i < freed_pages; i++) 362 TAILQ_REMOVE(&pool->list, (struct vm_page *)pages_to_free[i], pageq); 363 364 ttm_pool_update_free_locked(pool, freed_pages); 365 nr_free -= freed_pages; 366 } 367 368 spin_unlock_irqrestore(&pool->lock, irq_flags); 369 370 if (freed_pages) 371 ttm_pages_put(pages_to_free, freed_pages, pool->order); 372 out: 373 if (pages_to_free != static_buf) 374 kfree(pages_to_free); 375 return nr_free; 376 } 377 378 /** 379 * Callback for mm to request pool to reduce number of page held. 380 * 381 * XXX: (dchinner) Deadlock warning! 382 * 383 * This code is crying out for a shrinker per pool.... 384 */ 385 static unsigned long 386 ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) 387 { 388 static DEFINE_MUTEX(lock); 389 static unsigned start_pool; 390 unsigned i; 391 unsigned pool_offset; 392 struct ttm_page_pool *pool; 393 int shrink_pages = 100; /* XXXKIB */ 394 unsigned long freed = 0; 395 unsigned int nr_free_pool; 396 397 #ifdef __DragonFly__ 398 sc->gfp_mask = M_WAITOK; 399 #endif 400 401 if (!mutex_trylock(&lock)) 402 return SHRINK_STOP; 403 pool_offset = ++start_pool % NUM_POOLS; 404 /* select start pool in round robin fashion */ 405 for (i = 0; i < NUM_POOLS; ++i) { 406 unsigned nr_free = shrink_pages; 407 unsigned page_nr; 408 409 if (shrink_pages == 0) 410 break; 411 412 pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; 413 page_nr = (1 << pool->order); 414 /* OK to use static buffer since global mutex is held. */ 415 nr_free_pool = roundup(nr_free, page_nr) >> pool->order; 416 shrink_pages = ttm_page_pool_free(pool, nr_free_pool, true); 417 freed += (nr_free_pool - shrink_pages) << pool->order; 418 if (freed >= sc->nr_to_scan) 419 break; 420 shrink_pages <<= pool->order; 421 } 422 mutex_unlock(&lock); 423 return freed; 424 } 425 426 427 static unsigned long 428 ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) 429 { 430 unsigned i; 431 unsigned long count = 0; 432 struct ttm_page_pool *pool; 433 434 for (i = 0; i < NUM_POOLS; ++i) { 435 pool = &_manager->pools[i]; 436 count += (pool->npages << pool->order); 437 } 438 439 return count; 440 } 441 442 static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) 443 { 444 manager->mm_shrink.count_objects = ttm_pool_shrink_count; 445 manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; 446 manager->mm_shrink.seeks = 1; 447 manager->lowmem_handler = EVENTHANDLER_REGISTER(vm_lowmem, 448 ttm_pool_shrink_scan, manager, EVENTHANDLER_PRI_ANY); 449 } 450 451 static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) 452 { 453 EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler); 454 } 455 456 static int ttm_set_pages_caching(struct page **pages, 457 enum ttm_caching_state cstate, unsigned cpages) 458 { 459 int r = 0; 460 /* Set page caching */ 461 switch (cstate) { 462 case tt_uncached: 463 r = ttm_set_pages_array_uc(pages, cpages); 464 if (r) 465 pr_err("Failed to set %d pages to uc!\n", cpages); 466 break; 467 case tt_wc: 468 r = ttm_set_pages_array_wc(pages, cpages); 469 if (r) 470 pr_err("Failed to set %d pages to wc!\n", cpages); 471 break; 472 default: 473 break; 474 } 475 return r; 476 } 477 478 /** 479 * Free pages the pages that failed to change the caching state. If there is 480 * any pages that have changed their caching state already put them to the 481 * pool. 482 */ 483 static void ttm_handle_caching_state_failure(struct pglist *pages, 484 int ttm_flags, enum ttm_caching_state cstate, 485 struct page **failed_pages, unsigned cpages) 486 { 487 unsigned i; 488 /* Failed pages have to be freed */ 489 for (i = 0; i < cpages; ++i) { 490 TAILQ_REMOVE(pages, (struct vm_page *)failed_pages[i], pageq); 491 __free_page(failed_pages[i]); 492 } 493 } 494 495 /** 496 * Allocate new pages with correct caching. 497 * 498 * This function is reentrant if caller updates count depending on number of 499 * pages returned in pages array. 500 */ 501 static int ttm_alloc_new_pages(struct pglist *pages, gfp_t gfp_flags, 502 int ttm_flags, enum ttm_caching_state cstate, 503 unsigned count, unsigned order) 504 { 505 struct page **caching_array; 506 struct page *p; 507 int r = 0; 508 unsigned i, j, cpages; 509 unsigned npages = 1 << order; 510 unsigned max_cpages = min(count << order, (unsigned)NUM_PAGES_TO_ALLOC); 511 512 /* allocate array for page caching change */ 513 caching_array = kmalloc(max_cpages*sizeof(struct page *), M_DRM, M_WAITOK); 514 515 if (!caching_array) { 516 pr_debug("Unable to allocate table for new pages\n"); 517 return -ENOMEM; 518 } 519 520 for (i = 0, cpages = 0; i < count; ++i) { 521 p = alloc_pages(gfp_flags, order); 522 523 if (!p) { 524 pr_debug("Unable to get page %u\n", i); 525 526 /* store already allocated pages in the pool after 527 * setting the caching state */ 528 if (cpages) { 529 r = ttm_set_pages_caching(caching_array, 530 cstate, cpages); 531 if (r) 532 ttm_handle_caching_state_failure(pages, 533 ttm_flags, cstate, 534 caching_array, cpages); 535 } 536 r = -ENOMEM; 537 goto out; 538 } 539 540 TAILQ_INSERT_HEAD(pages, (struct vm_page *)p, pageq); 541 542 #ifdef CONFIG_HIGHMEM 543 /* gfp flags of highmem page should never be dma32 so we 544 * we should be fine in such case 545 */ 546 if (PageHighMem(p)) 547 continue; 548 549 #endif 550 for (j = 0; j < npages; ++j) { 551 caching_array[cpages++] = p++; 552 if (cpages == max_cpages) { 553 554 r = ttm_set_pages_caching(caching_array, 555 cstate, cpages); 556 if (r) { 557 ttm_handle_caching_state_failure(pages, 558 ttm_flags, cstate, 559 caching_array, cpages); 560 goto out; 561 } 562 cpages = 0; 563 } 564 } 565 } 566 567 if (cpages) { 568 r = ttm_set_pages_caching(caching_array, cstate, cpages); 569 if (r) 570 ttm_handle_caching_state_failure(pages, 571 ttm_flags, cstate, 572 caching_array, cpages); 573 } 574 out: 575 kfree(caching_array); 576 577 return r; 578 } 579 580 /** 581 * Fill the given pool if there aren't enough pages and the requested number of 582 * pages is small. 583 */ 584 static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags, 585 enum ttm_caching_state cstate, 586 unsigned count, unsigned long *irq_flags) 587 { 588 vm_page_t p; 589 int r; 590 unsigned cpages = 0; 591 /** 592 * Only allow one pool fill operation at a time. 593 * If pool doesn't have enough pages for the allocation new pages are 594 * allocated from outside of pool. 595 */ 596 if (pool->fill_lock) 597 return; 598 599 pool->fill_lock = true; 600 601 /* If allocation request is small and there are not enough 602 * pages in a pool we fill the pool up first. */ 603 if (count < _manager->options.small 604 && count > pool->npages) { 605 struct pglist new_pages; 606 unsigned alloc_size = _manager->options.alloc_size; 607 608 /** 609 * Can't change page caching if in irqsave context. We have to 610 * drop the pool->lock. 611 */ 612 spin_unlock_irqrestore(&pool->lock, *irq_flags); 613 614 TAILQ_INIT(&new_pages); 615 r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, 616 cstate, alloc_size, 0); 617 spin_lock_irqsave(&pool->lock, *irq_flags); 618 619 if (!r) { 620 TAILQ_CONCAT(&pool->list, &new_pages, pageq); 621 ++pool->nrefills; 622 pool->npages += alloc_size; 623 } else { 624 pr_debug("Failed to fill pool (%p)\n", pool); 625 /* If we have any pages left put them to the pool. */ 626 TAILQ_FOREACH(p, &new_pages, pageq) { 627 ++cpages; 628 } 629 TAILQ_CONCAT(&pool->list, &new_pages, pageq); 630 pool->npages += cpages; 631 } 632 633 } 634 pool->fill_lock = false; 635 } 636 637 /** 638 * Cut 'count' number of pages from the pool and put them on the return list. 639 * 640 * @return count of pages still required to fulfill the request. 641 */ 642 static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, 643 struct pglist *pages, 644 int ttm_flags, 645 enum ttm_caching_state cstate, 646 unsigned count, unsigned order) 647 { 648 unsigned long irq_flags; 649 vm_page_t p; 650 unsigned i; 651 652 spin_lock_irqsave(&pool->lock, irq_flags); 653 if (!order) 654 ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, 655 &irq_flags); 656 657 if (count >= pool->npages) { 658 /* take all pages from the pool */ 659 TAILQ_CONCAT(pages, &pool->list, pageq); 660 count -= pool->npages; 661 pool->npages = 0; 662 goto out; 663 } 664 for (i = 0; i < count; i++) { 665 p = TAILQ_FIRST(&pool->list); 666 TAILQ_REMOVE(&pool->list, p, pageq); 667 TAILQ_INSERT_TAIL(pages, p, pageq); 668 } 669 pool->npages -= count; 670 count = 0; 671 out: 672 spin_unlock_irqrestore(&pool->lock, irq_flags); 673 return count; 674 } 675 676 /* Put all pages in pages list to correct pool to wait for reuse */ 677 static void ttm_put_pages(struct page **pages, unsigned npages, int flags, 678 enum ttm_caching_state cstate) 679 { 680 struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); 681 unsigned long irq_flags; 682 unsigned i; 683 struct vm_page *page; 684 685 if (pool == NULL) { 686 /* No pool for this memory type so free the pages */ 687 for (i = 0; i < npages; i++) { 688 if (pages[i]) { 689 #if 0 690 if (page_count(pages[i]) != 1) 691 pr_err("Erroneous page count. Leaking pages.\n"); 692 #endif 693 __free_page(pages[i]); 694 pages[i] = NULL; 695 } 696 } 697 return; 698 } 699 700 spin_lock_irqsave(&pool->lock, irq_flags); 701 for (i = 0; i < npages; i++) { 702 if (pages[i]) { 703 page = (struct vm_page *)pages[i]; 704 TAILQ_INSERT_TAIL(&pool->list, page, pageq); 705 pages[i] = NULL; 706 pool->npages++; 707 } 708 } 709 /* Check that we don't go over the pool limit */ 710 npages = 0; 711 if (pool->npages > _manager->options.max_size) { 712 npages = pool->npages - _manager->options.max_size; 713 /* free at least NUM_PAGES_TO_ALLOC number of pages 714 * to reduce calls to set_memory_wb */ 715 if (npages < NUM_PAGES_TO_ALLOC) 716 npages = NUM_PAGES_TO_ALLOC; 717 } 718 spin_unlock_irqrestore(&pool->lock, irq_flags); 719 if (npages) 720 ttm_page_pool_free(pool, npages, false); 721 } 722 723 /* 724 * On success pages list will hold count number of correctly 725 * cached pages. 726 */ 727 static int ttm_get_pages(struct page **pages, unsigned npages, int flags, 728 enum ttm_caching_state cstate) 729 { 730 struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); 731 struct pglist plist; 732 struct vm_page *p = NULL; 733 gfp_t gfp_flags = GFP_USER; 734 unsigned count; 735 int r; 736 737 /* set zero flag for page allocation if required */ 738 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) 739 gfp_flags |= __GFP_ZERO; 740 741 /* No pool for cached pages */ 742 if (pool == NULL) { 743 if (flags & TTM_PAGE_FLAG_DMA32) 744 gfp_flags |= GFP_DMA32; 745 else 746 gfp_flags |= GFP_HIGHUSER; 747 748 for (r = 0; r < npages; ++r) { 749 p = (struct vm_page *)alloc_page(gfp_flags); 750 if (!p) { 751 752 pr_err("Unable to allocate page\n"); 753 return -ENOMEM; 754 } 755 pages[r] = (struct page *)p; 756 } 757 return 0; 758 } 759 760 /* combine zero flag to pool flags */ 761 gfp_flags |= pool->gfp_flags; 762 763 /* First we take pages from the pool */ 764 TAILQ_INIT(&plist); 765 npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages, 0); 766 count = 0; 767 TAILQ_FOREACH(p, &plist, pageq) { 768 pages[count++] = (struct page *)p; 769 } 770 771 /* clear the pages coming from the pool if requested */ 772 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { 773 TAILQ_FOREACH(p, &plist, pageq) { 774 pmap_zero_page(VM_PAGE_TO_PHYS(p)); 775 } 776 } 777 778 /* If pool didn't have enough pages allocate new one. */ 779 if (npages > 0) { 780 /* ttm_alloc_new_pages doesn't reference pool so we can run 781 * multiple requests in parallel. 782 **/ 783 TAILQ_INIT(&plist); 784 r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages, 0); 785 TAILQ_FOREACH(p, &plist, pageq) { 786 pages[count++] = (struct page *)p; 787 } 788 if (r) { 789 /* If there is any pages in the list put them back to 790 * the pool. */ 791 pr_err("Failed to allocate extra pages for large request\n"); 792 ttm_put_pages(pages, count, flags, cstate); 793 return r; 794 } 795 } 796 797 return 0; 798 } 799 800 static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, 801 char *name, unsigned int order) 802 { 803 lockinit(&pool->lock, "ttmpool", 0, LK_CANRECURSE); 804 pool->fill_lock = false; 805 TAILQ_INIT(&pool->list); 806 pool->npages = pool->nfrees = 0; 807 pool->gfp_flags = flags; 808 pool->name = name; 809 pool->order = order; 810 } 811 812 int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) 813 { 814 int ret; 815 816 WARN_ON(_manager); 817 818 pr_info("Initializing pool allocator\n"); 819 820 _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); 821 if (!_manager) 822 return -ENOMEM; 823 824 ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc", 0); 825 826 ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc", 0); 827 828 ttm_page_pool_init_locked(&_manager->wc_pool_dma32, 829 GFP_USER | GFP_DMA32, "wc dma", 0); 830 831 ttm_page_pool_init_locked(&_manager->uc_pool_dma32, 832 GFP_USER | GFP_DMA32, "uc dma", 0); 833 834 _manager->options.max_size = max_pages; 835 _manager->options.small = SMALL_ALLOCATION; 836 _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; 837 838 ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, 839 &glob->kobj, "pool"); 840 if (unlikely(ret != 0)) 841 goto error; 842 843 ttm_pool_mm_shrink_init(_manager); 844 845 return 0; 846 847 error: 848 kobject_put(&_manager->kobj); 849 _manager = NULL; 850 return ret; 851 } 852 853 void ttm_page_alloc_fini(void) 854 { 855 int i; 856 857 pr_info("Finalizing pool allocator\n"); 858 ttm_pool_mm_shrink_fini(_manager); 859 860 /* OK to use static buffer since global mutex is no longer used. */ 861 for (i = 0; i < NUM_POOLS; ++i) 862 ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES, true); 863 864 kobject_put(&_manager->kobj); 865 _manager = NULL; 866 } 867 868 static void 869 ttm_pool_unpopulate_helper(struct ttm_tt *ttm, unsigned mem_count_update) 870 { 871 struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob; 872 unsigned i; 873 874 if (mem_count_update == 0) 875 goto put_pages; 876 877 for (i = 0; i < mem_count_update; ++i) { 878 if (!ttm->pages[i]) 879 continue; 880 881 ttm_mem_global_free_page(mem_glob, ttm->pages[i], PAGE_SIZE); 882 } 883 884 put_pages: 885 ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags, 886 ttm->caching_state); 887 ttm->state = tt_unpopulated; 888 } 889 890 int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) 891 { 892 struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob; 893 unsigned i; 894 int ret; 895 896 if (ttm->state != tt_unpopulated) 897 return 0; 898 899 if (ttm_check_under_lowerlimit(mem_glob, ttm->num_pages, ctx)) 900 return -ENOMEM; 901 902 ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags, 903 ttm->caching_state); 904 if (unlikely(ret != 0)) { 905 ttm_pool_unpopulate_helper(ttm, 0); 906 return ret; 907 } 908 909 for (i = 0; i < ttm->num_pages; ++i) { 910 ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], 911 PAGE_SIZE, ctx); 912 if (unlikely(ret != 0)) { 913 ttm_pool_unpopulate_helper(ttm, i); 914 return -ENOMEM; 915 } 916 } 917 918 if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { 919 ret = ttm_tt_swapin(ttm); 920 if (unlikely(ret != 0)) { 921 ttm_pool_unpopulate(ttm); 922 return ret; 923 } 924 } 925 926 ttm->state = tt_unbound; 927 return 0; 928 } 929 EXPORT_SYMBOL(ttm_pool_populate); 930 931 void ttm_pool_unpopulate(struct ttm_tt *ttm) 932 { 933 ttm_pool_unpopulate_helper(ttm, ttm->num_pages); 934 } 935 EXPORT_SYMBOL(ttm_pool_unpopulate); 936 937 int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, 938 struct ttm_operation_ctx *ctx) 939 { 940 unsigned i, j; 941 int r; 942 943 r = ttm_pool_populate(&tt->ttm, ctx); 944 if (r) 945 return r; 946 947 for (i = 0; i < tt->ttm.num_pages; ++i) { 948 struct page *p = tt->ttm.pages[i]; 949 size_t num_pages = 1; 950 951 for (j = i + 1; j < tt->ttm.num_pages; ++j) { 952 if (++p != tt->ttm.pages[j]) 953 break; 954 955 ++num_pages; 956 } 957 958 tt->dma_address[i] = dma_map_page(dev, tt->ttm.pages[i], 959 0, num_pages * PAGE_SIZE, 960 DMA_BIDIRECTIONAL); 961 if (dma_mapping_error(dev, tt->dma_address[i])) { 962 while (i--) { 963 dma_unmap_page(dev, tt->dma_address[i], 964 PAGE_SIZE, DMA_BIDIRECTIONAL); 965 tt->dma_address[i] = 0; 966 } 967 ttm_pool_unpopulate(&tt->ttm); 968 return -EFAULT; 969 } 970 971 for (j = 1; j < num_pages; ++j) { 972 tt->dma_address[i + 1] = tt->dma_address[i] + PAGE_SIZE; 973 ++i; 974 } 975 } 976 return 0; 977 } 978 EXPORT_SYMBOL(ttm_populate_and_map_pages); 979 980 void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_dma_tt *tt) 981 { 982 unsigned i, j; 983 984 for (i = 0; i < tt->ttm.num_pages;) { 985 struct page *p = tt->ttm.pages[i]; 986 size_t num_pages = 1; 987 988 if (!tt->dma_address[i] || !tt->ttm.pages[i]) { 989 ++i; 990 continue; 991 } 992 993 for (j = i + 1; j < tt->ttm.num_pages; ++j) { 994 if (++p != tt->ttm.pages[j]) 995 break; 996 997 ++num_pages; 998 } 999 1000 dma_unmap_page(dev, tt->dma_address[i], num_pages * PAGE_SIZE, 1001 DMA_BIDIRECTIONAL); 1002 1003 i += num_pages; 1004 } 1005 ttm_pool_unpopulate(&tt->ttm); 1006 } 1007 EXPORT_SYMBOL(ttm_unmap_and_unpopulate_pages); 1008 1009 #if 0 1010 int ttm_page_alloc_debugfs(struct seq_file *m, void *data) 1011 { 1012 struct ttm_page_pool *p; 1013 unsigned i; 1014 char *h[] = {"pool", "refills", "pages freed", "size"}; 1015 if (!_manager) { 1016 seq_printf(m, "No pool allocator running.\n"); 1017 return 0; 1018 } 1019 seq_printf(m, "%7s %12s %13s %8s\n", 1020 h[0], h[1], h[2], h[3]); 1021 for (i = 0; i < NUM_POOLS; ++i) { 1022 p = &_manager->pools[i]; 1023 1024 seq_printf(m, "%7s %12ld %13ld %8d\n", 1025 p->name, p->nrefills, 1026 p->nfrees, p->npages); 1027 } 1028 return 0; 1029 } 1030 #endif 1031 EXPORT_SYMBOL(ttm_page_alloc_debugfs); 1032