xref: /dragonfly/sys/dev/drm/ttm/ttm_page_alloc.c (revision 932d855e)
1 /*
2  * Copyright (c) Red Hat Inc.
3 
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sub license,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the
12  * next paragraph) shall be included in all copies or substantial portions
13  * of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  *
23  * Authors: Dave Airlie <airlied@redhat.com>
24  *          Jerome Glisse <jglisse@redhat.com>
25  *          Pauli Nieminen <suokkos@gmail.com>
26  */
27 /*
28  * Copyright (c) 2013 The FreeBSD Foundation
29  * All rights reserved.
30  *
31  * Portions of this software were developed by Konstantin Belousov
32  * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation.
33  */
34 
35 /* simple list based uncached page pool
36  * - Pool collects resently freed pages for reuse
37  * - Use page->lru to keep a free list
38  * - doesn't track currently in use pages
39  */
40 
41 #define pr_fmt(fmt) "[TTM] " fmt
42 
43 #include <linux/list.h>
44 #include <linux/spinlock.h>
45 #include <linux/highmem.h>
46 #include <linux/mm_types.h>
47 #include <linux/module.h>
48 #include <linux/mm.h>
49 #include <linux/seq_file.h> /* for seq_printf */
50 #include <linux/dma-mapping.h>
51 
52 #include <linux/atomic.h>
53 
54 #include <drm/ttm/ttm_bo_driver.h>
55 #include <drm/ttm/ttm_page_alloc.h>
56 #include <drm/ttm/ttm_set_memory.h>
57 
58 #define NUM_PAGES_TO_ALLOC		(PAGE_SIZE/sizeof(struct page *))
59 #define SMALL_ALLOCATION		16
60 #define FREE_ALL_PAGES			(~0U)
61 /* times are in msecs */
62 #define PAGE_FREE_INTERVAL		1000
63 
64 /**
65  * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages.
66  *
67  * @lock: Protects the shared pool from concurrnet access. Must be used with
68  * irqsave/irqrestore variants because pool allocator maybe called from
69  * delayed work.
70  * @fill_lock: Prevent concurrent calls to fill.
71  * @list: Pool of free uc/wc pages for fast reuse.
72  * @gfp_flags: Flags to pass for alloc_page.
73  * @npages: Number of pages in pool.
74  */
75 struct ttm_page_pool {
76 	struct lock		lock;
77 	bool			fill_lock;
78 	struct pglist		list;
79 	gfp_t			gfp_flags;
80 	unsigned		npages;
81 	char			*name;
82 	unsigned long		nfrees;
83 	unsigned long		nrefills;
84 	unsigned int		order;
85 };
86 
87 /**
88  * Limits for the pool. They are handled without locks because only place where
89  * they may change is in sysfs store. They won't have immediate effect anyway
90  * so forcing serialization to access them is pointless.
91  */
92 
93 struct ttm_pool_opts {
94 	unsigned	alloc_size;
95 	unsigned	max_size;
96 	unsigned	small;
97 };
98 
99 #define NUM_POOLS 4
100 
101 /**
102  * struct ttm_pool_manager - Holds memory pools for fst allocation
103  *
104  * Manager is read only object for pool code so it doesn't need locking.
105  *
106  * @free_interval: minimum number of jiffies between freeing pages from pool.
107  * @page_alloc_inited: reference counting for pool allocation.
108  * @work: Work that is used to shrink the pool. Work is only run when there is
109  * some pages to free.
110  * @small_allocation: Limit in number of pages what is small allocation.
111  *
112  * @pools: All pool objects in use.
113  **/
114 struct ttm_pool_manager {
115 	struct kobject		kobj;
116 	struct shrinker		mm_shrink;
117 	eventhandler_tag lowmem_handler;
118 	struct ttm_pool_opts	options;
119 
120 	union {
121 		struct ttm_page_pool	pools[NUM_POOLS];
122 		struct {
123 			struct ttm_page_pool	wc_pool;
124 			struct ttm_page_pool	uc_pool;
125 			struct ttm_page_pool	wc_pool_dma32;
126 			struct ttm_page_pool	uc_pool_dma32;
127 			struct ttm_page_pool	wc_pool_huge;
128 			struct ttm_page_pool	uc_pool_huge;
129 		} ;
130 	};
131 };
132 
133 static struct attribute ttm_page_pool_max = {
134 	.name = "pool_max_size",
135 	.mode = S_IRUGO | S_IWUSR
136 };
137 static struct attribute ttm_page_pool_small = {
138 	.name = "pool_small_allocation",
139 	.mode = S_IRUGO | S_IWUSR
140 };
141 static struct attribute ttm_page_pool_alloc_size = {
142 	.name = "pool_allocation_size",
143 	.mode = S_IRUGO | S_IWUSR
144 };
145 
146 static struct attribute *ttm_pool_attrs[] = {
147 	&ttm_page_pool_max,
148 	&ttm_page_pool_small,
149 	&ttm_page_pool_alloc_size,
150 	NULL
151 };
152 
ttm_pool_kobj_release(struct kobject * kobj)153 static void ttm_pool_kobj_release(struct kobject *kobj)
154 {
155 	struct ttm_pool_manager *m =
156 		container_of(kobj, struct ttm_pool_manager, kobj);
157 	kfree(m);
158 }
159 
ttm_pool_store(struct kobject * kobj,struct attribute * attr,const char * buffer,size_t size)160 static ssize_t ttm_pool_store(struct kobject *kobj,
161 		struct attribute *attr, const char *buffer, size_t size)
162 {
163 	struct ttm_pool_manager *m =
164 		container_of(kobj, struct ttm_pool_manager, kobj);
165 	int chars;
166 	unsigned val;
167 	chars = ksscanf(buffer, "%u", &val);
168 	if (chars == 0)
169 		return size;
170 
171 	/* Convert kb to number of pages */
172 	val = val / (PAGE_SIZE >> 10);
173 
174 	if (attr == &ttm_page_pool_max)
175 		m->options.max_size = val;
176 	else if (attr == &ttm_page_pool_small)
177 		m->options.small = val;
178 	else if (attr == &ttm_page_pool_alloc_size) {
179 		if (val > NUM_PAGES_TO_ALLOC*8) {
180 			pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n",
181 			       NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
182 			       NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
183 			return size;
184 		} else if (val > NUM_PAGES_TO_ALLOC) {
185 			pr_warn("Setting allocation size to larger than %lu is not recommended\n",
186 				NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
187 		}
188 		m->options.alloc_size = val;
189 	}
190 
191 	return size;
192 }
193 
ttm_pool_show(struct kobject * kobj,struct attribute * attr,char * buffer)194 static ssize_t ttm_pool_show(struct kobject *kobj,
195 		struct attribute *attr, char *buffer)
196 {
197 	struct ttm_pool_manager *m =
198 		container_of(kobj, struct ttm_pool_manager, kobj);
199 	unsigned val = 0;
200 
201 	if (attr == &ttm_page_pool_max)
202 		val = m->options.max_size;
203 	else if (attr == &ttm_page_pool_small)
204 		val = m->options.small;
205 	else if (attr == &ttm_page_pool_alloc_size)
206 		val = m->options.alloc_size;
207 
208 	val = val * (PAGE_SIZE >> 10);
209 
210 	return ksnprintf(buffer, PAGE_SIZE, "%u\n", val);
211 }
212 
213 static const struct sysfs_ops ttm_pool_sysfs_ops = {
214 	.show = &ttm_pool_show,
215 	.store = &ttm_pool_store,
216 };
217 
218 static struct kobj_type ttm_pool_kobj_type = {
219 	.release = &ttm_pool_kobj_release,
220 	.sysfs_ops = &ttm_pool_sysfs_ops,
221 	.default_attrs = ttm_pool_attrs,
222 };
223 
224 static struct ttm_pool_manager *_manager;
225 
226 /**
227  * Select the right pool or requested caching state and ttm flags. */
ttm_get_pool(int flags,bool huge,enum ttm_caching_state cstate)228 static struct ttm_page_pool *ttm_get_pool(int flags, bool huge,
229 					  enum ttm_caching_state cstate)
230 {
231 	int pool_index;
232 
233 	if (cstate == tt_cached)
234 		return NULL;
235 
236 	if (cstate == tt_wc)
237 		pool_index = 0x0;
238 	else
239 		pool_index = 0x1;
240 
241 	if (flags & TTM_PAGE_FLAG_DMA32) {
242 		if (huge)
243 			return NULL;
244 		pool_index |= 0x2;
245 
246 	} else if (huge) {
247 		pool_index |= 0x4;
248 	}
249 
250 	return &_manager->pools[pool_index];
251 }
252 
253 /* set memory back to wb and free the pages. */
ttm_pages_put(struct page * pages[],unsigned npages,unsigned int order)254 static void ttm_pages_put(struct page *pages[], unsigned npages,
255 		unsigned int order)
256 {
257 	unsigned int i, pages_nr = (1 << order);
258 
259 	if (order == 0) {
260 		if (ttm_set_pages_array_wb(pages, npages))
261 			pr_err("Failed to set %d pages to wb!\n", npages);
262 	}
263 
264 	for (i = 0; i < npages; ++i) {
265 		if (order > 0) {
266 			if (ttm_set_pages_wb(pages[i], pages_nr))
267 				pr_err("Failed to set %d pages to wb!\n", pages_nr);
268 		}
269 		__free_pages(pages[i], order);
270 	}
271 }
272 
ttm_pool_update_free_locked(struct ttm_page_pool * pool,unsigned freed_pages)273 static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
274 		unsigned freed_pages)
275 {
276 	pool->npages -= freed_pages;
277 	pool->nfrees += freed_pages;
278 }
279 
280 /**
281  * Free pages from pool.
282  *
283  * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
284  * number of pages in one go.
285  *
286  * @pool: to free the pages from
287  * @free_all: If set to true will free all pages in pool
288  * @use_static: Safe to use static buffer
289  **/
ttm_page_pool_free(struct ttm_page_pool * pool,unsigned nr_free,bool use_static)290 static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free,
291 			      bool use_static)
292 {
293 	static struct page *static_buf[NUM_PAGES_TO_ALLOC];
294 	unsigned long irq_flags;
295 	struct vm_page *p, *p1;
296 	struct page **pages_to_free;
297 	unsigned freed_pages = 0,
298 		 npages_to_free = nr_free;
299 	unsigned i;
300 
301 	if (NUM_PAGES_TO_ALLOC < nr_free)
302 		npages_to_free = NUM_PAGES_TO_ALLOC;
303 
304 	if (use_static)
305 		pages_to_free = static_buf;
306 	else
307 		pages_to_free = kmalloc(npages_to_free * sizeof(struct page *),
308 					M_DRM, GFP_KERNEL);
309 	if (!pages_to_free) {
310 		pr_err("Failed to allocate memory for pool free operation\n");
311 		return 0;
312 	}
313 
314 restart:
315 	spin_lock_irqsave(&pool->lock, irq_flags);
316 
317 	TAILQ_FOREACH_REVERSE_MUTABLE(p, &pool->list, pglist, pageq, p1) {
318 		if (freed_pages >= npages_to_free)
319 			break;
320 
321 		pages_to_free[freed_pages++] = (struct page *)p;
322 		/* We can only remove NUM_PAGES_TO_ALLOC at a time. */
323 		if (freed_pages >= NUM_PAGES_TO_ALLOC) {
324 			/* remove range of pages from the pool */
325 			for (i = 0; i < freed_pages; i++)
326 				TAILQ_REMOVE(&pool->list, (struct vm_page *)pages_to_free[i], pageq);
327 
328 			ttm_pool_update_free_locked(pool, freed_pages);
329 			/**
330 			 * Because changing page caching is costly
331 			 * we unlock the pool to prevent stalling.
332 			 */
333 			spin_unlock_irqrestore(&pool->lock, irq_flags);
334 
335 			ttm_pages_put(pages_to_free, freed_pages, pool->order);
336 			if (likely(nr_free != FREE_ALL_PAGES))
337 				nr_free -= freed_pages;
338 
339 			if (NUM_PAGES_TO_ALLOC >= nr_free)
340 				npages_to_free = nr_free;
341 			else
342 				npages_to_free = NUM_PAGES_TO_ALLOC;
343 
344 			freed_pages = 0;
345 
346 			/* free all so restart the processing */
347 			if (nr_free)
348 				goto restart;
349 
350 			/* Not allowed to fall through or break because
351 			 * following context is inside spinlock while we are
352 			 * outside here.
353 			 */
354 			goto out;
355 
356 		}
357 	}
358 
359 	/* remove range of pages from the pool */
360 	if (freed_pages) {
361 		for (i = 0; i < freed_pages; i++)
362 			TAILQ_REMOVE(&pool->list, (struct vm_page *)pages_to_free[i], pageq);
363 
364 		ttm_pool_update_free_locked(pool, freed_pages);
365 		nr_free -= freed_pages;
366 	}
367 
368 	spin_unlock_irqrestore(&pool->lock, irq_flags);
369 
370 	if (freed_pages)
371 		ttm_pages_put(pages_to_free, freed_pages, pool->order);
372 out:
373 	if (pages_to_free != static_buf)
374 		kfree(pages_to_free);
375 	return nr_free;
376 }
377 
378 /**
379  * Callback for mm to request pool to reduce number of page held.
380  *
381  * XXX: (dchinner) Deadlock warning!
382  *
383  * This code is crying out for a shrinker per pool....
384  */
385 static unsigned long
ttm_pool_shrink_scan(struct shrinker * shrink,struct shrink_control * sc)386 ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
387 {
388 	static DEFINE_MUTEX(lock);
389 	static unsigned start_pool;
390 	unsigned i;
391 	unsigned pool_offset;
392 	struct ttm_page_pool *pool;
393 	int shrink_pages = 100; /* XXXKIB */
394 	unsigned long freed = 0;
395 	unsigned int nr_free_pool;
396 
397 #ifdef __DragonFly__
398 	sc->gfp_mask = M_WAITOK;
399 #endif
400 
401 	if (!mutex_trylock(&lock))
402 		return SHRINK_STOP;
403 	pool_offset = ++start_pool % NUM_POOLS;
404 	/* select start pool in round robin fashion */
405 	for (i = 0; i < NUM_POOLS; ++i) {
406 		unsigned nr_free = shrink_pages;
407 		unsigned page_nr;
408 
409 		if (shrink_pages == 0)
410 			break;
411 
412 		pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
413 		page_nr = (1 << pool->order);
414 		/* OK to use static buffer since global mutex is held. */
415 		nr_free_pool = roundup(nr_free, page_nr) >> pool->order;
416 		shrink_pages = ttm_page_pool_free(pool, nr_free_pool, true);
417 		freed += (nr_free_pool - shrink_pages) << pool->order;
418 		if (freed >= sc->nr_to_scan)
419 			break;
420 		shrink_pages <<= pool->order;
421 	}
422 	mutex_unlock(&lock);
423 	return freed;
424 }
425 
426 
427 static unsigned long
ttm_pool_shrink_count(struct shrinker * shrink,struct shrink_control * sc)428 ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
429 {
430 	unsigned i;
431 	unsigned long count = 0;
432 	struct ttm_page_pool *pool;
433 
434 	for (i = 0; i < NUM_POOLS; ++i) {
435 		pool = &_manager->pools[i];
436 		count += (pool->npages << pool->order);
437 	}
438 
439 	return count;
440 }
441 
ttm_pool_mm_shrink_init(struct ttm_pool_manager * manager)442 static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
443 {
444 	manager->mm_shrink.count_objects = ttm_pool_shrink_count;
445 	manager->mm_shrink.scan_objects = ttm_pool_shrink_scan;
446 	manager->mm_shrink.seeks = 1;
447 	manager->lowmem_handler = EVENTHANDLER_REGISTER(vm_lowmem,
448 	    ttm_pool_shrink_scan, manager, EVENTHANDLER_PRI_ANY);
449 }
450 
ttm_pool_mm_shrink_fini(struct ttm_pool_manager * manager)451 static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
452 {
453 	EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler);
454 }
455 
ttm_set_pages_caching(struct page ** pages,enum ttm_caching_state cstate,unsigned cpages)456 static int ttm_set_pages_caching(struct page **pages,
457 		enum ttm_caching_state cstate, unsigned cpages)
458 {
459 	int r = 0;
460 	/* Set page caching */
461 	switch (cstate) {
462 	case tt_uncached:
463 		r = ttm_set_pages_array_uc(pages, cpages);
464 		if (r)
465 			pr_err("Failed to set %d pages to uc!\n", cpages);
466 		break;
467 	case tt_wc:
468 		r = ttm_set_pages_array_wc(pages, cpages);
469 		if (r)
470 			pr_err("Failed to set %d pages to wc!\n", cpages);
471 		break;
472 	default:
473 		break;
474 	}
475 	return r;
476 }
477 
478 /**
479  * Free pages the pages that failed to change the caching state. If there is
480  * any pages that have changed their caching state already put them to the
481  * pool.
482  */
ttm_handle_caching_state_failure(struct pglist * pages,int ttm_flags,enum ttm_caching_state cstate,struct page ** failed_pages,unsigned cpages)483 static void ttm_handle_caching_state_failure(struct pglist *pages,
484 		int ttm_flags, enum ttm_caching_state cstate,
485 		struct page **failed_pages, unsigned cpages)
486 {
487 	unsigned i;
488 	/* Failed pages have to be freed */
489 	for (i = 0; i < cpages; ++i) {
490 		TAILQ_REMOVE(pages, (struct vm_page *)failed_pages[i], pageq);
491 		__free_page(failed_pages[i]);
492 	}
493 }
494 
495 /**
496  * Allocate new pages with correct caching.
497  *
498  * This function is reentrant if caller updates count depending on number of
499  * pages returned in pages array.
500  */
ttm_alloc_new_pages(struct pglist * pages,gfp_t gfp_flags,int ttm_flags,enum ttm_caching_state cstate,unsigned count,unsigned order)501 static int ttm_alloc_new_pages(struct pglist *pages, gfp_t gfp_flags,
502 			       int ttm_flags, enum ttm_caching_state cstate,
503 			       unsigned count, unsigned order)
504 {
505 	struct page **caching_array;
506 	struct page *p;
507 	int r = 0;
508 	unsigned i, j, cpages;
509 	unsigned npages = 1 << order;
510 	unsigned max_cpages = min(count << order, (unsigned)NUM_PAGES_TO_ALLOC);
511 
512 	/* allocate array for page caching change */
513 	caching_array = kmalloc(max_cpages*sizeof(struct page *), M_DRM, M_WAITOK);
514 
515 	if (!caching_array) {
516 		pr_debug("Unable to allocate table for new pages\n");
517 		return -ENOMEM;
518 	}
519 
520 	for (i = 0, cpages = 0; i < count; ++i) {
521 		p = alloc_pages(gfp_flags, order);
522 
523 		if (!p) {
524 			pr_debug("Unable to get page %u\n", i);
525 
526 			/* store already allocated pages in the pool after
527 			 * setting the caching state */
528 			if (cpages) {
529 				r = ttm_set_pages_caching(caching_array,
530 							  cstate, cpages);
531 				if (r)
532 					ttm_handle_caching_state_failure(pages,
533 						ttm_flags, cstate,
534 						caching_array, cpages);
535 			}
536 			r = -ENOMEM;
537 			goto out;
538 		}
539 
540 		TAILQ_INSERT_HEAD(pages, (struct vm_page *)p, pageq);
541 
542 #ifdef CONFIG_HIGHMEM
543 		/* gfp flags of highmem page should never be dma32 so we
544 		 * we should be fine in such case
545 		 */
546 		if (PageHighMem(p))
547 			continue;
548 
549 #endif
550 		for (j = 0; j < npages; ++j) {
551 			caching_array[cpages++] = p++;
552 			if (cpages == max_cpages) {
553 
554 				r = ttm_set_pages_caching(caching_array,
555 						cstate, cpages);
556 				if (r) {
557 					ttm_handle_caching_state_failure(pages,
558 						ttm_flags, cstate,
559 						caching_array, cpages);
560 					goto out;
561 				}
562 				cpages = 0;
563 			}
564 		}
565 	}
566 
567 	if (cpages) {
568 		r = ttm_set_pages_caching(caching_array, cstate, cpages);
569 		if (r)
570 			ttm_handle_caching_state_failure(pages,
571 					ttm_flags, cstate,
572 					caching_array, cpages);
573 	}
574 out:
575 	kfree(caching_array);
576 
577 	return r;
578 }
579 
580 /**
581  * Fill the given pool if there aren't enough pages and the requested number of
582  * pages is small.
583  */
ttm_page_pool_fill_locked(struct ttm_page_pool * pool,int ttm_flags,enum ttm_caching_state cstate,unsigned count,unsigned long * irq_flags)584 static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags,
585 				      enum ttm_caching_state cstate,
586 				      unsigned count, unsigned long *irq_flags)
587 {
588 	vm_page_t p;
589 	int r;
590 	unsigned cpages = 0;
591 	/**
592 	 * Only allow one pool fill operation at a time.
593 	 * If pool doesn't have enough pages for the allocation new pages are
594 	 * allocated from outside of pool.
595 	 */
596 	if (pool->fill_lock)
597 		return;
598 
599 	pool->fill_lock = true;
600 
601 	/* If allocation request is small and there are not enough
602 	 * pages in a pool we fill the pool up first. */
603 	if (count < _manager->options.small
604 		&& count > pool->npages) {
605 		struct pglist new_pages;
606 		unsigned alloc_size = _manager->options.alloc_size;
607 
608 		/**
609 		 * Can't change page caching if in irqsave context. We have to
610 		 * drop the pool->lock.
611 		 */
612 		spin_unlock_irqrestore(&pool->lock, *irq_flags);
613 
614 		TAILQ_INIT(&new_pages);
615 		r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
616 					cstate, alloc_size, 0);
617 		spin_lock_irqsave(&pool->lock, *irq_flags);
618 
619 		if (!r) {
620 			TAILQ_CONCAT(&pool->list, &new_pages, pageq);
621 			++pool->nrefills;
622 			pool->npages += alloc_size;
623 		} else {
624 			pr_debug("Failed to fill pool (%p)\n", pool);
625 			/* If we have any pages left put them to the pool. */
626 			TAILQ_FOREACH(p, &new_pages, pageq) {
627 				++cpages;
628 			}
629 			TAILQ_CONCAT(&pool->list, &new_pages, pageq);
630 			pool->npages += cpages;
631 		}
632 
633 	}
634 	pool->fill_lock = false;
635 }
636 
637 /**
638  * Cut 'count' number of pages from the pool and put them on the return list.
639  *
640  * @return count of pages still required to fulfill the request.
641  */
ttm_page_pool_get_pages(struct ttm_page_pool * pool,struct pglist * pages,int ttm_flags,enum ttm_caching_state cstate,unsigned count,unsigned order)642 static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
643 					struct pglist *pages,
644 					int ttm_flags,
645 					enum ttm_caching_state cstate,
646 					unsigned count, unsigned order)
647 {
648 	unsigned long irq_flags;
649 	vm_page_t p;
650 	unsigned i;
651 
652 	spin_lock_irqsave(&pool->lock, irq_flags);
653 	if (!order)
654 		ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count,
655 					  &irq_flags);
656 
657 	if (count >= pool->npages) {
658 		/* take all pages from the pool */
659 		TAILQ_CONCAT(pages, &pool->list, pageq);
660 		count -= pool->npages;
661 		pool->npages = 0;
662 		goto out;
663 	}
664 	for (i = 0; i < count; i++) {
665 		p = TAILQ_FIRST(&pool->list);
666 		TAILQ_REMOVE(&pool->list, p, pageq);
667 		TAILQ_INSERT_TAIL(pages, p, pageq);
668 	}
669 	pool->npages -= count;
670 	count = 0;
671 out:
672 	spin_unlock_irqrestore(&pool->lock, irq_flags);
673 	return count;
674 }
675 
676 /* Put all pages in pages list to correct pool to wait for reuse */
ttm_put_pages(struct page ** pages,unsigned npages,int flags,enum ttm_caching_state cstate)677 static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
678 			  enum ttm_caching_state cstate)
679 {
680 	struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
681 	unsigned long irq_flags;
682 	unsigned i;
683 	struct vm_page *page;
684 
685 	if (pool == NULL) {
686 		/* No pool for this memory type so free the pages */
687 		for (i = 0; i < npages; i++) {
688 			if (pages[i]) {
689 #if 0
690 				if (page_count(pages[i]) != 1)
691 					pr_err("Erroneous page count. Leaking pages.\n");
692 #endif
693 				__free_page(pages[i]);
694 				pages[i] = NULL;
695 			}
696 		}
697 		return;
698 	}
699 
700 	spin_lock_irqsave(&pool->lock, irq_flags);
701 	for (i = 0; i < npages; i++) {
702 		if (pages[i]) {
703 			page = (struct vm_page *)pages[i];
704 			TAILQ_INSERT_TAIL(&pool->list, page, pageq);
705 			pages[i] = NULL;
706 			pool->npages++;
707 		}
708 	}
709 	/* Check that we don't go over the pool limit */
710 	npages = 0;
711 	if (pool->npages > _manager->options.max_size) {
712 		npages = pool->npages - _manager->options.max_size;
713 		/* free at least NUM_PAGES_TO_ALLOC number of pages
714 		 * to reduce calls to set_memory_wb */
715 		if (npages < NUM_PAGES_TO_ALLOC)
716 			npages = NUM_PAGES_TO_ALLOC;
717 	}
718 	spin_unlock_irqrestore(&pool->lock, irq_flags);
719 	if (npages)
720 		ttm_page_pool_free(pool, npages, false);
721 }
722 
723 /*
724  * On success pages list will hold count number of correctly
725  * cached pages.
726  */
ttm_get_pages(struct page ** pages,unsigned npages,int flags,enum ttm_caching_state cstate)727 static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
728 			 enum ttm_caching_state cstate)
729 {
730 	struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
731 	struct pglist plist;
732 	struct vm_page *p = NULL;
733 	gfp_t gfp_flags = GFP_USER;
734 	unsigned count;
735 	int r;
736 
737 	/* set zero flag for page allocation if required */
738 	if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
739 		gfp_flags |= __GFP_ZERO;
740 
741 	/* No pool for cached pages */
742 	if (pool == NULL) {
743 		if (flags & TTM_PAGE_FLAG_DMA32)
744 			gfp_flags |= GFP_DMA32;
745 		else
746 			gfp_flags |= GFP_HIGHUSER;
747 
748 		for (r = 0; r < npages; ++r) {
749 			p = (struct vm_page *)alloc_page(gfp_flags);
750 			if (!p) {
751 
752 				pr_err("Unable to allocate page\n");
753 				return -ENOMEM;
754 			}
755 			pages[r] = (struct page *)p;
756 		}
757 		return 0;
758 	}
759 
760 	/* combine zero flag to pool flags */
761 	gfp_flags |= pool->gfp_flags;
762 
763 	/* First we take pages from the pool */
764 	TAILQ_INIT(&plist);
765 	npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages, 0);
766 	count = 0;
767 	TAILQ_FOREACH(p, &plist, pageq) {
768 		pages[count++] = (struct page *)p;
769 	}
770 
771 	/* clear the pages coming from the pool if requested */
772 	if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
773 		TAILQ_FOREACH(p, &plist, pageq) {
774 			pmap_zero_page(VM_PAGE_TO_PHYS(p));
775 		}
776 	}
777 
778 	/* If pool didn't have enough pages allocate new one. */
779 	if (npages > 0) {
780 		/* ttm_alloc_new_pages doesn't reference pool so we can run
781 		 * multiple requests in parallel.
782 		 **/
783 		TAILQ_INIT(&plist);
784 		r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages, 0);
785 		TAILQ_FOREACH(p, &plist, pageq) {
786 			pages[count++] = (struct page *)p;
787 		}
788 		if (r) {
789 			/* If there is any pages in the list put them back to
790 			 * the pool. */
791 			pr_err("Failed to allocate extra pages for large request\n");
792 			ttm_put_pages(pages, count, flags, cstate);
793 			return r;
794 		}
795 	}
796 
797 	return 0;
798 }
799 
ttm_page_pool_init_locked(struct ttm_page_pool * pool,gfp_t flags,char * name,unsigned int order)800 static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags,
801 		char *name, unsigned int order)
802 {
803 	lockinit(&pool->lock, "ttmpool", 0, LK_CANRECURSE);
804 	pool->fill_lock = false;
805 	TAILQ_INIT(&pool->list);
806 	pool->npages = pool->nfrees = 0;
807 	pool->gfp_flags = flags;
808 	pool->name = name;
809 	pool->order = order;
810 }
811 
ttm_page_alloc_init(struct ttm_mem_global * glob,unsigned max_pages)812 int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
813 {
814 	int ret;
815 
816 	WARN_ON(_manager);
817 
818 	pr_info("Initializing pool allocator\n");
819 
820 	_manager = kzalloc(sizeof(*_manager), GFP_KERNEL);
821 	if (!_manager)
822 		return -ENOMEM;
823 
824 	ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc", 0);
825 
826 	ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc", 0);
827 
828 	ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
829 				  GFP_USER | GFP_DMA32, "wc dma", 0);
830 
831 	ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
832 				  GFP_USER | GFP_DMA32, "uc dma", 0);
833 
834 	_manager->options.max_size = max_pages;
835 	_manager->options.small = SMALL_ALLOCATION;
836 	_manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
837 
838 	ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type,
839 				   &glob->kobj, "pool");
840 	if (unlikely(ret != 0))
841 		goto error;
842 
843 	ttm_pool_mm_shrink_init(_manager);
844 
845 	return 0;
846 
847 error:
848 	kobject_put(&_manager->kobj);
849 	_manager = NULL;
850 	return ret;
851 }
852 
ttm_page_alloc_fini(void)853 void ttm_page_alloc_fini(void)
854 {
855 	int i;
856 
857 	pr_info("Finalizing pool allocator\n");
858 	ttm_pool_mm_shrink_fini(_manager);
859 
860 	/* OK to use static buffer since global mutex is no longer used. */
861 	for (i = 0; i < NUM_POOLS; ++i)
862 		ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES, true);
863 
864 	kobject_put(&_manager->kobj);
865 	_manager = NULL;
866 }
867 
868 static void
ttm_pool_unpopulate_helper(struct ttm_tt * ttm,unsigned mem_count_update)869 ttm_pool_unpopulate_helper(struct ttm_tt *ttm, unsigned mem_count_update)
870 {
871 	struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob;
872 	unsigned i;
873 
874 	if (mem_count_update == 0)
875 		goto put_pages;
876 
877 	for (i = 0; i < mem_count_update; ++i) {
878 		if (!ttm->pages[i])
879 			continue;
880 
881 		ttm_mem_global_free_page(mem_glob, ttm->pages[i], PAGE_SIZE);
882 	}
883 
884 put_pages:
885 	ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
886 		      ttm->caching_state);
887 	ttm->state = tt_unpopulated;
888 }
889 
ttm_pool_populate(struct ttm_tt * ttm,struct ttm_operation_ctx * ctx)890 int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
891 {
892 	struct ttm_mem_global *mem_glob = ttm->bdev->glob->mem_glob;
893 	unsigned i;
894 	int ret;
895 
896 	if (ttm->state != tt_unpopulated)
897 		return 0;
898 
899 	if (ttm_check_under_lowerlimit(mem_glob, ttm->num_pages, ctx))
900 		return -ENOMEM;
901 
902 	ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
903 			    ttm->caching_state);
904 	if (unlikely(ret != 0)) {
905 		ttm_pool_unpopulate_helper(ttm, 0);
906 		return ret;
907 	}
908 
909 	for (i = 0; i < ttm->num_pages; ++i) {
910 		ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
911 						PAGE_SIZE, ctx);
912 		if (unlikely(ret != 0)) {
913 			ttm_pool_unpopulate_helper(ttm, i);
914 			return -ENOMEM;
915 		}
916 	}
917 
918 	if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
919 		ret = ttm_tt_swapin(ttm);
920 		if (unlikely(ret != 0)) {
921 			ttm_pool_unpopulate(ttm);
922 			return ret;
923 		}
924 	}
925 
926 	ttm->state = tt_unbound;
927 	return 0;
928 }
929 EXPORT_SYMBOL(ttm_pool_populate);
930 
ttm_pool_unpopulate(struct ttm_tt * ttm)931 void ttm_pool_unpopulate(struct ttm_tt *ttm)
932 {
933 	ttm_pool_unpopulate_helper(ttm, ttm->num_pages);
934 }
935 EXPORT_SYMBOL(ttm_pool_unpopulate);
936 
ttm_populate_and_map_pages(struct device * dev,struct ttm_dma_tt * tt,struct ttm_operation_ctx * ctx)937 int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt,
938 					struct ttm_operation_ctx *ctx)
939 {
940 	unsigned i, j;
941 	int r;
942 
943 	r = ttm_pool_populate(&tt->ttm, ctx);
944 	if (r)
945 		return r;
946 
947 	for (i = 0; i < tt->ttm.num_pages; ++i) {
948 		struct page *p = tt->ttm.pages[i];
949 		size_t num_pages = 1;
950 
951 		for (j = i + 1; j < tt->ttm.num_pages; ++j) {
952 			if (++p != tt->ttm.pages[j])
953 				break;
954 
955 			++num_pages;
956 		}
957 
958 		tt->dma_address[i] = dma_map_page(dev, tt->ttm.pages[i],
959 						  0, num_pages * PAGE_SIZE,
960 						  DMA_BIDIRECTIONAL);
961 		if (dma_mapping_error(dev, tt->dma_address[i])) {
962 			while (i--) {
963 				dma_unmap_page(dev, tt->dma_address[i],
964 					       PAGE_SIZE, DMA_BIDIRECTIONAL);
965 				tt->dma_address[i] = 0;
966 			}
967 			ttm_pool_unpopulate(&tt->ttm);
968 			return -EFAULT;
969 		}
970 
971 		for (j = 1; j < num_pages; ++j) {
972 			tt->dma_address[i + 1] = tt->dma_address[i] + PAGE_SIZE;
973 			++i;
974 		}
975 	}
976 	return 0;
977 }
978 EXPORT_SYMBOL(ttm_populate_and_map_pages);
979 
ttm_unmap_and_unpopulate_pages(struct device * dev,struct ttm_dma_tt * tt)980 void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_dma_tt *tt)
981 {
982 	unsigned i, j;
983 
984 	for (i = 0; i < tt->ttm.num_pages;) {
985 		struct page *p = tt->ttm.pages[i];
986 		size_t num_pages = 1;
987 
988 		if (!tt->dma_address[i] || !tt->ttm.pages[i]) {
989 			++i;
990 			continue;
991 		}
992 
993 		for (j = i + 1; j < tt->ttm.num_pages; ++j) {
994 			if (++p != tt->ttm.pages[j])
995 				break;
996 
997 			++num_pages;
998 		}
999 
1000 		dma_unmap_page(dev, tt->dma_address[i], num_pages * PAGE_SIZE,
1001 			       DMA_BIDIRECTIONAL);
1002 
1003 		i += num_pages;
1004 	}
1005 	ttm_pool_unpopulate(&tt->ttm);
1006 }
1007 EXPORT_SYMBOL(ttm_unmap_and_unpopulate_pages);
1008 
1009 #if 0
1010 int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
1011 {
1012 	struct ttm_page_pool *p;
1013 	unsigned i;
1014 	char *h[] = {"pool", "refills", "pages freed", "size"};
1015 	if (!_manager) {
1016 		seq_printf(m, "No pool allocator running.\n");
1017 		return 0;
1018 	}
1019 	seq_printf(m, "%7s %12s %13s %8s\n",
1020 			h[0], h[1], h[2], h[3]);
1021 	for (i = 0; i < NUM_POOLS; ++i) {
1022 		p = &_manager->pools[i];
1023 
1024 		seq_printf(m, "%7s %12ld %13ld %8d\n",
1025 				p->name, p->nrefills,
1026 				p->nfrees, p->npages);
1027 	}
1028 	return 0;
1029 }
1030 #endif
1031 EXPORT_SYMBOL(ttm_page_alloc_debugfs);
1032