xref: /dragonfly/sys/dev/drm/ttm/ttm_memory.c (revision 5ca0a96d) 
1 /**************************************************************************
2  *
3  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4  * All Rights Reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sub license, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the
15  * next paragraph) shall be included in all copies or substantial portions
16  * of the Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
25  *
26  **************************************************************************/
27 
28 #define pr_fmt(fmt) "[TTM] " fmt
29 
30 #include <drm/ttm/ttm_memory.h>
31 #include <drm/ttm/ttm_module.h>
32 #include <drm/ttm/ttm_page_alloc.h>
33 #include <linux/spinlock.h>
34 #include <linux/sched.h>
35 #include <linux/wait.h>
36 #include <linux/mm.h>
37 #include <linux/module.h>
38 #include <linux/slab.h>
39 
40 #define TTM_MEMORY_ALLOC_RETRIES 4
41 
42 struct ttm_mem_zone {
43 	struct kobject kobj;
44 	struct ttm_mem_global *glob;
45 	const char *name;
46 	uint64_t zone_mem;
47 	uint64_t emer_mem;
48 	uint64_t max_mem;
49 	uint64_t swap_limit;
50 	uint64_t used_mem;
51 };
52 
53 static struct attribute ttm_mem_sys = {
54 	.name = "zone_memory",
55 	.mode = S_IRUGO
56 };
57 static struct attribute ttm_mem_emer = {
58 	.name = "emergency_memory",
59 	.mode = S_IRUGO | S_IWUSR
60 };
61 static struct attribute ttm_mem_max = {
62 	.name = "available_memory",
63 	.mode = S_IRUGO | S_IWUSR
64 };
65 static struct attribute ttm_mem_swap = {
66 	.name = "swap_limit",
67 	.mode = S_IRUGO | S_IWUSR
68 };
69 static struct attribute ttm_mem_used = {
70 	.name = "used_memory",
71 	.mode = S_IRUGO
72 };
73 
74 static void ttm_mem_zone_kobj_release(struct kobject *kobj)
75 {
76 	struct ttm_mem_zone *zone =
77 		container_of(kobj, struct ttm_mem_zone, kobj);
78 
79 	pr_info("Zone %7s: Used memory at exit: %llu kiB\n",
80 		zone->name, (unsigned long long)zone->used_mem >> 10);
81 	kfree(zone);
82 }
83 
84 static ssize_t ttm_mem_zone_show(struct kobject *kobj,
85 				 struct attribute *attr,
86 				 char *buffer)
87 {
88 	struct ttm_mem_zone *zone =
89 		container_of(kobj, struct ttm_mem_zone, kobj);
90 	uint64_t val = 0;
91 
92 	lockmgr(&zone->glob->lock, LK_EXCLUSIVE);
93 	if (attr == &ttm_mem_sys)
94 		val = zone->zone_mem;
95 	else if (attr == &ttm_mem_emer)
96 		val = zone->emer_mem;
97 	else if (attr == &ttm_mem_max)
98 		val = zone->max_mem;
99 	else if (attr == &ttm_mem_swap)
100 		val = zone->swap_limit;
101 	else if (attr == &ttm_mem_used)
102 		val = zone->used_mem;
103 	lockmgr(&zone->glob->lock, LK_RELEASE);
104 
105 	return ksnprintf(buffer, PAGE_SIZE, "%llu\n",
106 			(unsigned long long) val >> 10);
107 }
108 
109 static void ttm_check_swapping(struct ttm_mem_global *glob);
110 
111 static ssize_t ttm_mem_zone_store(struct kobject *kobj,
112 				  struct attribute *attr,
113 				  const char *buffer,
114 				  size_t size)
115 {
116 	struct ttm_mem_zone *zone =
117 		container_of(kobj, struct ttm_mem_zone, kobj);
118 	int chars;
119 	unsigned long val;
120 	uint64_t val64;
121 
122 	chars = ksscanf(buffer, "%lu", &val);
123 	if (chars == 0)
124 		return size;
125 
126 	val64 = val;
127 	val64 <<= 10;
128 
129 	lockmgr(&zone->glob->lock, LK_EXCLUSIVE);
130 	if (val64 > zone->zone_mem)
131 		val64 = zone->zone_mem;
132 	if (attr == &ttm_mem_emer) {
133 		zone->emer_mem = val64;
134 		if (zone->max_mem > val64)
135 			zone->max_mem = val64;
136 	} else if (attr == &ttm_mem_max) {
137 		zone->max_mem = val64;
138 		if (zone->emer_mem < val64)
139 			zone->emer_mem = val64;
140 	} else if (attr == &ttm_mem_swap)
141 		zone->swap_limit = val64;
142 	lockmgr(&zone->glob->lock, LK_RELEASE);
143 
144 	ttm_check_swapping(zone->glob);
145 
146 	return size;
147 }
148 
149 static struct attribute *ttm_mem_zone_attrs[] = {
150 	&ttm_mem_sys,
151 	&ttm_mem_emer,
152 	&ttm_mem_max,
153 	&ttm_mem_swap,
154 	&ttm_mem_used,
155 	NULL
156 };
157 
158 static const struct sysfs_ops ttm_mem_zone_ops = {
159 	.show = &ttm_mem_zone_show,
160 	.store = &ttm_mem_zone_store
161 };
162 
163 static struct kobj_type ttm_mem_zone_kobj_type = {
164 	.release = &ttm_mem_zone_kobj_release,
165 	.sysfs_ops = &ttm_mem_zone_ops,
166 	.default_attrs = ttm_mem_zone_attrs,
167 };
168 
169 static void ttm_mem_global_kobj_release(struct kobject *kobj)
170 {
171 	struct ttm_mem_global *glob =
172 		container_of(kobj, struct ttm_mem_global, kobj);
173 
174 	kfree(glob);
175 }
176 
177 static struct kobj_type ttm_mem_glob_kobj_type = {
178 	.release = &ttm_mem_global_kobj_release,
179 };
180 
181 static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob,
182 					bool from_wq, uint64_t extra)
183 {
184 	unsigned int i;
185 	struct ttm_mem_zone *zone;
186 	uint64_t target;
187 
188 	for (i = 0; i < glob->num_zones; ++i) {
189 		zone = glob->zones[i];
190 
191 		if (from_wq)
192 			target = zone->swap_limit;
193 		else if (priv_check(curthread, PRIV_VM_MLOCK) == 0)
194 			target = zone->emer_mem;
195 		else
196 			target = zone->max_mem;
197 
198 		target = (extra > target) ? 0ULL : target;
199 
200 		if (zone->used_mem > target)
201 			return true;
202 	}
203 	return false;
204 }
205 
206 /**
207  * At this point we only support a single shrink callback.
208  * Extend this if needed, perhaps using a linked list of callbacks.
209  * Note that this function is reentrant:
210  * many threads may try to swap out at any given time.
211  */
212 
213 static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq,
214 		       uint64_t extra)
215 {
216 	int ret;
217 	struct ttm_mem_shrink *shrink;
218 
219 	lockmgr(&glob->lock, LK_EXCLUSIVE);
220 	if (glob->shrink == NULL)
221 		goto out;
222 
223 	while (ttm_zones_above_swap_target(glob, from_wq, extra)) {
224 		shrink = glob->shrink;
225 		lockmgr(&glob->lock, LK_RELEASE);
226 		ret = shrink->do_shrink(shrink);
227 		lockmgr(&glob->lock, LK_EXCLUSIVE);
228 		if (unlikely(ret != 0))
229 			goto out;
230 	}
231 out:
232 	lockmgr(&glob->lock, LK_RELEASE);
233 }
234 
235 
236 
237 static void ttm_shrink_work(struct work_struct *work)
238 {
239 	struct ttm_mem_global *glob =
240 	    container_of(work, struct ttm_mem_global, work);
241 
242 	ttm_shrink(glob, true, 0ULL);
243 }
244 
245 static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob,
246     uint64_t mem)
247 {
248 	struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL);
249 	int ret;
250 
251 	zone->name = "kernel";
252 	zone->zone_mem = mem;
253 	zone->max_mem = mem >> 1;
254 	zone->emer_mem = (mem >> 1) + (mem >> 2);
255 	zone->swap_limit = zone->max_mem - (mem >> 3);
256 	zone->used_mem = 0;
257 	zone->glob = glob;
258 	glob->zone_kernel = zone;
259 	ret = kobject_init_and_add(
260 		&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);
261 	if (unlikely(ret != 0)) {
262 		kobject_put(&zone->kobj);
263 		return ret;
264 	}
265 	glob->zones[glob->num_zones++] = zone;
266 	return 0;
267 }
268 
269 #ifdef CONFIG_HIGHMEM
270 #else
271 static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob,
272     uint64_t mem)
273 {
274 	struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL);
275 	int ret;
276 
277 	/**
278 	 * No special dma32 zone needed.
279 	 */
280 
281 	if ((physmem * PAGE_SIZE) <= ((uint64_t) 1ULL << 32)) {
282 		kfree(zone);
283 		return 0;
284 	}
285 
286 	/*
287 	 * Limit max dma32 memory to 4GB for now
288 	 * until we can figure out how big this
289 	 * zone really is.
290 	 */
291 	if (mem > ((uint64_t) 1ULL << 32))
292 		mem = ((uint64_t) 1ULL << 32);
293 
294 	zone->name = "dma32";
295 	zone->zone_mem = mem;
296 	zone->max_mem = mem >> 1;
297 	zone->emer_mem = (mem >> 1) + (mem >> 2);
298 	zone->swap_limit = zone->max_mem - (mem >> 3);
299 	zone->used_mem = 0;
300 	zone->glob = glob;
301 	glob->zone_dma32 = zone;
302 	ret = kobject_init_and_add(
303 		&zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);
304 	if (unlikely(ret != 0)) {
305 		kobject_put(&zone->kobj);
306 		return ret;
307 	}
308 	glob->zones[glob->num_zones++] = zone;
309 	return 0;
310 }
311 #endif
312 
313 int ttm_mem_global_init(struct ttm_mem_global *glob)
314 {
315 	u_int64_t mem;
316 	int ret;
317 	int i;
318 	struct ttm_mem_zone *zone;
319 
320 	lockinit(&glob->lock, "ttmemglob", 0, 0);
321 	glob->swap_queue = create_singlethread_workqueue("ttm_swap");
322 	INIT_WORK(&glob->work, ttm_shrink_work);
323 	ret = kobject_init_and_add(
324 		&glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting");
325 	if (unlikely(ret != 0)) {
326 		kobject_put(&glob->kobj);
327 		return ret;
328 	}
329 
330 	/*
331 	 * Managed contiguous memory for TTM.  Only use kernel-reserved
332 	 * dma memory for TTM, which can be controlled via /boot/loader.conf
333 	 * (e.g. vm.dma_reserved=256m).  This is the only truly dependable
334 	 * DMA memory.
335 	 */
336 	mem = (uint64_t)vm_contig_avail_pages() * PAGE_SIZE;
337 
338 	ret = ttm_mem_init_kernel_zone(glob, mem);
339 	if (unlikely(ret != 0))
340 		goto out_no_zone;
341 #ifdef CONFIG_HIGHMEM
342 	ret = ttm_mem_init_highmem_zone(glob, &si);
343 	if (unlikely(ret != 0))
344 		goto out_no_zone;
345 #else
346 	ret = ttm_mem_init_dma32_zone(glob, mem);
347 	if (unlikely(ret != 0))
348 		goto out_no_zone;
349 #endif
350 	for (i = 0; i < glob->num_zones; ++i) {
351 		zone = glob->zones[i];
352 		pr_info("Zone %7s: Available graphics memory: %llu kiB\n",
353 			zone->name, (unsigned long long)zone->max_mem >> 10);
354 	}
355 	ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE));
356 	ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE));
357 	return 0;
358 out_no_zone:
359 	ttm_mem_global_release(glob);
360 	return ret;
361 }
362 EXPORT_SYMBOL(ttm_mem_global_init);
363 
364 void ttm_mem_global_release(struct ttm_mem_global *glob)
365 {
366 	unsigned int i;
367 	struct ttm_mem_zone *zone;
368 
369 	/* let the page allocator first stop the shrink work. */
370 	ttm_page_alloc_fini();
371 	ttm_dma_page_alloc_fini();
372 
373 	flush_workqueue(glob->swap_queue);
374 	destroy_workqueue(glob->swap_queue);
375 	glob->swap_queue = NULL;
376 	for (i = 0; i < glob->num_zones; ++i) {
377 		zone = glob->zones[i];
378 		kobject_del(&zone->kobj);
379 		kobject_put(&zone->kobj);
380 	}
381 	kobject_del(&glob->kobj);
382 	kobject_put(&glob->kobj);
383 }
384 EXPORT_SYMBOL(ttm_mem_global_release);
385 
386 static void ttm_check_swapping(struct ttm_mem_global *glob)
387 {
388 	bool needs_swapping = false;
389 	unsigned int i;
390 	struct ttm_mem_zone *zone;
391 
392 	lockmgr(&glob->lock, LK_EXCLUSIVE);
393 	for (i = 0; i < glob->num_zones; ++i) {
394 		zone = glob->zones[i];
395 		if (zone->used_mem > zone->swap_limit) {
396 			needs_swapping = true;
397 			break;
398 		}
399 	}
400 	lockmgr(&glob->lock, LK_RELEASE);
401 
402 	if (unlikely(needs_swapping))
403 		(void)queue_work(glob->swap_queue, &glob->work);
404 
405 }
406 
407 static void ttm_mem_global_free_zone(struct ttm_mem_global *glob,
408 				     struct ttm_mem_zone *single_zone,
409 				     uint64_t amount)
410 {
411 	unsigned int i;
412 	struct ttm_mem_zone *zone;
413 
414 	lockmgr(&glob->lock, LK_EXCLUSIVE);
415 	for (i = 0; i < glob->num_zones; ++i) {
416 		zone = glob->zones[i];
417 		if (single_zone && zone != single_zone)
418 			continue;
419 		zone->used_mem -= amount;
420 	}
421 	lockmgr(&glob->lock, LK_RELEASE);
422 }
423 
424 void ttm_mem_global_free(struct ttm_mem_global *glob,
425 			 uint64_t amount)
426 {
427 	return ttm_mem_global_free_zone(glob, NULL, amount);
428 }
429 EXPORT_SYMBOL(ttm_mem_global_free);
430 
431 static int ttm_mem_global_reserve(struct ttm_mem_global *glob,
432 				  struct ttm_mem_zone *single_zone,
433 				  uint64_t amount, bool reserve)
434 {
435 	uint64_t limit;
436 	int ret = -ENOMEM;
437 	unsigned int i;
438 	struct ttm_mem_zone *zone;
439 
440 	lockmgr(&glob->lock, LK_EXCLUSIVE);
441 	for (i = 0; i < glob->num_zones; ++i) {
442 		zone = glob->zones[i];
443 		if (single_zone && zone != single_zone)
444 			continue;
445 
446 		limit = (priv_check(curthread, PRIV_VM_MLOCK) == 0) ?
447 			zone->emer_mem : zone->max_mem;
448 
449 		if (zone->used_mem > limit)
450 			goto out_unlock;
451 	}
452 
453 	if (reserve) {
454 		for (i = 0; i < glob->num_zones; ++i) {
455 			zone = glob->zones[i];
456 			if (single_zone && zone != single_zone)
457 				continue;
458 			zone->used_mem += amount;
459 		}
460 	}
461 
462 	ret = 0;
463 out_unlock:
464 	lockmgr(&glob->lock, LK_RELEASE);
465 	ttm_check_swapping(glob);
466 
467 	return ret;
468 }
469 
470 
471 static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob,
472 				     struct ttm_mem_zone *single_zone,
473 				     uint64_t memory,
474 				     bool no_wait, bool interruptible)
475 {
476 	int count = TTM_MEMORY_ALLOC_RETRIES;
477 
478 	while (unlikely(ttm_mem_global_reserve(glob,
479 					       single_zone,
480 					       memory, true)
481 			!= 0)) {
482 		if (no_wait)
483 			return -ENOMEM;
484 		if (unlikely(count-- == 0))
485 			return -ENOMEM;
486 		ttm_shrink(glob, false, memory + (memory >> 2) + 16);
487 	}
488 
489 	return 0;
490 }
491 
492 int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,
493 			 bool no_wait, bool interruptible)
494 {
495 	/**
496 	 * Normal allocations of kernel memory are registered in
497 	 * all zones.
498 	 */
499 
500 	return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait,
501 					 interruptible);
502 }
503 EXPORT_SYMBOL(ttm_mem_global_alloc);
504 
505 int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
506 			      struct page *page, uint64_t size)
507 {
508 
509 	struct ttm_mem_zone *zone = NULL;
510 
511 	/**
512 	 * Page allocations may be registed in a single zone
513 	 * only if highmem or !dma32.
514 	 */
515 
516 #ifdef CONFIG_HIGHMEM
517 	if (PageHighMem(page) && glob->zone_highmem != NULL)
518 		zone = glob->zone_highmem;
519 #else
520 	if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
521 		zone = glob->zone_kernel;
522 #endif
523 	return ttm_mem_global_alloc_zone(glob, zone, size, false, false);
524 }
525 
526 void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page,
527 			      uint64_t size)
528 {
529 	struct ttm_mem_zone *zone = NULL;
530 
531 #ifdef CONFIG_HIGHMEM
532 	if (PageHighMem(page) && glob->zone_highmem != NULL)
533 		zone = glob->zone_highmem;
534 #else
535 	if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
536 		zone = glob->zone_kernel;
537 #endif
538 	ttm_mem_global_free_zone(glob, zone, size);
539 }
540 
541 size_t ttm_round_pot(size_t size)
542 {
543 	if ((size & (size - 1)) == 0)
544 		return size;
545 	else if (size > PAGE_SIZE)
546 		return PAGE_ALIGN(size);
547 	else {
548 		size_t tmp_size = 4;
549 
550 		while (tmp_size < size)
551 			tmp_size <<= 1;
552 
553 		return tmp_size;
554 	}
555 	return 0;
556 }
557 EXPORT_SYMBOL(ttm_round_pot);
558 
559 uint64_t ttm_get_kernel_zone_memory_size(struct ttm_mem_global *glob)
560 {
561 	return glob->zone_kernel->max_mem;
562 }
563 EXPORT_SYMBOL(ttm_get_kernel_zone_memory_size);
564