1 /**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27 /*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 */
30
31 #define pr_fmt(fmt) "[TTM] " fmt
32
33 #include <linux/sched.h>
34 #include <linux/highmem.h>
35 #include <linux/pagemap.h>
36 #include <linux/shmem_fs.h>
37 #include <linux/file.h>
38 #include <linux/swap.h>
39 #include <linux/slab.h>
40 #include <linux/export.h>
41 #include <linux/printk.h>
42 #include <drm/drm_cache.h>
43 #include <drm/drm_mem_util.h>
44 #include <drm/ttm/ttm_module.h>
45 #include <drm/ttm/ttm_bo_driver.h>
46 #include <drm/ttm/ttm_placement.h>
47 #include <drm/ttm/ttm_page_alloc.h>
48 #include <drm/bus_dma_hacks.h>
49
50 /**
51 * Allocates storage for pointers to the pages that back the ttm.
52 */
ttm_tt_alloc_page_directory(struct ttm_tt * ttm)53 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
54 {
55 ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(void*));
56 }
57
ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt * ttm)58 static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm)
59 {
60 ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, sizeof(void*));
61 #ifndef __NetBSD__
62 ttm->dma_address = drm_calloc_large(ttm->ttm.num_pages,
63 sizeof(*ttm->dma_address));
64 #endif
65 }
66
67 #ifdef CONFIG_X86
ttm_tt_set_page_caching(struct page * p,enum ttm_caching_state c_old,enum ttm_caching_state c_new)68 static inline int ttm_tt_set_page_caching(struct page *p,
69 enum ttm_caching_state c_old,
70 enum ttm_caching_state c_new)
71 {
72 #ifdef __NetBSD__
73 return 0;
74 #else
75 int ret = 0;
76
77 if (PageHighMem(p))
78 return 0;
79
80 if (c_old != tt_cached) {
81 /* p isn't in the default caching state, set it to
82 * writeback first to free its current memtype. */
83
84 ret = set_pages_wb(p, 1);
85 if (ret)
86 return ret;
87 }
88
89 if (c_new == tt_wc)
90 ret = set_memory_wc((unsigned long) page_address(p), 1);
91 else if (c_new == tt_uncached)
92 ret = set_pages_uc(p, 1);
93
94 return ret;
95 #endif
96 }
97 #else /* CONFIG_X86 */
ttm_tt_set_page_caching(struct page * p,enum ttm_caching_state c_old,enum ttm_caching_state c_new)98 static inline int ttm_tt_set_page_caching(struct page *p,
99 enum ttm_caching_state c_old,
100 enum ttm_caching_state c_new)
101 {
102 return 0;
103 }
104 #endif /* CONFIG_X86 */
105
106 /*
107 * Change caching policy for the linear kernel map
108 * for range of pages in a ttm.
109 */
110
ttm_tt_set_caching(struct ttm_tt * ttm,enum ttm_caching_state c_state)111 static int ttm_tt_set_caching(struct ttm_tt *ttm,
112 enum ttm_caching_state c_state)
113 {
114 int i, j;
115 struct page *cur_page;
116 int ret;
117
118 if (ttm->caching_state == c_state)
119 return 0;
120
121 if (ttm->state == tt_unpopulated) {
122 /* Change caching but don't populate */
123 ttm->caching_state = c_state;
124 return 0;
125 }
126
127 if (ttm->caching_state == tt_cached)
128 drm_clflush_pages(ttm->pages, ttm->num_pages);
129
130 for (i = 0; i < ttm->num_pages; ++i) {
131 cur_page = ttm->pages[i];
132 if (likely(cur_page != NULL)) {
133 ret = ttm_tt_set_page_caching(cur_page,
134 ttm->caching_state,
135 c_state);
136 if (unlikely(ret != 0))
137 goto out_err;
138 }
139 }
140
141 ttm->caching_state = c_state;
142
143 return 0;
144
145 out_err:
146 for (j = 0; j < i; ++j) {
147 cur_page = ttm->pages[j];
148 if (likely(cur_page != NULL)) {
149 (void)ttm_tt_set_page_caching(cur_page, c_state,
150 ttm->caching_state);
151 }
152 }
153
154 return ret;
155 }
156
ttm_tt_set_placement_caching(struct ttm_tt * ttm,uint32_t placement)157 int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement)
158 {
159 enum ttm_caching_state state;
160
161 if (placement & TTM_PL_FLAG_WC)
162 state = tt_wc;
163 else if (placement & TTM_PL_FLAG_UNCACHED)
164 state = tt_uncached;
165 else
166 state = tt_cached;
167
168 return ttm_tt_set_caching(ttm, state);
169 }
170 EXPORT_SYMBOL(ttm_tt_set_placement_caching);
171
ttm_tt_destroy(struct ttm_tt * ttm)172 void ttm_tt_destroy(struct ttm_tt *ttm)
173 {
174 if (unlikely(ttm == NULL))
175 return;
176
177 if (ttm->state == tt_bound) {
178 ttm_tt_unbind(ttm);
179 }
180
181 if (ttm->state == tt_unbound)
182 ttm_tt_unpopulate(ttm);
183
184 #ifndef __NetBSD__
185 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) &&
186 ttm->swap_storage)
187 fput(ttm->swap_storage);
188
189 ttm->swap_storage = NULL;
190 #endif
191 ttm->func->destroy(ttm);
192 }
193
ttm_tt_init(struct ttm_tt * ttm,struct ttm_bo_device * bdev,unsigned long size,uint32_t page_flags,struct page * dummy_read_page)194 int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev,
195 unsigned long size, uint32_t page_flags,
196 struct page *dummy_read_page)
197 {
198 ttm->bdev = bdev;
199 ttm->glob = bdev->glob;
200 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
201 ttm->caching_state = tt_cached;
202 ttm->page_flags = page_flags;
203 ttm->dummy_read_page = dummy_read_page;
204 ttm->state = tt_unpopulated;
205 #ifdef __NetBSD__
206 ttm->swap_storage = uao_create(roundup2(size, PAGE_SIZE), 0);
207 uao_set_pgfl(ttm->swap_storage, bus_dmamem_pgfl(bdev->dmat));
208 #else
209 ttm->swap_storage = NULL;
210 #endif
211 TAILQ_INIT(&ttm->pglist);
212
213 ttm_tt_alloc_page_directory(ttm);
214 if (!ttm->pages) {
215 ttm_tt_destroy(ttm);
216 pr_err("Failed allocating page table\n");
217 return -ENOMEM;
218 }
219 return 0;
220 }
221 EXPORT_SYMBOL(ttm_tt_init);
222
ttm_tt_fini(struct ttm_tt * ttm)223 void ttm_tt_fini(struct ttm_tt *ttm)
224 {
225 #ifdef __NetBSD__
226 uao_detach(ttm->swap_storage);
227 ttm->swap_storage = NULL;
228 #endif
229 drm_free_large(ttm->pages);
230 ttm->pages = NULL;
231 }
232 EXPORT_SYMBOL(ttm_tt_fini);
233
ttm_dma_tt_init(struct ttm_dma_tt * ttm_dma,struct ttm_bo_device * bdev,unsigned long size,uint32_t page_flags,struct page * dummy_read_page)234 int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev,
235 unsigned long size, uint32_t page_flags,
236 struct page *dummy_read_page)
237 {
238 struct ttm_tt *ttm = &ttm_dma->ttm;
239
240 ttm->bdev = bdev;
241 ttm->glob = bdev->glob;
242 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
243 ttm->caching_state = tt_cached;
244 ttm->page_flags = page_flags;
245 ttm->dummy_read_page = dummy_read_page;
246 ttm->state = tt_unpopulated;
247 #ifdef __NetBSD__
248 ttm->swap_storage = uao_create(roundup2(size, PAGE_SIZE), 0);
249 uao_set_pgfl(ttm->swap_storage, bus_dmamem_pgfl(bdev->dmat));
250 #else
251 ttm->swap_storage = NULL;
252 #endif
253 TAILQ_INIT(&ttm->pglist);
254
255 INIT_LIST_HEAD(&ttm_dma->pages_list);
256 ttm_dma_tt_alloc_page_directory(ttm_dma);
257 #ifdef __NetBSD__
258 {
259 int error;
260
261 if (ttm->num_pages > (SIZE_MAX /
262 MIN(sizeof(ttm_dma->dma_segs[0]), PAGE_SIZE))) {
263 error = ENOMEM;
264 goto fail0;
265 }
266 ttm_dma->dma_segs = kmem_alloc((ttm->num_pages *
267 sizeof(ttm_dma->dma_segs[0])), KM_SLEEP);
268 error = bus_dmamap_create(ttm->bdev->dmat,
269 (ttm->num_pages * PAGE_SIZE), ttm->num_pages, PAGE_SIZE, 0,
270 BUS_DMA_WAITOK, &ttm_dma->dma_address);
271 if (error)
272 goto fail1;
273
274 return 0;
275
276 fail2: __unused
277 bus_dmamap_destroy(ttm->bdev->dmat, ttm_dma->dma_address);
278 fail1: kmem_free(ttm_dma->dma_segs, (ttm->num_pages *
279 sizeof(ttm_dma->dma_segs[0])));
280 fail0: KASSERT(error);
281 ttm_tt_destroy(ttm);
282 /* XXX errno NetBSD->Linux */
283 return -error;
284 }
285 #else
286 if (!ttm->pages || !ttm_dma->dma_address) {
287 ttm_tt_destroy(ttm);
288 pr_err("Failed allocating page table\n");
289 return -ENOMEM;
290 }
291 return 0;
292 #endif
293 }
294 EXPORT_SYMBOL(ttm_dma_tt_init);
295
ttm_dma_tt_fini(struct ttm_dma_tt * ttm_dma)296 void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma)
297 {
298 struct ttm_tt *ttm = &ttm_dma->ttm;
299
300 #ifdef __NetBSD__
301 uao_detach(ttm->swap_storage);
302 ttm->swap_storage = NULL;
303 #endif
304 drm_free_large(ttm->pages);
305 ttm->pages = NULL;
306 #ifdef __NetBSD__
307 bus_dmamap_destroy(ttm->bdev->dmat, ttm_dma->dma_address);
308 kmem_free(ttm_dma->dma_segs, (ttm->num_pages *
309 sizeof(ttm_dma->dma_segs[0])));
310 #else
311 drm_free_large(ttm_dma->dma_address);
312 ttm_dma->dma_address = NULL;
313 #endif
314 }
315 EXPORT_SYMBOL(ttm_dma_tt_fini);
316
ttm_tt_unbind(struct ttm_tt * ttm)317 void ttm_tt_unbind(struct ttm_tt *ttm)
318 {
319 int ret __diagused;
320
321 if (ttm->state == tt_bound) {
322 ret = ttm->func->unbind(ttm);
323 BUG_ON(ret);
324 ttm->state = tt_unbound;
325 }
326 }
327
ttm_tt_bind(struct ttm_tt * ttm,struct ttm_mem_reg * bo_mem)328 int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
329 {
330 int ret = 0;
331
332 if (!ttm)
333 return -EINVAL;
334
335 if (ttm->state == tt_bound)
336 return 0;
337
338 ret = ttm->bdev->driver->ttm_tt_populate(ttm);
339 if (ret)
340 return ret;
341
342 ret = ttm->func->bind(ttm, bo_mem);
343 if (unlikely(ret != 0))
344 return ret;
345
346 ttm->state = tt_bound;
347
348 return 0;
349 }
350 EXPORT_SYMBOL(ttm_tt_bind);
351
352 #ifdef __NetBSD__
353 /*
354 * ttm_tt_wire(ttm)
355 *
356 * Wire the uvm pages of ttm and fill the ttm page array. ttm
357 * must be unpopulated or unbound, and must be marked swapped.
358 * This does not change either state -- the caller is expected to
359 * include it among other operations for such a state transition.
360 */
361 int
ttm_tt_wire(struct ttm_tt * ttm)362 ttm_tt_wire(struct ttm_tt *ttm)
363 {
364 struct uvm_object *uobj = ttm->swap_storage;
365 struct vm_page *page;
366 unsigned i;
367 int error;
368
369 KASSERTMSG((ttm->state == tt_unpopulated || ttm->state == tt_unbound),
370 "ttm_tt %p must be unpopulated or unbound for wiring,"
371 " but state=%d",
372 ttm, (int)ttm->state);
373 KASSERT(ISSET(ttm->page_flags, TTM_PAGE_FLAG_SWAPPED));
374 KASSERT(uobj != NULL);
375
376 error = uvm_obj_wirepages(uobj, 0, (ttm->num_pages << PAGE_SHIFT),
377 &ttm->pglist);
378 if (error)
379 /* XXX errno NetBSD->Linux */
380 return -error;
381
382 i = 0;
383 TAILQ_FOREACH(page, &ttm->pglist, pageq.queue) {
384 KASSERT(i < ttm->num_pages);
385 KASSERT(ttm->pages[i] == NULL);
386 ttm->pages[i] = container_of(page, struct page, p_vmp);
387 i++;
388 }
389 KASSERT(i == ttm->num_pages);
390
391 /* Success! */
392 return 0;
393 }
394
395 /*
396 * ttm_tt_unwire(ttm)
397 *
398 * Nullify the ttm page array and unwire the uvm pages of ttm.
399 * ttm must be unbound and must be marked swapped. This does not
400 * change either state -- the caller is expected to include it
401 * among other operations for such a state transition.
402 */
403 void
ttm_tt_unwire(struct ttm_tt * ttm)404 ttm_tt_unwire(struct ttm_tt *ttm)
405 {
406 struct uvm_object *uobj = ttm->swap_storage;
407 unsigned i;
408
409 KASSERTMSG((ttm->state == tt_unbound),
410 "ttm_tt %p must be unbound for unwiring, but state=%d",
411 ttm, (int)ttm->state);
412 KASSERT(!ISSET(ttm->page_flags, TTM_PAGE_FLAG_SWAPPED));
413 KASSERT(uobj != NULL);
414
415 uvm_obj_unwirepages(uobj, 0, (ttm->num_pages << PAGE_SHIFT));
416 for (i = 0; i < ttm->num_pages; i++)
417 ttm->pages[i] = NULL;
418 }
419 #endif
420
421 #ifndef __NetBSD__
ttm_tt_swapin(struct ttm_tt * ttm)422 int ttm_tt_swapin(struct ttm_tt *ttm)
423 {
424 struct address_space *swap_space;
425 struct file *swap_storage;
426 struct page *from_page;
427 struct page *to_page;
428 int i;
429 int ret = -ENOMEM;
430
431 swap_storage = ttm->swap_storage;
432 BUG_ON(swap_storage == NULL);
433
434 swap_space = file_inode(swap_storage)->i_mapping;
435
436 for (i = 0; i < ttm->num_pages; ++i) {
437 from_page = shmem_read_mapping_page(swap_space, i);
438 if (IS_ERR(from_page)) {
439 ret = PTR_ERR(from_page);
440 goto out_err;
441 }
442 to_page = ttm->pages[i];
443 if (unlikely(to_page == NULL))
444 goto out_err;
445
446 copy_highpage(to_page, from_page);
447 page_cache_release(from_page);
448 }
449
450 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP))
451 fput(swap_storage);
452 ttm->swap_storage = NULL;
453 ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
454
455 return 0;
456 out_err:
457 return ret;
458 }
459 #endif
460
ttm_tt_swapout(struct ttm_tt * ttm,struct file * persistent_swap_storage)461 int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage)
462 {
463 #ifdef __NetBSD__
464
465 KASSERTMSG((ttm->state == tt_unpopulated || ttm->state == tt_unbound),
466 "ttm_tt %p must be unpopulated or unbound for swapout,"
467 " but state=%d",
468 ttm, (int)ttm->state);
469 KASSERTMSG((ttm->caching_state == tt_cached),
470 "ttm_tt %p must be cached for swapout, but caching_state=%d",
471 ttm, (int)ttm->caching_state);
472 KASSERT(persistent_swap_storage == NULL);
473
474 ttm->bdev->driver->ttm_tt_swapout(ttm);
475 return 0;
476 #else
477 struct address_space *swap_space;
478 struct file *swap_storage;
479 struct page *from_page;
480 struct page *to_page;
481 int i;
482 int ret = -ENOMEM;
483
484 BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated);
485 BUG_ON(ttm->caching_state != tt_cached);
486
487 if (!persistent_swap_storage) {
488 swap_storage = shmem_file_setup("ttm swap",
489 ttm->num_pages << PAGE_SHIFT,
490 0);
491 if (unlikely(IS_ERR(swap_storage))) {
492 pr_err("Failed allocating swap storage\n");
493 return PTR_ERR(swap_storage);
494 }
495 } else
496 swap_storage = persistent_swap_storage;
497
498 swap_space = file_inode(swap_storage)->i_mapping;
499
500 for (i = 0; i < ttm->num_pages; ++i) {
501 from_page = ttm->pages[i];
502 if (unlikely(from_page == NULL))
503 continue;
504 to_page = shmem_read_mapping_page(swap_space, i);
505 if (unlikely(IS_ERR(to_page))) {
506 ret = PTR_ERR(to_page);
507 goto out_err;
508 }
509 copy_highpage(to_page, from_page);
510 set_page_dirty(to_page);
511 mark_page_accessed(to_page);
512 page_cache_release(to_page);
513 }
514
515 ttm_tt_unpopulate(ttm);
516 ttm->swap_storage = swap_storage;
517 ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
518 if (persistent_swap_storage)
519 ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP;
520
521 return 0;
522 out_err:
523 if (!persistent_swap_storage)
524 fput(swap_storage);
525
526 return ret;
527 #endif
528 }
529
ttm_tt_clear_mapping(struct ttm_tt * ttm)530 static void ttm_tt_clear_mapping(struct ttm_tt *ttm)
531 {
532 #ifndef __NetBSD__
533 pgoff_t i;
534 struct page **page = ttm->pages;
535
536 if (ttm->page_flags & TTM_PAGE_FLAG_SG)
537 return;
538
539 for (i = 0; i < ttm->num_pages; ++i) {
540 (*page)->mapping = NULL;
541 (*page++)->index = 0;
542 }
543 #endif
544 }
545
ttm_tt_unpopulate(struct ttm_tt * ttm)546 void ttm_tt_unpopulate(struct ttm_tt *ttm)
547 {
548 if (ttm->state == tt_unpopulated)
549 return;
550
551 ttm_tt_clear_mapping(ttm);
552 ttm->bdev->driver->ttm_tt_unpopulate(ttm);
553 }
554