1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 /* 31 * Copyright (c) 2013 The FreeBSD Foundation 32 * All rights reserved. 33 * 34 * Portions of this software were developed by Konstantin Belousov 35 * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 36 * 37 * $FreeBSD: head/sys/dev/drm2/ttm/ttm_tt.c 251452 2013-06-06 06:17:20Z alc $ 38 */ 39 40 #include <drm/drmP.h> 41 42 #include <linux/export.h> 43 #include <drm/ttm/ttm_module.h> 44 #include <drm/ttm/ttm_bo_driver.h> 45 #include <drm/ttm/ttm_placement.h> 46 #include <drm/ttm/ttm_page_alloc.h> 47 48 MALLOC_DEFINE(M_TTM_PD, "ttm_pd", "TTM Page Directories"); 49 50 /** 51 * Allocates storage for pointers to the pages that back the ttm. 52 */ 53 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) 54 { 55 ttm->pages = kmalloc(ttm->num_pages * sizeof(void *), 56 M_TTM_PD, M_WAITOK | M_ZERO); 57 } 58 59 static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) 60 { 61 ttm->ttm.pages = kmalloc(ttm->ttm.num_pages * sizeof(void *), 62 M_TTM_PD, M_WAITOK | M_ZERO); 63 ttm->dma_address = kmalloc(ttm->ttm.num_pages * 64 sizeof(*ttm->dma_address), M_TTM_PD, M_WAITOK); 65 } 66 67 static inline int ttm_tt_set_page_caching(vm_page_t p, 68 enum ttm_caching_state c_old, 69 enum ttm_caching_state c_new) 70 { 71 72 /* XXXKIB our VM does not need this. */ 73 #if 0 74 if (c_old != tt_cached) { 75 /* p isn't in the default caching state, set it to 76 * writeback first to free its current memtype. */ 77 pmap_page_set_memattr(p, VM_MEMATTR_WRITE_BACK); 78 } 79 #endif 80 81 if (c_new == tt_wc) 82 pmap_page_set_memattr(p, VM_MEMATTR_WRITE_COMBINING); 83 else if (c_new == tt_uncached) 84 pmap_page_set_memattr(p, VM_MEMATTR_UNCACHEABLE); 85 86 return (0); 87 } 88 89 /* 90 * Change caching policy for the linear kernel map 91 * for range of pages in a ttm. 92 */ 93 94 static int ttm_tt_set_caching(struct ttm_tt *ttm, 95 enum ttm_caching_state c_state) 96 { 97 int i, j; 98 vm_page_t cur_page; 99 int ret; 100 101 if (ttm->caching_state == c_state) 102 return 0; 103 104 if (ttm->state == tt_unpopulated) { 105 /* Change caching but don't populate */ 106 ttm->caching_state = c_state; 107 return 0; 108 } 109 110 if (ttm->caching_state == tt_cached) 111 drm_clflush_pages(ttm->pages, ttm->num_pages); 112 113 for (i = 0; i < ttm->num_pages; ++i) { 114 cur_page = ttm->pages[i]; 115 if (likely(cur_page != NULL)) { 116 ret = ttm_tt_set_page_caching(cur_page, 117 ttm->caching_state, 118 c_state); 119 if (unlikely(ret != 0)) 120 goto out_err; 121 } 122 } 123 124 ttm->caching_state = c_state; 125 126 return 0; 127 128 out_err: 129 for (j = 0; j < i; ++j) { 130 cur_page = ttm->pages[j]; 131 if (likely(cur_page != NULL)) { 132 (void)ttm_tt_set_page_caching(cur_page, c_state, 133 ttm->caching_state); 134 } 135 } 136 137 return ret; 138 } 139 140 int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) 141 { 142 enum ttm_caching_state state; 143 144 if (placement & TTM_PL_FLAG_WC) 145 state = tt_wc; 146 else if (placement & TTM_PL_FLAG_UNCACHED) 147 state = tt_uncached; 148 else 149 state = tt_cached; 150 151 return ttm_tt_set_caching(ttm, state); 152 } 153 EXPORT_SYMBOL(ttm_tt_set_placement_caching); 154 155 void ttm_tt_destroy(struct ttm_tt *ttm) 156 { 157 if (unlikely(ttm == NULL)) 158 return; 159 160 if (ttm->state == tt_bound) { 161 ttm_tt_unbind(ttm); 162 } 163 164 if (likely(ttm->pages != NULL)) { 165 ttm->bdev->driver->ttm_tt_unpopulate(ttm); 166 } 167 168 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && 169 ttm->swap_storage) 170 vm_object_deallocate(ttm->swap_storage); 171 172 ttm->swap_storage = NULL; 173 ttm->func->destroy(ttm); 174 } 175 176 int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, 177 unsigned long size, uint32_t page_flags, 178 vm_page_t dummy_read_page) 179 { 180 ttm->bdev = bdev; 181 ttm->glob = bdev->glob; 182 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 183 ttm->caching_state = tt_cached; 184 ttm->page_flags = page_flags; 185 ttm->dummy_read_page = dummy_read_page; 186 ttm->state = tt_unpopulated; 187 ttm->swap_storage = NULL; 188 189 ttm_tt_alloc_page_directory(ttm); 190 if (!ttm->pages) { 191 ttm_tt_destroy(ttm); 192 kprintf("Failed allocating page table\n"); 193 return -ENOMEM; 194 } 195 return 0; 196 } 197 EXPORT_SYMBOL(ttm_tt_init); 198 199 void ttm_tt_fini(struct ttm_tt *ttm) 200 { 201 drm_free(ttm->pages, M_TTM_PD); 202 ttm->pages = NULL; 203 } 204 EXPORT_SYMBOL(ttm_tt_fini); 205 206 int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, 207 unsigned long size, uint32_t page_flags, 208 vm_page_t dummy_read_page) 209 { 210 struct ttm_tt *ttm = &ttm_dma->ttm; 211 212 ttm->bdev = bdev; 213 ttm->glob = bdev->glob; 214 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 215 ttm->caching_state = tt_cached; 216 ttm->page_flags = page_flags; 217 ttm->dummy_read_page = dummy_read_page; 218 ttm->state = tt_unpopulated; 219 ttm->swap_storage = NULL; 220 221 INIT_LIST_HEAD(&ttm_dma->pages_list); 222 ttm_dma_tt_alloc_page_directory(ttm_dma); 223 if (!ttm->pages || !ttm_dma->dma_address) { 224 ttm_tt_destroy(ttm); 225 kprintf("Failed allocating page table\n"); 226 return -ENOMEM; 227 } 228 return 0; 229 } 230 EXPORT_SYMBOL(ttm_dma_tt_init); 231 232 void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) 233 { 234 struct ttm_tt *ttm = &ttm_dma->ttm; 235 236 drm_free(ttm->pages, M_TTM_PD); 237 ttm->pages = NULL; 238 drm_free(ttm_dma->dma_address, M_TTM_PD); 239 ttm_dma->dma_address = NULL; 240 } 241 EXPORT_SYMBOL(ttm_dma_tt_fini); 242 243 void ttm_tt_unbind(struct ttm_tt *ttm) 244 { 245 int ret; 246 247 if (ttm->state == tt_bound) { 248 ret = ttm->func->unbind(ttm); 249 BUG_ON(ret); 250 ttm->state = tt_unbound; 251 } 252 } 253 254 int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) 255 { 256 int ret = 0; 257 258 if (!ttm) 259 return -EINVAL; 260 261 if (ttm->state == tt_bound) 262 return 0; 263 264 ret = ttm->bdev->driver->ttm_tt_populate(ttm); 265 if (ret) 266 return ret; 267 268 ret = ttm->func->bind(ttm, bo_mem); 269 if (unlikely(ret != 0)) 270 return ret; 271 272 ttm->state = tt_bound; 273 274 return 0; 275 } 276 EXPORT_SYMBOL(ttm_tt_bind); 277 278 int ttm_tt_swapin(struct ttm_tt *ttm) 279 { 280 vm_object_t obj; 281 vm_page_t from_page, to_page; 282 int i, ret, rv; 283 284 obj = ttm->swap_storage; 285 286 VM_OBJECT_WLOCK(obj); 287 vm_object_pip_add(obj, 1); 288 for (i = 0; i < ttm->num_pages; ++i) { 289 from_page = vm_page_grab(obj, i, VM_ALLOC_RETRY); 290 if (from_page->valid != VM_PAGE_BITS_ALL) { 291 vm_page_busy_try(from_page, FALSE); 292 if (vm_pager_has_page(obj, i)) { 293 rv = vm_pager_get_page(obj, &from_page, 1); 294 if (rv != VM_PAGER_OK) { 295 vm_page_free(from_page); 296 ret = -EIO; 297 goto err_ret; 298 } 299 } else 300 vm_page_zero_invalid(from_page, TRUE); 301 vm_page_wakeup(from_page); 302 } 303 to_page = ttm->pages[i]; 304 if (unlikely(to_page == NULL)) { 305 ret = -ENOMEM; 306 goto err_ret; 307 } 308 pmap_copy_page(VM_PAGE_TO_PHYS(from_page), 309 VM_PAGE_TO_PHYS(to_page)); 310 } 311 vm_object_pip_wakeup(obj); 312 VM_OBJECT_WUNLOCK(obj); 313 314 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) 315 vm_object_deallocate(obj); 316 ttm->swap_storage = NULL; 317 ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; 318 return (0); 319 320 err_ret: 321 vm_object_pip_wakeup(obj); 322 VM_OBJECT_WUNLOCK(obj); 323 return (ret); 324 } 325 326 int ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t persistent_swap_storage) 327 { 328 vm_object_t obj; 329 vm_page_t from_page, to_page; 330 int i; 331 332 BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); 333 BUG_ON(ttm->caching_state != tt_cached); 334 335 if (!persistent_swap_storage) { 336 obj = swap_pager_alloc(NULL, 337 IDX_TO_OFF(ttm->num_pages), VM_PROT_DEFAULT, 0); 338 if (obj == NULL) { 339 kprintf("[TTM] Failed allocating swap storage\n"); 340 return (-ENOMEM); 341 } 342 } else 343 obj = persistent_swap_storage; 344 345 VM_OBJECT_WLOCK(obj); 346 vm_object_pip_add(obj, 1); 347 for (i = 0; i < ttm->num_pages; ++i) { 348 from_page = ttm->pages[i]; 349 if (unlikely(from_page == NULL)) 350 continue; 351 to_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL); 352 pmap_copy_page(VM_PAGE_TO_PHYS(from_page), 353 VM_PAGE_TO_PHYS(to_page)); 354 to_page->valid = VM_PAGE_BITS_ALL; 355 vm_page_dirty(to_page); 356 vm_page_wakeup(to_page); 357 } 358 vm_object_pip_wakeup(obj); 359 VM_OBJECT_WUNLOCK(obj); 360 361 ttm->bdev->driver->ttm_tt_unpopulate(ttm); 362 ttm->swap_storage = obj; 363 ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; 364 if (persistent_swap_storage) 365 ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; 366 367 return 0; 368 } 369