1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 /* 31 * Copyright (c) 2013 The FreeBSD Foundation 32 * All rights reserved. 33 * 34 * Portions of this software were developed by Konstantin Belousov 35 * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 36 * 37 * $FreeBSD: head/sys/dev/drm2/ttm/ttm_tt.c 251452 2013-06-06 06:17:20Z alc $ 38 */ 39 40 #include <drm/drmP.h> 41 #include <dev/drm/ttm/ttm_module.h> 42 #include <dev/drm/ttm/ttm_bo_driver.h> 43 #include <dev/drm/ttm/ttm_placement.h> 44 #include <dev/drm/ttm/ttm_page_alloc.h> 45 46 MALLOC_DEFINE(M_TTM_PD, "ttm_pd", "TTM Page Directories"); 47 48 /** 49 * Allocates storage for pointers to the pages that back the ttm. 50 */ 51 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) 52 { 53 ttm->pages = kmalloc(ttm->num_pages * sizeof(void *), 54 M_TTM_PD, M_WAITOK | M_ZERO); 55 } 56 57 static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) 58 { 59 ttm->ttm.pages = kmalloc(ttm->ttm.num_pages * sizeof(void *), 60 M_TTM_PD, M_WAITOK | M_ZERO); 61 ttm->dma_address = kmalloc(ttm->ttm.num_pages * 62 sizeof(*ttm->dma_address), M_TTM_PD, M_WAITOK); 63 } 64 65 #if defined(__i386__) || defined(__amd64__) 66 static inline int ttm_tt_set_page_caching(vm_page_t p, 67 enum ttm_caching_state c_old, 68 enum ttm_caching_state c_new) 69 { 70 71 /* XXXKIB our VM does not need this. */ 72 #if 0 73 if (c_old != tt_cached) { 74 /* p isn't in the default caching state, set it to 75 * writeback first to free its current memtype. */ 76 pmap_page_set_memattr(p, VM_MEMATTR_WRITE_BACK); 77 } 78 #endif 79 80 if (c_new == tt_wc) 81 pmap_page_set_memattr(p, VM_MEMATTR_WRITE_COMBINING); 82 else if (c_new == tt_uncached) 83 pmap_page_set_memattr(p, VM_MEMATTR_UNCACHEABLE); 84 85 return (0); 86 } 87 #else 88 static inline int ttm_tt_set_page_caching(vm_page_t p, 89 enum ttm_caching_state c_old, 90 enum ttm_caching_state c_new) 91 { 92 return 0; 93 } 94 #endif 95 96 /* 97 * Change caching policy for the linear kernel map 98 * for range of pages in a ttm. 99 */ 100 101 static int ttm_tt_set_caching(struct ttm_tt *ttm, 102 enum ttm_caching_state c_state) 103 { 104 int i, j; 105 vm_page_t cur_page; 106 int ret; 107 108 if (ttm->caching_state == c_state) 109 return 0; 110 111 if (ttm->state == tt_unpopulated) { 112 /* Change caching but don't populate */ 113 ttm->caching_state = c_state; 114 return 0; 115 } 116 117 if (ttm->caching_state == tt_cached) 118 drm_clflush_pages(ttm->pages, ttm->num_pages); 119 120 for (i = 0; i < ttm->num_pages; ++i) { 121 cur_page = ttm->pages[i]; 122 if (likely(cur_page != NULL)) { 123 ret = ttm_tt_set_page_caching(cur_page, 124 ttm->caching_state, 125 c_state); 126 if (unlikely(ret != 0)) 127 goto out_err; 128 } 129 } 130 131 ttm->caching_state = c_state; 132 133 return 0; 134 135 out_err: 136 for (j = 0; j < i; ++j) { 137 cur_page = ttm->pages[j]; 138 if (cur_page != NULL) { 139 (void)ttm_tt_set_page_caching(cur_page, c_state, 140 ttm->caching_state); 141 } 142 } 143 144 return ret; 145 } 146 147 int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) 148 { 149 enum ttm_caching_state state; 150 151 if (placement & TTM_PL_FLAG_WC) 152 state = tt_wc; 153 else if (placement & TTM_PL_FLAG_UNCACHED) 154 state = tt_uncached; 155 else 156 state = tt_cached; 157 158 return ttm_tt_set_caching(ttm, state); 159 } 160 161 void ttm_tt_destroy(struct ttm_tt *ttm) 162 { 163 if (unlikely(ttm == NULL)) 164 return; 165 166 if (ttm->state == tt_bound) { 167 ttm_tt_unbind(ttm); 168 } 169 170 if (likely(ttm->pages != NULL)) { 171 ttm->bdev->driver->ttm_tt_unpopulate(ttm); 172 } 173 174 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && 175 ttm->swap_storage) 176 vm_object_deallocate(ttm->swap_storage); 177 178 ttm->swap_storage = NULL; 179 ttm->func->destroy(ttm); 180 } 181 182 int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, 183 unsigned long size, uint32_t page_flags, 184 vm_page_t dummy_read_page) 185 { 186 ttm->bdev = bdev; 187 ttm->glob = bdev->glob; 188 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 189 ttm->caching_state = tt_cached; 190 ttm->page_flags = page_flags; 191 ttm->dummy_read_page = dummy_read_page; 192 ttm->state = tt_unpopulated; 193 ttm->swap_storage = NULL; 194 195 ttm_tt_alloc_page_directory(ttm); 196 if (!ttm->pages) { 197 ttm_tt_destroy(ttm); 198 kprintf("Failed allocating page table\n"); 199 return -ENOMEM; 200 } 201 return 0; 202 } 203 204 void ttm_tt_fini(struct ttm_tt *ttm) 205 { 206 drm_free(ttm->pages, M_TTM_PD); 207 ttm->pages = NULL; 208 } 209 210 int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, 211 unsigned long size, uint32_t page_flags, 212 vm_page_t dummy_read_page) 213 { 214 struct ttm_tt *ttm = &ttm_dma->ttm; 215 216 ttm->bdev = bdev; 217 ttm->glob = bdev->glob; 218 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 219 ttm->caching_state = tt_cached; 220 ttm->page_flags = page_flags; 221 ttm->dummy_read_page = dummy_read_page; 222 ttm->state = tt_unpopulated; 223 ttm->swap_storage = NULL; 224 225 INIT_LIST_HEAD(&ttm_dma->pages_list); 226 ttm_dma_tt_alloc_page_directory(ttm_dma); 227 if (!ttm->pages || !ttm_dma->dma_address) { 228 ttm_tt_destroy(ttm); 229 kprintf("Failed allocating page table\n"); 230 return -ENOMEM; 231 } 232 return 0; 233 } 234 235 void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) 236 { 237 struct ttm_tt *ttm = &ttm_dma->ttm; 238 239 drm_free(ttm->pages, M_TTM_PD); 240 ttm->pages = NULL; 241 drm_free(ttm_dma->dma_address, M_TTM_PD); 242 ttm_dma->dma_address = NULL; 243 } 244 245 void ttm_tt_unbind(struct ttm_tt *ttm) 246 { 247 int ret; 248 249 if (ttm->state == tt_bound) { 250 ret = ttm->func->unbind(ttm); 251 KKASSERT(ret == 0); 252 ttm->state = tt_unbound; 253 } 254 } 255 256 int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) 257 { 258 int ret = 0; 259 260 if (!ttm) 261 return -EINVAL; 262 263 if (ttm->state == tt_bound) 264 return 0; 265 266 ret = ttm->bdev->driver->ttm_tt_populate(ttm); 267 if (ret) 268 return ret; 269 270 ret = ttm->func->bind(ttm, bo_mem); 271 if (unlikely(ret != 0)) 272 return ret; 273 274 ttm->state = tt_bound; 275 276 return 0; 277 } 278 279 int ttm_tt_swapin(struct ttm_tt *ttm) 280 { 281 vm_object_t obj; 282 vm_page_t from_page, to_page; 283 int i, ret, rv; 284 285 obj = ttm->swap_storage; 286 287 VM_OBJECT_WLOCK(obj); 288 vm_object_pip_add(obj, 1); 289 for (i = 0; i < ttm->num_pages; ++i) { 290 from_page = vm_page_grab(obj, i, VM_ALLOC_RETRY); 291 if (from_page->valid != VM_PAGE_BITS_ALL) { 292 vm_page_busy_try(from_page, FALSE); 293 if (vm_pager_has_page(obj, i)) { 294 rv = vm_pager_get_page(obj, &from_page, 1); 295 if (rv != VM_PAGER_OK) { 296 vm_page_free(from_page); 297 ret = -EIO; 298 goto err_ret; 299 } 300 } else 301 vm_page_zero_invalid(from_page, TRUE); 302 vm_page_wakeup(from_page); 303 } 304 to_page = ttm->pages[i]; 305 if (unlikely(to_page == NULL)) { 306 ret = -ENOMEM; 307 goto err_ret; 308 } 309 pmap_copy_page(VM_PAGE_TO_PHYS(from_page), 310 VM_PAGE_TO_PHYS(to_page)); 311 } 312 vm_object_pip_wakeup(obj); 313 VM_OBJECT_WUNLOCK(obj); 314 315 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) 316 vm_object_deallocate(obj); 317 ttm->swap_storage = NULL; 318 ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; 319 return (0); 320 321 err_ret: 322 vm_object_pip_wakeup(obj); 323 VM_OBJECT_WUNLOCK(obj); 324 return (ret); 325 } 326 327 int ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t persistent_swap_storage) 328 { 329 vm_object_t obj; 330 vm_page_t from_page, to_page; 331 int i; 332 333 KKASSERT(ttm->state == tt_unbound || ttm->state == tt_unpopulated); 334 KKASSERT(ttm->caching_state == tt_cached); 335 336 if (persistent_swap_storage == NULL) { 337 #if 0 338 obj = vm_pager_allocate(OBJT_SWAP, NULL, 339 IDX_TO_OFF(ttm->num_pages), VM_PROT_DEFAULT, 0, 340 curthread->td_ucred); 341 #else 342 obj = swap_pager_alloc(NULL, 343 IDX_TO_OFF(ttm->num_pages), VM_PROT_DEFAULT, 0); 344 #endif 345 if (obj == NULL) { 346 kprintf("[TTM] Failed allocating swap storage\n"); 347 return (-ENOMEM); 348 } 349 } else 350 obj = persistent_swap_storage; 351 352 VM_OBJECT_WLOCK(obj); 353 vm_object_pip_add(obj, 1); 354 for (i = 0; i < ttm->num_pages; ++i) { 355 from_page = ttm->pages[i]; 356 if (unlikely(from_page == NULL)) 357 continue; 358 to_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL); 359 pmap_copy_page(VM_PAGE_TO_PHYS(from_page), 360 VM_PAGE_TO_PHYS(to_page)); 361 to_page->valid = VM_PAGE_BITS_ALL; 362 vm_page_dirty(to_page); 363 vm_page_wakeup(to_page); 364 } 365 vm_object_pip_wakeup(obj); 366 VM_OBJECT_WUNLOCK(obj); 367 368 ttm->bdev->driver->ttm_tt_unpopulate(ttm); 369 ttm->swap_storage = obj; 370 ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; 371 if (persistent_swap_storage != NULL) 372 ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; 373 return (0); 374 } 375