1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 /* 31 * Copyright (c) 2013 The FreeBSD Foundation 32 * All rights reserved. 33 * 34 * Portions of this software were developed by Konstantin Belousov 35 * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 36 * 37 * $FreeBSD: head/sys/dev/drm2/ttm/ttm_tt.c 251452 2013-06-06 06:17:20Z alc $ 38 */ 39 40 #include <drm/drmP.h> 41 42 #include <linux/export.h> 43 #include <drm/drm_mem_util.h> 44 #include <drm/ttm/ttm_module.h> 45 #include <drm/ttm/ttm_bo_driver.h> 46 #include <drm/ttm/ttm_placement.h> 47 #include <drm/ttm/ttm_page_alloc.h> 48 49 /** 50 * Allocates storage for pointers to the pages that back the ttm. 51 */ 52 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) 53 { 54 ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(void*)); 55 } 56 57 static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) 58 { 59 ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages, sizeof(void*)); 60 ttm->dma_address = drm_calloc_large(ttm->ttm.num_pages, 61 sizeof(*ttm->dma_address)); 62 } 63 64 static inline int ttm_tt_set_page_caching(vm_page_t p, 65 enum ttm_caching_state c_old, 66 enum ttm_caching_state c_new) 67 { 68 69 /* XXXKIB our VM does not need this. */ 70 #if 0 71 if (c_old != tt_cached) { 72 /* p isn't in the default caching state, set it to 73 * writeback first to free its current memtype. */ 74 pmap_page_set_memattr(p, VM_MEMATTR_WRITE_BACK); 75 } 76 #endif 77 78 if (c_new == tt_wc) 79 pmap_page_set_memattr(p, VM_MEMATTR_WRITE_COMBINING); 80 else if (c_new == tt_uncached) 81 pmap_page_set_memattr(p, VM_MEMATTR_UNCACHEABLE); 82 83 return (0); 84 } 85 86 /* 87 * Change caching policy for the linear kernel map 88 * for range of pages in a ttm. 89 */ 90 91 static int ttm_tt_set_caching(struct ttm_tt *ttm, 92 enum ttm_caching_state c_state) 93 { 94 int i, j; 95 vm_page_t cur_page; 96 int ret; 97 98 if (ttm->caching_state == c_state) 99 return 0; 100 101 if (ttm->state == tt_unpopulated) { 102 /* Change caching but don't populate */ 103 ttm->caching_state = c_state; 104 return 0; 105 } 106 107 if (ttm->caching_state == tt_cached) 108 drm_clflush_pages(ttm->pages, ttm->num_pages); 109 110 for (i = 0; i < ttm->num_pages; ++i) { 111 cur_page = ttm->pages[i]; 112 if (likely(cur_page != NULL)) { 113 ret = ttm_tt_set_page_caching(cur_page, 114 ttm->caching_state, 115 c_state); 116 if (unlikely(ret != 0)) 117 goto out_err; 118 } 119 } 120 121 ttm->caching_state = c_state; 122 123 return 0; 124 125 out_err: 126 for (j = 0; j < i; ++j) { 127 cur_page = ttm->pages[j]; 128 if (likely(cur_page != NULL)) { 129 (void)ttm_tt_set_page_caching(cur_page, c_state, 130 ttm->caching_state); 131 } 132 } 133 134 return ret; 135 } 136 137 int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) 138 { 139 enum ttm_caching_state state; 140 141 if (placement & TTM_PL_FLAG_WC) 142 state = tt_wc; 143 else if (placement & TTM_PL_FLAG_UNCACHED) 144 state = tt_uncached; 145 else 146 state = tt_cached; 147 148 return ttm_tt_set_caching(ttm, state); 149 } 150 EXPORT_SYMBOL(ttm_tt_set_placement_caching); 151 152 void ttm_tt_destroy(struct ttm_tt *ttm) 153 { 154 if (unlikely(ttm == NULL)) 155 return; 156 157 if (ttm->state == tt_bound) { 158 ttm_tt_unbind(ttm); 159 } 160 161 if (likely(ttm->pages != NULL)) { 162 ttm->bdev->driver->ttm_tt_unpopulate(ttm); 163 } 164 165 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && 166 ttm->swap_storage) 167 vm_object_deallocate(ttm->swap_storage); 168 169 ttm->swap_storage = NULL; 170 ttm->func->destroy(ttm); 171 } 172 173 int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, 174 unsigned long size, uint32_t page_flags, 175 vm_page_t dummy_read_page) 176 { 177 ttm->bdev = bdev; 178 ttm->glob = bdev->glob; 179 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 180 ttm->caching_state = tt_cached; 181 ttm->page_flags = page_flags; 182 ttm->dummy_read_page = dummy_read_page; 183 ttm->state = tt_unpopulated; 184 ttm->swap_storage = NULL; 185 186 ttm_tt_alloc_page_directory(ttm); 187 if (!ttm->pages) { 188 ttm_tt_destroy(ttm); 189 kprintf("Failed allocating page table\n"); 190 return -ENOMEM; 191 } 192 return 0; 193 } 194 EXPORT_SYMBOL(ttm_tt_init); 195 196 void ttm_tt_fini(struct ttm_tt *ttm) 197 { 198 drm_free_large(ttm->pages); 199 ttm->pages = NULL; 200 } 201 EXPORT_SYMBOL(ttm_tt_fini); 202 203 int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, 204 unsigned long size, uint32_t page_flags, 205 vm_page_t dummy_read_page) 206 { 207 struct ttm_tt *ttm = &ttm_dma->ttm; 208 209 ttm->bdev = bdev; 210 ttm->glob = bdev->glob; 211 ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 212 ttm->caching_state = tt_cached; 213 ttm->page_flags = page_flags; 214 ttm->dummy_read_page = dummy_read_page; 215 ttm->state = tt_unpopulated; 216 ttm->swap_storage = NULL; 217 218 INIT_LIST_HEAD(&ttm_dma->pages_list); 219 ttm_dma_tt_alloc_page_directory(ttm_dma); 220 if (!ttm->pages || !ttm_dma->dma_address) { 221 ttm_tt_destroy(ttm); 222 kprintf("Failed allocating page table\n"); 223 return -ENOMEM; 224 } 225 return 0; 226 } 227 EXPORT_SYMBOL(ttm_dma_tt_init); 228 229 void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) 230 { 231 struct ttm_tt *ttm = &ttm_dma->ttm; 232 233 drm_free_large(ttm->pages); 234 ttm->pages = NULL; 235 drm_free_large(ttm_dma->dma_address); 236 ttm_dma->dma_address = NULL; 237 } 238 EXPORT_SYMBOL(ttm_dma_tt_fini); 239 240 void ttm_tt_unbind(struct ttm_tt *ttm) 241 { 242 int ret; 243 244 if (ttm->state == tt_bound) { 245 ret = ttm->func->unbind(ttm); 246 BUG_ON(ret); 247 ttm->state = tt_unbound; 248 } 249 } 250 251 int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) 252 { 253 int ret = 0; 254 255 if (!ttm) 256 return -EINVAL; 257 258 if (ttm->state == tt_bound) 259 return 0; 260 261 ret = ttm->bdev->driver->ttm_tt_populate(ttm); 262 if (ret) 263 return ret; 264 265 ret = ttm->func->bind(ttm, bo_mem); 266 if (unlikely(ret != 0)) 267 return ret; 268 269 ttm->state = tt_bound; 270 271 return 0; 272 } 273 EXPORT_SYMBOL(ttm_tt_bind); 274 275 int ttm_tt_swapin(struct ttm_tt *ttm) 276 { 277 vm_object_t obj; 278 vm_page_t from_page, to_page; 279 int i, ret, rv; 280 281 obj = ttm->swap_storage; 282 283 VM_OBJECT_WLOCK(obj); 284 vm_object_pip_add(obj, 1); 285 for (i = 0; i < ttm->num_pages; ++i) { 286 from_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL | 287 VM_ALLOC_RETRY); 288 if (from_page->valid != VM_PAGE_BITS_ALL) { 289 if (vm_pager_has_page(obj, i)) { 290 rv = vm_pager_get_page(obj, &from_page, 1); 291 if (rv != VM_PAGER_OK) { 292 vm_page_free(from_page); 293 ret = -EIO; 294 goto err_ret; 295 } 296 } else { 297 vm_page_zero_invalid(from_page, TRUE); 298 } 299 } 300 to_page = ttm->pages[i]; 301 if (unlikely(to_page == NULL)) { 302 ret = -ENOMEM; 303 vm_page_wakeup(from_page); 304 goto err_ret; 305 } 306 pmap_copy_page(VM_PAGE_TO_PHYS(from_page), 307 VM_PAGE_TO_PHYS(to_page)); 308 vm_page_wakeup(from_page); 309 } 310 vm_object_pip_wakeup(obj); 311 VM_OBJECT_WUNLOCK(obj); 312 313 if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) 314 vm_object_deallocate(obj); 315 ttm->swap_storage = NULL; 316 ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; 317 return (0); 318 319 err_ret: 320 vm_object_pip_wakeup(obj); 321 VM_OBJECT_WUNLOCK(obj); 322 return (ret); 323 } 324 325 int ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t persistent_swap_storage) 326 { 327 vm_object_t obj; 328 vm_page_t from_page, to_page; 329 int i; 330 331 BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); 332 BUG_ON(ttm->caching_state != tt_cached); 333 334 if (!persistent_swap_storage) { 335 obj = swap_pager_alloc(NULL, 336 IDX_TO_OFF(ttm->num_pages), VM_PROT_DEFAULT, 0); 337 if (obj == NULL) { 338 kprintf("[TTM] Failed allocating swap storage\n"); 339 return (-ENOMEM); 340 } 341 } else 342 obj = persistent_swap_storage; 343 344 VM_OBJECT_WLOCK(obj); 345 vm_object_pip_add(obj, 1); 346 for (i = 0; i < ttm->num_pages; ++i) { 347 from_page = ttm->pages[i]; 348 if (unlikely(from_page == NULL)) 349 continue; 350 to_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL | 351 VM_ALLOC_RETRY); 352 pmap_copy_page(VM_PAGE_TO_PHYS(from_page), 353 VM_PAGE_TO_PHYS(to_page)); 354 to_page->valid = VM_PAGE_BITS_ALL; 355 vm_page_dirty(to_page); 356 vm_page_wakeup(to_page); 357 } 358 vm_object_pip_wakeup(obj); 359 VM_OBJECT_WUNLOCK(obj); 360 361 ttm->bdev->driver->ttm_tt_unpopulate(ttm); 362 ttm->swap_storage = obj; 363 ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; 364 if (persistent_swap_storage) 365 ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; 366 367 return 0; 368 } 369