1 /* 2 * Copyright 2010 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Ben Skeggs 23 */ 24 #include "ummu.h" 25 #include "vmm.h" 26 27 #include <subdev/bar.h> 28 #include <subdev/fb.h> 29 30 #include <nvif/if500d.h> 31 #include <nvif/if900d.h> 32 33 struct nvkm_mmu_ptp { 34 struct nvkm_mmu_pt *pt; 35 struct list_head head; 36 u8 shift; 37 u16 mask; 38 u16 free; 39 }; 40 41 static void 42 nvkm_mmu_ptp_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt *pt) 43 { 44 const int slot = pt->base >> pt->ptp->shift; 45 struct nvkm_mmu_ptp *ptp = pt->ptp; 46 47 /* If there were no free slots in the parent allocation before, 48 * there will be now, so return PTP to the cache. 49 */ 50 if (!ptp->free) 51 list_add(&ptp->head, &mmu->ptp.list); 52 ptp->free |= BIT(slot); 53 54 /* If there's no more sub-allocations, destroy PTP. */ 55 if (ptp->free == ptp->mask) { 56 nvkm_mmu_ptc_put(mmu, force, &ptp->pt); 57 list_del(&ptp->head); 58 kfree(ptp); 59 } 60 61 kfree(pt); 62 } 63 64 struct nvkm_mmu_pt * 65 nvkm_mmu_ptp_get(struct nvkm_mmu *mmu, u32 size, bool zero) 66 { 67 struct nvkm_mmu_pt *pt; 68 struct nvkm_mmu_ptp *ptp; 69 int slot; 70 71 if (!(pt = kzalloc(sizeof(*pt), GFP_KERNEL))) 72 return NULL; 73 74 ptp = list_first_entry_or_null(&mmu->ptp.list, typeof(*ptp), head); 75 if (!ptp) { 76 /* Need to allocate a new parent to sub-allocate from. */ 77 if (!(ptp = kmalloc(sizeof(*ptp), GFP_KERNEL))) { 78 kfree(pt); 79 return NULL; 80 } 81 82 ptp->pt = nvkm_mmu_ptc_get(mmu, 0x1000, 0x1000, false); 83 if (!ptp->pt) { 84 kfree(ptp); 85 kfree(pt); 86 return NULL; 87 } 88 89 ptp->shift = order_base_2(size); 90 slot = nvkm_memory_size(ptp->pt->memory) >> ptp->shift; 91 ptp->mask = (1 << slot) - 1; 92 ptp->free = ptp->mask; 93 list_add(&ptp->head, &mmu->ptp.list); 94 } 95 pt->ptp = ptp; 96 pt->sub = true; 97 98 /* Sub-allocate from parent object, removing PTP from cache 99 * if there's no more free slots left. 100 */ 101 slot = __ffs(ptp->free); 102 ptp->free &= ~BIT(slot); 103 if (!ptp->free) 104 list_del(&ptp->head); 105 106 pt->memory = pt->ptp->pt->memory; 107 pt->base = slot << ptp->shift; 108 pt->addr = pt->ptp->pt->addr + pt->base; 109 return pt; 110 } 111 112 struct nvkm_mmu_ptc { 113 struct list_head head; 114 struct list_head item; 115 u32 size; 116 u32 refs; 117 }; 118 119 static inline struct nvkm_mmu_ptc * 120 nvkm_mmu_ptc_find(struct nvkm_mmu *mmu, u32 size) 121 { 122 struct nvkm_mmu_ptc *ptc; 123 124 list_for_each_entry(ptc, &mmu->ptc.list, head) { 125 if (ptc->size == size) 126 return ptc; 127 } 128 129 ptc = kmalloc(sizeof(*ptc), GFP_KERNEL); 130 if (ptc) { 131 INIT_LIST_HEAD(&ptc->item); 132 ptc->size = size; 133 ptc->refs = 0; 134 list_add(&ptc->head, &mmu->ptc.list); 135 } 136 137 return ptc; 138 } 139 140 void 141 nvkm_mmu_ptc_put(struct nvkm_mmu *mmu, bool force, struct nvkm_mmu_pt **ppt) 142 { 143 struct nvkm_mmu_pt *pt = *ppt; 144 if (pt) { 145 /* Handle sub-allocated page tables. */ 146 if (pt->sub) { 147 mutex_lock(&mmu->ptp.mutex); 148 nvkm_mmu_ptp_put(mmu, force, pt); 149 mutex_unlock(&mmu->ptp.mutex); 150 return; 151 } 152 153 /* Either cache or free the object. */ 154 mutex_lock(&mmu->ptc.mutex); 155 if (pt->ptc->refs < 8 /* Heuristic. */ && !force) { 156 list_add_tail(&pt->head, &pt->ptc->item); 157 pt->ptc->refs++; 158 } else { 159 nvkm_memory_unref(&pt->memory); 160 kfree(pt); 161 } 162 mutex_unlock(&mmu->ptc.mutex); 163 } 164 } 165 166 struct nvkm_mmu_pt * 167 nvkm_mmu_ptc_get(struct nvkm_mmu *mmu, u32 size, u32 align, bool zero) 168 { 169 struct nvkm_mmu_ptc *ptc; 170 struct nvkm_mmu_pt *pt; 171 int ret; 172 173 /* Sub-allocated page table (ie. GP100 LPT). */ 174 if (align < 0x1000) { 175 mutex_lock(&mmu->ptp.mutex); 176 pt = nvkm_mmu_ptp_get(mmu, align, zero); 177 mutex_unlock(&mmu->ptp.mutex); 178 return pt; 179 } 180 181 /* Lookup cache for this page table size. */ 182 mutex_lock(&mmu->ptc.mutex); 183 ptc = nvkm_mmu_ptc_find(mmu, size); 184 if (!ptc) { 185 mutex_unlock(&mmu->ptc.mutex); 186 return NULL; 187 } 188 189 /* If there's a free PT in the cache, reuse it. */ 190 pt = list_first_entry_or_null(&ptc->item, typeof(*pt), head); 191 if (pt) { 192 if (zero) 193 nvkm_fo64(pt->memory, 0, 0, size >> 3); 194 list_del(&pt->head); 195 ptc->refs--; 196 mutex_unlock(&mmu->ptc.mutex); 197 return pt; 198 } 199 mutex_unlock(&mmu->ptc.mutex); 200 201 /* No such luck, we need to allocate. */ 202 if (!(pt = kmalloc(sizeof(*pt), GFP_KERNEL))) 203 return NULL; 204 pt->ptc = ptc; 205 pt->sub = false; 206 207 ret = nvkm_memory_new(mmu->subdev.device, NVKM_MEM_TARGET_INST, 208 size, align, zero, &pt->memory); 209 if (ret) { 210 kfree(pt); 211 return NULL; 212 } 213 214 pt->base = 0; 215 pt->addr = nvkm_memory_addr(pt->memory); 216 return pt; 217 } 218 219 void 220 nvkm_mmu_ptc_dump(struct nvkm_mmu *mmu) 221 { 222 struct nvkm_mmu_ptc *ptc; 223 list_for_each_entry(ptc, &mmu->ptc.list, head) { 224 struct nvkm_mmu_pt *pt, *tt; 225 list_for_each_entry_safe(pt, tt, &ptc->item, head) { 226 nvkm_memory_unref(&pt->memory); 227 list_del(&pt->head); 228 kfree(pt); 229 } 230 } 231 } 232 233 static void 234 nvkm_mmu_ptc_fini(struct nvkm_mmu *mmu) 235 { 236 struct nvkm_mmu_ptc *ptc, *ptct; 237 238 list_for_each_entry_safe(ptc, ptct, &mmu->ptc.list, head) { 239 WARN_ON(!list_empty(&ptc->item)); 240 list_del(&ptc->head); 241 kfree(ptc); 242 } 243 } 244 245 static void 246 nvkm_mmu_ptc_init(struct nvkm_mmu *mmu) 247 { 248 mutex_init(&mmu->ptc.mutex); 249 INIT_LIST_HEAD(&mmu->ptc.list); 250 mutex_init(&mmu->ptp.mutex); 251 INIT_LIST_HEAD(&mmu->ptp.list); 252 } 253 254 static void 255 nvkm_mmu_type(struct nvkm_mmu *mmu, int heap, u8 type) 256 { 257 if (heap >= 0 && !WARN_ON(mmu->type_nr == ARRAY_SIZE(mmu->type))) { 258 mmu->type[mmu->type_nr].type = type | mmu->heap[heap].type; 259 mmu->type[mmu->type_nr].heap = heap; 260 mmu->type_nr++; 261 } 262 } 263 264 static int 265 nvkm_mmu_heap(struct nvkm_mmu *mmu, u8 type, u64 size) 266 { 267 if (size) { 268 if (!WARN_ON(mmu->heap_nr == ARRAY_SIZE(mmu->heap))) { 269 mmu->heap[mmu->heap_nr].type = type; 270 mmu->heap[mmu->heap_nr].size = size; 271 return mmu->heap_nr++; 272 } 273 } 274 return -EINVAL; 275 } 276 277 static void 278 nvkm_mmu_host(struct nvkm_mmu *mmu) 279 { 280 struct nvkm_device *device = mmu->subdev.device; 281 u8 type = NVKM_MEM_KIND * !!mmu->func->kind_sys; 282 int heap; 283 284 /* Non-mappable system memory. */ 285 heap = nvkm_mmu_heap(mmu, NVKM_MEM_HOST, ~0ULL); 286 nvkm_mmu_type(mmu, heap, type); 287 288 /* Non-coherent, cached, system memory. 289 * 290 * Block-linear mappings of system memory must be done through 291 * BAR1, and cannot be supported on systems where we're unable 292 * to map BAR1 with write-combining. 293 */ 294 type |= NVKM_MEM_MAPPABLE; 295 if (!device->bar || device->bar->iomap_uncached) 296 nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND); 297 else 298 nvkm_mmu_type(mmu, heap, type); 299 300 /* Coherent, cached, system memory. 301 * 302 * Unsupported on systems that aren't able to support snooped 303 * mappings, and also for block-linear mappings which must be 304 * done through BAR1. 305 */ 306 type |= NVKM_MEM_COHERENT; 307 if (device->func->cpu_coherent) 308 nvkm_mmu_type(mmu, heap, type & ~NVKM_MEM_KIND); 309 310 /* Uncached system memory. */ 311 nvkm_mmu_type(mmu, heap, type |= NVKM_MEM_UNCACHED); 312 } 313 314 static void 315 nvkm_mmu_vram(struct nvkm_mmu *mmu) 316 { 317 struct nvkm_device *device = mmu->subdev.device; 318 struct nvkm_mm *mm = &device->fb->ram->vram; 319 const u32 sizeN = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NORMAL); 320 const u32 sizeU = nvkm_mm_heap_size(mm, NVKM_RAM_MM_NOMAP); 321 const u32 sizeM = nvkm_mm_heap_size(mm, NVKM_RAM_MM_MIXED); 322 u8 type = NVKM_MEM_KIND * !!mmu->func->kind; 323 u8 heap = NVKM_MEM_VRAM; 324 int heapM, heapN, heapU; 325 326 /* Mixed-memory doesn't support compression or display. */ 327 heapM = nvkm_mmu_heap(mmu, heap, sizeM << NVKM_RAM_MM_SHIFT); 328 329 heap |= NVKM_MEM_COMP; 330 heap |= NVKM_MEM_DISP; 331 heapN = nvkm_mmu_heap(mmu, heap, sizeN << NVKM_RAM_MM_SHIFT); 332 heapU = nvkm_mmu_heap(mmu, heap, sizeU << NVKM_RAM_MM_SHIFT); 333 334 /* Add non-mappable VRAM types first so that they're preferred 335 * over anything else. Mixed-memory will be slower than other 336 * heaps, it's prioritised last. 337 */ 338 nvkm_mmu_type(mmu, heapU, type); 339 nvkm_mmu_type(mmu, heapN, type); 340 nvkm_mmu_type(mmu, heapM, type); 341 342 /* Add host memory types next, under the assumption that users 343 * wanting mappable memory want to use them as staging buffers 344 * or the like. 345 */ 346 nvkm_mmu_host(mmu); 347 348 /* Mappable VRAM types go last, as they're basically the worst 349 * possible type to ask for unless there's no other choice. 350 */ 351 if (device->bar) { 352 /* Write-combined BAR1 access. */ 353 type |= NVKM_MEM_MAPPABLE; 354 if (!device->bar->iomap_uncached) { 355 nvkm_mmu_type(mmu, heapN, type); 356 nvkm_mmu_type(mmu, heapM, type); 357 } 358 359 /* Uncached BAR1 access. */ 360 type |= NVKM_MEM_COHERENT; 361 type |= NVKM_MEM_UNCACHED; 362 nvkm_mmu_type(mmu, heapN, type); 363 nvkm_mmu_type(mmu, heapM, type); 364 } 365 } 366 367 static int 368 nvkm_mmu_oneinit(struct nvkm_subdev *subdev) 369 { 370 struct nvkm_mmu *mmu = nvkm_mmu(subdev); 371 372 /* Determine available memory types. */ 373 if (mmu->subdev.device->fb && mmu->subdev.device->fb->ram) 374 nvkm_mmu_vram(mmu); 375 else 376 nvkm_mmu_host(mmu); 377 378 if (mmu->func->vmm.global) { 379 int ret = nvkm_vmm_new(subdev->device, 0, 0, NULL, 0, NULL, 380 "gart", &mmu->vmm); 381 if (ret) 382 return ret; 383 } 384 385 return 0; 386 } 387 388 static int 389 nvkm_mmu_init(struct nvkm_subdev *subdev) 390 { 391 struct nvkm_mmu *mmu = nvkm_mmu(subdev); 392 if (mmu->func->init) 393 mmu->func->init(mmu); 394 return 0; 395 } 396 397 static void * 398 nvkm_mmu_dtor(struct nvkm_subdev *subdev) 399 { 400 struct nvkm_mmu *mmu = nvkm_mmu(subdev); 401 402 nvkm_vmm_unref(&mmu->vmm); 403 404 nvkm_mmu_ptc_fini(mmu); 405 return mmu; 406 } 407 408 static const struct nvkm_subdev_func 409 nvkm_mmu = { 410 .dtor = nvkm_mmu_dtor, 411 .oneinit = nvkm_mmu_oneinit, 412 .init = nvkm_mmu_init, 413 }; 414 415 void 416 nvkm_mmu_ctor(const struct nvkm_mmu_func *func, struct nvkm_device *device, 417 int index, struct nvkm_mmu *mmu) 418 { 419 nvkm_subdev_ctor(&nvkm_mmu, device, index, &mmu->subdev); 420 mmu->func = func; 421 mmu->dma_bits = func->dma_bits; 422 nvkm_mmu_ptc_init(mmu); 423 mmu->user.ctor = nvkm_ummu_new; 424 mmu->user.base = func->mmu.user; 425 } 426 427 int 428 nvkm_mmu_new_(const struct nvkm_mmu_func *func, struct nvkm_device *device, 429 int index, struct nvkm_mmu **pmmu) 430 { 431 if (!(*pmmu = kzalloc(sizeof(**pmmu), GFP_KERNEL))) 432 return -ENOMEM; 433 nvkm_mmu_ctor(func, device, index, *pmmu); 434 return 0; 435 } 436