15718399fSFrançois Tigeot /************************************************************************** 25718399fSFrançois Tigeot * 35718399fSFrançois Tigeot * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA 45718399fSFrançois Tigeot * All Rights Reserved. 55718399fSFrançois Tigeot * 65718399fSFrançois Tigeot * Permission is hereby granted, free of charge, to any person obtaining a 75718399fSFrançois Tigeot * copy of this software and associated documentation files (the 85718399fSFrançois Tigeot * "Software"), to deal in the Software without restriction, including 95718399fSFrançois Tigeot * without limitation the rights to use, copy, modify, merge, publish, 105718399fSFrançois Tigeot * distribute, sub license, and/or sell copies of the Software, and to 115718399fSFrançois Tigeot * permit persons to whom the Software is furnished to do so, subject to 125718399fSFrançois Tigeot * the following conditions: 135718399fSFrançois Tigeot * 145718399fSFrançois Tigeot * The above copyright notice and this permission notice (including the 155718399fSFrançois Tigeot * next paragraph) shall be included in all copies or substantial portions 165718399fSFrançois Tigeot * of the Software. 175718399fSFrançois Tigeot * 185718399fSFrançois Tigeot * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 195718399fSFrançois Tigeot * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 205718399fSFrançois Tigeot * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 215718399fSFrançois Tigeot * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 225718399fSFrançois Tigeot * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 235718399fSFrançois Tigeot * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 245718399fSFrançois Tigeot * USE OR OTHER DEALINGS IN THE SOFTWARE. 255718399fSFrançois Tigeot * 265718399fSFrançois Tigeot **************************************************************************/ 275718399fSFrançois Tigeot /* 285718399fSFrançois Tigeot * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 295718399fSFrançois Tigeot */ 305718399fSFrançois Tigeot 31216f7a2cSFrançois Tigeot #include <drm/ttm/ttm_bo_driver.h> 32216f7a2cSFrançois Tigeot #include <drm/ttm/ttm_placement.h> 335718399fSFrançois Tigeot #include <sys/sfbuf.h> 34*c19c6249SFrançois Tigeot #include <linux/export.h> 355718399fSFrançois Tigeot 365718399fSFrançois Tigeot void ttm_bo_free_old_node(struct ttm_buffer_object *bo) 375718399fSFrançois Tigeot { 385718399fSFrançois Tigeot ttm_bo_mem_put(bo, &bo->mem); 395718399fSFrançois Tigeot } 405718399fSFrançois Tigeot 415718399fSFrançois Tigeot int ttm_bo_move_ttm(struct ttm_buffer_object *bo, 425718399fSFrançois Tigeot bool evict, 435718399fSFrançois Tigeot bool no_wait_gpu, struct ttm_mem_reg *new_mem) 445718399fSFrançois Tigeot { 455718399fSFrançois Tigeot struct ttm_tt *ttm = bo->ttm; 465718399fSFrançois Tigeot struct ttm_mem_reg *old_mem = &bo->mem; 475718399fSFrançois Tigeot int ret; 485718399fSFrançois Tigeot 495718399fSFrançois Tigeot if (old_mem->mem_type != TTM_PL_SYSTEM) { 505718399fSFrançois Tigeot ttm_tt_unbind(ttm); 515718399fSFrançois Tigeot ttm_bo_free_old_node(bo); 525718399fSFrançois Tigeot ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, 535718399fSFrançois Tigeot TTM_PL_MASK_MEM); 545718399fSFrançois Tigeot old_mem->mem_type = TTM_PL_SYSTEM; 555718399fSFrançois Tigeot } 565718399fSFrançois Tigeot 575718399fSFrançois Tigeot ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); 585718399fSFrançois Tigeot if (unlikely(ret != 0)) 595718399fSFrançois Tigeot return ret; 605718399fSFrançois Tigeot 615718399fSFrançois Tigeot if (new_mem->mem_type != TTM_PL_SYSTEM) { 625718399fSFrançois Tigeot ret = ttm_tt_bind(ttm, new_mem); 635718399fSFrançois Tigeot if (unlikely(ret != 0)) 645718399fSFrançois Tigeot return ret; 655718399fSFrançois Tigeot } 665718399fSFrançois Tigeot 675718399fSFrançois Tigeot *old_mem = *new_mem; 685718399fSFrançois Tigeot new_mem->mm_node = NULL; 695718399fSFrançois Tigeot 705718399fSFrançois Tigeot return 0; 715718399fSFrançois Tigeot } 72*c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_bo_move_ttm); 735718399fSFrançois Tigeot 745718399fSFrançois Tigeot int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) 755718399fSFrançois Tigeot { 765718399fSFrançois Tigeot if (likely(man->io_reserve_fastpath)) 775718399fSFrançois Tigeot return 0; 785718399fSFrançois Tigeot 795718399fSFrançois Tigeot if (interruptible) { 805718399fSFrançois Tigeot if (lockmgr(&man->io_reserve_mutex, 815718399fSFrançois Tigeot LK_EXCLUSIVE | LK_SLEEPFAIL)) 825718399fSFrançois Tigeot return (-EINTR); 835718399fSFrançois Tigeot else 845718399fSFrançois Tigeot return (0); 855718399fSFrançois Tigeot } 865718399fSFrançois Tigeot 875718399fSFrançois Tigeot lockmgr(&man->io_reserve_mutex, LK_EXCLUSIVE); 885718399fSFrançois Tigeot return 0; 895718399fSFrançois Tigeot } 905718399fSFrançois Tigeot 915718399fSFrançois Tigeot void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) 925718399fSFrançois Tigeot { 935718399fSFrançois Tigeot if (likely(man->io_reserve_fastpath)) 945718399fSFrançois Tigeot return; 955718399fSFrançois Tigeot 965718399fSFrançois Tigeot lockmgr(&man->io_reserve_mutex, LK_RELEASE); 975718399fSFrançois Tigeot } 985718399fSFrançois Tigeot 995718399fSFrançois Tigeot static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) 1005718399fSFrançois Tigeot { 1015718399fSFrançois Tigeot struct ttm_buffer_object *bo; 1025718399fSFrançois Tigeot 1035718399fSFrançois Tigeot if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru)) 1045718399fSFrançois Tigeot return -EAGAIN; 1055718399fSFrançois Tigeot 1065718399fSFrançois Tigeot bo = list_first_entry(&man->io_reserve_lru, 1075718399fSFrançois Tigeot struct ttm_buffer_object, 1085718399fSFrançois Tigeot io_reserve_lru); 1095718399fSFrançois Tigeot list_del_init(&bo->io_reserve_lru); 1105718399fSFrançois Tigeot ttm_bo_unmap_virtual_locked(bo); 1115718399fSFrançois Tigeot 1125718399fSFrançois Tigeot return 0; 1135718399fSFrançois Tigeot } 1145718399fSFrançois Tigeot 1155718399fSFrançois Tigeot static int ttm_mem_io_reserve(struct ttm_bo_device *bdev, 1165718399fSFrançois Tigeot struct ttm_mem_reg *mem) 1175718399fSFrançois Tigeot { 1185718399fSFrançois Tigeot struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1195718399fSFrançois Tigeot int ret = 0; 1205718399fSFrançois Tigeot 1215718399fSFrançois Tigeot if (!bdev->driver->io_mem_reserve) 1225718399fSFrançois Tigeot return 0; 1235718399fSFrançois Tigeot if (likely(man->io_reserve_fastpath)) 1245718399fSFrançois Tigeot return bdev->driver->io_mem_reserve(bdev, mem); 1255718399fSFrançois Tigeot 1265718399fSFrançois Tigeot if (bdev->driver->io_mem_reserve && 1275718399fSFrançois Tigeot mem->bus.io_reserved_count++ == 0) { 1285718399fSFrançois Tigeot retry: 1295718399fSFrançois Tigeot ret = bdev->driver->io_mem_reserve(bdev, mem); 1305718399fSFrançois Tigeot if (ret == -EAGAIN) { 1315718399fSFrançois Tigeot ret = ttm_mem_io_evict(man); 1325718399fSFrançois Tigeot if (ret == 0) 1335718399fSFrançois Tigeot goto retry; 1345718399fSFrançois Tigeot } 1355718399fSFrançois Tigeot } 1365718399fSFrançois Tigeot return ret; 1375718399fSFrançois Tigeot } 1385718399fSFrançois Tigeot 1395718399fSFrançois Tigeot static void ttm_mem_io_free(struct ttm_bo_device *bdev, 1405718399fSFrançois Tigeot struct ttm_mem_reg *mem) 1415718399fSFrançois Tigeot { 1425718399fSFrançois Tigeot struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1435718399fSFrançois Tigeot 1445718399fSFrançois Tigeot if (likely(man->io_reserve_fastpath)) 1455718399fSFrançois Tigeot return; 1465718399fSFrançois Tigeot 1475718399fSFrançois Tigeot if (bdev->driver->io_mem_reserve && 1485718399fSFrançois Tigeot --mem->bus.io_reserved_count == 0 && 1495718399fSFrançois Tigeot bdev->driver->io_mem_free) 1505718399fSFrançois Tigeot bdev->driver->io_mem_free(bdev, mem); 1515718399fSFrançois Tigeot 1525718399fSFrançois Tigeot } 1535718399fSFrançois Tigeot 1545718399fSFrançois Tigeot int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) 1555718399fSFrançois Tigeot { 1565718399fSFrançois Tigeot struct ttm_mem_reg *mem = &bo->mem; 1575718399fSFrançois Tigeot int ret; 1585718399fSFrançois Tigeot 1595718399fSFrançois Tigeot if (!mem->bus.io_reserved_vm) { 1605718399fSFrançois Tigeot struct ttm_mem_type_manager *man = 1615718399fSFrançois Tigeot &bo->bdev->man[mem->mem_type]; 1625718399fSFrançois Tigeot 1635718399fSFrançois Tigeot ret = ttm_mem_io_reserve(bo->bdev, mem); 1645718399fSFrançois Tigeot if (unlikely(ret != 0)) 1655718399fSFrançois Tigeot return ret; 1665718399fSFrançois Tigeot mem->bus.io_reserved_vm = true; 1675718399fSFrançois Tigeot if (man->use_io_reserve_lru) 1685718399fSFrançois Tigeot list_add_tail(&bo->io_reserve_lru, 1695718399fSFrançois Tigeot &man->io_reserve_lru); 1705718399fSFrançois Tigeot } 1715718399fSFrançois Tigeot return 0; 1725718399fSFrançois Tigeot } 1735718399fSFrançois Tigeot 1745718399fSFrançois Tigeot void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) 1755718399fSFrançois Tigeot { 1765718399fSFrançois Tigeot struct ttm_mem_reg *mem = &bo->mem; 1775718399fSFrançois Tigeot 1785718399fSFrançois Tigeot if (mem->bus.io_reserved_vm) { 1795718399fSFrançois Tigeot mem->bus.io_reserved_vm = false; 1805718399fSFrançois Tigeot list_del_init(&bo->io_reserve_lru); 1815718399fSFrançois Tigeot ttm_mem_io_free(bo->bdev, mem); 1825718399fSFrançois Tigeot } 1835718399fSFrançois Tigeot } 1845718399fSFrançois Tigeot 1855718399fSFrançois Tigeot static 1865718399fSFrançois Tigeot int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 1875718399fSFrançois Tigeot void **virtual) 1885718399fSFrançois Tigeot { 1895718399fSFrançois Tigeot struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1905718399fSFrançois Tigeot int ret; 1915718399fSFrançois Tigeot void *addr; 1925718399fSFrançois Tigeot 1935718399fSFrançois Tigeot *virtual = NULL; 1945718399fSFrançois Tigeot (void) ttm_mem_io_lock(man, false); 1955718399fSFrançois Tigeot ret = ttm_mem_io_reserve(bdev, mem); 1965718399fSFrançois Tigeot ttm_mem_io_unlock(man); 1975718399fSFrançois Tigeot if (ret || !mem->bus.is_iomem) 1985718399fSFrançois Tigeot return ret; 1995718399fSFrançois Tigeot 2005718399fSFrançois Tigeot if (mem->bus.addr) { 2015718399fSFrançois Tigeot addr = mem->bus.addr; 2025718399fSFrançois Tigeot } else { 2035718399fSFrançois Tigeot addr = pmap_mapdev_attr(mem->bus.base + mem->bus.offset, 2045718399fSFrançois Tigeot mem->bus.size, (mem->placement & TTM_PL_FLAG_WC) ? 2055718399fSFrançois Tigeot VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE); 2065718399fSFrançois Tigeot if (!addr) { 2075718399fSFrançois Tigeot (void) ttm_mem_io_lock(man, false); 2085718399fSFrançois Tigeot ttm_mem_io_free(bdev, mem); 2095718399fSFrançois Tigeot ttm_mem_io_unlock(man); 2105718399fSFrançois Tigeot return -ENOMEM; 2115718399fSFrançois Tigeot } 2125718399fSFrançois Tigeot } 2135718399fSFrançois Tigeot *virtual = addr; 2145718399fSFrançois Tigeot return 0; 2155718399fSFrançois Tigeot } 2165718399fSFrançois Tigeot 2175718399fSFrançois Tigeot static 2185718399fSFrançois Tigeot void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 2195718399fSFrançois Tigeot void *virtual) 2205718399fSFrançois Tigeot { 2215718399fSFrançois Tigeot struct ttm_mem_type_manager *man; 2225718399fSFrançois Tigeot 2235718399fSFrançois Tigeot man = &bdev->man[mem->mem_type]; 2245718399fSFrançois Tigeot 2255718399fSFrançois Tigeot if (virtual && mem->bus.addr == NULL) 2265718399fSFrançois Tigeot pmap_unmapdev((vm_offset_t)virtual, mem->bus.size); 2275718399fSFrançois Tigeot (void) ttm_mem_io_lock(man, false); 2285718399fSFrançois Tigeot ttm_mem_io_free(bdev, mem); 2295718399fSFrançois Tigeot ttm_mem_io_unlock(man); 2305718399fSFrançois Tigeot } 2315718399fSFrançois Tigeot 2325718399fSFrançois Tigeot static int ttm_copy_io_page(void *dst, void *src, unsigned long page) 2335718399fSFrançois Tigeot { 2345718399fSFrançois Tigeot uint32_t *dstP = 2355718399fSFrançois Tigeot (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); 2365718399fSFrançois Tigeot uint32_t *srcP = 2375718399fSFrançois Tigeot (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); 2385718399fSFrançois Tigeot 2395718399fSFrançois Tigeot int i; 2405718399fSFrançois Tigeot for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) 2415718399fSFrançois Tigeot /* iowrite32(ioread32(srcP++), dstP++); */ 2425718399fSFrançois Tigeot *dstP++ = *srcP++; 2435718399fSFrançois Tigeot return 0; 2445718399fSFrançois Tigeot } 2455718399fSFrançois Tigeot 2465718399fSFrançois Tigeot static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, 2475718399fSFrançois Tigeot unsigned long page, 2485718399fSFrançois Tigeot vm_memattr_t prot) 2495718399fSFrançois Tigeot { 2505718399fSFrançois Tigeot vm_page_t d = ttm->pages[page]; 2515718399fSFrançois Tigeot void *dst; 2525718399fSFrançois Tigeot 2535718399fSFrançois Tigeot if (!d) 2545718399fSFrançois Tigeot return -ENOMEM; 2555718399fSFrançois Tigeot 2565718399fSFrançois Tigeot src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); 2575718399fSFrançois Tigeot 2585718399fSFrançois Tigeot /* XXXKIB can't sleep ? */ 2595718399fSFrançois Tigeot dst = pmap_mapdev_attr(VM_PAGE_TO_PHYS(d), PAGE_SIZE, prot); 2605718399fSFrançois Tigeot if (!dst) 2615718399fSFrançois Tigeot return -ENOMEM; 2625718399fSFrançois Tigeot 263*c19c6249SFrançois Tigeot memcpy_fromio(dst, src, PAGE_SIZE); 2645718399fSFrançois Tigeot 2655718399fSFrançois Tigeot pmap_unmapdev((vm_offset_t)dst, PAGE_SIZE); 2665718399fSFrançois Tigeot 2675718399fSFrançois Tigeot return 0; 2685718399fSFrançois Tigeot } 2695718399fSFrançois Tigeot 2705718399fSFrançois Tigeot static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, 2715718399fSFrançois Tigeot unsigned long page, 2725718399fSFrançois Tigeot vm_memattr_t prot) 2735718399fSFrançois Tigeot { 2745718399fSFrançois Tigeot vm_page_t s = ttm->pages[page]; 2755718399fSFrançois Tigeot void *src; 2765718399fSFrançois Tigeot 2775718399fSFrançois Tigeot if (!s) 2785718399fSFrançois Tigeot return -ENOMEM; 2795718399fSFrançois Tigeot 2805718399fSFrançois Tigeot dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); 2815718399fSFrançois Tigeot src = pmap_mapdev_attr(VM_PAGE_TO_PHYS(s), PAGE_SIZE, prot); 2825718399fSFrançois Tigeot if (!src) 2835718399fSFrançois Tigeot return -ENOMEM; 2845718399fSFrançois Tigeot 285*c19c6249SFrançois Tigeot memcpy_toio(dst, src, PAGE_SIZE); 2865718399fSFrançois Tigeot 2875718399fSFrançois Tigeot pmap_unmapdev((vm_offset_t)src, PAGE_SIZE); 2885718399fSFrançois Tigeot 2895718399fSFrançois Tigeot return 0; 2905718399fSFrançois Tigeot } 2915718399fSFrançois Tigeot 2925718399fSFrançois Tigeot int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, 2935718399fSFrançois Tigeot bool evict, bool no_wait_gpu, 2945718399fSFrançois Tigeot struct ttm_mem_reg *new_mem) 2955718399fSFrançois Tigeot { 2965718399fSFrançois Tigeot struct ttm_bo_device *bdev = bo->bdev; 2975718399fSFrançois Tigeot struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 2985718399fSFrançois Tigeot struct ttm_tt *ttm = bo->ttm; 2995718399fSFrançois Tigeot struct ttm_mem_reg *old_mem = &bo->mem; 3005718399fSFrançois Tigeot struct ttm_mem_reg old_copy = *old_mem; 3015718399fSFrançois Tigeot void *old_iomap; 3025718399fSFrançois Tigeot void *new_iomap; 3035718399fSFrançois Tigeot int ret; 3045718399fSFrançois Tigeot unsigned long i; 3055718399fSFrançois Tigeot unsigned long page; 3065718399fSFrançois Tigeot unsigned long add = 0; 3075718399fSFrançois Tigeot int dir; 3085718399fSFrançois Tigeot 3095718399fSFrançois Tigeot ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); 3105718399fSFrançois Tigeot if (ret) 3115718399fSFrançois Tigeot return ret; 3125718399fSFrançois Tigeot ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap); 3135718399fSFrançois Tigeot if (ret) 3145718399fSFrançois Tigeot goto out; 3155718399fSFrançois Tigeot 3165718399fSFrançois Tigeot if (old_iomap == NULL && new_iomap == NULL) 3175718399fSFrançois Tigeot goto out2; 3185718399fSFrançois Tigeot if (old_iomap == NULL && ttm == NULL) 3195718399fSFrançois Tigeot goto out2; 3205718399fSFrançois Tigeot 3215718399fSFrançois Tigeot if (ttm->state == tt_unpopulated) { 3225718399fSFrançois Tigeot ret = ttm->bdev->driver->ttm_tt_populate(ttm); 3236f486c69SFrançois Tigeot if (ret) { 3246f486c69SFrançois Tigeot /* if we fail here don't nuke the mm node 3256f486c69SFrançois Tigeot * as the bo still owns it */ 3266f486c69SFrançois Tigeot old_copy.mm_node = NULL; 3275718399fSFrançois Tigeot goto out1; 3285718399fSFrançois Tigeot } 3296f486c69SFrançois Tigeot } 3305718399fSFrançois Tigeot 3315718399fSFrançois Tigeot add = 0; 3325718399fSFrançois Tigeot dir = 1; 3335718399fSFrançois Tigeot 3345718399fSFrançois Tigeot if ((old_mem->mem_type == new_mem->mem_type) && 3355718399fSFrançois Tigeot (new_mem->start < old_mem->start + old_mem->size)) { 3365718399fSFrançois Tigeot dir = -1; 3375718399fSFrançois Tigeot add = new_mem->num_pages - 1; 3385718399fSFrançois Tigeot } 3395718399fSFrançois Tigeot 3405718399fSFrançois Tigeot for (i = 0; i < new_mem->num_pages; ++i) { 3415718399fSFrançois Tigeot page = i * dir + add; 3425718399fSFrançois Tigeot if (old_iomap == NULL) { 3435718399fSFrançois Tigeot vm_memattr_t prot = ttm_io_prot(old_mem->placement); 3445718399fSFrançois Tigeot ret = ttm_copy_ttm_io_page(ttm, new_iomap, page, 3455718399fSFrançois Tigeot prot); 3465718399fSFrançois Tigeot } else if (new_iomap == NULL) { 3475718399fSFrançois Tigeot vm_memattr_t prot = ttm_io_prot(new_mem->placement); 3485718399fSFrançois Tigeot ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, 3495718399fSFrançois Tigeot prot); 3505718399fSFrançois Tigeot } else 3515718399fSFrançois Tigeot ret = ttm_copy_io_page(new_iomap, old_iomap, page); 3526f486c69SFrançois Tigeot if (ret) { 3536f486c69SFrançois Tigeot /* failing here, means keep old copy as-is */ 3546f486c69SFrançois Tigeot old_copy.mm_node = NULL; 3555718399fSFrançois Tigeot goto out1; 3565718399fSFrançois Tigeot } 3576f486c69SFrançois Tigeot } 3585718399fSFrançois Tigeot cpu_mfence(); 3595718399fSFrançois Tigeot out2: 3605718399fSFrançois Tigeot old_copy = *old_mem; 3615718399fSFrançois Tigeot *old_mem = *new_mem; 3625718399fSFrançois Tigeot new_mem->mm_node = NULL; 3635718399fSFrançois Tigeot 3645718399fSFrançois Tigeot if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) { 3655718399fSFrançois Tigeot ttm_tt_unbind(ttm); 3665718399fSFrançois Tigeot ttm_tt_destroy(ttm); 3675718399fSFrançois Tigeot bo->ttm = NULL; 3685718399fSFrançois Tigeot } 3695718399fSFrançois Tigeot 3705718399fSFrançois Tigeot out1: 3715718399fSFrançois Tigeot ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); 3725718399fSFrançois Tigeot out: 3735718399fSFrançois Tigeot ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); 3745718399fSFrançois Tigeot ttm_bo_mem_put(bo, &old_copy); 3755718399fSFrançois Tigeot return ret; 3765718399fSFrançois Tigeot } 377*c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_bo_move_memcpy); 3785718399fSFrançois Tigeot 3795718399fSFrançois Tigeot MALLOC_DEFINE(M_TTM_TRANSF_OBJ, "ttm_transf_obj", "TTM Transfer Objects"); 3805718399fSFrançois Tigeot 3815718399fSFrançois Tigeot static void ttm_transfered_destroy(struct ttm_buffer_object *bo) 3825718399fSFrançois Tigeot { 383*c19c6249SFrançois Tigeot kfree(bo, M_TTM_TRANSF_OBJ); 3845718399fSFrançois Tigeot } 3855718399fSFrançois Tigeot 3865718399fSFrançois Tigeot /** 3875718399fSFrançois Tigeot * ttm_buffer_object_transfer 3885718399fSFrançois Tigeot * 3895718399fSFrançois Tigeot * @bo: A pointer to a struct ttm_buffer_object. 3905718399fSFrançois Tigeot * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, 3915718399fSFrançois Tigeot * holding the data of @bo with the old placement. 3925718399fSFrançois Tigeot * 3935718399fSFrançois Tigeot * This is a utility function that may be called after an accelerated move 3945718399fSFrançois Tigeot * has been scheduled. A new buffer object is created as a placeholder for 3955718399fSFrançois Tigeot * the old data while it's being copied. When that buffer object is idle, 3965718399fSFrançois Tigeot * it can be destroyed, releasing the space of the old placement. 3975718399fSFrançois Tigeot * Returns: 3985718399fSFrançois Tigeot * !0: Failure. 3995718399fSFrançois Tigeot */ 4005718399fSFrançois Tigeot 401*c19c6249SFrançois Tigeot static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 4026f486c69SFrançois Tigeot struct ttm_buffer_object **new_obj) 4035718399fSFrançois Tigeot { 4045718399fSFrançois Tigeot struct ttm_buffer_object *fbo; 4056f486c69SFrançois Tigeot struct ttm_bo_device *bdev = bo->bdev; 4066f486c69SFrançois Tigeot struct ttm_bo_driver *driver = bdev->driver; 4075718399fSFrançois Tigeot 4086f486c69SFrançois Tigeot fbo = kmalloc(sizeof(*fbo), M_TTM_TRANSF_OBJ, M_WAITOK); 409*c19c6249SFrançois Tigeot if (!fbo) 410*c19c6249SFrançois Tigeot return -ENOMEM; 411*c19c6249SFrançois Tigeot 4125718399fSFrançois Tigeot *fbo = *bo; 4135718399fSFrançois Tigeot 4145718399fSFrançois Tigeot /** 4155718399fSFrançois Tigeot * Fix up members that we shouldn't copy directly: 4165718399fSFrançois Tigeot * TODO: Explicit member copy would probably be better here. 4175718399fSFrançois Tigeot */ 4185718399fSFrançois Tigeot 4195718399fSFrançois Tigeot INIT_LIST_HEAD(&fbo->ddestroy); 4205718399fSFrançois Tigeot INIT_LIST_HEAD(&fbo->lru); 4215718399fSFrançois Tigeot INIT_LIST_HEAD(&fbo->swap); 4225718399fSFrançois Tigeot INIT_LIST_HEAD(&fbo->io_reserve_lru); 4235718399fSFrançois Tigeot fbo->vm_node = NULL; 4245718399fSFrançois Tigeot atomic_set(&fbo->cpu_writers, 0); 4255718399fSFrançois Tigeot 4266f486c69SFrançois Tigeot lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 4276f486c69SFrançois Tigeot if (bo->sync_obj) 4286f486c69SFrançois Tigeot fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); 4296f486c69SFrançois Tigeot else 4306f486c69SFrançois Tigeot fbo->sync_obj = NULL; 4316f486c69SFrançois Tigeot lockmgr(&bdev->fence_lock, LK_RELEASE); 432e3b244c9SFrançois Tigeot kref_init(&fbo->list_kref); 433e3b244c9SFrançois Tigeot kref_init(&fbo->kref); 4345718399fSFrançois Tigeot fbo->destroy = &ttm_transfered_destroy; 4355718399fSFrançois Tigeot fbo->acc_size = 0; 4365718399fSFrançois Tigeot 4375718399fSFrançois Tigeot *new_obj = fbo; 4385718399fSFrançois Tigeot return 0; 4395718399fSFrançois Tigeot } 4405718399fSFrançois Tigeot 4415718399fSFrançois Tigeot vm_memattr_t 4425718399fSFrançois Tigeot ttm_io_prot(uint32_t caching_flags) 4435718399fSFrançois Tigeot { 444*c19c6249SFrançois Tigeot #if defined(__i386__) || defined(__x86_64__) 4455718399fSFrançois Tigeot if (caching_flags & TTM_PL_FLAG_WC) 4465718399fSFrançois Tigeot return (VM_MEMATTR_WRITE_COMBINING); 4475718399fSFrançois Tigeot else 4485718399fSFrançois Tigeot /* 4495718399fSFrançois Tigeot * We do not support i386, look at the linux source 4505718399fSFrançois Tigeot * for the reason of the comment. 4515718399fSFrançois Tigeot */ 4525718399fSFrançois Tigeot return (VM_MEMATTR_UNCACHEABLE); 4535718399fSFrançois Tigeot #else 4545718399fSFrançois Tigeot #error Port me 4555718399fSFrançois Tigeot #endif 4565718399fSFrançois Tigeot } 457*c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_io_prot); 4585718399fSFrançois Tigeot 4595718399fSFrançois Tigeot static int ttm_bo_ioremap(struct ttm_buffer_object *bo, 4605718399fSFrançois Tigeot unsigned long offset, 4615718399fSFrançois Tigeot unsigned long size, 4625718399fSFrançois Tigeot struct ttm_bo_kmap_obj *map) 4635718399fSFrançois Tigeot { 4645718399fSFrançois Tigeot struct ttm_mem_reg *mem = &bo->mem; 4655718399fSFrançois Tigeot 4665718399fSFrançois Tigeot if (bo->mem.bus.addr) { 4675718399fSFrançois Tigeot map->bo_kmap_type = ttm_bo_map_premapped; 4685718399fSFrançois Tigeot map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); 4695718399fSFrançois Tigeot } else { 4705718399fSFrançois Tigeot map->bo_kmap_type = ttm_bo_map_iomap; 4715718399fSFrançois Tigeot map->virtual = pmap_mapdev_attr(bo->mem.bus.base + 4725718399fSFrançois Tigeot bo->mem.bus.offset + offset, size, 4735718399fSFrançois Tigeot (mem->placement & TTM_PL_FLAG_WC) ? 4745718399fSFrançois Tigeot VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE); 4755718399fSFrançois Tigeot map->size = size; 4765718399fSFrançois Tigeot } 4775718399fSFrançois Tigeot return (!map->virtual) ? -ENOMEM : 0; 4785718399fSFrançois Tigeot } 4795718399fSFrançois Tigeot 4805718399fSFrançois Tigeot static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, 4815718399fSFrançois Tigeot unsigned long start_page, 4825718399fSFrançois Tigeot unsigned long num_pages, 4835718399fSFrançois Tigeot struct ttm_bo_kmap_obj *map) 4845718399fSFrançois Tigeot { 4855718399fSFrançois Tigeot struct ttm_mem_reg *mem = &bo->mem; 4865718399fSFrançois Tigeot vm_memattr_t prot; 4875718399fSFrançois Tigeot struct ttm_tt *ttm = bo->ttm; 4885718399fSFrançois Tigeot int i, ret; 4895718399fSFrançois Tigeot 490*c19c6249SFrançois Tigeot BUG_ON(!ttm); 4915718399fSFrançois Tigeot 4925718399fSFrançois Tigeot if (ttm->state == tt_unpopulated) { 4935718399fSFrançois Tigeot ret = ttm->bdev->driver->ttm_tt_populate(ttm); 4945718399fSFrançois Tigeot if (ret) 4955718399fSFrançois Tigeot return ret; 4965718399fSFrançois Tigeot } 4975718399fSFrançois Tigeot 4985718399fSFrançois Tigeot if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { 4995718399fSFrançois Tigeot /* 5005718399fSFrançois Tigeot * We're mapping a single page, and the desired 5015718399fSFrançois Tigeot * page protection is consistent with the bo. 5025718399fSFrançois Tigeot */ 5035718399fSFrançois Tigeot 5045718399fSFrançois Tigeot map->bo_kmap_type = ttm_bo_map_kmap; 5055718399fSFrançois Tigeot map->page = ttm->pages[start_page]; 5065718399fSFrançois Tigeot map->sf = sf_buf_alloc(map->page); 5075718399fSFrançois Tigeot map->virtual = (void *)sf_buf_kva(map->sf); 5085718399fSFrançois Tigeot } else { 5095718399fSFrançois Tigeot /* 5105718399fSFrançois Tigeot * We need to use vmap to get the desired page protection 5115718399fSFrançois Tigeot * or to make the buffer object look contiguous. 5125718399fSFrançois Tigeot */ 5135718399fSFrançois Tigeot prot = (mem->placement & TTM_PL_FLAG_CACHED) ? 5145718399fSFrançois Tigeot VM_MEMATTR_WRITE_COMBINING : 5155718399fSFrançois Tigeot ttm_io_prot(mem->placement); 5165718399fSFrançois Tigeot map->bo_kmap_type = ttm_bo_map_vmap; 5175718399fSFrançois Tigeot map->num_pages = num_pages; 5185718399fSFrançois Tigeot map->virtual = (void *)kmem_alloc_nofault(&kernel_map, 5195718399fSFrançois Tigeot num_pages * PAGE_SIZE, PAGE_SIZE); 5205718399fSFrançois Tigeot if (map->virtual != NULL) { 5215718399fSFrançois Tigeot for (i = 0; i < num_pages; i++) { 5225718399fSFrançois Tigeot /* XXXKIB hack */ 5235718399fSFrançois Tigeot pmap_page_set_memattr(ttm->pages[start_page + 5245718399fSFrançois Tigeot i], prot); 5255718399fSFrançois Tigeot } 5265718399fSFrançois Tigeot pmap_qenter((vm_offset_t)map->virtual, 5275718399fSFrançois Tigeot &ttm->pages[start_page], num_pages); 5285718399fSFrançois Tigeot } 5295718399fSFrançois Tigeot } 5305718399fSFrançois Tigeot return (!map->virtual) ? -ENOMEM : 0; 5315718399fSFrançois Tigeot } 5325718399fSFrançois Tigeot 5335718399fSFrançois Tigeot int ttm_bo_kmap(struct ttm_buffer_object *bo, 5345718399fSFrançois Tigeot unsigned long start_page, unsigned long num_pages, 5355718399fSFrançois Tigeot struct ttm_bo_kmap_obj *map) 5365718399fSFrançois Tigeot { 5375718399fSFrançois Tigeot struct ttm_mem_type_manager *man = 5385718399fSFrançois Tigeot &bo->bdev->man[bo->mem.mem_type]; 5395718399fSFrançois Tigeot unsigned long offset, size; 5405718399fSFrançois Tigeot int ret; 5415718399fSFrançois Tigeot 542*c19c6249SFrançois Tigeot BUG_ON(!list_empty(&bo->swap)); 5435718399fSFrançois Tigeot map->virtual = NULL; 5445718399fSFrançois Tigeot map->bo = bo; 5455718399fSFrançois Tigeot if (num_pages > bo->num_pages) 5465718399fSFrançois Tigeot return -EINVAL; 5475718399fSFrançois Tigeot if (start_page > bo->num_pages) 5485718399fSFrançois Tigeot return -EINVAL; 5495718399fSFrançois Tigeot #if 0 5505718399fSFrançois Tigeot if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) 5515718399fSFrançois Tigeot return -EPERM; 5525718399fSFrançois Tigeot #endif 5535718399fSFrançois Tigeot (void) ttm_mem_io_lock(man, false); 5545718399fSFrançois Tigeot ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); 5555718399fSFrançois Tigeot ttm_mem_io_unlock(man); 5565718399fSFrançois Tigeot if (ret) 5575718399fSFrançois Tigeot return ret; 5585718399fSFrançois Tigeot if (!bo->mem.bus.is_iomem) { 5595718399fSFrançois Tigeot return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); 5605718399fSFrançois Tigeot } else { 5615718399fSFrançois Tigeot offset = start_page << PAGE_SHIFT; 5625718399fSFrançois Tigeot size = num_pages << PAGE_SHIFT; 5635718399fSFrançois Tigeot return ttm_bo_ioremap(bo, offset, size, map); 5645718399fSFrançois Tigeot } 5655718399fSFrançois Tigeot } 566*c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_bo_kmap); 5675718399fSFrançois Tigeot 5685718399fSFrançois Tigeot void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) 5695718399fSFrançois Tigeot { 5705718399fSFrançois Tigeot struct ttm_buffer_object *bo = map->bo; 5715718399fSFrançois Tigeot struct ttm_mem_type_manager *man = 5725718399fSFrançois Tigeot &bo->bdev->man[bo->mem.mem_type]; 5735718399fSFrançois Tigeot 5745718399fSFrançois Tigeot if (!map->virtual) 5755718399fSFrançois Tigeot return; 5765718399fSFrançois Tigeot switch (map->bo_kmap_type) { 5775718399fSFrançois Tigeot case ttm_bo_map_iomap: 5785718399fSFrançois Tigeot pmap_unmapdev((vm_offset_t)map->virtual, map->size); 5795718399fSFrançois Tigeot break; 5805718399fSFrançois Tigeot case ttm_bo_map_vmap: 5815718399fSFrançois Tigeot pmap_qremove((vm_offset_t)(map->virtual), map->num_pages); 5825718399fSFrançois Tigeot kmem_free(&kernel_map, (vm_offset_t)map->virtual, 5835718399fSFrançois Tigeot map->num_pages * PAGE_SIZE); 5845718399fSFrançois Tigeot break; 5855718399fSFrançois Tigeot case ttm_bo_map_kmap: 5865718399fSFrançois Tigeot sf_buf_free(map->sf); 5875718399fSFrançois Tigeot break; 5885718399fSFrançois Tigeot case ttm_bo_map_premapped: 5895718399fSFrançois Tigeot break; 5905718399fSFrançois Tigeot default: 591*c19c6249SFrançois Tigeot BUG(); 5925718399fSFrançois Tigeot } 5935718399fSFrançois Tigeot (void) ttm_mem_io_lock(man, false); 5945718399fSFrançois Tigeot ttm_mem_io_free(map->bo->bdev, &map->bo->mem); 5955718399fSFrançois Tigeot ttm_mem_io_unlock(man); 5965718399fSFrançois Tigeot map->virtual = NULL; 5975718399fSFrançois Tigeot map->page = NULL; 5985718399fSFrançois Tigeot map->sf = NULL; 5995718399fSFrançois Tigeot } 600*c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_bo_kunmap); 6015718399fSFrançois Tigeot 6025718399fSFrançois Tigeot int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, 6035718399fSFrançois Tigeot void *sync_obj, 6045718399fSFrançois Tigeot bool evict, 6055718399fSFrançois Tigeot bool no_wait_gpu, 6065718399fSFrançois Tigeot struct ttm_mem_reg *new_mem) 6075718399fSFrançois Tigeot { 6085718399fSFrançois Tigeot struct ttm_bo_device *bdev = bo->bdev; 6095718399fSFrançois Tigeot struct ttm_bo_driver *driver = bdev->driver; 6105718399fSFrançois Tigeot struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 6115718399fSFrançois Tigeot struct ttm_mem_reg *old_mem = &bo->mem; 6125718399fSFrançois Tigeot int ret; 6135718399fSFrançois Tigeot struct ttm_buffer_object *ghost_obj; 6145718399fSFrançois Tigeot void *tmp_obj = NULL; 6155718399fSFrançois Tigeot 6165718399fSFrançois Tigeot lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 6175718399fSFrançois Tigeot if (bo->sync_obj) { 6185718399fSFrançois Tigeot tmp_obj = bo->sync_obj; 6195718399fSFrançois Tigeot bo->sync_obj = NULL; 6205718399fSFrançois Tigeot } 6215718399fSFrançois Tigeot bo->sync_obj = driver->sync_obj_ref(sync_obj); 6225718399fSFrançois Tigeot if (evict) { 6235718399fSFrançois Tigeot ret = ttm_bo_wait(bo, false, false, false); 6245718399fSFrançois Tigeot lockmgr(&bdev->fence_lock, LK_RELEASE); 6255718399fSFrançois Tigeot if (tmp_obj) 6265718399fSFrançois Tigeot driver->sync_obj_unref(&tmp_obj); 6275718399fSFrançois Tigeot if (ret) 6285718399fSFrançois Tigeot return ret; 6295718399fSFrançois Tigeot 6305718399fSFrançois Tigeot if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && 6315718399fSFrançois Tigeot (bo->ttm != NULL)) { 6325718399fSFrançois Tigeot ttm_tt_unbind(bo->ttm); 6335718399fSFrançois Tigeot ttm_tt_destroy(bo->ttm); 6345718399fSFrançois Tigeot bo->ttm = NULL; 6355718399fSFrançois Tigeot } 6365718399fSFrançois Tigeot ttm_bo_free_old_node(bo); 6375718399fSFrançois Tigeot } else { 6385718399fSFrançois Tigeot /** 6395718399fSFrançois Tigeot * This should help pipeline ordinary buffer moves. 6405718399fSFrançois Tigeot * 6415718399fSFrançois Tigeot * Hang old buffer memory on a new buffer object, 6425718399fSFrançois Tigeot * and leave it to be released when the GPU 6435718399fSFrançois Tigeot * operation has completed. 6445718399fSFrançois Tigeot */ 6455718399fSFrançois Tigeot 6465718399fSFrançois Tigeot set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 6475718399fSFrançois Tigeot lockmgr(&bdev->fence_lock, LK_RELEASE); 6485718399fSFrançois Tigeot if (tmp_obj) 6495718399fSFrançois Tigeot driver->sync_obj_unref(&tmp_obj); 6505718399fSFrançois Tigeot 6516f486c69SFrançois Tigeot ret = ttm_buffer_object_transfer(bo, &ghost_obj); 6525718399fSFrançois Tigeot if (ret) 6535718399fSFrançois Tigeot return ret; 6545718399fSFrançois Tigeot 6555718399fSFrançois Tigeot /** 6565718399fSFrançois Tigeot * If we're not moving to fixed memory, the TTM object 6575718399fSFrançois Tigeot * needs to stay alive. Otherwhise hang it on the ghost 6585718399fSFrançois Tigeot * bo to be unbound and destroyed. 6595718399fSFrançois Tigeot */ 6605718399fSFrançois Tigeot 6615718399fSFrançois Tigeot if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) 6625718399fSFrançois Tigeot ghost_obj->ttm = NULL; 6635718399fSFrançois Tigeot else 6645718399fSFrançois Tigeot bo->ttm = NULL; 6655718399fSFrançois Tigeot 6665718399fSFrançois Tigeot ttm_bo_unreserve(ghost_obj); 6675718399fSFrançois Tigeot ttm_bo_unref(&ghost_obj); 6685718399fSFrançois Tigeot } 6695718399fSFrançois Tigeot 6705718399fSFrançois Tigeot *old_mem = *new_mem; 6715718399fSFrançois Tigeot new_mem->mm_node = NULL; 6725718399fSFrançois Tigeot 6735718399fSFrançois Tigeot return 0; 6745718399fSFrançois Tigeot } 675*c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); 676