15718399fSFrançois Tigeot /**************************************************************************
25718399fSFrançois Tigeot *
35718399fSFrançois Tigeot * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
45718399fSFrançois Tigeot * All Rights Reserved.
55718399fSFrançois Tigeot *
65718399fSFrançois Tigeot * Permission is hereby granted, free of charge, to any person obtaining a
75718399fSFrançois Tigeot * copy of this software and associated documentation files (the
85718399fSFrançois Tigeot * "Software"), to deal in the Software without restriction, including
95718399fSFrançois Tigeot * without limitation the rights to use, copy, modify, merge, publish,
105718399fSFrançois Tigeot * distribute, sub license, and/or sell copies of the Software, and to
115718399fSFrançois Tigeot * permit persons to whom the Software is furnished to do so, subject to
125718399fSFrançois Tigeot * the following conditions:
135718399fSFrançois Tigeot *
145718399fSFrançois Tigeot * The above copyright notice and this permission notice (including the
155718399fSFrançois Tigeot * next paragraph) shall be included in all copies or substantial portions
165718399fSFrançois Tigeot * of the Software.
175718399fSFrançois Tigeot *
185718399fSFrançois Tigeot * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
195718399fSFrançois Tigeot * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
205718399fSFrançois Tigeot * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
215718399fSFrançois Tigeot * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
225718399fSFrançois Tigeot * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
235718399fSFrançois Tigeot * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
245718399fSFrançois Tigeot * USE OR OTHER DEALINGS IN THE SOFTWARE.
255718399fSFrançois Tigeot *
265718399fSFrançois Tigeot **************************************************************************/
275718399fSFrançois Tigeot /*
285718399fSFrançois Tigeot * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
295718399fSFrançois Tigeot */
305718399fSFrançois Tigeot
31216f7a2cSFrançois Tigeot #include <drm/ttm/ttm_bo_driver.h>
32216f7a2cSFrançois Tigeot #include <drm/ttm/ttm_placement.h>
33a34b4168SMatthew Dillon #include <drm/drm_vma_manager.h>
3429bffcd3SFrançois Tigeot #include <linux/io.h>
354f125aeaSFrançois Tigeot #include <linux/highmem.h>
3629bffcd3SFrançois Tigeot #include <linux/wait.h>
374f125aeaSFrançois Tigeot #include <linux/slab.h>
384f125aeaSFrançois Tigeot #include <linux/vmalloc.h>
394f125aeaSFrançois Tigeot #include <linux/module.h>
401cfef1a5SFrançois Tigeot #include <linux/reservation.h>
415718399fSFrançois Tigeot
42*932d855eSSergey Zigachev struct ttm_transfer_obj {
43*932d855eSSergey Zigachev struct ttm_buffer_object base;
44*932d855eSSergey Zigachev struct ttm_buffer_object *bo;
45*932d855eSSergey Zigachev };
46*932d855eSSergey Zigachev
ttm_bo_free_old_node(struct ttm_buffer_object * bo)475718399fSFrançois Tigeot void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
485718399fSFrançois Tigeot {
495718399fSFrançois Tigeot ttm_bo_mem_put(bo, &bo->mem);
505718399fSFrançois Tigeot }
515718399fSFrançois Tigeot
ttm_bo_move_ttm(struct ttm_buffer_object * bo,struct ttm_operation_ctx * ctx,struct ttm_mem_reg * new_mem)525718399fSFrançois Tigeot int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
53*932d855eSSergey Zigachev struct ttm_operation_ctx *ctx,
541dedbd3bSFrançois Tigeot struct ttm_mem_reg *new_mem)
555718399fSFrançois Tigeot {
565718399fSFrançois Tigeot struct ttm_tt *ttm = bo->ttm;
575718399fSFrançois Tigeot struct ttm_mem_reg *old_mem = &bo->mem;
585718399fSFrançois Tigeot int ret;
595718399fSFrançois Tigeot
605718399fSFrançois Tigeot if (old_mem->mem_type != TTM_PL_SYSTEM) {
61*932d855eSSergey Zigachev ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
621dedbd3bSFrançois Tigeot
631dedbd3bSFrançois Tigeot if (unlikely(ret != 0)) {
641dedbd3bSFrançois Tigeot if (ret != -ERESTARTSYS)
651dedbd3bSFrançois Tigeot pr_err("Failed to expire sync object before unbinding TTM\n");
661dedbd3bSFrançois Tigeot return ret;
671dedbd3bSFrançois Tigeot }
681dedbd3bSFrançois Tigeot
695718399fSFrançois Tigeot ttm_tt_unbind(ttm);
705718399fSFrançois Tigeot ttm_bo_free_old_node(bo);
715718399fSFrançois Tigeot ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
725718399fSFrançois Tigeot TTM_PL_MASK_MEM);
735718399fSFrançois Tigeot old_mem->mem_type = TTM_PL_SYSTEM;
745718399fSFrançois Tigeot }
755718399fSFrançois Tigeot
765718399fSFrançois Tigeot ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
775718399fSFrançois Tigeot if (unlikely(ret != 0))
785718399fSFrançois Tigeot return ret;
795718399fSFrançois Tigeot
805718399fSFrançois Tigeot if (new_mem->mem_type != TTM_PL_SYSTEM) {
81*932d855eSSergey Zigachev ret = ttm_tt_bind(ttm, new_mem, ctx);
825718399fSFrançois Tigeot if (unlikely(ret != 0))
835718399fSFrançois Tigeot return ret;
845718399fSFrançois Tigeot }
855718399fSFrançois Tigeot
865718399fSFrançois Tigeot *old_mem = *new_mem;
875718399fSFrançois Tigeot new_mem->mm_node = NULL;
885718399fSFrançois Tigeot
895718399fSFrançois Tigeot return 0;
905718399fSFrançois Tigeot }
91c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_bo_move_ttm);
925718399fSFrançois Tigeot
ttm_mem_io_lock(struct ttm_mem_type_manager * man,bool interruptible)935718399fSFrançois Tigeot int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
945718399fSFrançois Tigeot {
955718399fSFrançois Tigeot if (likely(man->io_reserve_fastpath))
965718399fSFrançois Tigeot return 0;
975718399fSFrançois Tigeot
9843e748b9SFrançois Tigeot if (interruptible)
9943e748b9SFrançois Tigeot return mutex_lock_interruptible(&man->io_reserve_mutex);
1005718399fSFrançois Tigeot
10143e748b9SFrançois Tigeot mutex_lock(&man->io_reserve_mutex);
1025718399fSFrançois Tigeot return 0;
1035718399fSFrançois Tigeot }
1044cd92098Szrj EXPORT_SYMBOL(ttm_mem_io_lock);
1055718399fSFrançois Tigeot
ttm_mem_io_unlock(struct ttm_mem_type_manager * man)1065718399fSFrançois Tigeot void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
1075718399fSFrançois Tigeot {
1085718399fSFrançois Tigeot if (likely(man->io_reserve_fastpath))
1095718399fSFrançois Tigeot return;
1105718399fSFrançois Tigeot
11143e748b9SFrançois Tigeot mutex_unlock(&man->io_reserve_mutex);
1125718399fSFrançois Tigeot }
1134cd92098Szrj EXPORT_SYMBOL(ttm_mem_io_unlock);
1145718399fSFrançois Tigeot
ttm_mem_io_evict(struct ttm_mem_type_manager * man)1155718399fSFrançois Tigeot static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
1165718399fSFrançois Tigeot {
1175718399fSFrançois Tigeot struct ttm_buffer_object *bo;
1185718399fSFrançois Tigeot
1195718399fSFrançois Tigeot if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
1205718399fSFrançois Tigeot return -EAGAIN;
1215718399fSFrançois Tigeot
1225718399fSFrançois Tigeot bo = list_first_entry(&man->io_reserve_lru,
1235718399fSFrançois Tigeot struct ttm_buffer_object,
1245718399fSFrançois Tigeot io_reserve_lru);
1255718399fSFrançois Tigeot list_del_init(&bo->io_reserve_lru);
1265718399fSFrançois Tigeot ttm_bo_unmap_virtual_locked(bo);
1275718399fSFrançois Tigeot
1285718399fSFrançois Tigeot return 0;
1295718399fSFrançois Tigeot }
1305718399fSFrançois Tigeot
1314cd92098Szrj
ttm_mem_io_reserve(struct ttm_bo_device * bdev,struct ttm_mem_reg * mem)1324cd92098Szrj int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
1335718399fSFrançois Tigeot struct ttm_mem_reg *mem)
1345718399fSFrançois Tigeot {
1355718399fSFrançois Tigeot struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1365718399fSFrançois Tigeot int ret = 0;
1375718399fSFrançois Tigeot
1385718399fSFrançois Tigeot if (!bdev->driver->io_mem_reserve)
1395718399fSFrançois Tigeot return 0;
1405718399fSFrançois Tigeot if (likely(man->io_reserve_fastpath))
1415718399fSFrançois Tigeot return bdev->driver->io_mem_reserve(bdev, mem);
1425718399fSFrançois Tigeot
1435718399fSFrançois Tigeot if (bdev->driver->io_mem_reserve &&
1445718399fSFrançois Tigeot mem->bus.io_reserved_count++ == 0) {
1455718399fSFrançois Tigeot retry:
1465718399fSFrançois Tigeot ret = bdev->driver->io_mem_reserve(bdev, mem);
1475718399fSFrançois Tigeot if (ret == -EAGAIN) {
1485718399fSFrançois Tigeot ret = ttm_mem_io_evict(man);
1495718399fSFrançois Tigeot if (ret == 0)
1505718399fSFrançois Tigeot goto retry;
1515718399fSFrançois Tigeot }
1525718399fSFrançois Tigeot }
1535718399fSFrançois Tigeot return ret;
1545718399fSFrançois Tigeot }
1554cd92098Szrj EXPORT_SYMBOL(ttm_mem_io_reserve);
1565718399fSFrançois Tigeot
ttm_mem_io_free(struct ttm_bo_device * bdev,struct ttm_mem_reg * mem)1574cd92098Szrj void ttm_mem_io_free(struct ttm_bo_device *bdev,
1585718399fSFrançois Tigeot struct ttm_mem_reg *mem)
1595718399fSFrançois Tigeot {
1605718399fSFrançois Tigeot struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1615718399fSFrançois Tigeot
1625718399fSFrançois Tigeot if (likely(man->io_reserve_fastpath))
1635718399fSFrançois Tigeot return;
1645718399fSFrançois Tigeot
1655718399fSFrançois Tigeot if (bdev->driver->io_mem_reserve &&
1665718399fSFrançois Tigeot --mem->bus.io_reserved_count == 0 &&
1675718399fSFrançois Tigeot bdev->driver->io_mem_free)
1685718399fSFrançois Tigeot bdev->driver->io_mem_free(bdev, mem);
1695718399fSFrançois Tigeot
1705718399fSFrançois Tigeot }
1714cd92098Szrj EXPORT_SYMBOL(ttm_mem_io_free);
1725718399fSFrançois Tigeot
ttm_mem_io_reserve_vm(struct ttm_buffer_object * bo)1735718399fSFrançois Tigeot int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
1745718399fSFrançois Tigeot {
1755718399fSFrançois Tigeot struct ttm_mem_reg *mem = &bo->mem;
1765718399fSFrançois Tigeot int ret;
1775718399fSFrançois Tigeot
1785718399fSFrançois Tigeot if (!mem->bus.io_reserved_vm) {
1795718399fSFrançois Tigeot struct ttm_mem_type_manager *man =
1805718399fSFrançois Tigeot &bo->bdev->man[mem->mem_type];
1815718399fSFrançois Tigeot
1825718399fSFrançois Tigeot ret = ttm_mem_io_reserve(bo->bdev, mem);
1835718399fSFrançois Tigeot if (unlikely(ret != 0))
1845718399fSFrançois Tigeot return ret;
1855718399fSFrançois Tigeot mem->bus.io_reserved_vm = true;
1865718399fSFrançois Tigeot if (man->use_io_reserve_lru)
1875718399fSFrançois Tigeot list_add_tail(&bo->io_reserve_lru,
1885718399fSFrançois Tigeot &man->io_reserve_lru);
1895718399fSFrançois Tigeot }
1905718399fSFrançois Tigeot return 0;
1915718399fSFrançois Tigeot }
1925718399fSFrançois Tigeot
ttm_mem_io_free_vm(struct ttm_buffer_object * bo)1935718399fSFrançois Tigeot void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
1945718399fSFrançois Tigeot {
1955718399fSFrançois Tigeot struct ttm_mem_reg *mem = &bo->mem;
1965718399fSFrançois Tigeot
1975718399fSFrançois Tigeot if (mem->bus.io_reserved_vm) {
1985718399fSFrançois Tigeot mem->bus.io_reserved_vm = false;
1995718399fSFrançois Tigeot list_del_init(&bo->io_reserve_lru);
2005718399fSFrançois Tigeot ttm_mem_io_free(bo->bdev, mem);
2015718399fSFrançois Tigeot }
2025718399fSFrançois Tigeot }
2035718399fSFrançois Tigeot
ttm_mem_reg_ioremap(struct ttm_bo_device * bdev,struct ttm_mem_reg * mem,void ** virtual)204c6f73aabSFrançois Tigeot static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
2055718399fSFrançois Tigeot void **virtual)
2065718399fSFrançois Tigeot {
2075718399fSFrançois Tigeot struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
2085718399fSFrançois Tigeot int ret;
2095718399fSFrançois Tigeot void *addr;
2105718399fSFrançois Tigeot
2115718399fSFrançois Tigeot *virtual = NULL;
2125718399fSFrançois Tigeot (void) ttm_mem_io_lock(man, false);
2135718399fSFrançois Tigeot ret = ttm_mem_io_reserve(bdev, mem);
2145718399fSFrançois Tigeot ttm_mem_io_unlock(man);
2155718399fSFrançois Tigeot if (ret || !mem->bus.is_iomem)
2165718399fSFrançois Tigeot return ret;
2175718399fSFrançois Tigeot
2185718399fSFrançois Tigeot if (mem->bus.addr) {
2195718399fSFrançois Tigeot addr = mem->bus.addr;
2205718399fSFrançois Tigeot } else {
22143e748b9SFrançois Tigeot if (mem->placement & TTM_PL_FLAG_WC)
22243e748b9SFrançois Tigeot addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
22343e748b9SFrançois Tigeot else
22443e748b9SFrançois Tigeot addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
2255718399fSFrançois Tigeot if (!addr) {
2265718399fSFrançois Tigeot (void) ttm_mem_io_lock(man, false);
2275718399fSFrançois Tigeot ttm_mem_io_free(bdev, mem);
2285718399fSFrançois Tigeot ttm_mem_io_unlock(man);
2295718399fSFrançois Tigeot return -ENOMEM;
2305718399fSFrançois Tigeot }
2315718399fSFrançois Tigeot }
2325718399fSFrançois Tigeot *virtual = addr;
2335718399fSFrançois Tigeot return 0;
2345718399fSFrançois Tigeot }
2355718399fSFrançois Tigeot
ttm_mem_reg_iounmap(struct ttm_bo_device * bdev,struct ttm_mem_reg * mem,void * virtual)236c6f73aabSFrançois Tigeot static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
2375718399fSFrançois Tigeot void *virtual)
2385718399fSFrançois Tigeot {
2395718399fSFrançois Tigeot struct ttm_mem_type_manager *man;
2405718399fSFrançois Tigeot
2415718399fSFrançois Tigeot man = &bdev->man[mem->mem_type];
2425718399fSFrançois Tigeot
2435718399fSFrançois Tigeot if (virtual && mem->bus.addr == NULL)
24443e748b9SFrançois Tigeot iounmap(virtual);
2455718399fSFrançois Tigeot (void) ttm_mem_io_lock(man, false);
2465718399fSFrançois Tigeot ttm_mem_io_free(bdev, mem);
2475718399fSFrançois Tigeot ttm_mem_io_unlock(man);
2485718399fSFrançois Tigeot }
2495718399fSFrançois Tigeot
ttm_copy_io_page(void * dst,void * src,unsigned long page)2505718399fSFrançois Tigeot static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
2515718399fSFrançois Tigeot {
2525718399fSFrançois Tigeot uint32_t *dstP =
2535718399fSFrançois Tigeot (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
2545718399fSFrançois Tigeot uint32_t *srcP =
2555718399fSFrançois Tigeot (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
2565718399fSFrançois Tigeot
2575718399fSFrançois Tigeot int i;
2585718399fSFrançois Tigeot for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
25943e748b9SFrançois Tigeot iowrite32(ioread32(srcP++), dstP++);
2605718399fSFrançois Tigeot return 0;
2615718399fSFrançois Tigeot }
2625718399fSFrançois Tigeot
ttm_copy_io_ttm_page(struct ttm_tt * ttm,void * src,unsigned long page,pgprot_t prot)2635718399fSFrançois Tigeot static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
2645718399fSFrançois Tigeot unsigned long page,
265e5c1d8f1SFrançois Tigeot pgprot_t prot)
2665718399fSFrançois Tigeot {
267f0bba3d1SFrançois Tigeot struct page *d = ttm->pages[page];
2685718399fSFrançois Tigeot void *dst;
2695718399fSFrançois Tigeot
2705718399fSFrançois Tigeot if (!d)
2715718399fSFrançois Tigeot return -ENOMEM;
2725718399fSFrançois Tigeot
2735718399fSFrançois Tigeot src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
2745718399fSFrançois Tigeot
275e5c1d8f1SFrançois Tigeot #ifdef CONFIG_X86
276e5c1d8f1SFrançois Tigeot dst = kmap_atomic_prot(d, prot);
277e5c1d8f1SFrançois Tigeot #else
278e5c1d8f1SFrançois Tigeot if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
279e5c1d8f1SFrançois Tigeot dst = vmap(&d, 1, 0, prot);
280e5c1d8f1SFrançois Tigeot else
281e5c1d8f1SFrançois Tigeot dst = kmap(d);
282e5c1d8f1SFrançois Tigeot #endif
2835718399fSFrançois Tigeot if (!dst)
2845718399fSFrançois Tigeot return -ENOMEM;
2855718399fSFrançois Tigeot
286c19c6249SFrançois Tigeot memcpy_fromio(dst, src, PAGE_SIZE);
2875718399fSFrançois Tigeot
288e5c1d8f1SFrançois Tigeot #ifdef CONFIG_X86
289e5c1d8f1SFrançois Tigeot kunmap_atomic(dst);
290e5c1d8f1SFrançois Tigeot #else
291e5c1d8f1SFrançois Tigeot if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
292e5c1d8f1SFrançois Tigeot vunmap(dst);
293e5c1d8f1SFrançois Tigeot else
294e5c1d8f1SFrançois Tigeot kunmap(d);
295e5c1d8f1SFrançois Tigeot #endif
2965718399fSFrançois Tigeot
2975718399fSFrançois Tigeot return 0;
2985718399fSFrançois Tigeot }
2995718399fSFrançois Tigeot
ttm_copy_ttm_io_page(struct ttm_tt * ttm,void * dst,unsigned long page,pgprot_t prot)3005718399fSFrançois Tigeot static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
3015718399fSFrançois Tigeot unsigned long page,
302e5c1d8f1SFrançois Tigeot pgprot_t prot)
3035718399fSFrançois Tigeot {
304f0bba3d1SFrançois Tigeot struct page *s = ttm->pages[page];
3055718399fSFrançois Tigeot void *src;
3065718399fSFrançois Tigeot
3075718399fSFrançois Tigeot if (!s)
3085718399fSFrançois Tigeot return -ENOMEM;
3095718399fSFrançois Tigeot
3105718399fSFrançois Tigeot dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
311e5c1d8f1SFrançois Tigeot #ifdef CONFIG_X86
312e5c1d8f1SFrançois Tigeot src = kmap_atomic_prot(s, prot);
313e5c1d8f1SFrançois Tigeot #else
314e5c1d8f1SFrançois Tigeot if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
315e5c1d8f1SFrançois Tigeot src = vmap(&s, 1, 0, prot);
316e5c1d8f1SFrançois Tigeot else
317e5c1d8f1SFrançois Tigeot src = kmap(s);
318e5c1d8f1SFrançois Tigeot #endif
3195718399fSFrançois Tigeot if (!src)
3205718399fSFrançois Tigeot return -ENOMEM;
3215718399fSFrançois Tigeot
322c19c6249SFrançois Tigeot memcpy_toio(dst, src, PAGE_SIZE);
3235718399fSFrançois Tigeot
324e5c1d8f1SFrançois Tigeot #ifdef CONFIG_X86
325e5c1d8f1SFrançois Tigeot kunmap_atomic(src);
326e5c1d8f1SFrançois Tigeot #else
327e5c1d8f1SFrançois Tigeot if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
328e5c1d8f1SFrançois Tigeot vunmap(src);
329e5c1d8f1SFrançois Tigeot else
330e5c1d8f1SFrançois Tigeot kunmap(s);
331e5c1d8f1SFrançois Tigeot #endif
3325718399fSFrançois Tigeot
3335718399fSFrançois Tigeot return 0;
3345718399fSFrançois Tigeot }
3355718399fSFrançois Tigeot
ttm_bo_move_memcpy(struct ttm_buffer_object * bo,struct ttm_operation_ctx * ctx,struct ttm_mem_reg * new_mem)3365718399fSFrançois Tigeot int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
337*932d855eSSergey Zigachev struct ttm_operation_ctx *ctx,
3385718399fSFrançois Tigeot struct ttm_mem_reg *new_mem)
3395718399fSFrançois Tigeot {
3405718399fSFrançois Tigeot struct ttm_bo_device *bdev = bo->bdev;
3415718399fSFrançois Tigeot struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
3425718399fSFrançois Tigeot struct ttm_tt *ttm = bo->ttm;
3435718399fSFrançois Tigeot struct ttm_mem_reg *old_mem = &bo->mem;
3445718399fSFrançois Tigeot struct ttm_mem_reg old_copy = *old_mem;
3455718399fSFrançois Tigeot void *old_iomap;
3465718399fSFrançois Tigeot void *new_iomap;
3475718399fSFrançois Tigeot int ret;
3485718399fSFrançois Tigeot unsigned long i;
3495718399fSFrançois Tigeot unsigned long page;
3505718399fSFrançois Tigeot unsigned long add = 0;
3515718399fSFrançois Tigeot int dir;
3525718399fSFrançois Tigeot
353*932d855eSSergey Zigachev ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
3541dedbd3bSFrançois Tigeot if (ret)
3551dedbd3bSFrançois Tigeot return ret;
3561dedbd3bSFrançois Tigeot
3575718399fSFrançois Tigeot ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
3585718399fSFrançois Tigeot if (ret)
3595718399fSFrançois Tigeot return ret;
3605718399fSFrançois Tigeot ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
3615718399fSFrançois Tigeot if (ret)
3625718399fSFrançois Tigeot goto out;
3635718399fSFrançois Tigeot
364c6f73aabSFrançois Tigeot /*
365c6f73aabSFrançois Tigeot * Single TTM move. NOP.
366c6f73aabSFrançois Tigeot */
3675718399fSFrançois Tigeot if (old_iomap == NULL && new_iomap == NULL)
3685718399fSFrançois Tigeot goto out2;
3695718399fSFrançois Tigeot
370c6f73aabSFrançois Tigeot /*
371c6f73aabSFrançois Tigeot * Don't move nonexistent data. Clear destination instead.
372c6f73aabSFrançois Tigeot */
373c6f73aabSFrançois Tigeot if (old_iomap == NULL &&
374c6f73aabSFrançois Tigeot (ttm == NULL || (ttm->state == tt_unpopulated &&
375c6f73aabSFrançois Tigeot !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
376c6f73aabSFrançois Tigeot memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
377c6f73aabSFrançois Tigeot goto out2;
3785718399fSFrançois Tigeot }
379c6f73aabSFrançois Tigeot
380c6f73aabSFrançois Tigeot /*
381c6f73aabSFrançois Tigeot * TTM might be null for moves within the same region.
382c6f73aabSFrançois Tigeot */
383*932d855eSSergey Zigachev if (ttm) {
384*932d855eSSergey Zigachev ret = ttm_tt_populate(ttm, ctx);
385c6f73aabSFrançois Tigeot if (ret)
386c6f73aabSFrançois Tigeot goto out1;
3876f486c69SFrançois Tigeot }
3885718399fSFrançois Tigeot
3895718399fSFrançois Tigeot add = 0;
3905718399fSFrançois Tigeot dir = 1;
3915718399fSFrançois Tigeot
3925718399fSFrançois Tigeot if ((old_mem->mem_type == new_mem->mem_type) &&
3935718399fSFrançois Tigeot (new_mem->start < old_mem->start + old_mem->size)) {
3945718399fSFrançois Tigeot dir = -1;
3955718399fSFrançois Tigeot add = new_mem->num_pages - 1;
3965718399fSFrançois Tigeot }
3975718399fSFrançois Tigeot
3985718399fSFrançois Tigeot for (i = 0; i < new_mem->num_pages; ++i) {
3995718399fSFrançois Tigeot page = i * dir + add;
4005718399fSFrançois Tigeot if (old_iomap == NULL) {
4014f125aeaSFrançois Tigeot pgprot_t prot = ttm_io_prot(old_mem->placement,
4024f125aeaSFrançois Tigeot PAGE_KERNEL);
4035718399fSFrançois Tigeot ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
4045718399fSFrançois Tigeot prot);
4055718399fSFrançois Tigeot } else if (new_iomap == NULL) {
4064f125aeaSFrançois Tigeot pgprot_t prot = ttm_io_prot(new_mem->placement,
4074f125aeaSFrançois Tigeot PAGE_KERNEL);
4085718399fSFrançois Tigeot ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
4095718399fSFrançois Tigeot prot);
410*932d855eSSergey Zigachev } else {
4115718399fSFrançois Tigeot ret = ttm_copy_io_page(new_iomap, old_iomap, page);
412*932d855eSSergey Zigachev }
41343e748b9SFrançois Tigeot if (ret)
4145718399fSFrançois Tigeot goto out1;
4155718399fSFrançois Tigeot }
41643e748b9SFrançois Tigeot mb();
4175718399fSFrançois Tigeot out2:
4185718399fSFrançois Tigeot old_copy = *old_mem;
4195718399fSFrançois Tigeot *old_mem = *new_mem;
4205718399fSFrançois Tigeot new_mem->mm_node = NULL;
4215718399fSFrançois Tigeot
4221dedbd3bSFrançois Tigeot if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
4235718399fSFrançois Tigeot ttm_tt_destroy(ttm);
4245718399fSFrançois Tigeot bo->ttm = NULL;
4255718399fSFrançois Tigeot }
4265718399fSFrançois Tigeot
4275718399fSFrançois Tigeot out1:
4285718399fSFrançois Tigeot ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
4295718399fSFrançois Tigeot out:
4305718399fSFrançois Tigeot ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
431c6f73aabSFrançois Tigeot
432c6f73aabSFrançois Tigeot /*
433c6f73aabSFrançois Tigeot * On error, keep the mm node!
434c6f73aabSFrançois Tigeot */
435c6f73aabSFrançois Tigeot if (!ret)
4365718399fSFrançois Tigeot ttm_bo_mem_put(bo, &old_copy);
4375718399fSFrançois Tigeot return ret;
4385718399fSFrançois Tigeot }
439c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_bo_move_memcpy);
4405718399fSFrançois Tigeot
ttm_transfered_destroy(struct ttm_buffer_object * bo)4415718399fSFrançois Tigeot static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
4425718399fSFrançois Tigeot {
443*932d855eSSergey Zigachev struct ttm_transfer_obj *fbo;
444*932d855eSSergey Zigachev
445*932d855eSSergey Zigachev fbo = container_of(bo, struct ttm_transfer_obj, base);
446*932d855eSSergey Zigachev ttm_bo_put(fbo->bo);
447*932d855eSSergey Zigachev kfree(fbo);
4485718399fSFrançois Tigeot }
4495718399fSFrançois Tigeot
4505718399fSFrançois Tigeot /**
4515718399fSFrançois Tigeot * ttm_buffer_object_transfer
4525718399fSFrançois Tigeot *
4535718399fSFrançois Tigeot * @bo: A pointer to a struct ttm_buffer_object.
4545718399fSFrançois Tigeot * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
4555718399fSFrançois Tigeot * holding the data of @bo with the old placement.
4565718399fSFrançois Tigeot *
4575718399fSFrançois Tigeot * This is a utility function that may be called after an accelerated move
4585718399fSFrançois Tigeot * has been scheduled. A new buffer object is created as a placeholder for
4595718399fSFrançois Tigeot * the old data while it's being copied. When that buffer object is idle,
4605718399fSFrançois Tigeot * it can be destroyed, releasing the space of the old placement.
4615718399fSFrançois Tigeot * Returns:
4625718399fSFrançois Tigeot * !0: Failure.
4635718399fSFrançois Tigeot */
4645718399fSFrançois Tigeot
ttm_buffer_object_transfer(struct ttm_buffer_object * bo,struct ttm_buffer_object ** new_obj)465c19c6249SFrançois Tigeot static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
4666f486c69SFrançois Tigeot struct ttm_buffer_object **new_obj)
4675718399fSFrançois Tigeot {
468*932d855eSSergey Zigachev struct ttm_transfer_obj *fbo;
4693c3b358eSFrançois Tigeot int ret;
4705718399fSFrançois Tigeot
4711dedbd3bSFrançois Tigeot fbo = kmalloc(sizeof(*fbo), M_DRM, GFP_KERNEL);
472c19c6249SFrançois Tigeot if (!fbo)
473c19c6249SFrançois Tigeot return -ENOMEM;
474c19c6249SFrançois Tigeot
475*932d855eSSergey Zigachev fbo->base = *bo;
476*932d855eSSergey Zigachev fbo->base.mem.placement |= TTM_PL_FLAG_NO_EVICT;
477*932d855eSSergey Zigachev
478*932d855eSSergey Zigachev ttm_bo_get(bo);
479*932d855eSSergey Zigachev fbo->bo = bo;
4805718399fSFrançois Tigeot
4815718399fSFrançois Tigeot /**
4825718399fSFrançois Tigeot * Fix up members that we shouldn't copy directly:
4835718399fSFrançois Tigeot * TODO: Explicit member copy would probably be better here.
4845718399fSFrançois Tigeot */
4855718399fSFrançois Tigeot
486*932d855eSSergey Zigachev atomic_inc(&bo->bdev->glob->bo_count);
487*932d855eSSergey Zigachev INIT_LIST_HEAD(&fbo->base.ddestroy);
488*932d855eSSergey Zigachev INIT_LIST_HEAD(&fbo->base.lru);
489*932d855eSSergey Zigachev INIT_LIST_HEAD(&fbo->base.swap);
490*932d855eSSergey Zigachev INIT_LIST_HEAD(&fbo->base.io_reserve_lru);
491*932d855eSSergey Zigachev lockinit(&fbo->base.wu_mutex, "dtfbwm", 0, LK_CANRECURSE);
492*932d855eSSergey Zigachev fbo->base.moving = NULL;
493*932d855eSSergey Zigachev drm_vma_node_reset(&fbo->base.vma_node);
494*932d855eSSergey Zigachev atomic_set(&fbo->base.cpu_writers, 0);
4955718399fSFrançois Tigeot
496*932d855eSSergey Zigachev kref_init(&fbo->base.list_kref);
497*932d855eSSergey Zigachev kref_init(&fbo->base.kref);
498*932d855eSSergey Zigachev fbo->base.destroy = &ttm_transfered_destroy;
499*932d855eSSergey Zigachev fbo->base.acc_size = 0;
500*932d855eSSergey Zigachev fbo->base.resv = &fbo->base.ttm_resv;
501*932d855eSSergey Zigachev reservation_object_init(fbo->base.resv);
502*932d855eSSergey Zigachev ret = ww_mutex_trylock(&fbo->base.resv->lock);
5033c3b358eSFrançois Tigeot WARN_ON(!ret);
5045718399fSFrançois Tigeot
505*932d855eSSergey Zigachev *new_obj = &fbo->base;
5065718399fSFrançois Tigeot return 0;
5075718399fSFrançois Tigeot }
5085718399fSFrançois Tigeot
ttm_io_prot(uint32_t caching_flags,pgprot_t tmp)5094f125aeaSFrançois Tigeot pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
5105718399fSFrançois Tigeot {
5111cfef1a5SFrançois Tigeot /* Cached mappings need no adjustment */
5121cfef1a5SFrançois Tigeot if (caching_flags & TTM_PL_FLAG_CACHED)
5131cfef1a5SFrançois Tigeot return tmp;
5141cfef1a5SFrançois Tigeot
5154f125aeaSFrançois Tigeot #if defined(__i386__) || defined(__x86_64__)
5165718399fSFrançois Tigeot if (caching_flags & TTM_PL_FLAG_WC)
5174f125aeaSFrançois Tigeot tmp = pgprot_writecombine(tmp);
5185718399fSFrançois Tigeot else
5194f125aeaSFrançois Tigeot tmp = pgprot_noncached(tmp);
5205718399fSFrançois Tigeot #endif
521c59a5c48SFrançois Tigeot #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
522c59a5c48SFrançois Tigeot defined(__powerpc__)
5234f125aeaSFrançois Tigeot if (caching_flags & TTM_PL_FLAG_WC)
5244f125aeaSFrançois Tigeot tmp = pgprot_writecombine(tmp);
5254f125aeaSFrançois Tigeot else
5264f125aeaSFrançois Tigeot tmp = pgprot_noncached(tmp);
5274f125aeaSFrançois Tigeot #endif
5284f125aeaSFrançois Tigeot #if defined(__sparc__) || defined(__mips__)
5294f125aeaSFrançois Tigeot tmp = pgprot_noncached(tmp);
5304f125aeaSFrançois Tigeot #endif
5314f125aeaSFrançois Tigeot return tmp;
5325718399fSFrançois Tigeot }
533c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_io_prot);
5345718399fSFrançois Tigeot
ttm_bo_ioremap(struct ttm_buffer_object * bo,unsigned long offset,unsigned long size,struct ttm_bo_kmap_obj * map)5355718399fSFrançois Tigeot static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
5365718399fSFrançois Tigeot unsigned long offset,
5375718399fSFrançois Tigeot unsigned long size,
5385718399fSFrançois Tigeot struct ttm_bo_kmap_obj *map)
5395718399fSFrançois Tigeot {
5405718399fSFrançois Tigeot struct ttm_mem_reg *mem = &bo->mem;
5415718399fSFrançois Tigeot
5425718399fSFrançois Tigeot if (bo->mem.bus.addr) {
5435718399fSFrançois Tigeot map->bo_kmap_type = ttm_bo_map_premapped;
5445718399fSFrançois Tigeot map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
5455718399fSFrançois Tigeot } else {
5465718399fSFrançois Tigeot map->bo_kmap_type = ttm_bo_map_iomap;
547f146ed7fSFrançois Tigeot if (mem->placement & TTM_PL_FLAG_WC)
548f146ed7fSFrançois Tigeot map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
549f146ed7fSFrançois Tigeot size);
550f146ed7fSFrançois Tigeot else
551f146ed7fSFrançois Tigeot map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
552f146ed7fSFrançois Tigeot size);
5535718399fSFrançois Tigeot }
5545718399fSFrançois Tigeot return (!map->virtual) ? -ENOMEM : 0;
5555718399fSFrançois Tigeot }
5565718399fSFrançois Tigeot
ttm_bo_kmap_ttm(struct ttm_buffer_object * bo,unsigned long start_page,unsigned long num_pages,struct ttm_bo_kmap_obj * map)5575718399fSFrançois Tigeot static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
5585718399fSFrançois Tigeot unsigned long start_page,
5595718399fSFrançois Tigeot unsigned long num_pages,
5605718399fSFrançois Tigeot struct ttm_bo_kmap_obj *map)
5615718399fSFrançois Tigeot {
562*932d855eSSergey Zigachev struct ttm_mem_reg *mem = &bo->mem;
563*932d855eSSergey Zigachev struct ttm_operation_ctx ctx = {
564*932d855eSSergey Zigachev .interruptible = false,
565*932d855eSSergey Zigachev .no_wait_gpu = false
566*932d855eSSergey Zigachev };
5675718399fSFrançois Tigeot struct ttm_tt *ttm = bo->ttm;
568*932d855eSSergey Zigachev pgprot_t prot;
569f146ed7fSFrançois Tigeot int ret;
5705718399fSFrançois Tigeot
571c19c6249SFrançois Tigeot BUG_ON(!ttm);
5725718399fSFrançois Tigeot
573*932d855eSSergey Zigachev ret = ttm_tt_populate(ttm, &ctx);
5745718399fSFrançois Tigeot if (ret)
5755718399fSFrançois Tigeot return ret;
5765718399fSFrançois Tigeot
5775718399fSFrançois Tigeot if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
5785718399fSFrançois Tigeot /*
5795718399fSFrançois Tigeot * We're mapping a single page, and the desired
5805718399fSFrançois Tigeot * page protection is consistent with the bo.
5815718399fSFrançois Tigeot */
5825718399fSFrançois Tigeot
5835718399fSFrançois Tigeot map->bo_kmap_type = ttm_bo_map_kmap;
5845718399fSFrançois Tigeot map->page = ttm->pages[start_page];
585f146ed7fSFrançois Tigeot map->virtual = kmap(map->page);
5865718399fSFrançois Tigeot } else {
5875718399fSFrançois Tigeot /*
5885718399fSFrançois Tigeot * We need to use vmap to get the desired page protection
5895718399fSFrançois Tigeot * or to make the buffer object look contiguous.
5905718399fSFrançois Tigeot */
5911cfef1a5SFrançois Tigeot prot = ttm_io_prot(mem->placement, PAGE_KERNEL);
5925718399fSFrançois Tigeot map->bo_kmap_type = ttm_bo_map_vmap;
593f146ed7fSFrançois Tigeot map->virtual = vmap(ttm->pages + start_page, num_pages,
594f146ed7fSFrançois Tigeot 0, prot);
5955718399fSFrançois Tigeot }
5965718399fSFrançois Tigeot return (!map->virtual) ? -ENOMEM : 0;
5975718399fSFrançois Tigeot }
5985718399fSFrançois Tigeot
ttm_bo_kmap(struct ttm_buffer_object * bo,unsigned long start_page,unsigned long num_pages,struct ttm_bo_kmap_obj * map)5995718399fSFrançois Tigeot int ttm_bo_kmap(struct ttm_buffer_object *bo,
6005718399fSFrançois Tigeot unsigned long start_page, unsigned long num_pages,
6015718399fSFrançois Tigeot struct ttm_bo_kmap_obj *map)
6025718399fSFrançois Tigeot {
6035718399fSFrançois Tigeot struct ttm_mem_type_manager *man =
6045718399fSFrançois Tigeot &bo->bdev->man[bo->mem.mem_type];
6055718399fSFrançois Tigeot unsigned long offset, size;
6065718399fSFrançois Tigeot int ret;
6075718399fSFrançois Tigeot
6085718399fSFrançois Tigeot map->virtual = NULL;
6095718399fSFrançois Tigeot map->bo = bo;
6105718399fSFrançois Tigeot if (num_pages > bo->num_pages)
6115718399fSFrançois Tigeot return -EINVAL;
6125718399fSFrançois Tigeot if (start_page > bo->num_pages)
6135718399fSFrançois Tigeot return -EINVAL;
6145718399fSFrançois Tigeot #if 0
615c6f73aabSFrançois Tigeot if (num_pages > 1 && !capable(CAP_SYS_ADMIN))
6165718399fSFrançois Tigeot return -EPERM;
6175718399fSFrançois Tigeot #endif
6185718399fSFrançois Tigeot (void) ttm_mem_io_lock(man, false);
6195718399fSFrançois Tigeot ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
6205718399fSFrançois Tigeot ttm_mem_io_unlock(man);
6215718399fSFrançois Tigeot if (ret)
6225718399fSFrançois Tigeot return ret;
6235718399fSFrançois Tigeot if (!bo->mem.bus.is_iomem) {
6245718399fSFrançois Tigeot return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
6255718399fSFrançois Tigeot } else {
6265718399fSFrançois Tigeot offset = start_page << PAGE_SHIFT;
6275718399fSFrançois Tigeot size = num_pages << PAGE_SHIFT;
6285718399fSFrançois Tigeot return ttm_bo_ioremap(bo, offset, size, map);
6295718399fSFrançois Tigeot }
6305718399fSFrançois Tigeot }
631c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_bo_kmap);
6325718399fSFrançois Tigeot
ttm_bo_kunmap(struct ttm_bo_kmap_obj * map)6335718399fSFrançois Tigeot void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
6345718399fSFrançois Tigeot {
6355718399fSFrançois Tigeot struct ttm_buffer_object *bo = map->bo;
6365718399fSFrançois Tigeot struct ttm_mem_type_manager *man =
6375718399fSFrançois Tigeot &bo->bdev->man[bo->mem.mem_type];
6385718399fSFrançois Tigeot
6395718399fSFrançois Tigeot if (!map->virtual)
6405718399fSFrançois Tigeot return;
6415718399fSFrançois Tigeot switch (map->bo_kmap_type) {
6425718399fSFrançois Tigeot case ttm_bo_map_iomap:
643f146ed7fSFrançois Tigeot iounmap(map->virtual);
6445718399fSFrançois Tigeot break;
6455718399fSFrançois Tigeot case ttm_bo_map_vmap:
646f146ed7fSFrançois Tigeot vunmap(map->virtual);
6475718399fSFrançois Tigeot break;
6485718399fSFrançois Tigeot case ttm_bo_map_kmap:
649f146ed7fSFrançois Tigeot kunmap(map->page);
6505718399fSFrançois Tigeot break;
6515718399fSFrançois Tigeot case ttm_bo_map_premapped:
6525718399fSFrançois Tigeot break;
6535718399fSFrançois Tigeot default:
654c19c6249SFrançois Tigeot BUG();
6555718399fSFrançois Tigeot }
6565718399fSFrançois Tigeot (void) ttm_mem_io_lock(man, false);
6575718399fSFrançois Tigeot ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
6585718399fSFrançois Tigeot ttm_mem_io_unlock(man);
6595718399fSFrançois Tigeot map->virtual = NULL;
6605718399fSFrançois Tigeot map->page = NULL;
6615718399fSFrançois Tigeot }
662c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_bo_kunmap);
6635718399fSFrançois Tigeot
ttm_bo_move_accel_cleanup(struct ttm_buffer_object * bo,struct dma_fence * fence,bool evict,struct ttm_mem_reg * new_mem)6645718399fSFrançois Tigeot int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
6656559babbSFrançois Tigeot struct dma_fence *fence,
6665718399fSFrançois Tigeot bool evict,
6675718399fSFrançois Tigeot struct ttm_mem_reg *new_mem)
6685718399fSFrançois Tigeot {
6695718399fSFrançois Tigeot struct ttm_bo_device *bdev = bo->bdev;
6705718399fSFrançois Tigeot struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
6715718399fSFrançois Tigeot struct ttm_mem_reg *old_mem = &bo->mem;
6725718399fSFrançois Tigeot int ret;
6735718399fSFrançois Tigeot struct ttm_buffer_object *ghost_obj;
6745718399fSFrançois Tigeot
6751cfef1a5SFrançois Tigeot reservation_object_add_excl_fence(bo->resv, fence);
6765718399fSFrançois Tigeot if (evict) {
677d78d3a22SFrançois Tigeot ret = ttm_bo_wait(bo, false, false);
6785718399fSFrançois Tigeot if (ret)
6795718399fSFrançois Tigeot return ret;
6805718399fSFrançois Tigeot
6811dedbd3bSFrançois Tigeot if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
6825718399fSFrançois Tigeot ttm_tt_destroy(bo->ttm);
6835718399fSFrançois Tigeot bo->ttm = NULL;
6845718399fSFrançois Tigeot }
6855718399fSFrançois Tigeot ttm_bo_free_old_node(bo);
6865718399fSFrançois Tigeot } else {
6875718399fSFrançois Tigeot /**
6885718399fSFrançois Tigeot * This should help pipeline ordinary buffer moves.
6895718399fSFrançois Tigeot *
6905718399fSFrançois Tigeot * Hang old buffer memory on a new buffer object,
6915718399fSFrançois Tigeot * and leave it to be released when the GPU
6925718399fSFrançois Tigeot * operation has completed.
6935718399fSFrançois Tigeot */
6945718399fSFrançois Tigeot
6956559babbSFrançois Tigeot dma_fence_put(bo->moving);
6966559babbSFrançois Tigeot bo->moving = dma_fence_get(fence);
6975718399fSFrançois Tigeot
6986f486c69SFrançois Tigeot ret = ttm_buffer_object_transfer(bo, &ghost_obj);
6995718399fSFrançois Tigeot if (ret)
7005718399fSFrançois Tigeot return ret;
7015718399fSFrançois Tigeot
7021cfef1a5SFrançois Tigeot reservation_object_add_excl_fence(ghost_obj->resv, fence);
7031cfef1a5SFrançois Tigeot
7045718399fSFrançois Tigeot /**
7055718399fSFrançois Tigeot * If we're not moving to fixed memory, the TTM object
7065718399fSFrançois Tigeot * needs to stay alive. Otherwhise hang it on the ghost
7075718399fSFrançois Tigeot * bo to be unbound and destroyed.
7085718399fSFrançois Tigeot */
7095718399fSFrançois Tigeot
7105718399fSFrançois Tigeot if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
7115718399fSFrançois Tigeot ghost_obj->ttm = NULL;
7125718399fSFrançois Tigeot else
7135718399fSFrançois Tigeot bo->ttm = NULL;
7145718399fSFrançois Tigeot
7155718399fSFrançois Tigeot ttm_bo_unreserve(ghost_obj);
716*932d855eSSergey Zigachev ttm_bo_put(ghost_obj);
7175718399fSFrançois Tigeot }
7185718399fSFrançois Tigeot
7195718399fSFrançois Tigeot *old_mem = *new_mem;
7205718399fSFrançois Tigeot new_mem->mm_node = NULL;
7215718399fSFrançois Tigeot
7225718399fSFrançois Tigeot return 0;
7235718399fSFrançois Tigeot }
724c19c6249SFrançois Tigeot EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
7251dedbd3bSFrançois Tigeot
ttm_bo_pipeline_move(struct ttm_buffer_object * bo,struct dma_fence * fence,bool evict,struct ttm_mem_reg * new_mem)7261dedbd3bSFrançois Tigeot int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
7276559babbSFrançois Tigeot struct dma_fence *fence, bool evict,
7281dedbd3bSFrançois Tigeot struct ttm_mem_reg *new_mem)
7291dedbd3bSFrançois Tigeot {
7301dedbd3bSFrançois Tigeot struct ttm_bo_device *bdev = bo->bdev;
7311dedbd3bSFrançois Tigeot struct ttm_mem_reg *old_mem = &bo->mem;
7321dedbd3bSFrançois Tigeot
7331dedbd3bSFrançois Tigeot struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type];
7341dedbd3bSFrançois Tigeot struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type];
7351dedbd3bSFrançois Tigeot
7361dedbd3bSFrançois Tigeot int ret;
7371dedbd3bSFrançois Tigeot
7381dedbd3bSFrançois Tigeot reservation_object_add_excl_fence(bo->resv, fence);
7391dedbd3bSFrançois Tigeot
7401dedbd3bSFrançois Tigeot if (!evict) {
7411dedbd3bSFrançois Tigeot struct ttm_buffer_object *ghost_obj;
7421dedbd3bSFrançois Tigeot
7431dedbd3bSFrançois Tigeot /**
7441dedbd3bSFrançois Tigeot * This should help pipeline ordinary buffer moves.
7451dedbd3bSFrançois Tigeot *
7461dedbd3bSFrançois Tigeot * Hang old buffer memory on a new buffer object,
7471dedbd3bSFrançois Tigeot * and leave it to be released when the GPU
7481dedbd3bSFrançois Tigeot * operation has completed.
7491dedbd3bSFrançois Tigeot */
7501dedbd3bSFrançois Tigeot
7516559babbSFrançois Tigeot dma_fence_put(bo->moving);
7526559babbSFrançois Tigeot bo->moving = dma_fence_get(fence);
7531dedbd3bSFrançois Tigeot
7541dedbd3bSFrançois Tigeot ret = ttm_buffer_object_transfer(bo, &ghost_obj);
7551dedbd3bSFrançois Tigeot if (ret)
7561dedbd3bSFrançois Tigeot return ret;
7571dedbd3bSFrançois Tigeot
7581dedbd3bSFrançois Tigeot reservation_object_add_excl_fence(ghost_obj->resv, fence);
7591dedbd3bSFrançois Tigeot
7601dedbd3bSFrançois Tigeot /**
7611dedbd3bSFrançois Tigeot * If we're not moving to fixed memory, the TTM object
7621dedbd3bSFrançois Tigeot * needs to stay alive. Otherwhise hang it on the ghost
7631dedbd3bSFrançois Tigeot * bo to be unbound and destroyed.
7641dedbd3bSFrançois Tigeot */
7651dedbd3bSFrançois Tigeot
7661dedbd3bSFrançois Tigeot if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED))
7671dedbd3bSFrançois Tigeot ghost_obj->ttm = NULL;
7681dedbd3bSFrançois Tigeot else
7691dedbd3bSFrançois Tigeot bo->ttm = NULL;
7701dedbd3bSFrançois Tigeot
7711dedbd3bSFrançois Tigeot ttm_bo_unreserve(ghost_obj);
772*932d855eSSergey Zigachev ttm_bo_put(ghost_obj);
7731dedbd3bSFrançois Tigeot
7741dedbd3bSFrançois Tigeot } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) {
7751dedbd3bSFrançois Tigeot
7761dedbd3bSFrançois Tigeot /**
7771dedbd3bSFrançois Tigeot * BO doesn't have a TTM we need to bind/unbind. Just remember
7781dedbd3bSFrançois Tigeot * this eviction and free up the allocation
7791dedbd3bSFrançois Tigeot */
7801dedbd3bSFrançois Tigeot
781ec5b6af4SFrançois Tigeot lockmgr(&from->move_lock, LK_EXCLUSIVE);
7826559babbSFrançois Tigeot if (!from->move || dma_fence_is_later(fence, from->move)) {
7836559babbSFrançois Tigeot dma_fence_put(from->move);
7846559babbSFrançois Tigeot from->move = dma_fence_get(fence);
7851dedbd3bSFrançois Tigeot }
786ec5b6af4SFrançois Tigeot lockmgr(&from->move_lock, LK_RELEASE);
7871dedbd3bSFrançois Tigeot
7881dedbd3bSFrançois Tigeot ttm_bo_free_old_node(bo);
7891dedbd3bSFrançois Tigeot
7906559babbSFrançois Tigeot dma_fence_put(bo->moving);
7916559babbSFrançois Tigeot bo->moving = dma_fence_get(fence);
7921dedbd3bSFrançois Tigeot
7931dedbd3bSFrançois Tigeot } else {
7941dedbd3bSFrançois Tigeot /**
7951dedbd3bSFrançois Tigeot * Last resort, wait for the move to be completed.
7961dedbd3bSFrançois Tigeot *
7971dedbd3bSFrançois Tigeot * Should never happen in pratice.
7981dedbd3bSFrançois Tigeot */
7991dedbd3bSFrançois Tigeot
8001dedbd3bSFrançois Tigeot ret = ttm_bo_wait(bo, false, false);
8011dedbd3bSFrançois Tigeot if (ret)
8021dedbd3bSFrançois Tigeot return ret;
8031dedbd3bSFrançois Tigeot
8041dedbd3bSFrançois Tigeot if (to->flags & TTM_MEMTYPE_FLAG_FIXED) {
8051dedbd3bSFrançois Tigeot ttm_tt_destroy(bo->ttm);
8061dedbd3bSFrançois Tigeot bo->ttm = NULL;
8071dedbd3bSFrançois Tigeot }
8081dedbd3bSFrançois Tigeot ttm_bo_free_old_node(bo);
8091dedbd3bSFrançois Tigeot }
8101dedbd3bSFrançois Tigeot
8111dedbd3bSFrançois Tigeot *old_mem = *new_mem;
8121dedbd3bSFrançois Tigeot new_mem->mm_node = NULL;
8131dedbd3bSFrançois Tigeot
8141dedbd3bSFrançois Tigeot return 0;
8151dedbd3bSFrançois Tigeot }
8161dedbd3bSFrançois Tigeot EXPORT_SYMBOL(ttm_bo_pipeline_move);
817*932d855eSSergey Zigachev
ttm_bo_pipeline_gutting(struct ttm_buffer_object * bo)818*932d855eSSergey Zigachev int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
819*932d855eSSergey Zigachev {
820*932d855eSSergey Zigachev struct ttm_buffer_object *ghost;
821*932d855eSSergey Zigachev int ret;
822*932d855eSSergey Zigachev
823*932d855eSSergey Zigachev ret = ttm_buffer_object_transfer(bo, &ghost);
824*932d855eSSergey Zigachev if (ret)
825*932d855eSSergey Zigachev return ret;
826*932d855eSSergey Zigachev
827*932d855eSSergey Zigachev ret = reservation_object_copy_fences(ghost->resv, bo->resv);
828*932d855eSSergey Zigachev /* Last resort, wait for the BO to be idle when we are OOM */
829*932d855eSSergey Zigachev if (ret)
830*932d855eSSergey Zigachev ttm_bo_wait(bo, false, false);
831*932d855eSSergey Zigachev
832*932d855eSSergey Zigachev memset(&bo->mem, 0, sizeof(bo->mem));
833*932d855eSSergey Zigachev bo->mem.mem_type = TTM_PL_SYSTEM;
834*932d855eSSergey Zigachev bo->ttm = NULL;
835*932d855eSSergey Zigachev
836*932d855eSSergey Zigachev ttm_bo_unreserve(ghost);
837*932d855eSSergey Zigachev ttm_bo_put(ghost);
838*932d855eSSergey Zigachev
839*932d855eSSergey Zigachev return 0;
840*932d855eSSergey Zigachev }
841