77 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_mem_type_debug()  local
80 	pr_err("    has_type: %d\n", man->has_type);  in ttm_mem_type_debug()
81 	pr_err("    use_type: %d\n", man->use_type);  in ttm_mem_type_debug()
82 	pr_err("    flags: 0x%08X\n", man->flags);  in ttm_mem_type_debug()
83 	pr_err("    gpu_offset: 0x%08lX\n", man->gpu_offset);  in ttm_mem_type_debug()
84 	pr_err("    size: %ju\n", man->size);  in ttm_mem_type_debug()
85 	pr_err("    available_caching: 0x%08X\n", man->available_caching);  in ttm_mem_type_debug()
86 	pr_err("    default_caching: 0x%08X\n", man->default_caching);  in ttm_mem_type_debug()
88 		(*man->func->debug)(man, &p);  in ttm_mem_type_debug()
167 	struct ttm_mem_type_manager *man;  in ttm_bo_add_to_lru()  local
178 		man = &bdev->man[bo->mem.mem_type];  in ttm_bo_add_to_lru()
179 		list_add_tail(&bo->lru, &man->lru[bo->priority]);  in ttm_bo_add_to_lru()
240 	struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
241 	struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];  in ttm_bo_handle_move_mem()
317 		    bdev->man[bo->mem.mem_type].gpu_offset;  in ttm_bo_handle_move_mem()
325 	new_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
587 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_release()  local
590 	ttm_mem_io_lock(man, false);  in ttm_bo_release()
592 	ttm_mem_io_unlock(man);  in ttm_bo_release()
724 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_mem_evict_first()  local
732 		list_for_each_entry(bo, &man->lru[i], lru) {  in ttm_mem_evict_first()
746 		if (&bo->lru != &man->lru[i])  in ttm_mem_evict_first()
784 	struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];  in ttm_bo_mem_put()  local
787 		(*man->func->put_node)(man, mem);  in ttm_bo_mem_put()
795 				 struct ttm_mem_type_manager *man,  in ttm_bo_add_move_fence()  argument
801 	lockmgr(&man->move_lock, LK_EXCLUSIVE);  in ttm_bo_add_move_fence()
802 	fence = dma_fence_get(man->move);  in ttm_bo_add_move_fence()
803 	lockmgr(&man->move_lock, LK_RELEASE);  in ttm_bo_add_move_fence()
830 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_mem_force_space()  local
834 		ret = (*man->func->get_node)(man, bo, place, mem);  in ttm_bo_mem_force_space()
844 	return ttm_bo_add_move_fence(bo, man, mem);  in ttm_bo_mem_force_space()
847 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,  in ttm_bo_select_caching()  argument
860 	else if ((man->default_caching & caching) != 0)  in ttm_bo_select_caching()
861 		result |= man->default_caching;  in ttm_bo_select_caching()
872 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,  in ttm_bo_mt_compatible()  argument
882 	if ((place->flags & man->available_caching) == 0)  in ttm_bo_mt_compatible()
885 	cur_flags |= (place->flags & man->available_caching);  in ttm_bo_mt_compatible()
905 	struct ttm_mem_type_manager *man;  in ttm_bo_mem_space()  local
924 		man = &bdev->man[mem_type];  in ttm_bo_mem_space()
925 		if (!man->has_type || !man->use_type)  in ttm_bo_mem_space()
928 		type_ok = ttm_bo_mt_compatible(man, mem_type, place,  in ttm_bo_mem_space()
935 		cur_flags = ttm_bo_select_caching(man, bo->mem.placement,  in ttm_bo_mem_space()
947 		ret = (*man->func->get_node)(man, bo, place, mem);  in ttm_bo_mem_space()
952 			ret = ttm_bo_add_move_fence(bo, man, mem);  in ttm_bo_mem_space()
954 				(*man->func->put_node)(man, mem);  in ttm_bo_mem_space()
973 		man = &bdev->man[mem_type];  in ttm_bo_mem_space()
974 		if (!man->has_type || !man->use_type)  in ttm_bo_mem_space()
976 		if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))  in ttm_bo_mem_space()
980 		cur_flags = ttm_bo_select_caching(man, bo->mem.placement,  in ttm_bo_mem_space()
1320 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_force_list_clean()  local
1332 		while (!list_empty(&man->lru[i])) {  in ttm_bo_force_list_clean()
1342 	lockmgr(&man->move_lock, LK_EXCLUSIVE);  in ttm_bo_force_list_clean()
1343 	fence = dma_fence_get(man->move);  in ttm_bo_force_list_clean()
1344 	lockmgr(&man->move_lock, LK_RELEASE);  in ttm_bo_force_list_clean()
1358 	struct ttm_mem_type_manager *man;  in ttm_bo_clean_mm()  local
1365 	man = &bdev->man[mem_type];  in ttm_bo_clean_mm()
1367 	if (!man->has_type) {  in ttm_bo_clean_mm()
1373 	man->use_type = false;  in ttm_bo_clean_mm()
1374 	man->has_type = false;  in ttm_bo_clean_mm()
1384 		ret = (*man->func->takedown)(man);  in ttm_bo_clean_mm()
1387 	dma_fence_put(man->move);  in ttm_bo_clean_mm()
1388 	man->move = NULL;  in ttm_bo_clean_mm()
1396 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];  in ttm_bo_evict_mm()  local
1403 	if (!man->has_type) {  in ttm_bo_evict_mm()
1416 	struct ttm_mem_type_manager *man;  in ttm_bo_init_mm()  local
1420 	man = &bdev->man[type];  in ttm_bo_init_mm()
1421 	BUG_ON(man->has_type);  in ttm_bo_init_mm()
1422 	man->io_reserve_fastpath = true;  in ttm_bo_init_mm()
1423 	man->use_io_reserve_lru = false;  in ttm_bo_init_mm()
1424 	lockinit(&man->io_reserve_mutex, "ttmior", 0, 0);  in ttm_bo_init_mm()
1425 	lockinit(&man->move_lock, "ttmml", 0, 0);  in ttm_bo_init_mm()
1426 	INIT_LIST_HEAD(&man->io_reserve_lru);  in ttm_bo_init_mm()
1428 	ret = bdev->driver->init_mem_type(bdev, type, man);  in ttm_bo_init_mm()
1431 	man->bdev = bdev;  in ttm_bo_init_mm()
1434 		ret = (*man->func->init)(man, p_size);  in ttm_bo_init_mm()
1438 	man->has_type = true;  in ttm_bo_init_mm()
1439 	man->use_type = true;  in ttm_bo_init_mm()
1440 	man->size = p_size;  in ttm_bo_init_mm()
1443 		INIT_LIST_HEAD(&man->lru[i]);  in ttm_bo_init_mm()
1444 	man->move = NULL;  in ttm_bo_init_mm()
1507 	struct ttm_mem_type_manager *man;  in ttm_bo_device_release()  local
1511 		man = &bdev->man[i];  in ttm_bo_device_release()
1512 		if (man->has_type) {  in ttm_bo_device_release()
1513 			man->use_type = false;  in ttm_bo_device_release()
1519 			man->has_type = false;  in ttm_bo_device_release()
1536 		if (list_empty(&bdev->man[0].lru[0]))  in ttm_bo_device_release()
1557 	memset(bdev->man, 0, sizeof(bdev->man));  in ttm_bo_device_init()
1594 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];  in ttm_mem_reg_is_pci()  local
1596 	if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {  in ttm_mem_reg_is_pci()
1600 		if (man->flags & TTM_MEMTYPE_FLAG_CMA)  in ttm_mem_reg_is_pci()
1630 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_unmap_virtual()  local
1632 	ttm_mem_io_lock(man, false);  in ttm_bo_unmap_virtual()
1634 	ttm_mem_io_unlock(man);  in ttm_bo_unmap_virtual()