1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28 #include <drm/ttm/ttm_execbuf_util.h> 29 #include <drm/ttm/ttm_bo_driver.h> 30 #include <drm/ttm/ttm_placement.h> 31 #include <linux/export.h> 32 #include <linux/wait.h> 33 34 /* XXX this should go to dma-buf driver, for now just to avoid undef */ 35 DEFINE_WW_CLASS(reservation_ww_class); 36 37 static void ttm_eu_backoff_reservation_locked(struct list_head *list, 38 struct ww_acquire_ctx *ticket) 39 { 40 struct ttm_validate_buffer *entry; 41 42 list_for_each_entry(entry, list, head) { 43 struct ttm_buffer_object *bo = entry->bo; 44 if (!entry->reserved) 45 continue; 46 47 entry->reserved = false; 48 if (entry->removed) { 49 ttm_bo_unreserve_ticket_locked(bo, ticket); 50 entry->removed = false; 51 52 } else { 53 atomic_set(&bo->reserved, 0); 54 wake_up_all(&bo->event_queue); 55 } 56 } 57 } 58 59 static void ttm_eu_del_from_lru_locked(struct list_head *list) 60 { 61 struct ttm_validate_buffer *entry; 62 63 list_for_each_entry(entry, list, head) { 64 struct ttm_buffer_object *bo = entry->bo; 65 if (!entry->reserved) 66 continue; 67 68 if (!entry->removed) { 69 entry->put_count = ttm_bo_del_from_lru(bo); 70 entry->removed = true; 71 } 72 } 73 } 74 75 static void ttm_eu_list_ref_sub(struct list_head *list) 76 { 77 struct ttm_validate_buffer *entry; 78 79 list_for_each_entry(entry, list, head) { 80 struct ttm_buffer_object *bo = entry->bo; 81 82 if (entry->put_count) { 83 ttm_bo_list_ref_sub(bo, entry->put_count, true); 84 entry->put_count = 0; 85 } 86 } 87 } 88 89 void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, 90 struct list_head *list) 91 { 92 struct ttm_validate_buffer *entry; 93 struct ttm_bo_global *glob; 94 95 if (list_empty(list)) 96 return; 97 98 entry = list_first_entry(list, struct ttm_validate_buffer, head); 99 glob = entry->bo->glob; 100 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 101 ttm_eu_backoff_reservation_locked(list, ticket); 102 ww_acquire_fini(ticket); 103 lockmgr(&glob->lru_lock, LK_RELEASE); 104 } 105 EXPORT_SYMBOL(ttm_eu_backoff_reservation); 106 107 /* 108 * Reserve buffers for validation. 109 * 110 * If a buffer in the list is marked for CPU access, we back off and 111 * wait for that buffer to become free for GPU access. 112 * 113 * If a buffer is reserved for another validation, the validator with 114 * the highest validation sequence backs off and waits for that buffer 115 * to become unreserved. This prevents deadlocks when validating multiple 116 * buffers in different orders. 117 */ 118 119 int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, 120 struct list_head *list) 121 { 122 struct ttm_bo_global *glob; 123 struct ttm_validate_buffer *entry; 124 int ret; 125 126 if (list_empty(list)) 127 return 0; 128 129 list_for_each_entry(entry, list, head) { 130 entry->reserved = false; 131 entry->put_count = 0; 132 entry->removed = false; 133 } 134 135 entry = list_first_entry(list, struct ttm_validate_buffer, head); 136 glob = entry->bo->glob; 137 138 ww_acquire_init(ticket, &reservation_ww_class); 139 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 140 141 retry: 142 list_for_each_entry(entry, list, head) { 143 struct ttm_buffer_object *bo = entry->bo; 144 int owned; 145 146 /* already slowpath reserved? */ 147 if (entry->reserved) 148 continue; 149 150 ret = ttm_bo_reserve_nolru(bo, true, true, true, ticket); 151 switch (ret) { 152 case 0: 153 break; 154 case -EBUSY: 155 ttm_eu_del_from_lru_locked(list); 156 owned = lockstatus(&glob->lru_lock, curthread); 157 if (owned == LK_EXCLUSIVE) 158 lockmgr(&glob->lru_lock, LK_RELEASE); 159 ret = ttm_bo_reserve_nolru(bo, true, false, 160 true, ticket); 161 if (owned == LK_EXCLUSIVE) 162 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 163 if (!ret) 164 break; 165 166 if (unlikely(ret != -EAGAIN)) 167 goto err; 168 169 /* fallthrough */ 170 case -EAGAIN: 171 ttm_eu_backoff_reservation_locked(list, ticket); 172 lockmgr(&glob->lru_lock, LK_RELEASE); 173 ttm_eu_list_ref_sub(list); 174 ret = ttm_bo_reserve_slowpath_nolru(bo, true, ticket); 175 if (unlikely(ret != 0)) 176 goto err_fini; 177 178 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 179 entry->reserved = true; 180 if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { 181 ret = -EBUSY; 182 goto err; 183 } 184 goto retry; 185 default: 186 goto err; 187 } 188 189 entry->reserved = true; 190 if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { 191 ret = -EBUSY; 192 goto err; 193 } 194 } 195 196 ww_acquire_done(ticket); 197 ttm_eu_del_from_lru_locked(list); 198 lockmgr(&glob->lru_lock, LK_RELEASE); 199 ttm_eu_list_ref_sub(list); 200 return 0; 201 202 err: 203 ttm_eu_backoff_reservation_locked(list, ticket); 204 lockmgr(&glob->lru_lock, LK_RELEASE); 205 ttm_eu_list_ref_sub(list); 206 err_fini: 207 ww_acquire_done(ticket); 208 ww_acquire_fini(ticket); 209 return ret; 210 } 211 EXPORT_SYMBOL(ttm_eu_reserve_buffers); 212 213 void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, 214 struct list_head *list, void *sync_obj) 215 { 216 struct ttm_validate_buffer *entry; 217 struct ttm_buffer_object *bo; 218 struct ttm_bo_global *glob; 219 struct ttm_bo_device *bdev; 220 struct ttm_bo_driver *driver; 221 222 if (list_empty(list)) 223 return; 224 225 bo = list_first_entry(list, struct ttm_validate_buffer, head)->bo; 226 bdev = bo->bdev; 227 driver = bdev->driver; 228 glob = bo->glob; 229 230 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 231 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 232 233 list_for_each_entry(entry, list, head) { 234 bo = entry->bo; 235 entry->old_sync_obj = bo->sync_obj; 236 bo->sync_obj = driver->sync_obj_ref(sync_obj); 237 ttm_bo_unreserve_ticket_locked(bo, ticket); 238 entry->reserved = false; 239 } 240 lockmgr(&bdev->fence_lock, LK_RELEASE); 241 lockmgr(&glob->lru_lock, LK_RELEASE); 242 ww_acquire_fini(ticket); 243 244 list_for_each_entry(entry, list, head) { 245 if (entry->old_sync_obj) 246 driver->sync_obj_unref(&entry->old_sync_obj); 247 } 248 } 249 EXPORT_SYMBOL(ttm_eu_fence_buffer_objects); 250