1 /* 2 * Copyright (c) 2019 Jonathan Gray <jsg@openbsd.org> 3 * Copyright (c) 2020 François Tigeot <ftigeot@wolfpond.org> 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 * SOFTWARE. 23 */ 24 25 #include <linux/slab.h> 26 #include <linux/dma-fence.h> 27 28 void 29 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, 30 spinlock_t *lock, u64 context, unsigned seqno) 31 { 32 fence->ops = ops; 33 fence->lock = lock; 34 fence->context = context; 35 fence->seqno = seqno; 36 fence->flags = 0; 37 fence->error = 0; 38 kref_init(&fence->refcount); 39 INIT_LIST_HEAD(&fence->cb_list); 40 } 41 42 void 43 dma_fence_release(struct kref *ref) 44 { 45 struct dma_fence *fence = container_of(ref, struct dma_fence, refcount); 46 47 if (fence->ops && fence->ops->release) 48 fence->ops->release(fence); 49 else 50 kfree(fence); 51 } 52 53 long 54 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, long timeout) 55 { 56 if (timeout < 0) 57 return -EINVAL; 58 59 if (fence->ops->wait) 60 return fence->ops->wait(fence, intr, timeout); 61 else 62 return dma_fence_default_wait(fence, intr, timeout); 63 } 64 65 u64 66 dma_fence_context_alloc(unsigned n) 67 { 68 static atomic64_t next_context = { 0 }; 69 70 return atomic64_add_return(n, &next_context) - n; 71 } 72 73 struct default_wait_cb { 74 struct dma_fence_cb base; 75 struct task_struct *task; 76 }; 77 78 static void 79 dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb) 80 { 81 struct default_wait_cb *wait = 82 container_of(cb, struct default_wait_cb, base); 83 84 wake_up_process(wait->task); 85 } 86 87 /* This version a mix from OpenBSD and Linux */ 88 long 89 dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout) 90 { 91 long ret = timeout ? timeout : 1; 92 int err; 93 struct default_wait_cb cb; 94 bool was_set; 95 96 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 97 return ret; 98 99 if (intr && signal_pending(current)) { 100 ret = -ERESTARTSYS; 101 goto out; 102 } 103 104 lockmgr(fence->lock, LK_EXCLUSIVE); 105 106 was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, 107 &fence->flags); 108 109 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 110 goto out; 111 112 if (!was_set && fence->ops->enable_signaling) { 113 if (!fence->ops->enable_signaling(fence)) { 114 dma_fence_signal_locked(fence); 115 goto out; 116 } 117 } 118 119 cb.base.func = dma_fence_default_wait_cb; 120 cb.task = current; 121 list_add(&cb.base.node, &fence->cb_list); 122 123 if (timeout <= 0) 124 timeout = 1; 125 126 while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { 127 if (intr) { 128 __set_current_state(TASK_INTERRUPTIBLE); 129 } else { 130 __set_current_state(TASK_UNINTERRUPTIBLE); 131 } 132 /* wake_up_process() directly uses task_struct pointers as sleep identifiers */ 133 err = lksleep(current, fence->lock, intr ? PCATCH : 0, "dmafence", 134 timeout); 135 if (err == EINTR || err == ERESTART) { 136 ret = -ERESTARTSYS; 137 break; 138 } else if (err == EWOULDBLOCK) { 139 ret = 0; 140 break; 141 } 142 } 143 144 if (!list_empty(&cb.base.node)) 145 list_del(&cb.base.node); 146 __set_current_state(TASK_RUNNING); 147 148 out: 149 lockmgr(fence->lock, LK_RELEASE); 150 151 return ret; 152 } 153 154 /* This version from Linux 4.12 */ 155 #if 0 156 signed long 157 dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout) 158 { 159 struct default_wait_cb cb; 160 unsigned long flags; 161 signed long ret = timeout ? timeout : 1; 162 bool was_set; 163 164 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 165 return ret; 166 167 spin_lock_irqsave(fence->lock, flags); 168 169 if (intr && signal_pending(current)) { 170 ret = -ERESTARTSYS; 171 goto out; 172 } 173 174 was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, 175 &fence->flags); 176 177 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 178 goto out; 179 180 if (!was_set) { 181 trace_dma_fence_enable_signal(fence); 182 183 if (!fence->ops->enable_signaling(fence)) { 184 dma_fence_signal_locked(fence); 185 goto out; 186 } 187 } 188 189 cb.base.func = dma_fence_default_wait_cb; 190 cb.task = current; 191 list_add(&cb.base.node, &fence->cb_list); 192 193 while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) { 194 if (intr) 195 __set_current_state(TASK_INTERRUPTIBLE); 196 else 197 __set_current_state(TASK_UNINTERRUPTIBLE); 198 spin_unlock_irqrestore(fence->lock, flags); 199 200 ret = schedule_timeout(ret); 201 202 spin_lock_irqsave(fence->lock, flags); 203 if (ret > 0 && intr && signal_pending(current)) 204 ret = -ERESTARTSYS; 205 } 206 207 if (!list_empty(&cb.base.node)) 208 list_del(&cb.base.node); 209 __set_current_state(TASK_RUNNING); 210 211 out: 212 spin_unlock_irqrestore(fence->lock, flags); 213 return ret; 214 } 215 #endif 216 217 int 218 dma_fence_signal_locked(struct dma_fence *fence) 219 { 220 struct dma_fence_cb *cur, *tmp; 221 222 if (fence == NULL) 223 return -EINVAL; 224 225 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 226 return -EINVAL; 227 228 list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) { 229 list_del_init(&cur->node); 230 cur->func(fence, cur); 231 } 232 233 return 0; 234 } 235 236 int 237 dma_fence_signal(struct dma_fence *fence) 238 { 239 if (fence == NULL) 240 return -EINVAL; 241 242 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 243 return -EINVAL; 244 245 if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) { 246 struct dma_fence_cb *cur, *tmp; 247 248 lockmgr(fence->lock, LK_EXCLUSIVE); 249 list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) { 250 list_del_init(&cur->node); 251 cur->func(fence, cur); 252 } 253 lockmgr(fence->lock, LK_RELEASE); 254 } 255 256 return 0; 257 } 258 259 void 260 dma_fence_enable_sw_signaling(struct dma_fence *fence) 261 { 262 if (!test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags) && 263 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && 264 fence->ops->enable_signaling) { 265 lockmgr(fence->lock, LK_EXCLUSIVE); 266 if (!fence->ops->enable_signaling(fence)) 267 dma_fence_signal_locked(fence); 268 lockmgr(fence->lock, LK_RELEASE); 269 } 270 } 271 272 int 273 dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb, 274 dma_fence_func_t func) 275 { 276 int ret = 0; 277 bool was_set; 278 279 if (WARN_ON(!fence || !func)) 280 return -EINVAL; 281 282 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { 283 INIT_LIST_HEAD(&cb->node); 284 return -ENOENT; 285 } 286 287 lockmgr(fence->lock, LK_EXCLUSIVE); 288 289 was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags); 290 291 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 292 ret = -ENOENT; 293 else if (!was_set && fence->ops->enable_signaling) { 294 if (!fence->ops->enable_signaling(fence)) { 295 dma_fence_signal_locked(fence); 296 ret = -ENOENT; 297 } 298 } 299 300 if (!ret) { 301 cb->func = func; 302 list_add_tail(&cb->node, &fence->cb_list); 303 } else 304 INIT_LIST_HEAD(&cb->node); 305 lockmgr(fence->lock, LK_RELEASE); 306 307 return ret; 308 } 309 310 bool 311 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb) 312 { 313 bool ret; 314 315 lockmgr(fence->lock, LK_EXCLUSIVE); 316 317 ret = !list_empty(&cb->node); 318 if (ret) 319 list_del_init(&cb->node); 320 321 lockmgr(fence->lock, LK_RELEASE); 322 323 return ret; 324 } 325