xref: /dragonfly/sys/dev/drm/i915/i915_sw_fence.c (revision 2b57e6df) 
1 /*
2  * (C) Copyright 2016 Intel Corporation
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; version 2
7  * of the License.
8  */
9 
10 #include <linux/slab.h>
11 #include <linux/dma-fence.h>
12 #include <linux/reservation.h>
13 
14 #include "i915_sw_fence.h"
15 
16 #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */
17 
18 static DEFINE_SPINLOCK(i915_sw_fence_lock);
19 
20 enum {
21 	DEBUG_FENCE_IDLE = 0,
22 	DEBUG_FENCE_NOTIFY,
23 };
24 
25 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
26 
27 static void *i915_sw_fence_debug_hint(void *addr)
28 {
29 	return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
30 }
31 
32 static struct debug_obj_descr i915_sw_fence_debug_descr = {
33 	.name = "i915_sw_fence",
34 	.debug_hint = i915_sw_fence_debug_hint,
35 };
36 
37 static inline void debug_fence_init(struct i915_sw_fence *fence)
38 {
39 	debug_object_init(fence, &i915_sw_fence_debug_descr);
40 }
41 
42 static inline void debug_fence_activate(struct i915_sw_fence *fence)
43 {
44 	debug_object_activate(fence, &i915_sw_fence_debug_descr);
45 }
46 
47 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
48 					 int old, int new)
49 {
50 	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
51 }
52 
53 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
54 {
55 	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
56 }
57 
58 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
59 {
60 	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
61 }
62 
63 static inline void debug_fence_free(struct i915_sw_fence *fence)
64 {
65 	debug_object_free(fence, &i915_sw_fence_debug_descr);
66 	smp_wmb(); /* flush the change in state before reallocation */
67 }
68 
69 static inline void debug_fence_assert(struct i915_sw_fence *fence)
70 {
71 	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
72 }
73 
74 #else
75 
76 static inline void debug_fence_init(struct i915_sw_fence *fence)
77 {
78 }
79 
80 static inline void debug_fence_activate(struct i915_sw_fence *fence)
81 {
82 }
83 
84 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
85 					 int old, int new)
86 {
87 }
88 
89 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
90 {
91 }
92 
93 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
94 {
95 }
96 
97 static inline void debug_fence_free(struct i915_sw_fence *fence)
98 {
99 }
100 
101 static inline void debug_fence_assert(struct i915_sw_fence *fence)
102 {
103 }
104 
105 #endif
106 
107 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
108 				  enum i915_sw_fence_notify state)
109 {
110 	i915_sw_fence_notify_t fn;
111 
112 	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
113 	return fn(fence, state);
114 }
115 
116 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
117 void i915_sw_fence_fini(struct i915_sw_fence *fence)
118 {
119 	debug_fence_free(fence);
120 }
121 #endif
122 
123 static void i915_sw_fence_release(struct kref *kref)
124 {
125 	struct i915_sw_fence *fence = container_of(kref, typeof(*fence), kref);
126 
127 	WARN_ON(atomic_read(&fence->pending) > 0);
128 	debug_fence_destroy(fence);
129 
130 	if (fence->flags & I915_SW_FENCE_MASK) {
131 		__i915_sw_fence_notify(fence, FENCE_FREE);
132 	} else {
133 		i915_sw_fence_fini(fence);
134 		kfree(fence);
135 	}
136 }
137 
138 static void i915_sw_fence_put(struct i915_sw_fence *fence)
139 {
140 	debug_fence_assert(fence);
141 	kref_put(&fence->kref, i915_sw_fence_release);
142 }
143 
144 static struct i915_sw_fence *i915_sw_fence_get(struct i915_sw_fence *fence)
145 {
146 	debug_fence_assert(fence);
147 	kref_get(&fence->kref);
148 	return fence;
149 }
150 
151 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
152 					struct list_head *continuation)
153 {
154 	wait_queue_head_t *x = &fence->wait;
155 	wait_queue_t *pos, *next;
156 	unsigned long flags;
157 
158 	debug_fence_deactivate(fence);
159 	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
160 
161 	/*
162 	 * To prevent unbounded recursion as we traverse the graph of
163 	 * i915_sw_fences, we move the task_list from this, the next ready
164 	 * fence, to the tail of the original fence's task_list
165 	 * (and so added to the list to be woken).
166 	 */
167 
168 	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
169 	if (continuation) {
170 		list_for_each_entry_safe(pos, next, &x->task_list, task_list) {
171 			if (pos->func == autoremove_wake_function)
172 				pos->func(pos, TASK_NORMAL, 0, continuation);
173 			else
174 				list_move_tail(&pos->task_list, continuation);
175 		}
176 	} else {
177 		LINUX_LIST_HEAD(extra);
178 
179 		do {
180 			list_for_each_entry_safe(pos, next,
181 						 &x->task_list, task_list)
182 				pos->func(pos, TASK_NORMAL, 0, &extra);
183 
184 			if (list_empty(&extra))
185 				break;
186 
187 			list_splice_tail_init(&extra, &x->task_list);
188 		} while (1);
189 	}
190 	spin_unlock_irqrestore(&x->lock, flags);
191 
192 	debug_fence_assert(fence);
193 }
194 
195 static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
196 				     struct list_head *continuation)
197 {
198 	debug_fence_assert(fence);
199 
200 	if (!atomic_dec_and_test(&fence->pending))
201 		return;
202 
203 	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
204 
205 	if (fence->flags & I915_SW_FENCE_MASK &&
206 	    __i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
207 		return;
208 
209 	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
210 
211 	__i915_sw_fence_wake_up_all(fence, continuation);
212 }
213 
214 static void i915_sw_fence_complete(struct i915_sw_fence *fence)
215 {
216 	debug_fence_assert(fence);
217 
218 	if (WARN_ON(i915_sw_fence_done(fence)))
219 		return;
220 
221 	__i915_sw_fence_complete(fence, NULL);
222 }
223 
224 static void i915_sw_fence_await(struct i915_sw_fence *fence)
225 {
226 	debug_fence_assert(fence);
227 	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
228 }
229 
230 void __i915_sw_fence_init(struct i915_sw_fence *fence,
231 			  i915_sw_fence_notify_t fn,
232 			  const char *name,
233 			  struct lock_class_key *key)
234 {
235 	BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK);
236 
237 	debug_fence_init(fence);
238 
239 	__init_waitqueue_head(&fence->wait, name, key);
240 	kref_init(&fence->kref);
241 	atomic_set(&fence->pending, 1);
242 	fence->flags = (unsigned long)fn;
243 }
244 
245 static void __i915_sw_fence_commit(struct i915_sw_fence *fence)
246 {
247 	i915_sw_fence_complete(fence);
248 	i915_sw_fence_put(fence);
249 }
250 
251 void i915_sw_fence_commit(struct i915_sw_fence *fence)
252 {
253 	debug_fence_activate(fence);
254 	__i915_sw_fence_commit(fence);
255 }
256 
257 static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void *key)
258 {
259 	list_del(&wq->task_list);
260 	__i915_sw_fence_complete(wq->private, key);
261 	i915_sw_fence_put(wq->private);
262 	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
263 		kfree(wq);
264 	return 0;
265 }
266 
267 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
268 				    const struct i915_sw_fence * const signaler)
269 {
270 	wait_queue_t *wq;
271 
272 	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
273 		return false;
274 
275 	if (fence == signaler)
276 		return true;
277 
278 	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
279 		if (wq->func != i915_sw_fence_wake)
280 			continue;
281 
282 		if (__i915_sw_fence_check_if_after(wq->private, signaler))
283 			return true;
284 	}
285 
286 	return false;
287 }
288 
289 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
290 {
291 	wait_queue_t *wq;
292 
293 	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
294 		return;
295 
296 	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
297 		if (wq->func != i915_sw_fence_wake)
298 			continue;
299 
300 		__i915_sw_fence_clear_checked_bit(wq->private);
301 	}
302 }
303 
304 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
305 				  const struct i915_sw_fence * const signaler)
306 {
307 	unsigned long flags;
308 	bool err;
309 
310 	if (!IS_ENABLED(CONFIG_I915_SW_FENCE_CHECK_DAG))
311 		return false;
312 
313 	spin_lock_irqsave(&i915_sw_fence_lock, flags);
314 	err = __i915_sw_fence_check_if_after(fence, signaler);
315 	__i915_sw_fence_clear_checked_bit(fence);
316 	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
317 
318 	return err;
319 }
320 
321 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
322 					  struct i915_sw_fence *signaler,
323 					  wait_queue_t *wq, gfp_t gfp)
324 {
325 	unsigned long flags;
326 	int pending;
327 
328 	debug_fence_assert(fence);
329 
330 	if (i915_sw_fence_done(signaler))
331 		return 0;
332 
333 	debug_fence_assert(signaler);
334 
335 	/* The dependency graph must be acyclic. */
336 	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
337 		return -EINVAL;
338 
339 	pending = 0;
340 	if (!wq) {
341 		wq = kmalloc(sizeof(*wq), M_DRM, gfp);
342 		if (!wq) {
343 			if (!gfpflags_allow_blocking(gfp))
344 				return -ENOMEM;
345 
346 			i915_sw_fence_wait(signaler);
347 			return 0;
348 		}
349 
350 		pending |= I915_SW_FENCE_FLAG_ALLOC;
351 	}
352 
353 	INIT_LIST_HEAD(&wq->task_list);
354 	wq->flags = pending;
355 	wq->func = i915_sw_fence_wake;
356 	wq->private = i915_sw_fence_get(fence);
357 
358 	i915_sw_fence_await(fence);
359 
360 	spin_lock_irqsave(&signaler->wait.lock, flags);
361 	if (likely(!i915_sw_fence_done(signaler))) {
362 		__add_wait_queue_tail(&signaler->wait, wq);
363 		pending = 1;
364 	} else {
365 		i915_sw_fence_wake(wq, 0, 0, NULL);
366 		pending = 0;
367 	}
368 	spin_unlock_irqrestore(&signaler->wait.lock, flags);
369 
370 	return pending;
371 }
372 
373 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
374 				 struct i915_sw_fence *signaler,
375 				 wait_queue_t *wq)
376 {
377 	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
378 }
379 
380 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
381 				     struct i915_sw_fence *signaler,
382 				     gfp_t gfp)
383 {
384 	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
385 }
386 
387 struct i915_sw_dma_fence_cb {
388 	struct dma_fence_cb base;
389 	struct i915_sw_fence *fence;
390 	struct dma_fence *dma;
391 	struct timer_list timer;
392 };
393 
394 static void timer_i915_sw_fence_wake(unsigned long data)
395 {
396 	struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data;
397 
398 	pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
399 		cb->dma->ops->get_driver_name(cb->dma),
400 		cb->dma->ops->get_timeline_name(cb->dma),
401 		cb->dma->seqno);
402 	dma_fence_put(cb->dma);
403 	cb->dma = NULL;
404 
405 	__i915_sw_fence_commit(cb->fence);
406 	cb->timer.function = NULL;
407 }
408 
409 static void dma_i915_sw_fence_wake(struct dma_fence *dma,
410 				   struct dma_fence_cb *data)
411 {
412 	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
413 
414 	del_timer_sync(&cb->timer);
415 	if (cb->timer.function)
416 		__i915_sw_fence_commit(cb->fence);
417 	dma_fence_put(cb->dma);
418 
419 	kfree(cb);
420 }
421 
422 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
423 				  struct dma_fence *dma,
424 				  unsigned long timeout,
425 				  gfp_t gfp)
426 {
427 	struct i915_sw_dma_fence_cb *cb;
428 	int ret;
429 
430 	debug_fence_assert(fence);
431 
432 	if (dma_fence_is_signaled(dma))
433 		return 0;
434 
435 	cb = kmalloc(sizeof(*cb), M_DRM, gfp);
436 	if (!cb) {
437 		if (!gfpflags_allow_blocking(gfp))
438 			return -ENOMEM;
439 
440 		return dma_fence_wait(dma, false);
441 	}
442 
443 	cb->fence = i915_sw_fence_get(fence);
444 	i915_sw_fence_await(fence);
445 
446 	cb->dma = NULL;
447 	__setup_timer(&cb->timer,
448 		      timer_i915_sw_fence_wake, (unsigned long)cb,
449 		      TIMER_IRQSAFE);
450 	if (timeout) {
451 		cb->dma = dma_fence_get(dma);
452 		mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
453 	}
454 
455 	ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
456 	if (ret == 0) {
457 		ret = 1;
458 	} else {
459 		dma_i915_sw_fence_wake(dma, &cb->base);
460 		if (ret == -ENOENT) /* fence already signaled */
461 			ret = 0;
462 	}
463 
464 	return ret;
465 }
466 
467 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
468 				    struct reservation_object *resv,
469 				    const struct dma_fence_ops *exclude,
470 				    bool write,
471 				    unsigned long timeout,
472 				    gfp_t gfp)
473 {
474 	struct dma_fence *excl;
475 	int ret = 0, pending;
476 
477 	debug_fence_assert(fence);
478 
479 	if (write) {
480 		struct dma_fence **shared;
481 		unsigned int count, i;
482 
483 		ret = reservation_object_get_fences_rcu(resv,
484 							&excl, &count, &shared);
485 		if (ret)
486 			return ret;
487 
488 		for (i = 0; i < count; i++) {
489 			if (shared[i]->ops == exclude)
490 				continue;
491 
492 			pending = i915_sw_fence_await_dma_fence(fence,
493 								shared[i],
494 								timeout,
495 								gfp);
496 			if (pending < 0) {
497 				ret = pending;
498 				break;
499 			}
500 
501 			ret |= pending;
502 		}
503 
504 		for (i = 0; i < count; i++)
505 			dma_fence_put(shared[i]);
506 		kfree(shared);
507 	} else {
508 		excl = reservation_object_get_excl_rcu(resv);
509 	}
510 
511 	if (ret >= 0 && excl && excl->ops != exclude) {
512 		pending = i915_sw_fence_await_dma_fence(fence,
513 							excl,
514 							timeout,
515 							gfp);
516 		if (pending < 0)
517 			ret = pending;
518 		else
519 			ret |= pending;
520 	}
521 
522 	dma_fence_put(excl);
523 
524 	return ret;
525 }
526