49 static inline void debug_fence_init(struct i915_sw_fence *fence)  in debug_fence_init()  argument
51 	debug_object_init(fence, &i915_sw_fence_debug_descr);  in debug_fence_init()
54 static inline __maybe_unused void debug_fence_init_onstack(struct i915_sw_fence *fence)  in debug_fence_init_onstack()  argument
56 	debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);  in debug_fence_init_onstack()
59 static inline void debug_fence_activate(struct i915_sw_fence *fence)  in debug_fence_activate()  argument
61 	debug_object_activate(fence, &i915_sw_fence_debug_descr);  in debug_fence_activate()
64 static inline void debug_fence_set_state(struct i915_sw_fence *fence,  in debug_fence_set_state()  argument
67 	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);  in debug_fence_set_state()
70 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)  in debug_fence_deactivate()  argument
72 	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);  in debug_fence_deactivate()
75 static inline void debug_fence_destroy(struct i915_sw_fence *fence)  in debug_fence_destroy()  argument
77 	debug_object_destroy(fence, &i915_sw_fence_debug_descr);  in debug_fence_destroy()
80 static inline __maybe_unused void debug_fence_free(struct i915_sw_fence *fence)  in debug_fence_free()  argument
82 	debug_object_free(fence, &i915_sw_fence_debug_descr);  in debug_fence_free()
86 static inline void debug_fence_assert(struct i915_sw_fence *fence)  in debug_fence_assert()  argument
88 	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);  in debug_fence_assert()
93 static inline void debug_fence_init(struct i915_sw_fence *fence)  in debug_fence_init()  argument
97 static inline __maybe_unused void debug_fence_init_onstack(struct i915_sw_fence *fence)  in debug_fence_init_onstack()  argument
101 static inline void debug_fence_activate(struct i915_sw_fence *fence)  in debug_fence_activate()  argument
105 static inline void debug_fence_set_state(struct i915_sw_fence *fence,  in debug_fence_set_state()  argument
110 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)  in debug_fence_deactivate()  argument
114 static inline void debug_fence_destroy(struct i915_sw_fence *fence)  in debug_fence_destroy()  argument
118 static inline __maybe_unused void debug_fence_free(struct i915_sw_fence *fence)  in debug_fence_free()  argument
122 static inline void debug_fence_assert(struct i915_sw_fence *fence)  in debug_fence_assert()  argument
128 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,  in __i915_sw_fence_notify()  argument
131 	return fence->fn(fence, state);  in __i915_sw_fence_notify()
135 void i915_sw_fence_fini(struct i915_sw_fence *fence)  in i915_sw_fence_fini()  argument
137 	debug_fence_free(fence);  in i915_sw_fence_fini()
141 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,  in __i915_sw_fence_wake_up_all()  argument
144 	wait_queue_head_t *x = &fence->wait;  in __i915_sw_fence_wake_up_all()
148 	debug_fence_deactivate(fence);  in __i915_sw_fence_wake_up_all()
149 	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */  in __i915_sw_fence_wake_up_all()
175 					wake_flags = fence->error;  in __i915_sw_fence_wake_up_all()
188 	debug_fence_assert(fence);  in __i915_sw_fence_wake_up_all()
191 static void __i915_sw_fence_complete(struct i915_sw_fence *fence,  in __i915_sw_fence_complete()  argument
194 	debug_fence_assert(fence);  in __i915_sw_fence_complete()
196 	if (!atomic_dec_and_test(&fence->pending))  in __i915_sw_fence_complete()
199 	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);  in __i915_sw_fence_complete()
201 	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)  in __i915_sw_fence_complete()
204 	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);  in __i915_sw_fence_complete()
206 	__i915_sw_fence_wake_up_all(fence, continuation);  in __i915_sw_fence_complete()
208 	debug_fence_destroy(fence);  in __i915_sw_fence_complete()
209 	__i915_sw_fence_notify(fence, FENCE_FREE);  in __i915_sw_fence_complete()
212 void i915_sw_fence_complete(struct i915_sw_fence *fence)  in i915_sw_fence_complete()  argument
214 	debug_fence_assert(fence);  in i915_sw_fence_complete()
216 	if (WARN_ON(i915_sw_fence_done(fence)))  in i915_sw_fence_complete()
219 	__i915_sw_fence_complete(fence, NULL);  in i915_sw_fence_complete()
222 bool i915_sw_fence_await(struct i915_sw_fence *fence)  in i915_sw_fence_await()  argument
230 	pending = atomic_read(&fence->pending);  in i915_sw_fence_await()
234 	} while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1));  in i915_sw_fence_await()
239 void __i915_sw_fence_init(struct i915_sw_fence *fence,  in __i915_sw_fence_init()  argument
244 	__init_waitqueue_head(&fence->wait, name, key);  in __i915_sw_fence_init()
245 	fence->fn = fn;  in __i915_sw_fence_init()
247 	fence->flags = 0;  in __i915_sw_fence_init()
250 	i915_sw_fence_reinit(fence);  in __i915_sw_fence_init()
253 void i915_sw_fence_reinit(struct i915_sw_fence *fence)  in i915_sw_fence_reinit()  argument
255 	debug_fence_init(fence);  in i915_sw_fence_reinit()
257 	atomic_set(&fence->pending, 1);  in i915_sw_fence_reinit()
258 	fence->error = 0;  in i915_sw_fence_reinit()
260 	I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head));  in i915_sw_fence_reinit()
263 void i915_sw_fence_commit(struct i915_sw_fence *fence)  in i915_sw_fence_commit()  argument
265 	debug_fence_activate(fence);  in i915_sw_fence_commit()
266 	i915_sw_fence_complete(fence);  in i915_sw_fence_commit()
282 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,  in __i915_sw_fence_check_if_after()  argument
287 	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))  in __i915_sw_fence_check_if_after()
290 	if (fence == signaler)  in __i915_sw_fence_check_if_after()
293 	list_for_each_entry(wq, &fence->wait.head, entry) {  in __i915_sw_fence_check_if_after()
304 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)  in __i915_sw_fence_clear_checked_bit()  argument
308 	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))  in __i915_sw_fence_clear_checked_bit()
311 	list_for_each_entry(wq, &fence->wait.head, entry) {  in __i915_sw_fence_clear_checked_bit()
319 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,  in i915_sw_fence_check_if_after()  argument
326 	err = __i915_sw_fence_check_if_after(fence, signaler);  in i915_sw_fence_check_if_after()
327 	__i915_sw_fence_clear_checked_bit(fence);  in i915_sw_fence_check_if_after()
333 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,  in i915_sw_fence_check_if_after()  argument
340 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,  in __i915_sw_fence_await_sw_fence()  argument
347 	debug_fence_assert(fence);  in __i915_sw_fence_await_sw_fence()
351 		i915_sw_fence_set_error_once(fence, signaler->error);  in __i915_sw_fence_await_sw_fence()
358 	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))  in __i915_sw_fence_await_sw_fence()
369 			i915_sw_fence_set_error_once(fence, signaler->error);  in __i915_sw_fence_await_sw_fence()
379 	wq->private = fence;  in __i915_sw_fence_await_sw_fence()
381 	i915_sw_fence_await(fence);  in __i915_sw_fence_await_sw_fence()
396 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,  in i915_sw_fence_await_sw_fence()  argument
400 	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);  in i915_sw_fence_await_sw_fence()
403 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,  in i915_sw_fence_await_sw_fence_gfp()  argument
407 	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);  in i915_sw_fence_await_sw_fence_gfp()
423 	i915_sw_fence_set_error_once(cb->fence, dma->error);  in dma_i915_sw_fence_wake()
424 	i915_sw_fence_complete(cb->fence);  in dma_i915_sw_fence_wake()
431 	struct i915_sw_fence *fence;  in timer_i915_sw_fence_wake()  local
433 	fence = xchg(&cb->base.fence, NULL);  in timer_i915_sw_fence_wake()
434 	if (!fence)  in timer_i915_sw_fence_wake()
441 		  i915_sw_fence_debug_hint(fence));  in timer_i915_sw_fence_wake()
443 	i915_sw_fence_set_error_once(fence, -ETIMEDOUT);  in timer_i915_sw_fence_wake()
444 	i915_sw_fence_complete(fence);  in timer_i915_sw_fence_wake()
452 	struct i915_sw_fence *fence;  in dma_i915_sw_fence_wake_timer()  local
454 	fence = xchg(&cb->base.fence, NULL);  in dma_i915_sw_fence_wake_timer()
455 	if (fence) {  in dma_i915_sw_fence_wake_timer()
456 		i915_sw_fence_set_error_once(fence, dma->error);  in dma_i915_sw_fence_wake_timer()
457 		i915_sw_fence_complete(fence);  in dma_i915_sw_fence_wake_timer()
474 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,  in i915_sw_fence_await_dma_fence()  argument
483 	debug_fence_assert(fence);  in i915_sw_fence_await_dma_fence()
487 		i915_sw_fence_set_error_once(fence, dma->error);  in i915_sw_fence_await_dma_fence()
503 		i915_sw_fence_set_error_once(fence, dma->error);  in i915_sw_fence_await_dma_fence()
507 	cb->fence = fence;  in i915_sw_fence_await_dma_fence()
508 	i915_sw_fence_await(fence);  in i915_sw_fence_await_dma_fence()
547 	i915_sw_fence_set_error_once(cb->fence, dma->error);  in __dma_i915_sw_fence_wake()
548 	i915_sw_fence_complete(cb->fence);  in __dma_i915_sw_fence_wake()
551 int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,  in __i915_sw_fence_await_dma_fence()  argument
557 	debug_fence_assert(fence);  in __i915_sw_fence_await_dma_fence()
560 		i915_sw_fence_set_error_once(fence, dma->error);  in __i915_sw_fence_await_dma_fence()
564 	cb->fence = fence;  in __i915_sw_fence_await_dma_fence()
565 	i915_sw_fence_await(fence);  in __i915_sw_fence_await_dma_fence()
577 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,  in i915_sw_fence_await_reservation()  argument
587 	debug_fence_assert(fence);  in i915_sw_fence_await_reservation()
592 		pending = i915_sw_fence_await_dma_fence(fence, f, timeout,  in i915_sw_fence_await_reservation()