47 unsigned int intel_engine_wakeup(struct intel_engine_cs *engine)  in intel_engine_wakeup()  argument
49 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_wakeup()
65 static noinline void missed_breadcrumb(struct intel_engine_cs *engine)  in missed_breadcrumb()  argument
68 			 engine->name, __builtin_return_address(0),  in missed_breadcrumb()
70 					&engine->irq_posted)),  in missed_breadcrumb()
71 			 intel_engine_get_seqno(engine),  in missed_breadcrumb()
72 			 intel_engine_last_submit(engine));  in missed_breadcrumb()
74 	set_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);  in missed_breadcrumb()
79 	struct intel_engine_cs *engine = from_timer(engine, t,  in intel_breadcrumbs_hangcheck()  local
81 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_breadcrumbs_hangcheck()
86 	if (b->hangcheck_interrupts != atomic_read(&engine->irq_count)) {  in intel_breadcrumbs_hangcheck()
87 		b->hangcheck_interrupts = atomic_read(&engine->irq_count);  in intel_breadcrumbs_hangcheck()
104 	if (intel_engine_wakeup(engine) & ENGINE_WAKEUP_ASLEEP) {  in intel_breadcrumbs_hangcheck()
105 		missed_breadcrumb(engine);  in intel_breadcrumbs_hangcheck()
106 		mod_timer(&engine->breadcrumbs.fake_irq, jiffies + 1);  in intel_breadcrumbs_hangcheck()
114 	struct intel_engine_cs *engine = from_timer(engine, t,  in intel_breadcrumbs_fake_irq()  local
116 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_breadcrumbs_fake_irq()
127 		__intel_engine_disarm_breadcrumbs(engine);  in intel_breadcrumbs_fake_irq()
143 	i915_queue_hangcheck(engine->i915);  in intel_breadcrumbs_fake_irq()
146 static void irq_enable(struct intel_engine_cs *engine)  in irq_enable()  argument
152 	set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);  in irq_enable()
155 	lockmgr(&engine->i915->irq_lock, LK_EXCLUSIVE);  in irq_enable()
156 	engine->irq_enable(engine);  in irq_enable()
157 	lockmgr(&engine->i915->irq_lock, LK_RELEASE);  in irq_enable()
160 static void irq_disable(struct intel_engine_cs *engine)  in irq_disable()  argument
163 	lockmgr(&engine->i915->irq_lock, LK_EXCLUSIVE);  in irq_disable()
164 	engine->irq_disable(engine);  in irq_disable()
165 	lockmgr(&engine->i915->irq_lock, LK_RELEASE);  in irq_disable()
170 void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)  in __intel_engine_disarm_breadcrumbs()  argument
172 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in __intel_engine_disarm_breadcrumbs()
178 		irq_disable(engine);  in __intel_engine_disarm_breadcrumbs()
185 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)  in intel_engine_disarm_breadcrumbs()  argument
187 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_disarm_breadcrumbs()
202 	__intel_engine_disarm_breadcrumbs(engine);  in intel_engine_disarm_breadcrumbs()
208 			missed_breadcrumb(engine);  in intel_engine_disarm_breadcrumbs()
225 	const struct intel_engine_cs *engine =  in use_fake_irq()  local
228 	if (!test_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings))  in use_fake_irq()
237 	return atomic_read(&engine->irq_count) == b->hangcheck_interrupts;  in use_fake_irq()
251 	struct intel_engine_cs *engine =  in __intel_breadcrumbs_enable_irq()  local
253 	struct drm_i915_private *i915 = engine->i915;  in __intel_breadcrumbs_enable_irq()
287 		if (!test_bit(engine->id, &i915->gpu_error.test_irq_rings))  in __intel_breadcrumbs_enable_irq()
288 			irq_enable(engine);  in __intel_breadcrumbs_enable_irq()
321 static inline void __intel_breadcrumbs_next(struct intel_engine_cs *engine,  in __intel_breadcrumbs_next()  argument
324 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in __intel_breadcrumbs_next()
340 static bool __intel_engine_add_wait(struct intel_engine_cs *engine,  in __intel_engine_add_wait()  argument
343 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in __intel_engine_add_wait()
364 	seqno = intel_engine_get_seqno(engine);  in __intel_engine_add_wait()
429 			__intel_breadcrumbs_next(engine, next);  in __intel_engine_add_wait()
446 bool intel_engine_add_wait(struct intel_engine_cs *engine,  in intel_engine_add_wait()  argument
449 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_add_wait()
453 	armed = __intel_engine_add_wait(engine, wait);  in intel_engine_add_wait()
459 	return i915_seqno_passed(intel_engine_get_seqno(engine),  in intel_engine_add_wait()
477 static void __intel_engine_remove_wait(struct intel_engine_cs *engine,  in __intel_engine_remove_wait()  argument
480 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in __intel_engine_remove_wait()
511 			u32 seqno = intel_engine_get_seqno(engine);  in __intel_engine_remove_wait()
523 		__intel_breadcrumbs_next(engine, next);  in __intel_engine_remove_wait()
538 void intel_engine_remove_wait(struct intel_engine_cs *engine,  in intel_engine_remove_wait()  argument
541 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_remove_wait()
553 	__intel_engine_remove_wait(engine, wait);  in intel_engine_remove_wait()
583 	struct intel_engine_cs *engine = arg;  in intel_breadcrumbs_signaler()  local
584 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_breadcrumbs_signaler()
622 			__intel_engine_remove_wait(engine,  in intel_breadcrumbs_signaler()
675 	struct intel_engine_cs *engine = request->engine;  in intel_engine_enable_signaling()  local
676 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_enable_signaling()
709 	wakeup &= __intel_engine_add_wait(engine, &request->signaling.wait);  in intel_engine_enable_signaling()
737 		__intel_engine_remove_wait(engine, &request->signaling.wait);  in intel_engine_enable_signaling()
750 	struct intel_engine_cs *engine = request->engine;  in intel_engine_cancel_signaling()  local
751 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_cancel_signaling()
771 	__intel_engine_remove_wait(engine, &request->signaling.wait);  in intel_engine_cancel_signaling()
778 int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)  in intel_engine_init_breadcrumbs()  argument
780 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_init_breadcrumbs()
795 	tsk = kthread_run(intel_breadcrumbs_signaler, engine,  in intel_engine_init_breadcrumbs()
796 			  "i915/signal:%d", engine->id);  in intel_engine_init_breadcrumbs()
805 static void cancel_fake_irq(struct intel_engine_cs *engine)  in cancel_fake_irq()  argument
807 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in cancel_fake_irq()
811 	clear_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);  in cancel_fake_irq()
814 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)  in intel_engine_reset_breadcrumbs()  argument
816 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_reset_breadcrumbs()
818 	cancel_fake_irq(engine);  in intel_engine_reset_breadcrumbs()
822 		irq_enable(engine);  in intel_engine_reset_breadcrumbs()
824 		irq_disable(engine);  in intel_engine_reset_breadcrumbs()
833 	clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);  in intel_engine_reset_breadcrumbs()
841 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)  in intel_engine_fini_breadcrumbs()  argument
843 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_engine_fini_breadcrumbs()
854 	cancel_fake_irq(engine);  in intel_engine_fini_breadcrumbs()
857 bool intel_breadcrumbs_busy(struct intel_engine_cs *engine)  in intel_breadcrumbs_busy()  argument
859 	struct intel_breadcrumbs *b = &engine->breadcrumbs;  in intel_breadcrumbs_busy()