59 int drm_sched_entity_init(struct drm_sched_entity *entity,  in drm_sched_entity_init()  argument
65 	if (!(entity && sched_list && (num_sched_list == 0 || sched_list[0])))  in drm_sched_entity_init()
68 	memset(entity, 0, sizeof(struct drm_sched_entity));  in drm_sched_entity_init()
69 	INIT_LIST_HEAD(&entity->list);  in drm_sched_entity_init()
70 	entity->rq = NULL;  in drm_sched_entity_init()
71 	entity->guilty = guilty;  in drm_sched_entity_init()
72 	entity->num_sched_list = num_sched_list;  in drm_sched_entity_init()
73 	entity->priority = priority;  in drm_sched_entity_init()
74 	entity->sched_list = num_sched_list > 1 ? sched_list : NULL;  in drm_sched_entity_init()
75 	RCU_INIT_POINTER(entity->last_scheduled, NULL);  in drm_sched_entity_init()
76 	RB_CLEAR_NODE(&entity->rb_tree_node);  in drm_sched_entity_init()
79 		entity->rq = &sched_list[0]->sched_rq[entity->priority];  in drm_sched_entity_init()
81 	init_completion(&entity->entity_idle);  in drm_sched_entity_init()
84 	complete_all(&entity->entity_idle);  in drm_sched_entity_init()
86 	mtx_init(&entity->rq_lock, IPL_NONE);  in drm_sched_entity_init()
87 	spsc_queue_init(&entity->job_queue);  in drm_sched_entity_init()
89 	atomic_set(&entity->fence_seq, 0);  in drm_sched_entity_init()
90 	entity->fence_context = dma_fence_context_alloc(2);  in drm_sched_entity_init()
108 void drm_sched_entity_modify_sched(struct drm_sched_entity *entity,  in drm_sched_entity_modify_sched()  argument
114 	spin_lock(&entity->rq_lock);  in drm_sched_entity_modify_sched()
115 	entity->sched_list = sched_list;  in drm_sched_entity_modify_sched()
116 	entity->num_sched_list = num_sched_list;  in drm_sched_entity_modify_sched()
117 	spin_unlock(&entity->rq_lock);  in drm_sched_entity_modify_sched()
121 static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity)  in drm_sched_entity_is_idle()  argument
125 	if (list_empty(&entity->list) ||  in drm_sched_entity_is_idle()
126 	    spsc_queue_count(&entity->job_queue) == 0 ||  in drm_sched_entity_is_idle()
127 	    entity->stopped)  in drm_sched_entity_is_idle()
134 bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)  in drm_sched_entity_is_ready()  argument
136 	if (spsc_queue_peek(&entity->job_queue) == NULL)  in drm_sched_entity_is_ready()
139 	if (READ_ONCE(entity->dependency))  in drm_sched_entity_is_ready()
152 int drm_sched_entity_error(struct drm_sched_entity *entity)  in drm_sched_entity_error()  argument
158 	fence = rcu_dereference(entity->last_scheduled);  in drm_sched_entity_error()
217 static void drm_sched_entity_kill(struct drm_sched_entity *entity)  in drm_sched_entity_kill()  argument
222 	if (!entity->rq)  in drm_sched_entity_kill()
225 	spin_lock(&entity->rq_lock);  in drm_sched_entity_kill()
226 	entity->stopped = true;  in drm_sched_entity_kill()
227 	drm_sched_rq_remove_entity(entity->rq, entity);  in drm_sched_entity_kill()
228 	spin_unlock(&entity->rq_lock);  in drm_sched_entity_kill()
231 	wait_for_completion(&entity->entity_idle);  in drm_sched_entity_kill()
234 	prev = rcu_dereference_check(entity->last_scheduled, true);  in drm_sched_entity_kill()
236 	while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) {  in drm_sched_entity_kill()
261 long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout)  argument
271 	if (!entity->rq)
274 	sched = entity->rq->sched;
288 					drm_sched_entity_is_idle(entity),
292 				    drm_sched_entity_is_idle(entity));
297 	last_user = cmpxchg(&entity->last_user, current->group_leader, NULL);
301 	last_user = cmpxchg(&entity->last_user, curpr, NULL);
306 		drm_sched_entity_kill(entity);
323 void drm_sched_entity_fini(struct drm_sched_entity *entity)  argument
330 	drm_sched_entity_kill(entity);
332 	if (entity->dependency) {
333 		dma_fence_remove_callback(entity->dependency, &entity->cb);
334 		dma_fence_put(entity->dependency);
335 		entity->dependency = NULL;
338 	dma_fence_put(rcu_dereference_check(entity->last_scheduled, true));
339 	RCU_INIT_POINTER(entity->last_scheduled, NULL);
350 void drm_sched_entity_destroy(struct drm_sched_entity *entity)  argument
352 	drm_sched_entity_flush(entity, MAX_WAIT_SCHED_ENTITY_Q_EMPTY);
353 	drm_sched_entity_fini(entity);
361 	struct drm_sched_entity *entity =  local
364 	entity->dependency = NULL;
375 	struct drm_sched_entity *entity =  local
379 	drm_sched_wakeup_if_can_queue(entity->rq->sched);
390 void drm_sched_entity_set_priority(struct drm_sched_entity *entity,  argument
393 	spin_lock(&entity->rq_lock);
394 	entity->priority = priority;
395 	spin_unlock(&entity->rq_lock);
403 static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity)  argument
405 	struct drm_gpu_scheduler *sched = entity->rq->sched;
406 	struct dma_fence *fence = entity->dependency;
409 	if (fence->context == entity->fence_context ||
410 	    fence->context == entity->fence_context + 1) {
416 		dma_fence_put(entity->dependency);
429 		dma_fence_put(entity->dependency);
430 		entity->dependency = fence;
431 		if (!dma_fence_add_callback(fence, &entity->cb,
440 	if (!dma_fence_add_callback(entity->dependency, &entity->cb,
444 	dma_fence_put(entity->dependency);
450 			 struct drm_sched_entity *entity)  argument
465 		return job->sched->ops->prepare_job(job, entity);
470 struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity)  argument
474 	sched_job = to_drm_sched_job(spsc_queue_peek(&entity->job_queue));
478 	while ((entity->dependency =
479 			drm_sched_job_dependency(sched_job, entity))) {
480 		trace_drm_sched_job_wait_dep(sched_job, entity->dependency);
482 		if (drm_sched_entity_add_dependency_cb(entity))
487 	if (entity->guilty && atomic_read(entity->guilty))
490 	dma_fence_put(rcu_dereference_check(entity->last_scheduled, true));
491 	rcu_assign_pointer(entity->last_scheduled,
501 	spsc_queue_pop(&entity->job_queue);
510 		next = to_drm_sched_job(spsc_queue_peek(&entity->job_queue));
512 			drm_sched_rq_update_fifo(entity, next->submit_ts);
519 	sched_job->entity = NULL;
524 void drm_sched_entity_select_rq(struct drm_sched_entity *entity)  argument
531 	if (!entity->sched_list)
535 	if (spsc_queue_count(&entity->job_queue))
546 	fence = rcu_dereference_check(entity->last_scheduled, true);
552 	spin_lock(&entity->rq_lock);
553 	sched = drm_sched_pick_best(entity->sched_list, entity->num_sched_list);
554 	rq = sched ? &sched->sched_rq[entity->priority] : NULL;
555 	if (rq != entity->rq) {
556 		drm_sched_rq_remove_entity(entity->rq, entity);
557 		entity->rq = rq;
559 	spin_unlock(&entity->rq_lock);
561 	if (entity->num_sched_list == 1)
562 		entity->sched_list = NULL;
578 	struct drm_sched_entity *entity = sched_job->entity;  local
582 	trace_drm_sched_job(sched_job, entity);
583 	atomic_inc(entity->rq->sched->score);
585 	WRITE_ONCE(entity->last_user, current->group_leader);
587 	WRITE_ONCE(entity->last_user, curproc->p_p);
596 	first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
601 		spin_lock(&entity->rq_lock);
602 		if (entity->stopped) {
603 			spin_unlock(&entity->rq_lock);
609 		drm_sched_rq_add_entity(entity->rq, entity);
610 		spin_unlock(&entity->rq_lock);
613 			drm_sched_rq_update_fifo(entity, submit_ts);
615 		drm_sched_wakeup_if_can_queue(entity->rq->sched);