1b843c749SSergey Zigachev /*
2b843c749SSergey Zigachev * Copyright 2015 Advanced Micro Devices, Inc.
3b843c749SSergey Zigachev *
4b843c749SSergey Zigachev * Permission is hereby granted, free of charge, to any person obtaining a
5b843c749SSergey Zigachev * copy of this software and associated documentation files (the "Software"),
6b843c749SSergey Zigachev * to deal in the Software without restriction, including without limitation
7b843c749SSergey Zigachev * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8b843c749SSergey Zigachev * and/or sell copies of the Software, and to permit persons to whom the
9b843c749SSergey Zigachev * Software is furnished to do so, subject to the following conditions:
10b843c749SSergey Zigachev *
11b843c749SSergey Zigachev * The above copyright notice and this permission notice shall be included in
12b843c749SSergey Zigachev * all copies or substantial portions of the Software.
13b843c749SSergey Zigachev *
14b843c749SSergey Zigachev * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15b843c749SSergey Zigachev * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16b843c749SSergey Zigachev * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17b843c749SSergey Zigachev * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18b843c749SSergey Zigachev * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19b843c749SSergey Zigachev * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20b843c749SSergey Zigachev * OTHER DEALINGS IN THE SOFTWARE.
21b843c749SSergey Zigachev *
22b843c749SSergey Zigachev */
23b843c749SSergey Zigachev
24b843c749SSergey Zigachev #ifndef _DRM_GPU_SCHEDULER_H_
25b843c749SSergey Zigachev #define _DRM_GPU_SCHEDULER_H_
26b843c749SSergey Zigachev
27b843c749SSergey Zigachev #include <drm/spsc_queue.h>
28b843c749SSergey Zigachev #include <linux/dma-fence.h>
29b843c749SSergey Zigachev
30b843c749SSergey Zigachev #define MAX_WAIT_SCHED_ENTITY_Q_EMPTY msecs_to_jiffies(1000)
31b843c749SSergey Zigachev
32b843c749SSergey Zigachev struct drm_gpu_scheduler;
33b843c749SSergey Zigachev struct drm_sched_rq;
34b843c749SSergey Zigachev
35b843c749SSergey Zigachev enum drm_sched_priority {
36b843c749SSergey Zigachev DRM_SCHED_PRIORITY_MIN,
37b843c749SSergey Zigachev DRM_SCHED_PRIORITY_LOW = DRM_SCHED_PRIORITY_MIN,
38b843c749SSergey Zigachev DRM_SCHED_PRIORITY_NORMAL,
39b843c749SSergey Zigachev DRM_SCHED_PRIORITY_HIGH_SW,
40b843c749SSergey Zigachev DRM_SCHED_PRIORITY_HIGH_HW,
41b843c749SSergey Zigachev DRM_SCHED_PRIORITY_KERNEL,
42b843c749SSergey Zigachev DRM_SCHED_PRIORITY_MAX,
43b843c749SSergey Zigachev DRM_SCHED_PRIORITY_INVALID = -1,
44b843c749SSergey Zigachev DRM_SCHED_PRIORITY_UNSET = -2
45b843c749SSergey Zigachev };
46b843c749SSergey Zigachev
47b843c749SSergey Zigachev /**
48b843c749SSergey Zigachev * struct drm_sched_entity - A wrapper around a job queue (typically
49b843c749SSergey Zigachev * attached to the DRM file_priv).
50b843c749SSergey Zigachev *
51b843c749SSergey Zigachev * @list: used to append this struct to the list of entities in the
52b843c749SSergey Zigachev * runqueue.
53b843c749SSergey Zigachev * @rq: runqueue to which this entity belongs.
54b843c749SSergey Zigachev * @rq_lock: lock to modify the runqueue to which this entity belongs.
55b843c749SSergey Zigachev * @job_queue: the list of jobs of this entity.
56b843c749SSergey Zigachev * @fence_seq: a linearly increasing seqno incremented with each
57b843c749SSergey Zigachev * new &drm_sched_fence which is part of the entity.
58b843c749SSergey Zigachev * @fence_context: a unique context for all the fences which belong
59b843c749SSergey Zigachev * to this entity.
60b843c749SSergey Zigachev * The &drm_sched_fence.scheduled uses the
61b843c749SSergey Zigachev * fence_context but &drm_sched_fence.finished uses
62b843c749SSergey Zigachev * fence_context + 1.
63b843c749SSergey Zigachev * @dependency: the dependency fence of the job which is on the top
64b843c749SSergey Zigachev * of the job queue.
65b843c749SSergey Zigachev * @cb: callback for the dependency fence above.
66b843c749SSergey Zigachev * @guilty: points to ctx's guilty.
67b843c749SSergey Zigachev * @fini_status: contains the exit status in case the process was signalled.
68b843c749SSergey Zigachev * @last_scheduled: points to the finished fence of the last scheduled job.
69b843c749SSergey Zigachev * @last_user: last group leader pushing a job into the entity.
70b843c749SSergey Zigachev *
71b843c749SSergey Zigachev * Entities will emit jobs in order to their corresponding hardware
72b843c749SSergey Zigachev * ring, and the scheduler will alternate between entities based on
73b843c749SSergey Zigachev * scheduling policy.
74b843c749SSergey Zigachev */
75b843c749SSergey Zigachev struct drm_sched_entity {
76b843c749SSergey Zigachev struct list_head list;
77b843c749SSergey Zigachev struct drm_sched_rq *rq;
78*78973132SSergey Zigachev struct spinlock rq_lock;
79b843c749SSergey Zigachev
80b843c749SSergey Zigachev struct spsc_queue job_queue;
81b843c749SSergey Zigachev
82b843c749SSergey Zigachev atomic_t fence_seq;
83b843c749SSergey Zigachev uint64_t fence_context;
84b843c749SSergey Zigachev
85b843c749SSergey Zigachev struct dma_fence *dependency;
86b843c749SSergey Zigachev struct dma_fence_cb cb;
87b843c749SSergey Zigachev atomic_t *guilty;
88b843c749SSergey Zigachev struct dma_fence *last_scheduled;
89b843c749SSergey Zigachev struct task_struct *last_user;
90b843c749SSergey Zigachev };
91b843c749SSergey Zigachev
92b843c749SSergey Zigachev /**
93b843c749SSergey Zigachev * struct drm_sched_rq - queue of entities to be scheduled.
94b843c749SSergey Zigachev *
95b843c749SSergey Zigachev * @lock: to modify the entities list.
96b843c749SSergey Zigachev * @sched: the scheduler to which this rq belongs to.
97b843c749SSergey Zigachev * @entities: list of the entities to be scheduled.
98b843c749SSergey Zigachev * @current_entity: the entity which is to be scheduled.
99b843c749SSergey Zigachev *
100b843c749SSergey Zigachev * Run queue is a set of entities scheduling command submissions for
101b843c749SSergey Zigachev * one specific ring. It implements the scheduling policy that selects
102b843c749SSergey Zigachev * the next entity to emit commands from.
103b843c749SSergey Zigachev */
104b843c749SSergey Zigachev struct drm_sched_rq {
105*78973132SSergey Zigachev struct spinlock lock;
106b843c749SSergey Zigachev struct drm_gpu_scheduler *sched;
107b843c749SSergey Zigachev struct list_head entities;
108b843c749SSergey Zigachev struct drm_sched_entity *current_entity;
109b843c749SSergey Zigachev };
110b843c749SSergey Zigachev
111b843c749SSergey Zigachev /**
112b843c749SSergey Zigachev * struct drm_sched_fence - fences corresponding to the scheduling of a job.
113b843c749SSergey Zigachev */
114b843c749SSergey Zigachev struct drm_sched_fence {
115b843c749SSergey Zigachev /**
116b843c749SSergey Zigachev * @scheduled: this fence is what will be signaled by the scheduler
117b843c749SSergey Zigachev * when the job is scheduled.
118b843c749SSergey Zigachev */
119b843c749SSergey Zigachev struct dma_fence scheduled;
120b843c749SSergey Zigachev
121b843c749SSergey Zigachev /**
122b843c749SSergey Zigachev * @finished: this fence is what will be signaled by the scheduler
123b843c749SSergey Zigachev * when the job is completed.
124b843c749SSergey Zigachev *
125b843c749SSergey Zigachev * When setting up an out fence for the job, you should use
126b843c749SSergey Zigachev * this, since it's available immediately upon
127b843c749SSergey Zigachev * drm_sched_job_init(), and the fence returned by the driver
128b843c749SSergey Zigachev * from run_job() won't be created until the dependencies have
129b843c749SSergey Zigachev * resolved.
130b843c749SSergey Zigachev */
131b843c749SSergey Zigachev struct dma_fence finished;
132b843c749SSergey Zigachev
133b843c749SSergey Zigachev /**
134b843c749SSergey Zigachev * @cb: the callback for the parent fence below.
135b843c749SSergey Zigachev */
136b843c749SSergey Zigachev struct dma_fence_cb cb;
137b843c749SSergey Zigachev /**
138b843c749SSergey Zigachev * @parent: the fence returned by &drm_sched_backend_ops.run_job
139b843c749SSergey Zigachev * when scheduling the job on hardware. We signal the
140b843c749SSergey Zigachev * &drm_sched_fence.finished fence once parent is signalled.
141b843c749SSergey Zigachev */
142b843c749SSergey Zigachev struct dma_fence *parent;
143b843c749SSergey Zigachev /**
144b843c749SSergey Zigachev * @sched: the scheduler instance to which the job having this struct
145b843c749SSergey Zigachev * belongs to.
146b843c749SSergey Zigachev */
147b843c749SSergey Zigachev struct drm_gpu_scheduler *sched;
148b843c749SSergey Zigachev /**
149b843c749SSergey Zigachev * @lock: the lock used by the scheduled and the finished fences.
150b843c749SSergey Zigachev */
151b843c749SSergey Zigachev spinlock_t lock;
152b843c749SSergey Zigachev /**
153b843c749SSergey Zigachev * @owner: job owner for debugging
154b843c749SSergey Zigachev */
155b843c749SSergey Zigachev void *owner;
156b843c749SSergey Zigachev };
157b843c749SSergey Zigachev
158b843c749SSergey Zigachev struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f);
159b843c749SSergey Zigachev
160b843c749SSergey Zigachev /**
161b843c749SSergey Zigachev * struct drm_sched_job - A job to be run by an entity.
162b843c749SSergey Zigachev *
163b843c749SSergey Zigachev * @queue_node: used to append this struct to the queue of jobs in an entity.
164b843c749SSergey Zigachev * @sched: the scheduler instance on which this job is scheduled.
165b843c749SSergey Zigachev * @s_fence: contains the fences for the scheduling of job.
166b843c749SSergey Zigachev * @finish_cb: the callback for the finished fence.
167b843c749SSergey Zigachev * @finish_work: schedules the function @drm_sched_job_finish once the job has
168b843c749SSergey Zigachev * finished to remove the job from the
169b843c749SSergey Zigachev * @drm_gpu_scheduler.ring_mirror_list.
170b843c749SSergey Zigachev * @node: used to append this struct to the @drm_gpu_scheduler.ring_mirror_list.
171b843c749SSergey Zigachev * @work_tdr: schedules a delayed call to @drm_sched_job_timedout after the timeout
172b843c749SSergey Zigachev * interval is over.
173b843c749SSergey Zigachev * @id: a unique id assigned to each job scheduled on the scheduler.
174b843c749SSergey Zigachev * @karma: increment on every hang caused by this job. If this exceeds the hang
175b843c749SSergey Zigachev * limit of the scheduler then the job is marked guilty and will not
176b843c749SSergey Zigachev * be scheduled further.
177b843c749SSergey Zigachev * @s_priority: the priority of the job.
178b843c749SSergey Zigachev * @entity: the entity to which this job belongs.
179b843c749SSergey Zigachev *
180b843c749SSergey Zigachev * A job is created by the driver using drm_sched_job_init(), and
181b843c749SSergey Zigachev * should call drm_sched_entity_push_job() once it wants the scheduler
182b843c749SSergey Zigachev * to schedule the job.
183b843c749SSergey Zigachev */
184b843c749SSergey Zigachev struct drm_sched_job {
185b843c749SSergey Zigachev struct spsc_node queue_node;
186b843c749SSergey Zigachev struct drm_gpu_scheduler *sched;
187b843c749SSergey Zigachev struct drm_sched_fence *s_fence;
188b843c749SSergey Zigachev struct dma_fence_cb finish_cb;
189b843c749SSergey Zigachev struct work_struct finish_work;
190b843c749SSergey Zigachev struct list_head node;
191b843c749SSergey Zigachev struct delayed_work work_tdr;
192b843c749SSergey Zigachev uint64_t id;
193b843c749SSergey Zigachev atomic_t karma;
194b843c749SSergey Zigachev enum drm_sched_priority s_priority;
195b843c749SSergey Zigachev struct drm_sched_entity *entity;
196b843c749SSergey Zigachev };
197b843c749SSergey Zigachev
drm_sched_invalidate_job(struct drm_sched_job * s_job,int threshold)198b843c749SSergey Zigachev static inline bool drm_sched_invalidate_job(struct drm_sched_job *s_job,
199b843c749SSergey Zigachev int threshold)
200b843c749SSergey Zigachev {
201b843c749SSergey Zigachev return (s_job && atomic_inc_return(&s_job->karma) > threshold);
202b843c749SSergey Zigachev }
203b843c749SSergey Zigachev
204b843c749SSergey Zigachev /**
205b843c749SSergey Zigachev * struct drm_sched_backend_ops
206b843c749SSergey Zigachev *
207b843c749SSergey Zigachev * Define the backend operations called by the scheduler,
208b843c749SSergey Zigachev * these functions should be implemented in driver side.
209b843c749SSergey Zigachev */
210b843c749SSergey Zigachev struct drm_sched_backend_ops {
211b843c749SSergey Zigachev /**
212b843c749SSergey Zigachev * @dependency: Called when the scheduler is considering scheduling
213b843c749SSergey Zigachev * this job next, to get another struct dma_fence for this job to
214b843c749SSergey Zigachev * block on. Once it returns NULL, run_job() may be called.
215b843c749SSergey Zigachev */
216b843c749SSergey Zigachev struct dma_fence *(*dependency)(struct drm_sched_job *sched_job,
217b843c749SSergey Zigachev struct drm_sched_entity *s_entity);
218b843c749SSergey Zigachev
219b843c749SSergey Zigachev /**
220b843c749SSergey Zigachev * @run_job: Called to execute the job once all of the dependencies
221b843c749SSergey Zigachev * have been resolved. This may be called multiple times, if
222b843c749SSergey Zigachev * timedout_job() has happened and drm_sched_job_recovery()
223b843c749SSergey Zigachev * decides to try it again.
224b843c749SSergey Zigachev */
225b843c749SSergey Zigachev struct dma_fence *(*run_job)(struct drm_sched_job *sched_job);
226b843c749SSergey Zigachev
227b843c749SSergey Zigachev /**
228b843c749SSergey Zigachev * @timedout_job: Called when a job has taken too long to execute,
229b843c749SSergey Zigachev * to trigger GPU recovery.
230b843c749SSergey Zigachev */
231b843c749SSergey Zigachev void (*timedout_job)(struct drm_sched_job *sched_job);
232b843c749SSergey Zigachev
233b843c749SSergey Zigachev /**
234b843c749SSergey Zigachev * @free_job: Called once the job's finished fence has been signaled
235b843c749SSergey Zigachev * and it's time to clean it up.
236b843c749SSergey Zigachev */
237b843c749SSergey Zigachev void (*free_job)(struct drm_sched_job *sched_job);
238b843c749SSergey Zigachev };
239b843c749SSergey Zigachev
240b843c749SSergey Zigachev /**
241b843c749SSergey Zigachev * struct drm_gpu_scheduler
242b843c749SSergey Zigachev *
243b843c749SSergey Zigachev * @ops: backend operations provided by the driver.
244b843c749SSergey Zigachev * @hw_submission_limit: the max size of the hardware queue.
245b843c749SSergey Zigachev * @timeout: the time after which a job is removed from the scheduler.
246b843c749SSergey Zigachev * @name: name of the ring for which this scheduler is being used.
247b843c749SSergey Zigachev * @sched_rq: priority wise array of run queues.
248b843c749SSergey Zigachev * @wake_up_worker: the wait queue on which the scheduler sleeps until a job
249b843c749SSergey Zigachev * is ready to be scheduled.
250b843c749SSergey Zigachev * @job_scheduled: once @drm_sched_entity_do_release is called the scheduler
251b843c749SSergey Zigachev * waits on this wait queue until all the scheduled jobs are
252b843c749SSergey Zigachev * finished.
253b843c749SSergey Zigachev * @hw_rq_count: the number of jobs currently in the hardware queue.
254b843c749SSergey Zigachev * @job_id_count: used to assign unique id to the each job.
255b843c749SSergey Zigachev * @thread: the kthread on which the scheduler which run.
256b843c749SSergey Zigachev * @ring_mirror_list: the list of jobs which are currently in the job queue.
257b843c749SSergey Zigachev * @job_list_lock: lock to protect the ring_mirror_list.
258b843c749SSergey Zigachev * @hang_limit: once the hangs by a job crosses this limit then it is marked
259b843c749SSergey Zigachev * guilty and it will be considered for scheduling further.
260b843c749SSergey Zigachev *
261b843c749SSergey Zigachev * One scheduler is implemented for each hardware ring.
262b843c749SSergey Zigachev */
263b843c749SSergey Zigachev struct drm_gpu_scheduler {
264b843c749SSergey Zigachev const struct drm_sched_backend_ops *ops;
265b843c749SSergey Zigachev uint32_t hw_submission_limit;
266b843c749SSergey Zigachev long timeout;
267b843c749SSergey Zigachev const char *name;
268b843c749SSergey Zigachev struct drm_sched_rq sched_rq[DRM_SCHED_PRIORITY_MAX];
269b843c749SSergey Zigachev wait_queue_head_t wake_up_worker;
270b843c749SSergey Zigachev wait_queue_head_t job_scheduled;
271b843c749SSergey Zigachev atomic_t hw_rq_count;
272b843c749SSergey Zigachev atomic64_t job_id_count;
273b843c749SSergey Zigachev struct task_struct *thread;
274b843c749SSergey Zigachev struct list_head ring_mirror_list;
275*78973132SSergey Zigachev struct spinlock job_list_lock;
276b843c749SSergey Zigachev int hang_limit;
277b843c749SSergey Zigachev };
278b843c749SSergey Zigachev
279b843c749SSergey Zigachev int drm_sched_init(struct drm_gpu_scheduler *sched,
280b843c749SSergey Zigachev const struct drm_sched_backend_ops *ops,
281b843c749SSergey Zigachev uint32_t hw_submission, unsigned hang_limit, long timeout,
282b843c749SSergey Zigachev const char *name);
283b843c749SSergey Zigachev void drm_sched_fini(struct drm_gpu_scheduler *sched);
284b843c749SSergey Zigachev
285b843c749SSergey Zigachev int drm_sched_entity_init(struct drm_sched_entity *entity,
286b843c749SSergey Zigachev struct drm_sched_rq **rq_list,
287b843c749SSergey Zigachev unsigned int num_rq_list,
288b843c749SSergey Zigachev atomic_t *guilty);
289b843c749SSergey Zigachev long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout);
290b843c749SSergey Zigachev void drm_sched_entity_fini(struct drm_sched_entity *entity);
291b843c749SSergey Zigachev void drm_sched_entity_destroy(struct drm_sched_entity *entity);
292b843c749SSergey Zigachev void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
293b843c749SSergey Zigachev struct drm_sched_entity *entity);
294b843c749SSergey Zigachev void drm_sched_entity_set_rq(struct drm_sched_entity *entity,
295b843c749SSergey Zigachev struct drm_sched_rq *rq);
296b843c749SSergey Zigachev
297b843c749SSergey Zigachev struct drm_sched_fence *drm_sched_fence_create(
298b843c749SSergey Zigachev struct drm_sched_entity *s_entity, void *owner);
299b843c749SSergey Zigachev void drm_sched_fence_scheduled(struct drm_sched_fence *fence);
300b843c749SSergey Zigachev void drm_sched_fence_finished(struct drm_sched_fence *fence);
301b843c749SSergey Zigachev int drm_sched_job_init(struct drm_sched_job *job,
302b843c749SSergey Zigachev struct drm_sched_entity *entity,
303b843c749SSergey Zigachev void *owner);
304b843c749SSergey Zigachev void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
305b843c749SSergey Zigachev struct drm_sched_job *job);
306b843c749SSergey Zigachev void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
307b843c749SSergey Zigachev bool drm_sched_dependency_optimized(struct dma_fence* fence,
308b843c749SSergey Zigachev struct drm_sched_entity *entity);
309b843c749SSergey Zigachev void drm_sched_job_kickout(struct drm_sched_job *s_job);
310b843c749SSergey Zigachev
311b843c749SSergey Zigachev #endif
312