1b843c749SSergey Zigachev /*
2b843c749SSergey Zigachev * Copyright 2015 Advanced Micro Devices, Inc.
3b843c749SSergey Zigachev *
4b843c749SSergey Zigachev * Permission is hereby granted, free of charge, to any person obtaining a
5b843c749SSergey Zigachev * copy of this software and associated documentation files (the "Software"),
6b843c749SSergey Zigachev * to deal in the Software without restriction, including without limitation
7b843c749SSergey Zigachev * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8b843c749SSergey Zigachev * and/or sell copies of the Software, and to permit persons to whom the
9b843c749SSergey Zigachev * Software is furnished to do so, subject to the following conditions:
10b843c749SSergey Zigachev *
11b843c749SSergey Zigachev * The above copyright notice and this permission notice shall be included in
12b843c749SSergey Zigachev * all copies or substantial portions of the Software.
13b843c749SSergey Zigachev *
14b843c749SSergey Zigachev * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15b843c749SSergey Zigachev * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16b843c749SSergey Zigachev * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17b843c749SSergey Zigachev * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18b843c749SSergey Zigachev * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19b843c749SSergey Zigachev * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20b843c749SSergey Zigachev * OTHER DEALINGS IN THE SOFTWARE.
21b843c749SSergey Zigachev *
22b843c749SSergey Zigachev */
23b843c749SSergey Zigachev
24b843c749SSergey Zigachev #include <linux/kthread.h>
25b843c749SSergey Zigachev #include <linux/wait.h>
26b843c749SSergey Zigachev #include <linux/sched.h>
27b843c749SSergey Zigachev #include <drm/drmP.h>
28b843c749SSergey Zigachev #include <drm/gpu_scheduler.h>
29b843c749SSergey Zigachev
30b843c749SSergey Zigachev static struct kmem_cache *sched_fence_slab;
31b843c749SSergey Zigachev
drm_sched_fence_slab_init(void)32b843c749SSergey Zigachev static int __init drm_sched_fence_slab_init(void)
33b843c749SSergey Zigachev {
34b843c749SSergey Zigachev sched_fence_slab = kmem_cache_create(
35b843c749SSergey Zigachev "drm_sched_fence", sizeof(struct drm_sched_fence), 0,
36b843c749SSergey Zigachev SLAB_HWCACHE_ALIGN, NULL);
37b843c749SSergey Zigachev if (!sched_fence_slab)
38b843c749SSergey Zigachev return -ENOMEM;
39b843c749SSergey Zigachev
40b843c749SSergey Zigachev return 0;
41b843c749SSergey Zigachev }
42b843c749SSergey Zigachev
drm_sched_fence_slab_fini(void)43b843c749SSergey Zigachev static void __exit drm_sched_fence_slab_fini(void)
44b843c749SSergey Zigachev {
45b843c749SSergey Zigachev rcu_barrier();
46b843c749SSergey Zigachev kmem_cache_destroy(sched_fence_slab);
47b843c749SSergey Zigachev }
48b843c749SSergey Zigachev
drm_sched_fence_scheduled(struct drm_sched_fence * fence)49b843c749SSergey Zigachev void drm_sched_fence_scheduled(struct drm_sched_fence *fence)
50b843c749SSergey Zigachev {
51b843c749SSergey Zigachev int ret = dma_fence_signal(&fence->scheduled);
52b843c749SSergey Zigachev
53b843c749SSergey Zigachev if (!ret)
54b843c749SSergey Zigachev DMA_FENCE_TRACE(&fence->scheduled,
55b843c749SSergey Zigachev "signaled from irq context\n");
56b843c749SSergey Zigachev else
57b843c749SSergey Zigachev DMA_FENCE_TRACE(&fence->scheduled,
58b843c749SSergey Zigachev "was already signaled\n");
59b843c749SSergey Zigachev }
60b843c749SSergey Zigachev
drm_sched_fence_finished(struct drm_sched_fence * fence)61b843c749SSergey Zigachev void drm_sched_fence_finished(struct drm_sched_fence *fence)
62b843c749SSergey Zigachev {
63b843c749SSergey Zigachev int ret = dma_fence_signal(&fence->finished);
64b843c749SSergey Zigachev
65b843c749SSergey Zigachev if (!ret)
66b843c749SSergey Zigachev DMA_FENCE_TRACE(&fence->finished,
67b843c749SSergey Zigachev "signaled from irq context\n");
68b843c749SSergey Zigachev else
69b843c749SSergey Zigachev DMA_FENCE_TRACE(&fence->finished,
70b843c749SSergey Zigachev "was already signaled\n");
71b843c749SSergey Zigachev }
72b843c749SSergey Zigachev
drm_sched_fence_get_driver_name(struct dma_fence * fence)73b843c749SSergey Zigachev static const char *drm_sched_fence_get_driver_name(struct dma_fence *fence)
74b843c749SSergey Zigachev {
75b843c749SSergey Zigachev return "drm_sched";
76b843c749SSergey Zigachev }
77b843c749SSergey Zigachev
drm_sched_fence_get_timeline_name(struct dma_fence * f)78b843c749SSergey Zigachev static const char *drm_sched_fence_get_timeline_name(struct dma_fence *f)
79b843c749SSergey Zigachev {
80b843c749SSergey Zigachev struct drm_sched_fence *fence = to_drm_sched_fence(f);
81b843c749SSergey Zigachev return (const char *)fence->sched->name;
82b843c749SSergey Zigachev }
83b843c749SSergey Zigachev
84b843c749SSergey Zigachev /**
85b843c749SSergey Zigachev * drm_sched_fence_free - free up the fence memory
86b843c749SSergey Zigachev *
87b843c749SSergey Zigachev * @rcu: RCU callback head
88b843c749SSergey Zigachev *
89b843c749SSergey Zigachev * Free up the fence memory after the RCU grace period.
90b843c749SSergey Zigachev */
drm_sched_fence_free(struct rcu_head * rcu)91b843c749SSergey Zigachev static void drm_sched_fence_free(struct rcu_head *rcu)
92b843c749SSergey Zigachev {
93b843c749SSergey Zigachev struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
94b843c749SSergey Zigachev struct drm_sched_fence *fence = to_drm_sched_fence(f);
95b843c749SSergey Zigachev
96b843c749SSergey Zigachev kmem_cache_free(sched_fence_slab, fence);
97b843c749SSergey Zigachev }
98b843c749SSergey Zigachev
99b843c749SSergey Zigachev /**
100b843c749SSergey Zigachev * drm_sched_fence_release_scheduled - callback that fence can be freed
101b843c749SSergey Zigachev *
102b843c749SSergey Zigachev * @fence: fence
103b843c749SSergey Zigachev *
104b843c749SSergey Zigachev * This function is called when the reference count becomes zero.
105b843c749SSergey Zigachev * It just RCU schedules freeing up the fence.
106b843c749SSergey Zigachev */
drm_sched_fence_release_scheduled(struct dma_fence * f)107b843c749SSergey Zigachev static void drm_sched_fence_release_scheduled(struct dma_fence *f)
108b843c749SSergey Zigachev {
109b843c749SSergey Zigachev struct drm_sched_fence *fence = to_drm_sched_fence(f);
110b843c749SSergey Zigachev
111b843c749SSergey Zigachev dma_fence_put(fence->parent);
112b843c749SSergey Zigachev call_rcu(&fence->finished.rcu, drm_sched_fence_free);
113b843c749SSergey Zigachev }
114b843c749SSergey Zigachev
115b843c749SSergey Zigachev /**
116b843c749SSergey Zigachev * drm_sched_fence_release_finished - drop extra reference
117b843c749SSergey Zigachev *
118b843c749SSergey Zigachev * @f: fence
119b843c749SSergey Zigachev *
120b843c749SSergey Zigachev * Drop the extra reference from the scheduled fence to the base fence.
121b843c749SSergey Zigachev */
drm_sched_fence_release_finished(struct dma_fence * f)122b843c749SSergey Zigachev static void drm_sched_fence_release_finished(struct dma_fence *f)
123b843c749SSergey Zigachev {
124b843c749SSergey Zigachev struct drm_sched_fence *fence = to_drm_sched_fence(f);
125b843c749SSergey Zigachev
126b843c749SSergey Zigachev dma_fence_put(&fence->scheduled);
127b843c749SSergey Zigachev }
128b843c749SSergey Zigachev
129b843c749SSergey Zigachev const struct dma_fence_ops drm_sched_fence_ops_scheduled = {
130b843c749SSergey Zigachev .get_driver_name = drm_sched_fence_get_driver_name,
131b843c749SSergey Zigachev .get_timeline_name = drm_sched_fence_get_timeline_name,
132b843c749SSergey Zigachev .release = drm_sched_fence_release_scheduled,
133b843c749SSergey Zigachev };
134b843c749SSergey Zigachev
135b843c749SSergey Zigachev const struct dma_fence_ops drm_sched_fence_ops_finished = {
136b843c749SSergey Zigachev .get_driver_name = drm_sched_fence_get_driver_name,
137b843c749SSergey Zigachev .get_timeline_name = drm_sched_fence_get_timeline_name,
138b843c749SSergey Zigachev .release = drm_sched_fence_release_finished,
139b843c749SSergey Zigachev };
140b843c749SSergey Zigachev
to_drm_sched_fence(struct dma_fence * f)141b843c749SSergey Zigachev struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f)
142b843c749SSergey Zigachev {
143b843c749SSergey Zigachev if (f->ops == &drm_sched_fence_ops_scheduled)
144b843c749SSergey Zigachev return container_of(f, struct drm_sched_fence, scheduled);
145b843c749SSergey Zigachev
146b843c749SSergey Zigachev if (f->ops == &drm_sched_fence_ops_finished)
147b843c749SSergey Zigachev return container_of(f, struct drm_sched_fence, finished);
148b843c749SSergey Zigachev
149b843c749SSergey Zigachev return NULL;
150b843c749SSergey Zigachev }
151b843c749SSergey Zigachev EXPORT_SYMBOL(to_drm_sched_fence);
152b843c749SSergey Zigachev
drm_sched_fence_create(struct drm_sched_entity * entity,void * owner)153b843c749SSergey Zigachev struct drm_sched_fence *drm_sched_fence_create(struct drm_sched_entity *entity,
154b843c749SSergey Zigachev void *owner)
155b843c749SSergey Zigachev {
156b843c749SSergey Zigachev struct drm_sched_fence *fence = NULL;
157b843c749SSergey Zigachev unsigned seq;
158b843c749SSergey Zigachev
159*78973132SSergey Zigachev fence = kzalloc(sizeof(struct drm_sched_fence), GFP_KERNEL);
160b843c749SSergey Zigachev if (fence == NULL)
161b843c749SSergey Zigachev return NULL;
162b843c749SSergey Zigachev
163b843c749SSergey Zigachev fence->owner = owner;
164b843c749SSergey Zigachev fence->sched = entity->rq->sched;
165*78973132SSergey Zigachev lockinit(&fence->lock, "dscl", 0, LK_CANRECURSE);
166b843c749SSergey Zigachev
167b843c749SSergey Zigachev seq = atomic_inc_return(&entity->fence_seq);
168b843c749SSergey Zigachev dma_fence_init(&fence->scheduled, &drm_sched_fence_ops_scheduled,
169b843c749SSergey Zigachev &fence->lock, entity->fence_context, seq);
170b843c749SSergey Zigachev dma_fence_init(&fence->finished, &drm_sched_fence_ops_finished,
171b843c749SSergey Zigachev &fence->lock, entity->fence_context + 1, seq);
172b843c749SSergey Zigachev
173b843c749SSergey Zigachev return fence;
174b843c749SSergey Zigachev }
175b843c749SSergey Zigachev
176b843c749SSergey Zigachev module_init(drm_sched_fence_slab_init);
177b843c749SSergey Zigachev module_exit(drm_sched_fence_slab_fini);
178b843c749SSergey Zigachev
179b843c749SSergey Zigachev MODULE_DESCRIPTION("DRM GPU scheduler");
180b843c749SSergey Zigachev MODULE_LICENSE("GPL and additional rights");
181