1 /* 2 * Copyright © 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/kthread.h> 26 #include <trace/events/dma_fence.h> 27 #ifdef notyet 28 #include <uapi/linux/sched/types.h> 29 #endif 30 31 #include "i915_drv.h" 32 #include "i915_trace.h" 33 #include "intel_gt_pm.h" 34 #include "intel_gt_requests.h" 35 36 static void irq_enable(struct intel_engine_cs *engine) 37 { 38 if (!engine->irq_enable) 39 return; 40 41 /* Caller disables interrupts */ 42 spin_lock(&engine->gt->irq_lock); 43 engine->irq_enable(engine); 44 spin_unlock(&engine->gt->irq_lock); 45 } 46 47 static void irq_disable(struct intel_engine_cs *engine) 48 { 49 if (!engine->irq_disable) 50 return; 51 52 /* Caller disables interrupts */ 53 spin_lock(&engine->gt->irq_lock); 54 engine->irq_disable(engine); 55 spin_unlock(&engine->gt->irq_lock); 56 } 57 58 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) 59 { 60 struct intel_engine_cs *engine = 61 container_of(b, struct intel_engine_cs, breadcrumbs); 62 63 lockdep_assert_held(&b->irq_lock); 64 65 GEM_BUG_ON(!b->irq_enabled); 66 if (!--b->irq_enabled) 67 irq_disable(engine); 68 69 b->irq_armed = false; 70 intel_gt_pm_put_async(engine->gt); 71 } 72 73 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) 74 { 75 struct intel_breadcrumbs *b = &engine->breadcrumbs; 76 unsigned long flags; 77 78 if (!b->irq_armed) 79 return; 80 81 spin_lock_irqsave(&b->irq_lock, flags); 82 if (b->irq_armed) 83 __intel_breadcrumbs_disarm_irq(b); 84 spin_unlock_irqrestore(&b->irq_lock, flags); 85 } 86 87 static inline bool __request_completed(const struct i915_request *rq) 88 { 89 return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno); 90 } 91 92 __maybe_unused static bool 93 check_signal_order(struct intel_context *ce, struct i915_request *rq) 94 { 95 if (!list_is_last(&rq->signal_link, &ce->signals) && 96 i915_seqno_passed(rq->fence.seqno, 97 list_next_entry(rq, signal_link)->fence.seqno)) 98 return false; 99 100 if (!list_is_first(&rq->signal_link, &ce->signals) && 101 i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno, 102 rq->fence.seqno)) 103 return false; 104 105 return true; 106 } 107 108 static bool 109 __dma_fence_signal(struct dma_fence *fence) 110 { 111 return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags); 112 } 113 114 static void 115 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp) 116 { 117 fence->timestamp = timestamp; 118 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); 119 trace_dma_fence_signaled(fence); 120 } 121 122 static void 123 __dma_fence_signal__notify(struct dma_fence *fence, 124 const struct list_head *list) 125 { 126 struct dma_fence_cb *cur, *tmp; 127 128 lockdep_assert_held(fence->lock); 129 130 list_for_each_entry_safe(cur, tmp, list, node) { 131 INIT_LIST_HEAD(&cur->node); 132 cur->func(fence, cur); 133 } 134 } 135 136 static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl) 137 { 138 struct intel_engine_cs *engine = 139 container_of(b, struct intel_engine_cs, breadcrumbs); 140 141 if (unlikely(intel_engine_is_virtual(engine))) 142 engine = intel_virtual_engine_get_sibling(engine, 0); 143 144 intel_engine_add_retire(engine, tl); 145 } 146 147 static void signal_irq_work(struct irq_work *work) 148 { 149 struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work); 150 const ktime_t timestamp = ktime_get(); 151 struct intel_context *ce, *cn; 152 struct list_head *pos, *next; 153 DRM_LIST_HEAD(signal); 154 155 spin_lock(&b->irq_lock); 156 157 if (b->irq_armed && list_empty(&b->signalers)) 158 __intel_breadcrumbs_disarm_irq(b); 159 160 list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) { 161 GEM_BUG_ON(list_empty(&ce->signals)); 162 163 list_for_each_safe(pos, next, &ce->signals) { 164 struct i915_request *rq = 165 list_entry(pos, typeof(*rq), signal_link); 166 167 GEM_BUG_ON(!check_signal_order(ce, rq)); 168 169 if (!__request_completed(rq)) 170 break; 171 172 GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, 173 &rq->fence.flags)); 174 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 175 176 if (!__dma_fence_signal(&rq->fence)) 177 continue; 178 179 /* 180 * Queue for execution after dropping the signaling 181 * spinlock as the callback chain may end up adding 182 * more signalers to the same context or engine. 183 */ 184 i915_request_get(rq); 185 list_add_tail(&rq->signal_link, &signal); 186 } 187 188 /* 189 * We process the list deletion in bulk, only using a list_add 190 * (not list_move) above but keeping the status of 191 * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit. 192 */ 193 if (!list_is_first(pos, &ce->signals)) { 194 /* Advance the list to the first incomplete request */ 195 __list_del_many(&ce->signals, pos); 196 if (&ce->signals == pos) { /* now empty */ 197 list_del_init(&ce->signal_link); 198 add_retire(b, ce->timeline); 199 } 200 } 201 } 202 203 spin_unlock(&b->irq_lock); 204 205 list_for_each_safe(pos, next, &signal) { 206 struct i915_request *rq = 207 list_entry(pos, typeof(*rq), signal_link); 208 struct list_head cb_list; 209 210 spin_lock(&rq->lock); 211 list_replace(&rq->fence.cb_list, &cb_list); 212 __dma_fence_signal__timestamp(&rq->fence, timestamp); 213 __dma_fence_signal__notify(&rq->fence, &cb_list); 214 spin_unlock(&rq->lock); 215 216 i915_request_put(rq); 217 } 218 } 219 220 static bool __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) 221 { 222 struct intel_engine_cs *engine = 223 container_of(b, struct intel_engine_cs, breadcrumbs); 224 225 lockdep_assert_held(&b->irq_lock); 226 if (b->irq_armed) 227 return true; 228 229 if (!intel_gt_pm_get_if_awake(engine->gt)) 230 return false; 231 232 /* 233 * The breadcrumb irq will be disarmed on the interrupt after the 234 * waiters are signaled. This gives us a single interrupt window in 235 * which we can add a new waiter and avoid the cost of re-enabling 236 * the irq. 237 */ 238 b->irq_armed = true; 239 240 /* 241 * Since we are waiting on a request, the GPU should be busy 242 * and should have its own rpm reference. This is tracked 243 * by i915->gt.awake, we can forgo holding our own wakref 244 * for the interrupt as before i915->gt.awake is released (when 245 * the driver is idle) we disarm the breadcrumbs. 246 */ 247 248 if (!b->irq_enabled++) 249 irq_enable(engine); 250 251 return true; 252 } 253 254 void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) 255 { 256 struct intel_breadcrumbs *b = &engine->breadcrumbs; 257 258 mtx_init(&b->irq_lock, IPL_TTY); 259 INIT_LIST_HEAD(&b->signalers); 260 261 init_irq_work(&b->irq_work, signal_irq_work); 262 } 263 264 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) 265 { 266 struct intel_breadcrumbs *b = &engine->breadcrumbs; 267 unsigned long flags; 268 269 spin_lock_irqsave(&b->irq_lock, flags); 270 271 if (b->irq_enabled) 272 irq_enable(engine); 273 else 274 irq_disable(engine); 275 276 spin_unlock_irqrestore(&b->irq_lock, flags); 277 } 278 279 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) 280 { 281 } 282 283 bool i915_request_enable_breadcrumb(struct i915_request *rq) 284 { 285 lockdep_assert_held(&rq->lock); 286 287 if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) { 288 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; 289 struct intel_context *ce = rq->context; 290 struct list_head *pos; 291 292 spin_lock(&b->irq_lock); 293 GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); 294 295 if (!__intel_breadcrumbs_arm_irq(b)) 296 goto unlock; 297 298 /* 299 * We keep the seqno in retirement order, so we can break 300 * inside intel_engine_signal_breadcrumbs as soon as we've 301 * passed the last completed request (or seen a request that 302 * hasn't event started). We could walk the timeline->requests, 303 * but keeping a separate signalers_list has the advantage of 304 * hopefully being much smaller than the full list and so 305 * provides faster iteration and detection when there are no 306 * more interrupts required for this context. 307 * 308 * We typically expect to add new signalers in order, so we 309 * start looking for our insertion point from the tail of 310 * the list. 311 */ 312 list_for_each_prev(pos, &ce->signals) { 313 struct i915_request *it = 314 list_entry(pos, typeof(*it), signal_link); 315 316 if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno)) 317 break; 318 } 319 list_add(&rq->signal_link, pos); 320 if (pos == &ce->signals) /* catch transitions from empty list */ 321 list_move_tail(&ce->signal_link, &b->signalers); 322 GEM_BUG_ON(!check_signal_order(ce, rq)); 323 324 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 325 unlock: 326 spin_unlock(&b->irq_lock); 327 } 328 329 return !__request_completed(rq); 330 } 331 332 void i915_request_cancel_breadcrumb(struct i915_request *rq) 333 { 334 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; 335 336 lockdep_assert_held(&rq->lock); 337 338 /* 339 * We must wait for b->irq_lock so that we know the interrupt handler 340 * has released its reference to the intel_context and has completed 341 * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if 342 * required). 343 */ 344 spin_lock(&b->irq_lock); 345 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) { 346 struct intel_context *ce = rq->context; 347 348 list_del(&rq->signal_link); 349 if (list_empty(&ce->signals)) 350 list_del_init(&ce->signal_link); 351 352 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 353 } 354 spin_unlock(&b->irq_lock); 355 } 356 357 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, 358 struct drm_printer *p) 359 { 360 struct intel_breadcrumbs *b = &engine->breadcrumbs; 361 struct intel_context *ce; 362 struct i915_request *rq; 363 364 if (list_empty(&b->signalers)) 365 return; 366 367 drm_printf(p, "Signals:\n"); 368 369 spin_lock_irq(&b->irq_lock); 370 list_for_each_entry(ce, &b->signalers, signal_link) { 371 list_for_each_entry(rq, &ce->signals, signal_link) { 372 drm_printf(p, "\t[%llx:%llx%s] @ %dms\n", 373 rq->fence.context, rq->fence.seqno, 374 i915_request_completed(rq) ? "!" : 375 i915_request_started(rq) ? "*" : 376 "", 377 jiffies_to_msecs(jiffies - rq->emitted_jiffies)); 378 } 379 } 380 spin_unlock_irq(&b->irq_lock); 381 } 382