1 /* 2 * Copyright © 2008-2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #ifndef I915_GEM_REQUEST_H 26 #define I915_GEM_REQUEST_H 27 28 #include <linux/dma-fence.h> 29 30 #include "i915_gem.h" 31 #include "i915_sw_fence.h" 32 33 struct drm_file; 34 struct drm_i915_gem_object; 35 struct drm_i915_gem_request; 36 37 struct intel_wait { 38 struct rb_node node; 39 struct task_struct *tsk; 40 struct drm_i915_gem_request *request; 41 u32 seqno; 42 }; 43 44 struct intel_signal_node { 45 struct rb_node node; 46 struct intel_wait wait; 47 }; 48 49 struct i915_dependency { 50 struct i915_priotree *signaler; 51 struct list_head signal_link; 52 struct list_head wait_link; 53 struct list_head dfs_link; 54 unsigned long flags; 55 #define I915_DEPENDENCY_ALLOC BIT(0) 56 }; 57 58 /* Requests exist in a complex web of interdependencies. Each request 59 * has to wait for some other request to complete before it is ready to be run 60 * (e.g. we have to wait until the pixels have been rendering into a texture 61 * before we can copy from it). We track the readiness of a request in terms 62 * of fences, but we also need to keep the dependency tree for the lifetime 63 * of the request (beyond the life of an individual fence). We use the tree 64 * at various points to reorder the requests whilst keeping the requests 65 * in order with respect to their various dependencies. 66 */ 67 struct i915_priotree { 68 struct list_head signalers_list; /* those before us, we depend upon */ 69 struct list_head waiters_list; /* those after us, they depend upon us */ 70 struct rb_node node; 71 int priority; 72 #define I915_PRIORITY_MAX 1024 73 #define I915_PRIORITY_MIN (-I915_PRIORITY_MAX) 74 }; 75 76 /** 77 * Request queue structure. 78 * 79 * The request queue allows us to note sequence numbers that have been emitted 80 * and may be associated with active buffers to be retired. 81 * 82 * By keeping this list, we can avoid having to do questionable sequence 83 * number comparisons on buffer last_read|write_seqno. It also allows an 84 * emission time to be associated with the request for tracking how far ahead 85 * of the GPU the submission is. 86 * 87 * When modifying this structure be very aware that we perform a lockless 88 * RCU lookup of it that may race against reallocation of the struct 89 * from the slab freelist. We intentionally do not zero the structure on 90 * allocation so that the lookup can use the dangling pointers (and is 91 * cogniscent that those pointers may be wrong). Instead, everything that 92 * needs to be initialised must be done so explicitly. 93 * 94 * The requests are reference counted. 95 */ 96 struct drm_i915_gem_request { 97 struct dma_fence fence; 98 spinlock_t lock; 99 100 /** On Which ring this request was generated */ 101 struct drm_i915_private *i915; 102 103 /** 104 * Context and ring buffer related to this request 105 * Contexts are refcounted, so when this request is associated with a 106 * context, we must increment the context's refcount, to guarantee that 107 * it persists while any request is linked to it. Requests themselves 108 * are also refcounted, so the request will only be freed when the last 109 * reference to it is dismissed, and the code in 110 * i915_gem_request_free() will then decrement the refcount on the 111 * context. 112 */ 113 struct i915_gem_context *ctx; 114 struct intel_engine_cs *engine; 115 struct intel_ring *ring; 116 struct intel_timeline *timeline; 117 struct intel_signal_node signaling; 118 119 /* Fences for the various phases in the request's lifetime. 120 * 121 * The submit fence is used to await upon all of the request's 122 * dependencies. When it is signaled, the request is ready to run. 123 * It is used by the driver to then queue the request for execution. 124 */ 125 struct i915_sw_fence submit; 126 wait_queue_t submitq; 127 wait_queue_head_t execute; 128 129 /* A list of everyone we wait upon, and everyone who waits upon us. 130 * Even though we will not be submitted to the hardware before the 131 * submit fence is signaled (it waits for all external events as well 132 * as our own requests), the scheduler still needs to know the 133 * dependency tree for the lifetime of the request (from execbuf 134 * to retirement), i.e. bidirectional dependency information for the 135 * request not tied to individual fences. 136 */ 137 struct i915_priotree priotree; 138 struct i915_dependency dep; 139 140 /** GEM sequence number associated with this request on the 141 * global execution timeline. It is zero when the request is not 142 * on the HW queue (i.e. not on the engine timeline list). 143 * Its value is guarded by the timeline spinlock. 144 */ 145 u32 global_seqno; 146 147 /** Position in the ring of the start of the request */ 148 u32 head; 149 150 /** 151 * Position in the ring of the start of the postfix. 152 * This is required to calculate the maximum available ring space 153 * without overwriting the postfix. 154 */ 155 u32 postfix; 156 157 /** Position in the ring of the end of the whole request */ 158 u32 tail; 159 160 /** Position in the ring of the end of any workarounds after the tail */ 161 u32 wa_tail; 162 163 /** Preallocate space in the ring for the emitting the request */ 164 u32 reserved_space; 165 166 /** Batch buffer related to this request if any (used for 167 * error state dump only). 168 */ 169 struct i915_vma *batch; 170 struct list_head active_list; 171 172 /** Time at which this request was emitted, in jiffies. */ 173 unsigned long emitted_jiffies; 174 175 /** engine->request_list entry for this request */ 176 struct list_head link; 177 178 /** ring->request_list entry for this request */ 179 struct list_head ring_link; 180 181 struct drm_i915_file_private *file_priv; 182 /** file_priv list entry for this request */ 183 struct list_head client_link; 184 }; 185 186 extern const struct dma_fence_ops i915_fence_ops; 187 188 static inline bool dma_fence_is_i915(const struct dma_fence *fence) 189 { 190 return fence->ops == &i915_fence_ops; 191 } 192 193 struct drm_i915_gem_request * __must_check 194 i915_gem_request_alloc(struct intel_engine_cs *engine, 195 struct i915_gem_context *ctx); 196 void i915_gem_request_retire_upto(struct drm_i915_gem_request *req); 197 198 static inline struct drm_i915_gem_request * 199 to_request(struct dma_fence *fence) 200 { 201 /* We assume that NULL fence/request are interoperable */ 202 BUILD_BUG_ON(offsetof(struct drm_i915_gem_request, fence) != 0); 203 GEM_BUG_ON(fence && !dma_fence_is_i915(fence)); 204 return container_of(fence, struct drm_i915_gem_request, fence); 205 } 206 207 static inline struct drm_i915_gem_request * 208 i915_gem_request_get(struct drm_i915_gem_request *req) 209 { 210 return to_request(dma_fence_get(&req->fence)); 211 } 212 213 static inline struct drm_i915_gem_request * 214 i915_gem_request_get_rcu(struct drm_i915_gem_request *req) 215 { 216 return to_request(dma_fence_get_rcu(&req->fence)); 217 } 218 219 static inline void 220 i915_gem_request_put(struct drm_i915_gem_request *req) 221 { 222 dma_fence_put(&req->fence); 223 } 224 225 static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst, 226 struct drm_i915_gem_request *src) 227 { 228 if (src) 229 i915_gem_request_get(src); 230 231 if (*pdst) 232 i915_gem_request_put(*pdst); 233 234 *pdst = src; 235 } 236 237 /** 238 * i915_gem_request_global_seqno - report the current global seqno 239 * @request - the request 240 * 241 * A request is assigned a global seqno only when it is on the hardware 242 * execution queue. The global seqno can be used to maintain a list of 243 * requests on the same engine in retirement order, for example for 244 * constructing a priority queue for waiting. Prior to its execution, or 245 * if it is subsequently removed in the event of preemption, its global 246 * seqno is zero. As both insertion and removal from the execution queue 247 * may operate in IRQ context, it is not guarded by the usual struct_mutex 248 * BKL. Instead those relying on the global seqno must be prepared for its 249 * value to change between reads. Only when the request is complete can 250 * the global seqno be stable (due to the memory barriers on submitting 251 * the commands to the hardware to write the breadcrumb, if the HWS shows 252 * that it has passed the global seqno and the global seqno is unchanged 253 * after the read, it is indeed complete). 254 */ 255 static u32 256 i915_gem_request_global_seqno(const struct drm_i915_gem_request *request) 257 { 258 return READ_ONCE(request->global_seqno); 259 } 260 261 int 262 i915_gem_request_await_object(struct drm_i915_gem_request *to, 263 struct drm_i915_gem_object *obj, 264 bool write); 265 int i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req, 266 struct dma_fence *fence); 267 268 void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches); 269 #define i915_add_request(req) \ 270 __i915_add_request(req, false) 271 272 void __i915_gem_request_submit(struct drm_i915_gem_request *request); 273 void i915_gem_request_submit(struct drm_i915_gem_request *request); 274 275 void __i915_gem_request_unsubmit(struct drm_i915_gem_request *request); 276 void i915_gem_request_unsubmit(struct drm_i915_gem_request *request); 277 278 struct intel_rps_client; 279 #define NO_WAITBOOST ERR_PTR(-1) 280 #define IS_RPS_CLIENT(p) (!IS_ERR(p)) 281 #define IS_RPS_USER(p) (!IS_ERR_OR_NULL(p)) 282 283 long i915_wait_request(struct drm_i915_gem_request *req, 284 unsigned int flags, 285 long timeout) 286 __attribute__((nonnull(1))); 287 #define I915_WAIT_INTERRUPTIBLE BIT(0) 288 #define I915_WAIT_LOCKED BIT(1) /* struct_mutex held, handle GPU reset */ 289 #define I915_WAIT_ALL BIT(2) /* used by i915_gem_object_wait() */ 290 291 static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine); 292 293 /** 294 * Returns true if seq1 is later than seq2. 295 */ 296 static inline bool i915_seqno_passed(u32 seq1, u32 seq2) 297 { 298 return (s32)(seq1 - seq2) >= 0; 299 } 300 301 static inline bool 302 __i915_gem_request_started(const struct drm_i915_gem_request *req, u32 seqno) 303 { 304 GEM_BUG_ON(!seqno); 305 return i915_seqno_passed(intel_engine_get_seqno(req->engine), 306 seqno - 1); 307 } 308 309 static inline bool 310 i915_gem_request_started(const struct drm_i915_gem_request *req) 311 { 312 u32 seqno; 313 314 seqno = i915_gem_request_global_seqno(req); 315 if (!seqno) 316 return false; 317 318 return __i915_gem_request_started(req, seqno); 319 } 320 321 static inline bool 322 __i915_gem_request_completed(const struct drm_i915_gem_request *req, u32 seqno) 323 { 324 GEM_BUG_ON(!seqno); 325 return i915_seqno_passed(intel_engine_get_seqno(req->engine), seqno) && 326 seqno == i915_gem_request_global_seqno(req); 327 } 328 329 static inline bool 330 i915_gem_request_completed(const struct drm_i915_gem_request *req) 331 { 332 u32 seqno; 333 334 seqno = i915_gem_request_global_seqno(req); 335 if (!seqno) 336 return false; 337 338 return __i915_gem_request_completed(req, seqno); 339 } 340 341 bool __i915_spin_request(const struct drm_i915_gem_request *request, 342 u32 seqno, int state, unsigned long timeout_us); 343 static inline bool i915_spin_request(const struct drm_i915_gem_request *request, 344 int state, unsigned long timeout_us) 345 { 346 u32 seqno; 347 348 seqno = i915_gem_request_global_seqno(request); 349 if (!seqno) 350 return 0; 351 352 return (__i915_gem_request_started(request, seqno) && 353 __i915_spin_request(request, seqno, state, timeout_us)); 354 } 355 356 /* We treat requests as fences. This is not be to confused with our 357 * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync. 358 * We use the fences to synchronize access from the CPU with activity on the 359 * GPU, for example, we should not rewrite an object's PTE whilst the GPU 360 * is reading them. We also track fences at a higher level to provide 361 * implicit synchronisation around GEM objects, e.g. set-domain will wait 362 * for outstanding GPU rendering before marking the object ready for CPU 363 * access, or a pageflip will wait until the GPU is complete before showing 364 * the frame on the scanout. 365 * 366 * In order to use a fence, the object must track the fence it needs to 367 * serialise with. For example, GEM objects want to track both read and 368 * write access so that we can perform concurrent read operations between 369 * the CPU and GPU engines, as well as waiting for all rendering to 370 * complete, or waiting for the last GPU user of a "fence register". The 371 * object then embeds a #i915_gem_active to track the most recent (in 372 * retirement order) request relevant for the desired mode of access. 373 * The #i915_gem_active is updated with i915_gem_active_set() to track the 374 * most recent fence request, typically this is done as part of 375 * i915_vma_move_to_active(). 376 * 377 * When the #i915_gem_active completes (is retired), it will 378 * signal its completion to the owner through a callback as well as mark 379 * itself as idle (i915_gem_active.request == NULL). The owner 380 * can then perform any action, such as delayed freeing of an active 381 * resource including itself. 382 */ 383 struct i915_gem_active; 384 385 typedef void (*i915_gem_retire_fn)(struct i915_gem_active *, 386 struct drm_i915_gem_request *); 387 388 struct i915_gem_active { 389 struct drm_i915_gem_request __rcu *request; 390 struct list_head link; 391 i915_gem_retire_fn retire; 392 }; 393 394 void i915_gem_retire_noop(struct i915_gem_active *, 395 struct drm_i915_gem_request *request); 396 397 /** 398 * init_request_active - prepares the activity tracker for use 399 * @active - the active tracker 400 * @func - a callback when then the tracker is retired (becomes idle), 401 * can be NULL 402 * 403 * init_request_active() prepares the embedded @active struct for use as 404 * an activity tracker, that is for tracking the last known active request 405 * associated with it. When the last request becomes idle, when it is retired 406 * after completion, the optional callback @func is invoked. 407 */ 408 static inline void 409 init_request_active(struct i915_gem_active *active, 410 i915_gem_retire_fn retire) 411 { 412 INIT_LIST_HEAD(&active->link); 413 active->retire = retire ?: i915_gem_retire_noop; 414 } 415 416 /** 417 * i915_gem_active_set - updates the tracker to watch the current request 418 * @active - the active tracker 419 * @request - the request to watch 420 * 421 * i915_gem_active_set() watches the given @request for completion. Whilst 422 * that @request is busy, the @active reports busy. When that @request is 423 * retired, the @active tracker is updated to report idle. 424 */ 425 static inline void 426 i915_gem_active_set(struct i915_gem_active *active, 427 struct drm_i915_gem_request *request) 428 { 429 list_move(&active->link, &request->active_list); 430 rcu_assign_pointer(active->request, request); 431 } 432 433 /** 434 * i915_gem_active_set_retire_fn - updates the retirement callback 435 * @active - the active tracker 436 * @fn - the routine called when the request is retired 437 * @mutex - struct_mutex used to guard retirements 438 * 439 * i915_gem_active_set_retire_fn() updates the function pointer that 440 * is called when the final request associated with the @active tracker 441 * is retired. 442 */ 443 static inline void 444 i915_gem_active_set_retire_fn(struct i915_gem_active *active, 445 i915_gem_retire_fn fn, 446 struct lock *mutex) 447 { 448 lockdep_assert_held(mutex); 449 active->retire = fn ?: i915_gem_retire_noop; 450 } 451 452 static inline struct drm_i915_gem_request * 453 __i915_gem_active_peek(const struct i915_gem_active *active) 454 { 455 /* Inside the error capture (running with the driver in an unknown 456 * state), we want to bend the rules slightly (a lot). 457 * 458 * Work is in progress to make it safer, in the meantime this keeps 459 * the known issue from spamming the logs. 460 */ 461 return rcu_dereference_protected(active->request, 1); 462 } 463 464 /** 465 * i915_gem_active_raw - return the active request 466 * @active - the active tracker 467 * 468 * i915_gem_active_raw() returns the current request being tracked, or NULL. 469 * It does not obtain a reference on the request for the caller, so the caller 470 * must hold struct_mutex. 471 */ 472 static inline struct drm_i915_gem_request * 473 i915_gem_active_raw(const struct i915_gem_active *active, struct lock *mutex) 474 { 475 return rcu_dereference_protected(active->request, 476 lockdep_is_held(mutex)); 477 } 478 479 /** 480 * i915_gem_active_peek - report the active request being monitored 481 * @active - the active tracker 482 * 483 * i915_gem_active_peek() returns the current request being tracked if 484 * still active, or NULL. It does not obtain a reference on the request 485 * for the caller, so the caller must hold struct_mutex. 486 */ 487 static inline struct drm_i915_gem_request * 488 i915_gem_active_peek(const struct i915_gem_active *active, struct lock *mutex) 489 { 490 struct drm_i915_gem_request *request; 491 492 request = i915_gem_active_raw(active, mutex); 493 if (!request || i915_gem_request_completed(request)) 494 return NULL; 495 496 return request; 497 } 498 499 /** 500 * i915_gem_active_get - return a reference to the active request 501 * @active - the active tracker 502 * 503 * i915_gem_active_get() returns a reference to the active request, or NULL 504 * if the active tracker is idle. The caller must hold struct_mutex. 505 */ 506 static inline struct drm_i915_gem_request * 507 i915_gem_active_get(const struct i915_gem_active *active, struct lock *mutex) 508 { 509 return i915_gem_request_get(i915_gem_active_peek(active, mutex)); 510 } 511 512 /** 513 * __i915_gem_active_get_rcu - return a reference to the active request 514 * @active - the active tracker 515 * 516 * __i915_gem_active_get() returns a reference to the active request, or NULL 517 * if the active tracker is idle. The caller must hold the RCU read lock, but 518 * the returned pointer is safe to use outside of RCU. 519 */ 520 static inline struct drm_i915_gem_request * 521 __i915_gem_active_get_rcu(const struct i915_gem_active *active) 522 { 523 /* Performing a lockless retrieval of the active request is super 524 * tricky. SLAB_TYPESAFE_BY_RCU merely guarantees that the backing 525 * slab of request objects will not be freed whilst we hold the 526 * RCU read lock. It does not guarantee that the request itself 527 * will not be freed and then *reused*. Viz, 528 * 529 * Thread A Thread B 530 * 531 * req = active.request 532 * retire(req) -> free(req); 533 * (req is now first on the slab freelist) 534 * active.request = NULL 535 * 536 * req = new submission on a new object 537 * ref(req) 538 * 539 * To prevent the request from being reused whilst the caller 540 * uses it, we take a reference like normal. Whilst acquiring 541 * the reference we check that it is not in a destroyed state 542 * (refcnt == 0). That prevents the request being reallocated 543 * whilst the caller holds on to it. To check that the request 544 * was not reallocated as we acquired the reference we have to 545 * check that our request remains the active request across 546 * the lookup, in the same manner as a seqlock. The visibility 547 * of the pointer versus the reference counting is controlled 548 * by using RCU barriers (rcu_dereference and rcu_assign_pointer). 549 * 550 * In the middle of all that, we inspect whether the request is 551 * complete. Retiring is lazy so the request may be completed long 552 * before the active tracker is updated. Querying whether the 553 * request is complete is far cheaper (as it involves no locked 554 * instructions setting cachelines to exclusive) than acquiring 555 * the reference, so we do it first. The RCU read lock ensures the 556 * pointer dereference is valid, but does not ensure that the 557 * seqno nor HWS is the right one! However, if the request was 558 * reallocated, that means the active tracker's request was complete. 559 * If the new request is also complete, then both are and we can 560 * just report the active tracker is idle. If the new request is 561 * incomplete, then we acquire a reference on it and check that 562 * it remained the active request. 563 * 564 * It is then imperative that we do not zero the request on 565 * reallocation, so that we can chase the dangling pointers! 566 * See i915_gem_request_alloc(). 567 */ 568 do { 569 struct drm_i915_gem_request *request; 570 571 request = rcu_dereference(active->request); 572 if (!request || i915_gem_request_completed(request)) 573 return NULL; 574 575 /* An especially silly compiler could decide to recompute the 576 * result of i915_gem_request_completed, more specifically 577 * re-emit the load for request->fence.seqno. A race would catch 578 * a later seqno value, which could flip the result from true to 579 * false. Which means part of the instructions below might not 580 * be executed, while later on instructions are executed. Due to 581 * barriers within the refcounting the inconsistency can't reach 582 * past the call to i915_gem_request_get_rcu, but not executing 583 * that while still executing i915_gem_request_put() creates 584 * havoc enough. Prevent this with a compiler barrier. 585 */ 586 barrier(); 587 588 request = i915_gem_request_get_rcu(request); 589 590 /* What stops the following rcu_access_pointer() from occurring 591 * before the above i915_gem_request_get_rcu()? If we were 592 * to read the value before pausing to get the reference to 593 * the request, we may not notice a change in the active 594 * tracker. 595 * 596 * The rcu_access_pointer() is a mere compiler barrier, which 597 * means both the CPU and compiler are free to perform the 598 * memory read without constraint. The compiler only has to 599 * ensure that any operations after the rcu_access_pointer() 600 * occur afterwards in program order. This means the read may 601 * be performed earlier by an out-of-order CPU, or adventurous 602 * compiler. 603 * 604 * The atomic operation at the heart of 605 * i915_gem_request_get_rcu(), see dma_fence_get_rcu(), is 606 * atomic_inc_not_zero() which is only a full memory barrier 607 * when successful. That is, if i915_gem_request_get_rcu() 608 * returns the request (and so with the reference counted 609 * incremented) then the following read for rcu_access_pointer() 610 * must occur after the atomic operation and so confirm 611 * that this request is the one currently being tracked. 612 * 613 * The corresponding write barrier is part of 614 * rcu_assign_pointer(). 615 */ 616 if (!request || request == rcu_access_pointer(active->request)) 617 return rcu_pointer_handoff(request); 618 619 i915_gem_request_put(request); 620 } while (1); 621 } 622 623 /** 624 * i915_gem_active_get_unlocked - return a reference to the active request 625 * @active - the active tracker 626 * 627 * i915_gem_active_get_unlocked() returns a reference to the active request, 628 * or NULL if the active tracker is idle. The reference is obtained under RCU, 629 * so no locking is required by the caller. 630 * 631 * The reference should be freed with i915_gem_request_put(). 632 */ 633 static inline struct drm_i915_gem_request * 634 i915_gem_active_get_unlocked(const struct i915_gem_active *active) 635 { 636 struct drm_i915_gem_request *request; 637 638 rcu_read_lock(); 639 request = __i915_gem_active_get_rcu(active); 640 rcu_read_unlock(); 641 642 return request; 643 } 644 645 /** 646 * i915_gem_active_isset - report whether the active tracker is assigned 647 * @active - the active tracker 648 * 649 * i915_gem_active_isset() returns true if the active tracker is currently 650 * assigned to a request. Due to the lazy retiring, that request may be idle 651 * and this may report stale information. 652 */ 653 static inline bool 654 i915_gem_active_isset(const struct i915_gem_active *active) 655 { 656 return rcu_access_pointer(active->request); 657 } 658 659 /** 660 * i915_gem_active_wait - waits until the request is completed 661 * @active - the active request on which to wait 662 * @flags - how to wait 663 * @timeout - how long to wait at most 664 * @rps - userspace client to charge for a waitboost 665 * 666 * i915_gem_active_wait() waits until the request is completed before 667 * returning, without requiring any locks to be held. Note that it does not 668 * retire any requests before returning. 669 * 670 * This function relies on RCU in order to acquire the reference to the active 671 * request without holding any locks. See __i915_gem_active_get_rcu() for the 672 * glory details on how that is managed. Once the reference is acquired, we 673 * can then wait upon the request, and afterwards release our reference, 674 * free of any locking. 675 * 676 * This function wraps i915_wait_request(), see it for the full details on 677 * the arguments. 678 * 679 * Returns 0 if successful, or a negative error code. 680 */ 681 static inline int 682 i915_gem_active_wait(const struct i915_gem_active *active, unsigned int flags) 683 { 684 struct drm_i915_gem_request *request; 685 long ret = 0; 686 687 request = i915_gem_active_get_unlocked(active); 688 if (request) { 689 ret = i915_wait_request(request, flags, MAX_SCHEDULE_TIMEOUT); 690 i915_gem_request_put(request); 691 } 692 693 return ret < 0 ? ret : 0; 694 } 695 696 /** 697 * i915_gem_active_retire - waits until the request is retired 698 * @active - the active request on which to wait 699 * 700 * i915_gem_active_retire() waits until the request is completed, 701 * and then ensures that at least the retirement handler for this 702 * @active tracker is called before returning. If the @active 703 * tracker is idle, the function returns immediately. 704 */ 705 static inline int __must_check 706 i915_gem_active_retire(struct i915_gem_active *active, 707 struct lock *mutex) 708 { 709 struct drm_i915_gem_request *request; 710 long ret; 711 712 request = i915_gem_active_raw(active, mutex); 713 if (!request) 714 return 0; 715 716 ret = i915_wait_request(request, 717 I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, 718 MAX_SCHEDULE_TIMEOUT); 719 if (ret < 0) 720 return ret; 721 722 list_del_init(&active->link); 723 RCU_INIT_POINTER(active->request, NULL); 724 725 active->retire(active, request); 726 727 return 0; 728 } 729 730 #define for_each_active(mask, idx) \ 731 for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx)) 732 733 #endif /* I915_GEM_REQUEST_H */ 734