1 /* 2 * Copyright © 2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Ben Widawsky <ben@bwidawsk.net> 25 * Michel Thierry <michel.thierry@intel.com> 26 * Thomas Daniel <thomas.daniel@intel.com> 27 * Oscar Mateo <oscar.mateo@intel.com> 28 * 29 */ 30 31 /** 32 * DOC: Logical Rings, Logical Ring Contexts and Execlists 33 * 34 * Motivation: 35 * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts". 36 * These expanded contexts enable a number of new abilities, especially 37 * "Execlists" (also implemented in this file). 38 * 39 * One of the main differences with the legacy HW contexts is that logical 40 * ring contexts incorporate many more things to the context's state, like 41 * PDPs or ringbuffer control registers: 42 * 43 * The reason why PDPs are included in the context is straightforward: as 44 * PPGTTs (per-process GTTs) are actually per-context, having the PDPs 45 * contained there mean you don't need to do a ppgtt->switch_mm yourself, 46 * instead, the GPU will do it for you on the context switch. 47 * 48 * But, what about the ringbuffer control registers (head, tail, etc..)? 49 * shouldn't we just need a set of those per engine command streamer? This is 50 * where the name "Logical Rings" starts to make sense: by virtualizing the 51 * rings, the engine cs shifts to a new "ring buffer" with every context 52 * switch. When you want to submit a workload to the GPU you: A) choose your 53 * context, B) find its appropriate virtualized ring, C) write commands to it 54 * and then, finally, D) tell the GPU to switch to that context. 55 * 56 * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch 57 * to a contexts is via a context execution list, ergo "Execlists". 58 * 59 * LRC implementation: 60 * Regarding the creation of contexts, we have: 61 * 62 * - One global default context. 63 * - One local default context for each opened fd. 64 * - One local extra context for each context create ioctl call. 65 * 66 * Now that ringbuffers belong per-context (and not per-engine, like before) 67 * and that contexts are uniquely tied to a given engine (and not reusable, 68 * like before) we need: 69 * 70 * - One ringbuffer per-engine inside each context. 71 * - One backing object per-engine inside each context. 72 * 73 * The global default context starts its life with these new objects fully 74 * allocated and populated. The local default context for each opened fd is 75 * more complex, because we don't know at creation time which engine is going 76 * to use them. To handle this, we have implemented a deferred creation of LR 77 * contexts: 78 * 79 * The local context starts its life as a hollow or blank holder, that only 80 * gets populated for a given engine once we receive an execbuffer. If later 81 * on we receive another execbuffer ioctl for the same context but a different 82 * engine, we allocate/populate a new ringbuffer and context backing object and 83 * so on. 84 * 85 * Finally, regarding local contexts created using the ioctl call: as they are 86 * only allowed with the render ring, we can allocate & populate them right 87 * away (no need to defer anything, at least for now). 88 * 89 * Execlists implementation: 90 * Execlists are the new method by which, on gen8+ hardware, workloads are 91 * submitted for execution (as opposed to the legacy, ringbuffer-based, method). 92 * This method works as follows: 93 * 94 * When a request is committed, its commands (the BB start and any leading or 95 * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer 96 * for the appropriate context. The tail pointer in the hardware context is not 97 * updated at this time, but instead, kept by the driver in the ringbuffer 98 * structure. A structure representing this request is added to a request queue 99 * for the appropriate engine: this structure contains a copy of the context's 100 * tail after the request was written to the ring buffer and a pointer to the 101 * context itself. 102 * 103 * If the engine's request queue was empty before the request was added, the 104 * queue is processed immediately. Otherwise the queue will be processed during 105 * a context switch interrupt. In any case, elements on the queue will get sent 106 * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a 107 * globally unique 20-bits submission ID. 108 * 109 * When execution of a request completes, the GPU updates the context status 110 * buffer with a context complete event and generates a context switch interrupt. 111 * During the interrupt handling, the driver examines the events in the buffer: 112 * for each context complete event, if the announced ID matches that on the head 113 * of the request queue, then that request is retired and removed from the queue. 114 * 115 * After processing, if any requests were retired and the queue is not empty 116 * then a new execution list can be submitted. The two requests at the front of 117 * the queue are next to be submitted but since a context may not occur twice in 118 * an execution list, if subsequent requests have the same ID as the first then 119 * the two requests must be combined. This is done simply by discarding requests 120 * at the head of the queue until either only one requests is left (in which case 121 * we use a NULL second context) or the first two requests have unique IDs. 122 * 123 * By always executing the first two requests in the queue the driver ensures 124 * that the GPU is kept as busy as possible. In the case where a single context 125 * completes but a second context is still executing, the request for this second 126 * context will be at the head of the queue when we remove the first one. This 127 * request will then be resubmitted along with a new request for a different context, 128 * which will cause the hardware to continue executing the second request and queue 129 * the new request (the GPU detects the condition of a context getting preempted 130 * with the same context and optimizes the context switch flow by not doing 131 * preemption, but just sampling the new tail pointer). 132 * 133 */ 134 #include <linux/interrupt.h> 135 136 #include <drm/drmP.h> 137 #include <drm/i915_drm.h> 138 #include "i915_drv.h" 139 #include "intel_mocs.h" 140 141 #define RING_EXECLIST_QFULL (1 << 0x2) 142 #define RING_EXECLIST1_VALID (1 << 0x3) 143 #define RING_EXECLIST0_VALID (1 << 0x4) 144 #define RING_EXECLIST_ACTIVE_STATUS (3 << 0xE) 145 #define RING_EXECLIST1_ACTIVE (1 << 0x11) 146 #define RING_EXECLIST0_ACTIVE (1 << 0x12) 147 148 #define GEN8_CTX_STATUS_IDLE_ACTIVE (1 << 0) 149 #define GEN8_CTX_STATUS_PREEMPTED (1 << 1) 150 #define GEN8_CTX_STATUS_ELEMENT_SWITCH (1 << 2) 151 #define GEN8_CTX_STATUS_ACTIVE_IDLE (1 << 3) 152 #define GEN8_CTX_STATUS_COMPLETE (1 << 4) 153 #define GEN8_CTX_STATUS_LITE_RESTORE (1 << 15) 154 155 #define GEN8_CTX_STATUS_COMPLETED_MASK \ 156 (GEN8_CTX_STATUS_ACTIVE_IDLE | \ 157 GEN8_CTX_STATUS_PREEMPTED | \ 158 GEN8_CTX_STATUS_ELEMENT_SWITCH) 159 160 #define CTX_LRI_HEADER_0 0x01 161 #define CTX_CONTEXT_CONTROL 0x02 162 #define CTX_RING_HEAD 0x04 163 #define CTX_RING_TAIL 0x06 164 #define CTX_RING_BUFFER_START 0x08 165 #define CTX_RING_BUFFER_CONTROL 0x0a 166 #define CTX_BB_HEAD_U 0x0c 167 #define CTX_BB_HEAD_L 0x0e 168 #define CTX_BB_STATE 0x10 169 #define CTX_SECOND_BB_HEAD_U 0x12 170 #define CTX_SECOND_BB_HEAD_L 0x14 171 #define CTX_SECOND_BB_STATE 0x16 172 #define CTX_BB_PER_CTX_PTR 0x18 173 #define CTX_RCS_INDIRECT_CTX 0x1a 174 #define CTX_RCS_INDIRECT_CTX_OFFSET 0x1c 175 #define CTX_LRI_HEADER_1 0x21 176 #define CTX_CTX_TIMESTAMP 0x22 177 #define CTX_PDP3_UDW 0x24 178 #define CTX_PDP3_LDW 0x26 179 #define CTX_PDP2_UDW 0x28 180 #define CTX_PDP2_LDW 0x2a 181 #define CTX_PDP1_UDW 0x2c 182 #define CTX_PDP1_LDW 0x2e 183 #define CTX_PDP0_UDW 0x30 184 #define CTX_PDP0_LDW 0x32 185 #define CTX_LRI_HEADER_2 0x41 186 #define CTX_R_PWR_CLK_STATE 0x42 187 #define CTX_GPGPU_CSR_BASE_ADDRESS 0x44 188 189 #define CTX_REG(reg_state, pos, reg, val) do { \ 190 (reg_state)[(pos)+0] = i915_mmio_reg_offset(reg); \ 191 (reg_state)[(pos)+1] = (val); \ 192 } while (0) 193 194 #define ASSIGN_CTX_PDP(ppgtt, reg_state, n) do { \ 195 const u64 _addr = i915_page_dir_dma_addr((ppgtt), (n)); \ 196 reg_state[CTX_PDP ## n ## _UDW+1] = upper_32_bits(_addr); \ 197 reg_state[CTX_PDP ## n ## _LDW+1] = lower_32_bits(_addr); \ 198 } while (0) 199 200 #define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \ 201 reg_state[CTX_PDP0_UDW + 1] = upper_32_bits(px_dma(&ppgtt->pml4)); \ 202 reg_state[CTX_PDP0_LDW + 1] = lower_32_bits(px_dma(&ppgtt->pml4)); \ 203 } while (0) 204 205 #define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17 206 #define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x26 207 #define GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x19 208 209 /* Typical size of the average request (2 pipecontrols and a MI_BB) */ 210 #define EXECLISTS_REQUEST_SIZE 64 /* bytes */ 211 #define WA_TAIL_DWORDS 2 212 #define WA_TAIL_BYTES (sizeof(u32) * WA_TAIL_DWORDS) 213 #define PREEMPT_ID 0x1 214 215 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx, 216 struct intel_engine_cs *engine); 217 static void execlists_init_reg_state(u32 *reg_state, 218 struct i915_gem_context *ctx, 219 struct intel_engine_cs *engine, 220 struct intel_ring *ring); 221 222 /** 223 * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists 224 * @dev_priv: i915 device private 225 * @enable_execlists: value of i915.enable_execlists module parameter. 226 * 227 * Only certain platforms support Execlists (the prerequisites being 228 * support for Logical Ring Contexts and Aliasing PPGTT or better). 229 * 230 * Return: 1 if Execlists is supported and has to be enabled. 231 */ 232 int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, int enable_execlists) 233 { 234 /* On platforms with execlist available, vGPU will only 235 * support execlist mode, no ring buffer mode. 236 */ 237 if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && intel_vgpu_active(dev_priv)) 238 return 1; 239 240 if (INTEL_GEN(dev_priv) >= 9) 241 return 1; 242 243 if (enable_execlists == 0) 244 return 0; 245 246 if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && 247 USES_PPGTT(dev_priv)) 248 return 1; 249 250 return 0; 251 } 252 253 /** 254 * intel_lr_context_descriptor_update() - calculate & cache the descriptor 255 * descriptor for a pinned context 256 * @ctx: Context to work on 257 * @engine: Engine the descriptor will be used with 258 * 259 * The context descriptor encodes various attributes of a context, 260 * including its GTT address and some flags. Because it's fairly 261 * expensive to calculate, we'll just do it once and cache the result, 262 * which remains valid until the context is unpinned. 263 * 264 * This is what a descriptor looks like, from LSB to MSB:: 265 * 266 * bits 0-11: flags, GEN8_CTX_* (cached in ctx->desc_template) 267 * bits 12-31: LRCA, GTT address of (the HWSP of) this context 268 * bits 32-52: ctx ID, a globally unique tag 269 * bits 53-54: mbz, reserved for use by hardware 270 * bits 55-63: group ID, currently unused and set to 0 271 */ 272 static void 273 intel_lr_context_descriptor_update(struct i915_gem_context *ctx, 274 struct intel_engine_cs *engine) 275 { 276 struct intel_context *ce = &ctx->engine[engine->id]; 277 u64 desc; 278 279 BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (1<<GEN8_CTX_ID_WIDTH)); 280 281 desc = ctx->desc_template; /* bits 0-11 */ 282 desc |= i915_ggtt_offset(ce->state) + LRC_HEADER_PAGES * PAGE_SIZE; 283 /* bits 12-31 */ 284 desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT; /* bits 32-52 */ 285 286 ce->lrc_desc = desc; 287 } 288 289 static struct i915_priolist * 290 lookup_priolist(struct intel_engine_cs *engine, 291 struct i915_priotree *pt, 292 int prio) 293 { 294 struct intel_engine_execlists * const execlists = &engine->execlists; 295 struct i915_priolist *p; 296 struct rb_node **parent, *rb; 297 bool first = true; 298 299 if (unlikely(execlists->no_priolist)) 300 prio = I915_PRIORITY_NORMAL; 301 302 find_priolist: 303 /* most positive priority is scheduled first, equal priorities fifo */ 304 rb = NULL; 305 parent = &execlists->queue.rb_node; 306 while (*parent) { 307 rb = *parent; 308 p = rb_entry(rb, typeof(*p), node); 309 if (prio > p->priority) { 310 parent = &rb->rb_left; 311 } else if (prio < p->priority) { 312 parent = &rb->rb_right; 313 first = false; 314 } else { 315 return p; 316 } 317 } 318 319 if (prio == I915_PRIORITY_NORMAL) { 320 p = &execlists->default_priolist; 321 } else { 322 p = kmem_cache_alloc(engine->i915->priorities, GFP_ATOMIC); 323 /* Convert an allocation failure to a priority bump */ 324 if (unlikely(!p)) { 325 prio = I915_PRIORITY_NORMAL; /* recurses just once */ 326 327 /* To maintain ordering with all rendering, after an 328 * allocation failure we have to disable all scheduling. 329 * Requests will then be executed in fifo, and schedule 330 * will ensure that dependencies are emitted in fifo. 331 * There will be still some reordering with existing 332 * requests, so if userspace lied about their 333 * dependencies that reordering may be visible. 334 */ 335 execlists->no_priolist = true; 336 goto find_priolist; 337 } 338 } 339 340 p->priority = prio; 341 INIT_LIST_HEAD(&p->requests); 342 rb_link_node(&p->node, rb, parent); 343 rb_insert_color(&p->node, &execlists->queue); 344 345 if (first) 346 execlists->first = &p->node; 347 348 return ptr_pack_bits(p, first, 1); 349 } 350 351 static void unwind_wa_tail(struct drm_i915_gem_request *rq) 352 { 353 rq->tail = intel_ring_wrap(rq->ring, rq->wa_tail - WA_TAIL_BYTES); 354 assert_ring_tail_valid(rq->ring, rq->tail); 355 } 356 357 static void unwind_incomplete_requests(struct intel_engine_cs *engine) 358 { 359 struct drm_i915_gem_request *rq, *rn; 360 struct i915_priolist *p = NULL; 361 int last_prio = I915_PRIORITY_INVALID; 362 363 lockdep_assert_held(&engine->timeline->lock); 364 365 list_for_each_entry_safe_reverse(rq, rn, 366 &engine->timeline->requests, 367 link) { 368 if (i915_gem_request_completed(rq)) 369 return; 370 371 __i915_gem_request_unsubmit(rq); 372 unwind_wa_tail(rq); 373 374 GEM_BUG_ON(rq->priotree.priority == I915_PRIORITY_INVALID); 375 if (rq->priotree.priority != last_prio) { 376 p = lookup_priolist(engine, 377 &rq->priotree, 378 rq->priotree.priority); 379 p = ptr_mask_bits(p, 1); 380 381 last_prio = rq->priotree.priority; 382 } 383 384 list_add(&rq->priotree.link, &p->requests); 385 } 386 } 387 388 static inline void 389 execlists_context_status_change(struct drm_i915_gem_request *rq, 390 unsigned long status) 391 { 392 /* 393 * Only used when GVT-g is enabled now. When GVT-g is disabled, 394 * The compiler should eliminate this function as dead-code. 395 */ 396 if (!IS_ENABLED(CONFIG_DRM_I915_GVT)) 397 return; 398 399 atomic_notifier_call_chain(&rq->engine->context_status_notifier, 400 status, rq); 401 } 402 403 static void 404 execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state) 405 { 406 ASSIGN_CTX_PDP(ppgtt, reg_state, 3); 407 ASSIGN_CTX_PDP(ppgtt, reg_state, 2); 408 ASSIGN_CTX_PDP(ppgtt, reg_state, 1); 409 ASSIGN_CTX_PDP(ppgtt, reg_state, 0); 410 } 411 412 static u64 execlists_update_context(struct drm_i915_gem_request *rq) 413 { 414 struct intel_context *ce = &rq->ctx->engine[rq->engine->id]; 415 struct i915_hw_ppgtt *ppgtt = 416 rq->ctx->ppgtt ?: rq->i915->mm.aliasing_ppgtt; 417 u32 *reg_state = ce->lrc_reg_state; 418 419 reg_state[CTX_RING_TAIL+1] = intel_ring_set_tail(rq->ring, rq->tail); 420 421 /* True 32b PPGTT with dynamic page allocation: update PDP 422 * registers and point the unallocated PDPs to scratch page. 423 * PML4 is allocated during ppgtt init, so this is not needed 424 * in 48-bit mode. 425 */ 426 if (ppgtt && !i915_vm_is_48bit(&ppgtt->base)) 427 execlists_update_context_pdps(ppgtt, reg_state); 428 429 return ce->lrc_desc; 430 } 431 432 static inline void elsp_write(u64 desc, u32 __iomem *elsp) 433 { 434 writel(upper_32_bits(desc), elsp); 435 writel(lower_32_bits(desc), elsp); 436 } 437 438 static void execlists_submit_ports(struct intel_engine_cs *engine) 439 { 440 struct execlist_port *port = engine->execlists.port; 441 u32 __iomem *elsp = 442 engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine)); 443 unsigned int n; 444 445 for (n = execlists_num_ports(&engine->execlists); n--; ) { 446 struct drm_i915_gem_request *rq; 447 unsigned int count; 448 u64 desc; 449 450 rq = port_unpack(&port[n], &count); 451 if (rq) { 452 GEM_BUG_ON(count > !n); 453 if (!count++) 454 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN); 455 port_set(&port[n], port_pack(rq, count)); 456 desc = execlists_update_context(rq); 457 GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc)); 458 } else { 459 GEM_BUG_ON(!n); 460 desc = 0; 461 } 462 463 elsp_write(desc, elsp); 464 } 465 } 466 467 static bool ctx_single_port_submission(const struct i915_gem_context *ctx) 468 { 469 return (IS_ENABLED(CONFIG_DRM_I915_GVT) && 470 i915_gem_context_force_single_submission(ctx)); 471 } 472 473 static bool can_merge_ctx(const struct i915_gem_context *prev, 474 const struct i915_gem_context *next) 475 { 476 if (prev != next) 477 return false; 478 479 if (ctx_single_port_submission(prev)) 480 return false; 481 482 return true; 483 } 484 485 static void port_assign(struct execlist_port *port, 486 struct drm_i915_gem_request *rq) 487 { 488 GEM_BUG_ON(rq == port_request(port)); 489 490 if (port_isset(port)) 491 i915_gem_request_put(port_request(port)); 492 493 port_set(port, port_pack(i915_gem_request_get(rq), port_count(port))); 494 } 495 496 static void inject_preempt_context(struct intel_engine_cs *engine) 497 { 498 struct intel_context *ce = 499 &engine->i915->preempt_context->engine[engine->id]; 500 u32 __iomem *elsp = 501 engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine)); 502 unsigned int n; 503 504 GEM_BUG_ON(engine->i915->preempt_context->hw_id != PREEMPT_ID); 505 GEM_BUG_ON(!IS_ALIGNED(ce->ring->size, WA_TAIL_BYTES)); 506 507 memset(ce->ring->vaddr + ce->ring->tail, 0, WA_TAIL_BYTES); 508 ce->ring->tail += WA_TAIL_BYTES; 509 ce->ring->tail &= (ce->ring->size - 1); 510 ce->lrc_reg_state[CTX_RING_TAIL+1] = ce->ring->tail; 511 512 for (n = execlists_num_ports(&engine->execlists); --n; ) 513 elsp_write(0, elsp); 514 515 elsp_write(ce->lrc_desc, elsp); 516 } 517 518 static bool can_preempt(struct intel_engine_cs *engine) 519 { 520 return INTEL_INFO(engine->i915)->has_logical_ring_preemption; 521 } 522 523 static void execlists_dequeue(struct intel_engine_cs *engine) 524 { 525 struct intel_engine_execlists * const execlists = &engine->execlists; 526 struct execlist_port *port = execlists->port; 527 const struct execlist_port * const last_port = 528 &execlists->port[execlists->port_mask]; 529 struct drm_i915_gem_request *last = port_request(port); 530 struct rb_node *rb; 531 bool submit = false; 532 533 /* Hardware submission is through 2 ports. Conceptually each port 534 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is 535 * static for a context, and unique to each, so we only execute 536 * requests belonging to a single context from each ring. RING_HEAD 537 * is maintained by the CS in the context image, it marks the place 538 * where it got up to last time, and through RING_TAIL we tell the CS 539 * where we want to execute up to this time. 540 * 541 * In this list the requests are in order of execution. Consecutive 542 * requests from the same context are adjacent in the ringbuffer. We 543 * can combine these requests into a single RING_TAIL update: 544 * 545 * RING_HEAD...req1...req2 546 * ^- RING_TAIL 547 * since to execute req2 the CS must first execute req1. 548 * 549 * Our goal then is to point each port to the end of a consecutive 550 * sequence of requests as being the most optimal (fewest wake ups 551 * and context switches) submission. 552 */ 553 554 spin_lock_irq(&engine->timeline->lock); 555 rb = execlists->first; 556 GEM_BUG_ON(rb_first(&execlists->queue) != rb); 557 if (!rb) 558 goto unlock; 559 560 if (last) { 561 /* 562 * Don't resubmit or switch until all outstanding 563 * preemptions (lite-restore) are seen. Then we 564 * know the next preemption status we see corresponds 565 * to this ELSP update. 566 */ 567 if (port_count(&port[0]) > 1) 568 goto unlock; 569 570 if (can_preempt(engine) && 571 rb_entry(rb, struct i915_priolist, node)->priority > 572 max(last->priotree.priority, 0)) { 573 /* 574 * Switch to our empty preempt context so 575 * the state of the GPU is known (idle). 576 */ 577 inject_preempt_context(engine); 578 execlists_set_active(execlists, 579 EXECLISTS_ACTIVE_PREEMPT); 580 goto unlock; 581 } else { 582 /* 583 * In theory, we could coalesce more requests onto 584 * the second port (the first port is active, with 585 * no preemptions pending). However, that means we 586 * then have to deal with the possible lite-restore 587 * of the second port (as we submit the ELSP, there 588 * may be a context-switch) but also we may complete 589 * the resubmission before the context-switch. Ergo, 590 * coalescing onto the second port will cause a 591 * preemption event, but we cannot predict whether 592 * that will affect port[0] or port[1]. 593 * 594 * If the second port is already active, we can wait 595 * until the next context-switch before contemplating 596 * new requests. The GPU will be busy and we should be 597 * able to resubmit the new ELSP before it idles, 598 * avoiding pipeline bubbles (momentary pauses where 599 * the driver is unable to keep up the supply of new 600 * work). 601 */ 602 if (port_count(&port[1])) 603 goto unlock; 604 605 /* WaIdleLiteRestore:bdw,skl 606 * Apply the wa NOOPs to prevent 607 * ring:HEAD == req:TAIL as we resubmit the 608 * request. See gen8_emit_breadcrumb() for 609 * where we prepare the padding after the 610 * end of the request. 611 */ 612 last->tail = last->wa_tail; 613 } 614 } 615 616 do { 617 struct i915_priolist *p = rb_entry(rb, typeof(*p), node); 618 struct drm_i915_gem_request *rq, *rn; 619 620 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) { 621 /* 622 * Can we combine this request with the current port? 623 * It has to be the same context/ringbuffer and not 624 * have any exceptions (e.g. GVT saying never to 625 * combine contexts). 626 * 627 * If we can combine the requests, we can execute both 628 * by updating the RING_TAIL to point to the end of the 629 * second request, and so we never need to tell the 630 * hardware about the first. 631 */ 632 if (last && !can_merge_ctx(rq->ctx, last->ctx)) { 633 /* 634 * If we are on the second port and cannot 635 * combine this request with the last, then we 636 * are done. 637 */ 638 if (port == last_port) { 639 __list_del_many(&p->requests, 640 &rq->priotree.link); 641 goto done; 642 } 643 644 /* 645 * If GVT overrides us we only ever submit 646 * port[0], leaving port[1] empty. Note that we 647 * also have to be careful that we don't queue 648 * the same context (even though a different 649 * request) to the second port. 650 */ 651 if (ctx_single_port_submission(last->ctx) || 652 ctx_single_port_submission(rq->ctx)) { 653 __list_del_many(&p->requests, 654 &rq->priotree.link); 655 goto done; 656 } 657 658 GEM_BUG_ON(last->ctx == rq->ctx); 659 660 if (submit) 661 port_assign(port, last); 662 port++; 663 664 GEM_BUG_ON(port_isset(port)); 665 } 666 667 INIT_LIST_HEAD(&rq->priotree.link); 668 __i915_gem_request_submit(rq); 669 trace_i915_gem_request_in(rq, port_index(port, execlists)); 670 last = rq; 671 submit = true; 672 } 673 674 rb = rb_next(rb); 675 rb_erase(&p->node, &execlists->queue); 676 INIT_LIST_HEAD(&p->requests); 677 if (p->priority != I915_PRIORITY_NORMAL) 678 kmem_cache_free(engine->i915->priorities, p); 679 } while (rb); 680 done: 681 execlists->first = rb; 682 if (submit) 683 port_assign(port, last); 684 unlock: 685 spin_unlock_irq(&engine->timeline->lock); 686 687 if (submit) { 688 execlists_set_active(execlists, EXECLISTS_ACTIVE_USER); 689 execlists_submit_ports(engine); 690 } 691 } 692 693 static void 694 execlist_cancel_port_requests(struct intel_engine_execlists *execlists) 695 { 696 struct execlist_port *port = execlists->port; 697 unsigned int num_ports = execlists_num_ports(execlists); 698 699 while (num_ports-- && port_isset(port)) { 700 struct drm_i915_gem_request *rq = port_request(port); 701 702 GEM_BUG_ON(!execlists->active); 703 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_PREEMPTED); 704 i915_gem_request_put(rq); 705 706 memset(port, 0, sizeof(*port)); 707 port++; 708 } 709 } 710 711 static void execlists_cancel_requests(struct intel_engine_cs *engine) 712 { 713 struct intel_engine_execlists * const execlists = &engine->execlists; 714 struct drm_i915_gem_request *rq, *rn; 715 struct rb_node *rb; 716 unsigned long flags; 717 718 spin_lock_irqsave(&engine->timeline->lock, flags); 719 720 /* Cancel the requests on the HW and clear the ELSP tracker. */ 721 execlist_cancel_port_requests(execlists); 722 723 /* Mark all executing requests as skipped. */ 724 list_for_each_entry(rq, &engine->timeline->requests, link) { 725 GEM_BUG_ON(!rq->global_seqno); 726 if (!i915_gem_request_completed(rq)) 727 dma_fence_set_error(&rq->fence, -EIO); 728 } 729 730 /* Flush the queued requests to the timeline list (for retiring). */ 731 rb = execlists->first; 732 while (rb) { 733 struct i915_priolist *p = rb_entry(rb, typeof(*p), node); 734 735 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) { 736 INIT_LIST_HEAD(&rq->priotree.link); 737 738 dma_fence_set_error(&rq->fence, -EIO); 739 __i915_gem_request_submit(rq); 740 } 741 742 rb = rb_next(rb); 743 rb_erase(&p->node, &execlists->queue); 744 INIT_LIST_HEAD(&p->requests); 745 if (p->priority != I915_PRIORITY_NORMAL) 746 kmem_cache_free(engine->i915->priorities, p); 747 } 748 749 /* Remaining _unready_ requests will be nop'ed when submitted */ 750 751 752 execlists->queue = LINUX_RB_ROOT; 753 execlists->first = NULL; 754 GEM_BUG_ON(port_isset(execlists->port)); 755 756 /* 757 * The port is checked prior to scheduling a tasklet, but 758 * just in case we have suspended the tasklet to do the 759 * wedging make sure that when it wakes, it decides there 760 * is no work to do by clearing the irq_posted bit. 761 */ 762 clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted); 763 764 spin_unlock_irqrestore(&engine->timeline->lock, flags); 765 } 766 767 /* 768 * Check the unread Context Status Buffers and manage the submission of new 769 * contexts to the ELSP accordingly. 770 */ 771 static void intel_lrc_irq_handler(unsigned long data) 772 { 773 struct intel_engine_cs * const engine = (struct intel_engine_cs *)data; 774 struct intel_engine_execlists * const execlists = &engine->execlists; 775 struct execlist_port * const port = execlists->port; 776 struct drm_i915_private *dev_priv = engine->i915; 777 778 /* We can skip acquiring intel_runtime_pm_get() here as it was taken 779 * on our behalf by the request (see i915_gem_mark_busy()) and it will 780 * not be relinquished until the device is idle (see 781 * i915_gem_idle_work_handler()). As a precaution, we make sure 782 * that all ELSP are drained i.e. we have processed the CSB, 783 * before allowing ourselves to idle and calling intel_runtime_pm_put(). 784 */ 785 GEM_BUG_ON(!dev_priv->gt.awake); 786 787 intel_uncore_forcewake_get(dev_priv, execlists->fw_domains); 788 789 /* Prefer doing test_and_clear_bit() as a two stage operation to avoid 790 * imposing the cost of a locked atomic transaction when submitting a 791 * new request (outside of the context-switch interrupt). 792 */ 793 while (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted)) { 794 /* The HWSP contains a (cacheable) mirror of the CSB */ 795 const u32 *buf = 796 &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX]; 797 unsigned int head, tail; 798 799 if (unlikely(execlists->csb_use_mmio)) { 800 buf = (u32 * __force) 801 (dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0))); 802 execlists->csb_head = -1; /* force mmio read of CSB ptrs */ 803 } 804 805 /* The write will be ordered by the uncached read (itself 806 * a memory barrier), so we do not need another in the form 807 * of a locked instruction. The race between the interrupt 808 * handler and the split test/clear is harmless as we order 809 * our clear before the CSB read. If the interrupt arrived 810 * first between the test and the clear, we read the updated 811 * CSB and clear the bit. If the interrupt arrives as we read 812 * the CSB or later (i.e. after we had cleared the bit) the bit 813 * is set and we do a new loop. 814 */ 815 __clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted); 816 if (unlikely(execlists->csb_head == -1)) { /* following a reset */ 817 head = readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine))); 818 tail = GEN8_CSB_WRITE_PTR(head); 819 head = GEN8_CSB_READ_PTR(head); 820 execlists->csb_head = head; 821 } else { 822 const int write_idx = 823 intel_hws_csb_write_index(dev_priv) - 824 I915_HWS_CSB_BUF0_INDEX; 825 826 head = execlists->csb_head; 827 tail = READ_ONCE(buf[write_idx]); 828 } 829 830 while (head != tail) { 831 struct drm_i915_gem_request *rq; 832 unsigned int status; 833 unsigned int count; 834 835 if (++head == GEN8_CSB_ENTRIES) 836 head = 0; 837 838 /* We are flying near dragons again. 839 * 840 * We hold a reference to the request in execlist_port[] 841 * but no more than that. We are operating in softirq 842 * context and so cannot hold any mutex or sleep. That 843 * prevents us stopping the requests we are processing 844 * in port[] from being retired simultaneously (the 845 * breadcrumb will be complete before we see the 846 * context-switch). As we only hold the reference to the 847 * request, any pointer chasing underneath the request 848 * is subject to a potential use-after-free. Thus we 849 * store all of the bookkeeping within port[] as 850 * required, and avoid using unguarded pointers beneath 851 * request itself. The same applies to the atomic 852 * status notifier. 853 */ 854 855 status = READ_ONCE(buf[2 * head]); /* maybe mmio! */ 856 if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK)) 857 continue; 858 859 if (status & GEN8_CTX_STATUS_ACTIVE_IDLE && 860 buf[2*head + 1] == PREEMPT_ID) { 861 execlist_cancel_port_requests(execlists); 862 863 spin_lock_irq(&engine->timeline->lock); 864 unwind_incomplete_requests(engine); 865 spin_unlock_irq(&engine->timeline->lock); 866 867 GEM_BUG_ON(!execlists_is_active(execlists, 868 EXECLISTS_ACTIVE_PREEMPT)); 869 execlists_clear_active(execlists, 870 EXECLISTS_ACTIVE_PREEMPT); 871 continue; 872 } 873 874 if (status & GEN8_CTX_STATUS_PREEMPTED && 875 execlists_is_active(execlists, 876 EXECLISTS_ACTIVE_PREEMPT)) 877 continue; 878 879 GEM_BUG_ON(!execlists_is_active(execlists, 880 EXECLISTS_ACTIVE_USER)); 881 882 /* Check the context/desc id for this event matches */ 883 GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id); 884 885 rq = port_unpack(port, &count); 886 GEM_BUG_ON(count == 0); 887 if (--count == 0) { 888 GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED); 889 /* 890 * XXX DragonFly XXX 891 * 892 * This gets hit for me on an i5-6500 on X 893 * startup. Report and ignore for now. May 894 * be related to a ring timeout during early 895 * startup. 896 */ 897 //GEM_BUG_ON(!i915_gem_request_completed(rq)); 898 if (!i915_gem_request_completed(rq)) { 899 kprintf("i915: warning, request %p " 900 "not completed\n", rq); 901 } 902 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT); 903 904 trace_i915_gem_request_out(rq); 905 i915_gem_request_put(rq); 906 907 execlists_port_complete(execlists, port); 908 } else { 909 port_set(port, port_pack(rq, count)); 910 } 911 912 /* After the final element, the hw should be idle */ 913 GEM_BUG_ON(port_count(port) == 0 && 914 !(status & GEN8_CTX_STATUS_ACTIVE_IDLE)); 915 if (port_count(port) == 0) 916 execlists_clear_active(execlists, 917 EXECLISTS_ACTIVE_USER); 918 } 919 920 if (head != execlists->csb_head) { 921 execlists->csb_head = head; 922 writel(_MASKED_FIELD(GEN8_CSB_READ_PTR_MASK, head << 8), 923 dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine))); 924 } 925 } 926 927 if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT)) 928 execlists_dequeue(engine); 929 930 intel_uncore_forcewake_put(dev_priv, execlists->fw_domains); 931 } 932 933 static void insert_request(struct intel_engine_cs *engine, 934 struct i915_priotree *pt, 935 int prio) 936 { 937 struct i915_priolist *p = lookup_priolist(engine, pt, prio); 938 939 list_add_tail(&pt->link, &ptr_mask_bits(p, 1)->requests); 940 if (ptr_unmask_bits(p, 1)) 941 tasklet_hi_schedule(&engine->execlists.irq_tasklet); 942 } 943 944 static void execlists_submit_request(struct drm_i915_gem_request *request) 945 { 946 struct intel_engine_cs *engine = request->engine; 947 unsigned long flags; 948 949 /* Will be called from irq-context when using foreign fences. */ 950 spin_lock_irqsave(&engine->timeline->lock, flags); 951 952 insert_request(engine, &request->priotree, request->priotree.priority); 953 954 GEM_BUG_ON(!engine->execlists.first); 955 GEM_BUG_ON(list_empty(&request->priotree.link)); 956 957 spin_unlock_irqrestore(&engine->timeline->lock, flags); 958 } 959 960 static struct drm_i915_gem_request *pt_to_request(struct i915_priotree *pt) 961 { 962 return container_of(pt, struct drm_i915_gem_request, priotree); 963 } 964 965 static struct intel_engine_cs * 966 pt_lock_engine(struct i915_priotree *pt, struct intel_engine_cs *locked) 967 { 968 struct intel_engine_cs *engine = pt_to_request(pt)->engine; 969 970 GEM_BUG_ON(!locked); 971 972 if (engine != locked) { 973 lockmgr(&locked->timeline->lock, LK_RELEASE); 974 lockmgr(&engine->timeline->lock, LK_EXCLUSIVE); 975 } 976 977 return engine; 978 } 979 980 static void execlists_schedule(struct drm_i915_gem_request *request, int prio) 981 { 982 struct intel_engine_cs *engine; 983 struct i915_dependency *dep, *p; 984 struct i915_dependency stack; 985 LINUX_LIST_HEAD(dfs); 986 987 GEM_BUG_ON(prio == I915_PRIORITY_INVALID); 988 989 if (i915_gem_request_completed(request)) 990 return; 991 992 if (prio <= READ_ONCE(request->priotree.priority)) 993 return; 994 995 /* Need BKL in order to use the temporary link inside i915_dependency */ 996 lockdep_assert_held(&request->i915->drm.struct_mutex); 997 998 stack.signaler = &request->priotree; 999 list_add(&stack.dfs_link, &dfs); 1000 1001 /* Recursively bump all dependent priorities to match the new request. 1002 * 1003 * A naive approach would be to use recursion: 1004 * static void update_priorities(struct i915_priotree *pt, prio) { 1005 * list_for_each_entry(dep, &pt->signalers_list, signal_link) 1006 * update_priorities(dep->signal, prio) 1007 * insert_request(pt); 1008 * } 1009 * but that may have unlimited recursion depth and so runs a very 1010 * real risk of overunning the kernel stack. Instead, we build 1011 * a flat list of all dependencies starting with the current request. 1012 * As we walk the list of dependencies, we add all of its dependencies 1013 * to the end of the list (this may include an already visited 1014 * request) and continue to walk onwards onto the new dependencies. The 1015 * end result is a topological list of requests in reverse order, the 1016 * last element in the list is the request we must execute first. 1017 */ 1018 list_for_each_entry_safe(dep, p, &dfs, dfs_link) { 1019 struct i915_priotree *pt = dep->signaler; 1020 1021 /* Within an engine, there can be no cycle, but we may 1022 * refer to the same dependency chain multiple times 1023 * (redundant dependencies are not eliminated) and across 1024 * engines. 1025 */ 1026 list_for_each_entry(p, &pt->signalers_list, signal_link) { 1027 if (i915_gem_request_completed(pt_to_request(p->signaler))) 1028 continue; 1029 1030 GEM_BUG_ON(p->signaler->priority < pt->priority); 1031 if (prio > READ_ONCE(p->signaler->priority)) 1032 list_move_tail(&p->dfs_link, &dfs); 1033 } 1034 1035 list_safe_reset_next(dep, p, dfs_link); 1036 } 1037 1038 /* If we didn't need to bump any existing priorities, and we haven't 1039 * yet submitted this request (i.e. there is no potential race with 1040 * execlists_submit_request()), we can set our own priority and skip 1041 * acquiring the engine locks. 1042 */ 1043 if (request->priotree.priority == I915_PRIORITY_INVALID) { 1044 GEM_BUG_ON(!list_empty(&request->priotree.link)); 1045 request->priotree.priority = prio; 1046 if (stack.dfs_link.next == stack.dfs_link.prev) 1047 return; 1048 __list_del_entry(&stack.dfs_link); 1049 } 1050 1051 engine = request->engine; 1052 spin_lock_irq(&engine->timeline->lock); 1053 1054 /* Fifo and depth-first replacement ensure our deps execute before us */ 1055 list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) { 1056 struct i915_priotree *pt = dep->signaler; 1057 1058 INIT_LIST_HEAD(&dep->dfs_link); 1059 1060 engine = pt_lock_engine(pt, engine); 1061 1062 if (prio <= pt->priority) 1063 continue; 1064 1065 pt->priority = prio; 1066 if (!list_empty(&pt->link)) { 1067 __list_del_entry(&pt->link); 1068 insert_request(engine, pt, prio); 1069 } 1070 } 1071 1072 spin_unlock_irq(&engine->timeline->lock); 1073 } 1074 1075 static struct intel_ring * 1076 execlists_context_pin(struct intel_engine_cs *engine, 1077 struct i915_gem_context *ctx) 1078 { 1079 struct intel_context *ce = &ctx->engine[engine->id]; 1080 unsigned int flags; 1081 void *vaddr; 1082 int ret; 1083 1084 lockdep_assert_held(&ctx->i915->drm.struct_mutex); 1085 1086 if (likely(ce->pin_count++)) 1087 goto out; 1088 GEM_BUG_ON(!ce->pin_count); /* no overflow please! */ 1089 1090 if (!ce->state) { 1091 ret = execlists_context_deferred_alloc(ctx, engine); 1092 if (ret) 1093 goto err; 1094 } 1095 GEM_BUG_ON(!ce->state); 1096 1097 flags = PIN_GLOBAL | PIN_HIGH; 1098 if (ctx->ggtt_offset_bias) 1099 flags |= PIN_OFFSET_BIAS | ctx->ggtt_offset_bias; 1100 1101 ret = i915_vma_pin(ce->state, 0, GEN8_LR_CONTEXT_ALIGN, flags); 1102 if (ret) 1103 goto err; 1104 1105 vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB); 1106 if (IS_ERR(vaddr)) { 1107 ret = PTR_ERR(vaddr); 1108 goto unpin_vma; 1109 } 1110 1111 ret = intel_ring_pin(ce->ring, ctx->i915, ctx->ggtt_offset_bias); 1112 if (ret) 1113 goto unpin_map; 1114 1115 intel_lr_context_descriptor_update(ctx, engine); 1116 1117 ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE; 1118 ce->lrc_reg_state[CTX_RING_BUFFER_START+1] = 1119 i915_ggtt_offset(ce->ring->vma); 1120 1121 ce->state->obj->mm.dirty = true; 1122 ce->state->obj->pin_global++; 1123 1124 i915_gem_context_get(ctx); 1125 out: 1126 return ce->ring; 1127 1128 unpin_map: 1129 i915_gem_object_unpin_map(ce->state->obj); 1130 unpin_vma: 1131 __i915_vma_unpin(ce->state); 1132 err: 1133 ce->pin_count = 0; 1134 return ERR_PTR(ret); 1135 } 1136 1137 static void execlists_context_unpin(struct intel_engine_cs *engine, 1138 struct i915_gem_context *ctx) 1139 { 1140 struct intel_context *ce = &ctx->engine[engine->id]; 1141 1142 lockdep_assert_held(&ctx->i915->drm.struct_mutex); 1143 GEM_BUG_ON(ce->pin_count == 0); 1144 1145 if (--ce->pin_count) 1146 return; 1147 1148 intel_ring_unpin(ce->ring); 1149 1150 ce->state->obj->pin_global--; 1151 i915_gem_object_unpin_map(ce->state->obj); 1152 i915_vma_unpin(ce->state); 1153 1154 i915_gem_context_put(ctx); 1155 } 1156 1157 static int execlists_request_alloc(struct drm_i915_gem_request *request) 1158 { 1159 struct intel_engine_cs *engine = request->engine; 1160 struct intel_context *ce = &request->ctx->engine[engine->id]; 1161 u32 *cs; 1162 int ret; 1163 1164 GEM_BUG_ON(!ce->pin_count); 1165 1166 /* Flush enough space to reduce the likelihood of waiting after 1167 * we start building the request - in which case we will just 1168 * have to repeat work. 1169 */ 1170 request->reserved_space += EXECLISTS_REQUEST_SIZE; 1171 1172 cs = intel_ring_begin(request, 0); 1173 if (IS_ERR(cs)) 1174 return PTR_ERR(cs); 1175 1176 if (!ce->initialised) { 1177 ret = engine->init_context(request); 1178 if (ret) 1179 return ret; 1180 1181 ce->initialised = true; 1182 } 1183 1184 /* Note that after this point, we have committed to using 1185 * this request as it is being used to both track the 1186 * state of engine initialisation and liveness of the 1187 * golden renderstate above. Think twice before you try 1188 * to cancel/unwind this request now. 1189 */ 1190 1191 request->reserved_space -= EXECLISTS_REQUEST_SIZE; 1192 return 0; 1193 } 1194 1195 /* 1196 * In this WA we need to set GEN8_L3SQCREG4[21:21] and reset it after 1197 * PIPE_CONTROL instruction. This is required for the flush to happen correctly 1198 * but there is a slight complication as this is applied in WA batch where the 1199 * values are only initialized once so we cannot take register value at the 1200 * beginning and reuse it further; hence we save its value to memory, upload a 1201 * constant value with bit21 set and then we restore it back with the saved value. 1202 * To simplify the WA, a constant value is formed by using the default value 1203 * of this register. This shouldn't be a problem because we are only modifying 1204 * it for a short period and this batch in non-premptible. We can ofcourse 1205 * use additional instructions that read the actual value of the register 1206 * at that time and set our bit of interest but it makes the WA complicated. 1207 * 1208 * This WA is also required for Gen9 so extracting as a function avoids 1209 * code duplication. 1210 */ 1211 static u32 * 1212 gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, u32 *batch) 1213 { 1214 *batch++ = MI_STORE_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT; 1215 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4); 1216 *batch++ = i915_ggtt_offset(engine->scratch) + 256; 1217 *batch++ = 0; 1218 1219 *batch++ = MI_LOAD_REGISTER_IMM(1); 1220 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4); 1221 *batch++ = 0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES; 1222 1223 batch = gen8_emit_pipe_control(batch, 1224 PIPE_CONTROL_CS_STALL | 1225 PIPE_CONTROL_DC_FLUSH_ENABLE, 1226 0); 1227 1228 *batch++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT; 1229 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4); 1230 *batch++ = i915_ggtt_offset(engine->scratch) + 256; 1231 *batch++ = 0; 1232 1233 return batch; 1234 } 1235 1236 /* 1237 * Typically we only have one indirect_ctx and per_ctx batch buffer which are 1238 * initialized at the beginning and shared across all contexts but this field 1239 * helps us to have multiple batches at different offsets and select them based 1240 * on a criteria. At the moment this batch always start at the beginning of the page 1241 * and at this point we don't have multiple wa_ctx batch buffers. 1242 * 1243 * The number of WA applied are not known at the beginning; we use this field 1244 * to return the no of DWORDS written. 1245 * 1246 * It is to be noted that this batch does not contain MI_BATCH_BUFFER_END 1247 * so it adds NOOPs as padding to make it cacheline aligned. 1248 * MI_BATCH_BUFFER_END will be added to perctx batch and both of them together 1249 * makes a complete batch buffer. 1250 */ 1251 static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) 1252 { 1253 /* WaDisableCtxRestoreArbitration:bdw,chv */ 1254 *batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1255 1256 /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */ 1257 if (IS_BROADWELL(engine->i915)) 1258 batch = gen8_emit_flush_coherentl3_wa(engine, batch); 1259 1260 /* WaClearSlmSpaceAtContextSwitch:bdw,chv */ 1261 /* Actual scratch location is at 128 bytes offset */ 1262 batch = gen8_emit_pipe_control(batch, 1263 PIPE_CONTROL_FLUSH_L3 | 1264 PIPE_CONTROL_GLOBAL_GTT_IVB | 1265 PIPE_CONTROL_CS_STALL | 1266 PIPE_CONTROL_QW_WRITE, 1267 i915_ggtt_offset(engine->scratch) + 1268 2 * CACHELINE_BYTES); 1269 1270 *batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1271 1272 /* Pad to end of cacheline */ 1273 while ((unsigned long)batch % CACHELINE_BYTES) 1274 *batch++ = MI_NOOP; 1275 1276 /* 1277 * MI_BATCH_BUFFER_END is not required in Indirect ctx BB because 1278 * execution depends on the length specified in terms of cache lines 1279 * in the register CTX_RCS_INDIRECT_CTX 1280 */ 1281 1282 return batch; 1283 } 1284 1285 static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) 1286 { 1287 *batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1288 1289 /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt,glk */ 1290 batch = gen8_emit_flush_coherentl3_wa(engine, batch); 1291 1292 /* WaDisableGatherAtSetShaderCommonSlice:skl,bxt,kbl,glk */ 1293 *batch++ = MI_LOAD_REGISTER_IMM(1); 1294 *batch++ = i915_mmio_reg_offset(COMMON_SLICE_CHICKEN2); 1295 *batch++ = _MASKED_BIT_DISABLE( 1296 GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE); 1297 *batch++ = MI_NOOP; 1298 1299 /* WaClearSlmSpaceAtContextSwitch:kbl */ 1300 /* Actual scratch location is at 128 bytes offset */ 1301 if (IS_KBL_REVID(engine->i915, 0, KBL_REVID_A0)) { 1302 batch = gen8_emit_pipe_control(batch, 1303 PIPE_CONTROL_FLUSH_L3 | 1304 PIPE_CONTROL_GLOBAL_GTT_IVB | 1305 PIPE_CONTROL_CS_STALL | 1306 PIPE_CONTROL_QW_WRITE, 1307 i915_ggtt_offset(engine->scratch) 1308 + 2 * CACHELINE_BYTES); 1309 } 1310 1311 /* WaMediaPoolStateCmdInWABB:bxt,glk */ 1312 if (HAS_POOLED_EU(engine->i915)) { 1313 /* 1314 * EU pool configuration is setup along with golden context 1315 * during context initialization. This value depends on 1316 * device type (2x6 or 3x6) and needs to be updated based 1317 * on which subslice is disabled especially for 2x6 1318 * devices, however it is safe to load default 1319 * configuration of 3x6 device instead of masking off 1320 * corresponding bits because HW ignores bits of a disabled 1321 * subslice and drops down to appropriate config. Please 1322 * see render_state_setup() in i915_gem_render_state.c for 1323 * possible configurations, to avoid duplication they are 1324 * not shown here again. 1325 */ 1326 *batch++ = GEN9_MEDIA_POOL_STATE; 1327 *batch++ = GEN9_MEDIA_POOL_ENABLE; 1328 *batch++ = 0x00777000; 1329 *batch++ = 0; 1330 *batch++ = 0; 1331 *batch++ = 0; 1332 } 1333 1334 *batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1335 1336 /* Pad to end of cacheline */ 1337 while ((unsigned long)batch % CACHELINE_BYTES) 1338 *batch++ = MI_NOOP; 1339 1340 return batch; 1341 } 1342 1343 #define CTX_WA_BB_OBJ_SIZE (PAGE_SIZE) 1344 1345 static int lrc_setup_wa_ctx(struct intel_engine_cs *engine) 1346 { 1347 struct drm_i915_gem_object *obj; 1348 struct i915_vma *vma; 1349 int err; 1350 1351 obj = i915_gem_object_create(engine->i915, CTX_WA_BB_OBJ_SIZE); 1352 if (IS_ERR(obj)) 1353 return PTR_ERR(obj); 1354 1355 vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL); 1356 if (IS_ERR(vma)) { 1357 err = PTR_ERR(vma); 1358 goto err; 1359 } 1360 1361 err = i915_vma_pin(vma, 0, PAGE_SIZE, PIN_GLOBAL | PIN_HIGH); 1362 if (err) 1363 goto err; 1364 1365 engine->wa_ctx.vma = vma; 1366 return 0; 1367 1368 err: 1369 i915_gem_object_put(obj); 1370 return err; 1371 } 1372 1373 static void lrc_destroy_wa_ctx(struct intel_engine_cs *engine) 1374 { 1375 i915_vma_unpin_and_release(&engine->wa_ctx.vma); 1376 } 1377 1378 typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch); 1379 1380 static int intel_init_workaround_bb(struct intel_engine_cs *engine) 1381 { 1382 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx; 1383 struct i915_wa_ctx_bb *wa_bb[2] = { &wa_ctx->indirect_ctx, 1384 &wa_ctx->per_ctx }; 1385 wa_bb_func_t wa_bb_fn[2]; 1386 struct page *page; 1387 void *batch, *batch_ptr; 1388 unsigned int i; 1389 int ret; 1390 1391 if (WARN_ON(engine->id != RCS || !engine->scratch)) 1392 return -EINVAL; 1393 1394 switch (INTEL_GEN(engine->i915)) { 1395 case 10: 1396 return 0; 1397 case 9: 1398 wa_bb_fn[0] = gen9_init_indirectctx_bb; 1399 wa_bb_fn[1] = NULL; 1400 break; 1401 case 8: 1402 wa_bb_fn[0] = gen8_init_indirectctx_bb; 1403 wa_bb_fn[1] = NULL; 1404 break; 1405 default: 1406 MISSING_CASE(INTEL_GEN(engine->i915)); 1407 return 0; 1408 } 1409 1410 ret = lrc_setup_wa_ctx(engine); 1411 if (ret) { 1412 DRM_DEBUG_DRIVER("Failed to setup context WA page: %d\n", ret); 1413 return ret; 1414 } 1415 1416 page = i915_gem_object_get_dirty_page(wa_ctx->vma->obj, 0); 1417 batch = batch_ptr = kmap_atomic(page); 1418 1419 /* 1420 * Emit the two workaround batch buffers, recording the offset from the 1421 * start of the workaround batch buffer object for each and their 1422 * respective sizes. 1423 */ 1424 for (i = 0; i < ARRAY_SIZE(wa_bb_fn); i++) { 1425 wa_bb[i]->offset = batch_ptr - batch; 1426 if (WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, CACHELINE_BYTES))) { 1427 ret = -EINVAL; 1428 break; 1429 } 1430 if (wa_bb_fn[i]) 1431 batch_ptr = wa_bb_fn[i](engine, batch_ptr); 1432 wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset); 1433 } 1434 1435 BUG_ON(batch_ptr - batch > CTX_WA_BB_OBJ_SIZE); 1436 1437 kunmap_atomic(batch); 1438 if (ret) 1439 lrc_destroy_wa_ctx(engine); 1440 1441 return ret; 1442 } 1443 1444 static u8 gtiir[] = { 1445 [RCS] = 0, 1446 [BCS] = 0, 1447 [VCS] = 1, 1448 [VCS2] = 1, 1449 [VECS] = 3, 1450 }; 1451 1452 static int gen8_init_common_ring(struct intel_engine_cs *engine) 1453 { 1454 struct drm_i915_private *dev_priv = engine->i915; 1455 struct intel_engine_execlists * const execlists = &engine->execlists; 1456 int ret; 1457 1458 ret = intel_mocs_init_engine(engine); 1459 if (ret) 1460 return ret; 1461 1462 intel_engine_reset_breadcrumbs(engine); 1463 intel_engine_init_hangcheck(engine); 1464 1465 I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff); 1466 I915_WRITE(RING_MODE_GEN7(engine), 1467 _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE)); 1468 I915_WRITE(RING_HWS_PGA(engine->mmio_base), 1469 engine->status_page.ggtt_offset); 1470 POSTING_READ(RING_HWS_PGA(engine->mmio_base)); 1471 1472 DRM_DEBUG_DRIVER("Execlists enabled for %s\n", engine->name); 1473 1474 GEM_BUG_ON(engine->id >= ARRAY_SIZE(gtiir)); 1475 1476 /* 1477 * Clear any pending interrupt state. 1478 * 1479 * We do it twice out of paranoia that some of the IIR are double 1480 * buffered, and if we only reset it once there may still be 1481 * an interrupt pending. 1482 */ 1483 I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]), 1484 GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift); 1485 I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]), 1486 GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift); 1487 clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted); 1488 execlists->csb_head = -1; 1489 execlists->active = 0; 1490 1491 /* After a GPU reset, we may have requests to replay */ 1492 if (!i915_modparams.enable_guc_submission && execlists->first) 1493 tasklet_schedule(&execlists->irq_tasklet); 1494 1495 return 0; 1496 } 1497 1498 static int gen8_init_render_ring(struct intel_engine_cs *engine) 1499 { 1500 struct drm_i915_private *dev_priv = engine->i915; 1501 int ret; 1502 1503 ret = gen8_init_common_ring(engine); 1504 if (ret) 1505 return ret; 1506 1507 /* We need to disable the AsyncFlip performance optimisations in order 1508 * to use MI_WAIT_FOR_EVENT within the CS. It should already be 1509 * programmed to '1' on all products. 1510 * 1511 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv 1512 */ 1513 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); 1514 1515 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); 1516 1517 return init_workarounds_ring(engine); 1518 } 1519 1520 static int gen9_init_render_ring(struct intel_engine_cs *engine) 1521 { 1522 int ret; 1523 1524 ret = gen8_init_common_ring(engine); 1525 if (ret) 1526 return ret; 1527 1528 return init_workarounds_ring(engine); 1529 } 1530 1531 static void reset_common_ring(struct intel_engine_cs *engine, 1532 struct drm_i915_gem_request *request) 1533 { 1534 struct intel_engine_execlists * const execlists = &engine->execlists; 1535 struct intel_context *ce; 1536 unsigned long flags; 1537 1538 spin_lock_irqsave(&engine->timeline->lock, flags); 1539 1540 /* 1541 * Catch up with any missed context-switch interrupts. 1542 * 1543 * Ideally we would just read the remaining CSB entries now that we 1544 * know the gpu is idle. However, the CSB registers are sometimes^W 1545 * often trashed across a GPU reset! Instead we have to rely on 1546 * guessing the missed context-switch events by looking at what 1547 * requests were completed. 1548 */ 1549 execlist_cancel_port_requests(execlists); 1550 1551 /* Push back any incomplete requests for replay after the reset. */ 1552 unwind_incomplete_requests(engine); 1553 1554 spin_unlock_irqrestore(&engine->timeline->lock, flags); 1555 1556 /* If the request was innocent, we leave the request in the ELSP 1557 * and will try to replay it on restarting. The context image may 1558 * have been corrupted by the reset, in which case we may have 1559 * to service a new GPU hang, but more likely we can continue on 1560 * without impact. 1561 * 1562 * If the request was guilty, we presume the context is corrupt 1563 * and have to at least restore the RING register in the context 1564 * image back to the expected values to skip over the guilty request. 1565 */ 1566 if (!request || request->fence.error != -EIO) 1567 return; 1568 1569 /* We want a simple context + ring to execute the breadcrumb update. 1570 * We cannot rely on the context being intact across the GPU hang, 1571 * so clear it and rebuild just what we need for the breadcrumb. 1572 * All pending requests for this context will be zapped, and any 1573 * future request will be after userspace has had the opportunity 1574 * to recreate its own state. 1575 */ 1576 ce = &request->ctx->engine[engine->id]; 1577 execlists_init_reg_state(ce->lrc_reg_state, 1578 request->ctx, engine, ce->ring); 1579 1580 /* Move the RING_HEAD onto the breadcrumb, past the hanging batch */ 1581 ce->lrc_reg_state[CTX_RING_BUFFER_START+1] = 1582 i915_ggtt_offset(ce->ring->vma); 1583 ce->lrc_reg_state[CTX_RING_HEAD+1] = request->postfix; 1584 1585 request->ring->head = request->postfix; 1586 intel_ring_update_space(request->ring); 1587 1588 /* Reset WaIdleLiteRestore:bdw,skl as well */ 1589 unwind_wa_tail(request); 1590 } 1591 1592 static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req) 1593 { 1594 struct i915_hw_ppgtt *ppgtt = req->ctx->ppgtt; 1595 struct intel_engine_cs *engine = req->engine; 1596 const int num_lri_cmds = GEN8_3LVL_PDPES * 2; 1597 u32 *cs; 1598 int i; 1599 1600 cs = intel_ring_begin(req, num_lri_cmds * 2 + 2); 1601 if (IS_ERR(cs)) 1602 return PTR_ERR(cs); 1603 1604 *cs++ = MI_LOAD_REGISTER_IMM(num_lri_cmds); 1605 for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) { 1606 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 1607 1608 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, i)); 1609 *cs++ = upper_32_bits(pd_daddr); 1610 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, i)); 1611 *cs++ = lower_32_bits(pd_daddr); 1612 } 1613 1614 *cs++ = MI_NOOP; 1615 intel_ring_advance(req, cs); 1616 1617 return 0; 1618 } 1619 1620 static int gen8_emit_bb_start(struct drm_i915_gem_request *req, 1621 u64 offset, u32 len, 1622 const unsigned int flags) 1623 { 1624 u32 *cs; 1625 int ret; 1626 1627 /* Don't rely in hw updating PDPs, specially in lite-restore. 1628 * Ideally, we should set Force PD Restore in ctx descriptor, 1629 * but we can't. Force Restore would be a second option, but 1630 * it is unsafe in case of lite-restore (because the ctx is 1631 * not idle). PML4 is allocated during ppgtt init so this is 1632 * not needed in 48-bit.*/ 1633 if (req->ctx->ppgtt && 1634 (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings) && 1635 !i915_vm_is_48bit(&req->ctx->ppgtt->base) && 1636 !intel_vgpu_active(req->i915)) { 1637 ret = intel_logical_ring_emit_pdps(req); 1638 if (ret) 1639 return ret; 1640 1641 req->ctx->ppgtt->pd_dirty_rings &= ~intel_engine_flag(req->engine); 1642 } 1643 1644 cs = intel_ring_begin(req, 4); 1645 if (IS_ERR(cs)) 1646 return PTR_ERR(cs); 1647 1648 /* 1649 * WaDisableCtxRestoreArbitration:bdw,chv 1650 * 1651 * We don't need to perform MI_ARB_ENABLE as often as we do (in 1652 * particular all the gen that do not need the w/a at all!), if we 1653 * took care to make sure that on every switch into this context 1654 * (both ordinary and for preemption) that arbitrartion was enabled 1655 * we would be fine. However, there doesn't seem to be a downside to 1656 * being paranoid and making sure it is set before each batch and 1657 * every context-switch. 1658 * 1659 * Note that if we fail to enable arbitration before the request 1660 * is complete, then we do not see the context-switch interrupt and 1661 * the engine hangs (with RING_HEAD == RING_TAIL). 1662 * 1663 * That satisfies both the GPGPU w/a and our heavy-handed paranoia. 1664 */ 1665 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1666 1667 /* FIXME(BDW): Address space and security selectors. */ 1668 *cs++ = MI_BATCH_BUFFER_START_GEN8 | 1669 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)) | 1670 (flags & I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0); 1671 *cs++ = lower_32_bits(offset); 1672 *cs++ = upper_32_bits(offset); 1673 intel_ring_advance(req, cs); 1674 1675 return 0; 1676 } 1677 1678 static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine) 1679 { 1680 struct drm_i915_private *dev_priv = engine->i915; 1681 I915_WRITE_IMR(engine, 1682 ~(engine->irq_enable_mask | engine->irq_keep_mask)); 1683 POSTING_READ_FW(RING_IMR(engine->mmio_base)); 1684 } 1685 1686 static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine) 1687 { 1688 struct drm_i915_private *dev_priv = engine->i915; 1689 I915_WRITE_IMR(engine, ~engine->irq_keep_mask); 1690 } 1691 1692 static int gen8_emit_flush(struct drm_i915_gem_request *request, u32 mode) 1693 { 1694 u32 cmd, *cs; 1695 1696 cs = intel_ring_begin(request, 4); 1697 if (IS_ERR(cs)) 1698 return PTR_ERR(cs); 1699 1700 cmd = MI_FLUSH_DW + 1; 1701 1702 /* We always require a command barrier so that subsequent 1703 * commands, such as breadcrumb interrupts, are strictly ordered 1704 * wrt the contents of the write cache being flushed to memory 1705 * (and thus being coherent from the CPU). 1706 */ 1707 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW; 1708 1709 if (mode & EMIT_INVALIDATE) { 1710 cmd |= MI_INVALIDATE_TLB; 1711 if (request->engine->id == VCS) 1712 cmd |= MI_INVALIDATE_BSD; 1713 } 1714 1715 *cs++ = cmd; 1716 *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; 1717 *cs++ = 0; /* upper addr */ 1718 *cs++ = 0; /* value */ 1719 intel_ring_advance(request, cs); 1720 1721 return 0; 1722 } 1723 1724 static int gen8_emit_flush_render(struct drm_i915_gem_request *request, 1725 u32 mode) 1726 { 1727 struct intel_engine_cs *engine = request->engine; 1728 u32 scratch_addr = 1729 i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES; 1730 bool vf_flush_wa = false, dc_flush_wa = false; 1731 u32 *cs, flags = 0; 1732 int len; 1733 1734 flags |= PIPE_CONTROL_CS_STALL; 1735 1736 if (mode & EMIT_FLUSH) { 1737 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; 1738 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; 1739 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; 1740 flags |= PIPE_CONTROL_FLUSH_ENABLE; 1741 } 1742 1743 if (mode & EMIT_INVALIDATE) { 1744 flags |= PIPE_CONTROL_TLB_INVALIDATE; 1745 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; 1746 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 1747 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; 1748 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; 1749 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; 1750 flags |= PIPE_CONTROL_QW_WRITE; 1751 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; 1752 1753 /* 1754 * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL 1755 * pipe control. 1756 */ 1757 if (IS_GEN9(request->i915)) 1758 vf_flush_wa = true; 1759 1760 /* WaForGAMHang:kbl */ 1761 if (IS_KBL_REVID(request->i915, 0, KBL_REVID_B0)) 1762 dc_flush_wa = true; 1763 } 1764 1765 len = 6; 1766 1767 if (vf_flush_wa) 1768 len += 6; 1769 1770 if (dc_flush_wa) 1771 len += 12; 1772 1773 cs = intel_ring_begin(request, len); 1774 if (IS_ERR(cs)) 1775 return PTR_ERR(cs); 1776 1777 if (vf_flush_wa) 1778 cs = gen8_emit_pipe_control(cs, 0, 0); 1779 1780 if (dc_flush_wa) 1781 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE, 1782 0); 1783 1784 cs = gen8_emit_pipe_control(cs, flags, scratch_addr); 1785 1786 if (dc_flush_wa) 1787 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0); 1788 1789 intel_ring_advance(request, cs); 1790 1791 return 0; 1792 } 1793 1794 /* 1795 * Reserve space for 2 NOOPs at the end of each request to be 1796 * used as a workaround for not being allowed to do lite 1797 * restore with HEAD==TAIL (WaIdleLiteRestore). 1798 */ 1799 static void gen8_emit_wa_tail(struct drm_i915_gem_request *request, u32 *cs) 1800 { 1801 /* Ensure there's always at least one preemption point per-request. */ 1802 *cs++ = MI_ARB_CHECK; 1803 *cs++ = MI_NOOP; 1804 request->wa_tail = intel_ring_offset(request, cs); 1805 } 1806 1807 static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, u32 *cs) 1808 { 1809 /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */ 1810 BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5)); 1811 1812 *cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW; 1813 *cs++ = intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT; 1814 *cs++ = 0; 1815 *cs++ = request->global_seqno; 1816 *cs++ = MI_USER_INTERRUPT; 1817 *cs++ = MI_NOOP; 1818 request->tail = intel_ring_offset(request, cs); 1819 assert_ring_tail_valid(request->ring, request->tail); 1820 1821 gen8_emit_wa_tail(request, cs); 1822 } 1823 static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS; 1824 1825 static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request, 1826 u32 *cs) 1827 { 1828 /* We're using qword write, seqno should be aligned to 8 bytes. */ 1829 BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1); 1830 1831 /* w/a for post sync ops following a GPGPU operation we 1832 * need a prior CS_STALL, which is emitted by the flush 1833 * following the batch. 1834 */ 1835 *cs++ = GFX_OP_PIPE_CONTROL(6); 1836 *cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL | 1837 PIPE_CONTROL_QW_WRITE; 1838 *cs++ = intel_hws_seqno_address(request->engine); 1839 *cs++ = 0; 1840 *cs++ = request->global_seqno; 1841 /* We're thrashing one dword of HWS. */ 1842 *cs++ = 0; 1843 *cs++ = MI_USER_INTERRUPT; 1844 *cs++ = MI_NOOP; 1845 request->tail = intel_ring_offset(request, cs); 1846 assert_ring_tail_valid(request->ring, request->tail); 1847 1848 gen8_emit_wa_tail(request, cs); 1849 } 1850 static const int gen8_emit_breadcrumb_render_sz = 8 + WA_TAIL_DWORDS; 1851 1852 static int gen8_init_rcs_context(struct drm_i915_gem_request *req) 1853 { 1854 int ret; 1855 1856 ret = intel_ring_workarounds_emit(req); 1857 if (ret) 1858 return ret; 1859 1860 ret = intel_rcs_context_init_mocs(req); 1861 /* 1862 * Failing to program the MOCS is non-fatal.The system will not 1863 * run at peak performance. So generate an error and carry on. 1864 */ 1865 if (ret) 1866 DRM_ERROR("MOCS failed to program: expect performance issues.\n"); 1867 1868 return i915_gem_render_state_emit(req); 1869 } 1870 1871 /** 1872 * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer 1873 * @engine: Engine Command Streamer. 1874 */ 1875 void intel_logical_ring_cleanup(struct intel_engine_cs *engine) 1876 { 1877 struct drm_i915_private *dev_priv; 1878 1879 /* 1880 * Tasklet cannot be active at this point due intel_mark_active/idle 1881 * so this is just for documentation. 1882 */ 1883 if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->execlists.irq_tasklet.state))) 1884 tasklet_kill(&engine->execlists.irq_tasklet); 1885 1886 dev_priv = engine->i915; 1887 1888 if (engine->buffer) { 1889 WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0); 1890 } 1891 1892 if (engine->cleanup) 1893 engine->cleanup(engine); 1894 1895 intel_engine_cleanup_common(engine); 1896 1897 lrc_destroy_wa_ctx(engine); 1898 engine->i915 = NULL; 1899 dev_priv->engine[engine->id] = NULL; 1900 kfree(engine); 1901 } 1902 1903 static void execlists_set_default_submission(struct intel_engine_cs *engine) 1904 { 1905 engine->submit_request = execlists_submit_request; 1906 engine->cancel_requests = execlists_cancel_requests; 1907 engine->schedule = execlists_schedule; 1908 engine->execlists.irq_tasklet.func = intel_lrc_irq_handler; 1909 } 1910 1911 static void 1912 logical_ring_default_vfuncs(struct intel_engine_cs *engine) 1913 { 1914 /* Default vfuncs which can be overriden by each engine. */ 1915 engine->init_hw = gen8_init_common_ring; 1916 engine->reset_hw = reset_common_ring; 1917 1918 engine->context_pin = execlists_context_pin; 1919 engine->context_unpin = execlists_context_unpin; 1920 1921 engine->request_alloc = execlists_request_alloc; 1922 1923 engine->emit_flush = gen8_emit_flush; 1924 engine->emit_breadcrumb = gen8_emit_breadcrumb; 1925 engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz; 1926 1927 engine->set_default_submission = execlists_set_default_submission; 1928 1929 engine->irq_enable = gen8_logical_ring_enable_irq; 1930 engine->irq_disable = gen8_logical_ring_disable_irq; 1931 engine->emit_bb_start = gen8_emit_bb_start; 1932 } 1933 1934 static inline void 1935 logical_ring_default_irqs(struct intel_engine_cs *engine) 1936 { 1937 unsigned shift = engine->irq_shift; 1938 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift; 1939 engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift; 1940 } 1941 1942 static void 1943 logical_ring_setup(struct intel_engine_cs *engine) 1944 { 1945 struct drm_i915_private *dev_priv = engine->i915; 1946 enum forcewake_domains fw_domains; 1947 1948 intel_engine_setup_common(engine); 1949 1950 /* Intentionally left blank. */ 1951 engine->buffer = NULL; 1952 1953 fw_domains = intel_uncore_forcewake_for_reg(dev_priv, 1954 RING_ELSP(engine), 1955 FW_REG_WRITE); 1956 1957 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, 1958 RING_CONTEXT_STATUS_PTR(engine), 1959 FW_REG_READ | FW_REG_WRITE); 1960 1961 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, 1962 RING_CONTEXT_STATUS_BUF_BASE(engine), 1963 FW_REG_READ); 1964 1965 engine->execlists.fw_domains = fw_domains; 1966 1967 tasklet_init(&engine->execlists.irq_tasklet, 1968 intel_lrc_irq_handler, (unsigned long)engine); 1969 1970 logical_ring_default_vfuncs(engine); 1971 logical_ring_default_irqs(engine); 1972 } 1973 1974 static int logical_ring_init(struct intel_engine_cs *engine) 1975 { 1976 int ret; 1977 1978 ret = intel_engine_init_common(engine); 1979 if (ret) 1980 goto error; 1981 1982 return 0; 1983 1984 error: 1985 intel_logical_ring_cleanup(engine); 1986 return ret; 1987 } 1988 1989 int logical_render_ring_init(struct intel_engine_cs *engine) 1990 { 1991 struct drm_i915_private *dev_priv = engine->i915; 1992 int ret; 1993 1994 logical_ring_setup(engine); 1995 1996 if (HAS_L3_DPF(dev_priv)) 1997 engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; 1998 1999 /* Override some for render ring. */ 2000 if (INTEL_GEN(dev_priv) >= 9) 2001 engine->init_hw = gen9_init_render_ring; 2002 else 2003 engine->init_hw = gen8_init_render_ring; 2004 engine->init_context = gen8_init_rcs_context; 2005 engine->emit_flush = gen8_emit_flush_render; 2006 engine->emit_breadcrumb = gen8_emit_breadcrumb_render; 2007 engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_render_sz; 2008 2009 ret = intel_engine_create_scratch(engine, PAGE_SIZE); 2010 if (ret) 2011 return ret; 2012 2013 ret = intel_init_workaround_bb(engine); 2014 if (ret) { 2015 /* 2016 * We continue even if we fail to initialize WA batch 2017 * because we only expect rare glitches but nothing 2018 * critical to prevent us from using GPU 2019 */ 2020 DRM_ERROR("WA batch buffer initialization failed: %d\n", 2021 ret); 2022 } 2023 2024 return logical_ring_init(engine); 2025 } 2026 2027 int logical_xcs_ring_init(struct intel_engine_cs *engine) 2028 { 2029 logical_ring_setup(engine); 2030 2031 return logical_ring_init(engine); 2032 } 2033 2034 static u32 2035 make_rpcs(struct drm_i915_private *dev_priv) 2036 { 2037 u32 rpcs = 0; 2038 2039 /* 2040 * No explicit RPCS request is needed to ensure full 2041 * slice/subslice/EU enablement prior to Gen9. 2042 */ 2043 if (INTEL_GEN(dev_priv) < 9) 2044 return 0; 2045 2046 /* 2047 * Starting in Gen9, render power gating can leave 2048 * slice/subslice/EU in a partially enabled state. We 2049 * must make an explicit request through RPCS for full 2050 * enablement. 2051 */ 2052 if (INTEL_INFO(dev_priv)->sseu.has_slice_pg) { 2053 rpcs |= GEN8_RPCS_S_CNT_ENABLE; 2054 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.slice_mask) << 2055 GEN8_RPCS_S_CNT_SHIFT; 2056 rpcs |= GEN8_RPCS_ENABLE; 2057 } 2058 2059 if (INTEL_INFO(dev_priv)->sseu.has_subslice_pg) { 2060 rpcs |= GEN8_RPCS_SS_CNT_ENABLE; 2061 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.subslice_mask) << 2062 GEN8_RPCS_SS_CNT_SHIFT; 2063 rpcs |= GEN8_RPCS_ENABLE; 2064 } 2065 2066 if (INTEL_INFO(dev_priv)->sseu.has_eu_pg) { 2067 rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice << 2068 GEN8_RPCS_EU_MIN_SHIFT; 2069 rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice << 2070 GEN8_RPCS_EU_MAX_SHIFT; 2071 rpcs |= GEN8_RPCS_ENABLE; 2072 } 2073 2074 return rpcs; 2075 } 2076 2077 static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine) 2078 { 2079 u32 indirect_ctx_offset; 2080 2081 switch (INTEL_GEN(engine->i915)) { 2082 default: 2083 MISSING_CASE(INTEL_GEN(engine->i915)); 2084 /* fall through */ 2085 case 10: 2086 indirect_ctx_offset = 2087 GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 2088 break; 2089 case 9: 2090 indirect_ctx_offset = 2091 GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 2092 break; 2093 case 8: 2094 indirect_ctx_offset = 2095 GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 2096 break; 2097 } 2098 2099 return indirect_ctx_offset; 2100 } 2101 2102 static void execlists_init_reg_state(u32 *regs, 2103 struct i915_gem_context *ctx, 2104 struct intel_engine_cs *engine, 2105 struct intel_ring *ring) 2106 { 2107 struct drm_i915_private *dev_priv = engine->i915; 2108 struct i915_hw_ppgtt *ppgtt = ctx->ppgtt ?: dev_priv->mm.aliasing_ppgtt; 2109 u32 base = engine->mmio_base; 2110 bool rcs = engine->id == RCS; 2111 2112 /* A context is actually a big batch buffer with several 2113 * MI_LOAD_REGISTER_IMM commands followed by (reg, value) pairs. The 2114 * values we are setting here are only for the first context restore: 2115 * on a subsequent save, the GPU will recreate this batchbuffer with new 2116 * values (including all the missing MI_LOAD_REGISTER_IMM commands that 2117 * we are not initializing here). 2118 */ 2119 regs[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(rcs ? 14 : 11) | 2120 MI_LRI_FORCE_POSTED; 2121 2122 CTX_REG(regs, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(engine), 2123 _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | 2124 CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | 2125 (HAS_RESOURCE_STREAMER(dev_priv) ? 2126 CTX_CTRL_RS_CTX_ENABLE : 0))); 2127 CTX_REG(regs, CTX_RING_HEAD, RING_HEAD(base), 0); 2128 CTX_REG(regs, CTX_RING_TAIL, RING_TAIL(base), 0); 2129 CTX_REG(regs, CTX_RING_BUFFER_START, RING_START(base), 0); 2130 CTX_REG(regs, CTX_RING_BUFFER_CONTROL, RING_CTL(base), 2131 RING_CTL_SIZE(ring->size) | RING_VALID); 2132 CTX_REG(regs, CTX_BB_HEAD_U, RING_BBADDR_UDW(base), 0); 2133 CTX_REG(regs, CTX_BB_HEAD_L, RING_BBADDR(base), 0); 2134 CTX_REG(regs, CTX_BB_STATE, RING_BBSTATE(base), RING_BB_PPGTT); 2135 CTX_REG(regs, CTX_SECOND_BB_HEAD_U, RING_SBBADDR_UDW(base), 0); 2136 CTX_REG(regs, CTX_SECOND_BB_HEAD_L, RING_SBBADDR(base), 0); 2137 CTX_REG(regs, CTX_SECOND_BB_STATE, RING_SBBSTATE(base), 0); 2138 if (rcs) { 2139 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx; 2140 2141 CTX_REG(regs, CTX_RCS_INDIRECT_CTX, RING_INDIRECT_CTX(base), 0); 2142 CTX_REG(regs, CTX_RCS_INDIRECT_CTX_OFFSET, 2143 RING_INDIRECT_CTX_OFFSET(base), 0); 2144 if (wa_ctx->indirect_ctx.size) { 2145 u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); 2146 2147 regs[CTX_RCS_INDIRECT_CTX + 1] = 2148 (ggtt_offset + wa_ctx->indirect_ctx.offset) | 2149 (wa_ctx->indirect_ctx.size / CACHELINE_BYTES); 2150 2151 regs[CTX_RCS_INDIRECT_CTX_OFFSET + 1] = 2152 intel_lr_indirect_ctx_offset(engine) << 6; 2153 } 2154 2155 CTX_REG(regs, CTX_BB_PER_CTX_PTR, RING_BB_PER_CTX_PTR(base), 0); 2156 if (wa_ctx->per_ctx.size) { 2157 u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); 2158 2159 regs[CTX_BB_PER_CTX_PTR + 1] = 2160 (ggtt_offset + wa_ctx->per_ctx.offset) | 0x01; 2161 } 2162 } 2163 2164 regs[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9) | MI_LRI_FORCE_POSTED; 2165 2166 CTX_REG(regs, CTX_CTX_TIMESTAMP, RING_CTX_TIMESTAMP(base), 0); 2167 /* PDP values well be assigned later if needed */ 2168 CTX_REG(regs, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(engine, 3), 0); 2169 CTX_REG(regs, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(engine, 3), 0); 2170 CTX_REG(regs, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(engine, 2), 0); 2171 CTX_REG(regs, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(engine, 2), 0); 2172 CTX_REG(regs, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(engine, 1), 0); 2173 CTX_REG(regs, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(engine, 1), 0); 2174 CTX_REG(regs, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(engine, 0), 0); 2175 CTX_REG(regs, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(engine, 0), 0); 2176 2177 if (ppgtt && i915_vm_is_48bit(&ppgtt->base)) { 2178 /* 64b PPGTT (48bit canonical) 2179 * PDP0_DESCRIPTOR contains the base address to PML4 and 2180 * other PDP Descriptors are ignored. 2181 */ 2182 ASSIGN_CTX_PML4(ppgtt, regs); 2183 } 2184 2185 if (rcs) { 2186 regs[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1); 2187 CTX_REG(regs, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, 2188 make_rpcs(dev_priv)); 2189 2190 i915_oa_init_reg_state(engine, ctx, regs); 2191 } 2192 } 2193 2194 static int 2195 populate_lr_context(struct i915_gem_context *ctx, 2196 struct drm_i915_gem_object *ctx_obj, 2197 struct intel_engine_cs *engine, 2198 struct intel_ring *ring) 2199 { 2200 void *vaddr; 2201 int ret; 2202 2203 ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true); 2204 if (ret) { 2205 DRM_DEBUG_DRIVER("Could not set to CPU domain\n"); 2206 return ret; 2207 } 2208 2209 vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB); 2210 if (IS_ERR(vaddr)) { 2211 ret = PTR_ERR(vaddr); 2212 DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret); 2213 return ret; 2214 } 2215 ctx_obj->mm.dirty = true; 2216 2217 /* The second page of the context object contains some fields which must 2218 * be set up prior to the first execution. */ 2219 2220 execlists_init_reg_state(vaddr + LRC_STATE_PN * PAGE_SIZE, 2221 ctx, engine, ring); 2222 2223 i915_gem_object_unpin_map(ctx_obj); 2224 2225 return 0; 2226 } 2227 2228 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx, 2229 struct intel_engine_cs *engine) 2230 { 2231 struct drm_i915_gem_object *ctx_obj; 2232 struct intel_context *ce = &ctx->engine[engine->id]; 2233 struct i915_vma *vma; 2234 uint32_t context_size; 2235 struct intel_ring *ring; 2236 int ret; 2237 2238 WARN_ON(ce->state); 2239 2240 context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE); 2241 2242 /* 2243 * Before the actual start of the context image, we insert a few pages 2244 * for our own use and for sharing with the GuC. 2245 */ 2246 context_size += LRC_HEADER_PAGES * PAGE_SIZE; 2247 2248 ctx_obj = i915_gem_object_create(ctx->i915, context_size); 2249 if (IS_ERR(ctx_obj)) { 2250 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n"); 2251 return PTR_ERR(ctx_obj); 2252 } 2253 2254 vma = i915_vma_instance(ctx_obj, &ctx->i915->ggtt.base, NULL); 2255 if (IS_ERR(vma)) { 2256 ret = PTR_ERR(vma); 2257 goto error_deref_obj; 2258 } 2259 2260 ring = intel_engine_create_ring(engine, ctx->ring_size); 2261 if (IS_ERR(ring)) { 2262 ret = PTR_ERR(ring); 2263 goto error_deref_obj; 2264 } 2265 2266 ret = populate_lr_context(ctx, ctx_obj, engine, ring); 2267 if (ret) { 2268 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret); 2269 goto error_ring_free; 2270 } 2271 2272 ce->ring = ring; 2273 ce->state = vma; 2274 ce->initialised |= engine->init_context == NULL; 2275 2276 return 0; 2277 2278 error_ring_free: 2279 intel_ring_free(ring); 2280 error_deref_obj: 2281 i915_gem_object_put(ctx_obj); 2282 return ret; 2283 } 2284 2285 void intel_lr_context_resume(struct drm_i915_private *dev_priv) 2286 { 2287 struct intel_engine_cs *engine; 2288 struct i915_gem_context *ctx; 2289 enum intel_engine_id id; 2290 2291 /* Because we emit WA_TAIL_DWORDS there may be a disparity 2292 * between our bookkeeping in ce->ring->head and ce->ring->tail and 2293 * that stored in context. As we only write new commands from 2294 * ce->ring->tail onwards, everything before that is junk. If the GPU 2295 * starts reading from its RING_HEAD from the context, it may try to 2296 * execute that junk and die. 2297 * 2298 * So to avoid that we reset the context images upon resume. For 2299 * simplicity, we just zero everything out. 2300 */ 2301 list_for_each_entry(ctx, &dev_priv->contexts.list, link) { 2302 for_each_engine(engine, dev_priv, id) { 2303 struct intel_context *ce = &ctx->engine[engine->id]; 2304 u32 *reg; 2305 2306 if (!ce->state) 2307 continue; 2308 2309 reg = i915_gem_object_pin_map(ce->state->obj, 2310 I915_MAP_WB); 2311 if (WARN_ON(IS_ERR(reg))) 2312 continue; 2313 2314 reg += LRC_STATE_PN * PAGE_SIZE / sizeof(*reg); 2315 reg[CTX_RING_HEAD+1] = 0; 2316 reg[CTX_RING_TAIL+1] = 0; 2317 2318 ce->state->obj->mm.dirty = true; 2319 i915_gem_object_unpin_map(ce->state->obj); 2320 2321 intel_ring_reset(ce->ring, 0); 2322 } 2323 } 2324 } 2325