1 /* 2 * Copyright © 2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Ben Widawsky <ben@bwidawsk.net> 25 * Michel Thierry <michel.thierry@intel.com> 26 * Thomas Daniel <thomas.daniel@intel.com> 27 * Oscar Mateo <oscar.mateo@intel.com> 28 * 29 */ 30 31 /** 32 * DOC: Logical Rings, Logical Ring Contexts and Execlists 33 * 34 * Motivation: 35 * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts". 36 * These expanded contexts enable a number of new abilities, especially 37 * "Execlists" (also implemented in this file). 38 * 39 * One of the main differences with the legacy HW contexts is that logical 40 * ring contexts incorporate many more things to the context's state, like 41 * PDPs or ringbuffer control registers: 42 * 43 * The reason why PDPs are included in the context is straightforward: as 44 * PPGTTs (per-process GTTs) are actually per-context, having the PDPs 45 * contained there mean you don't need to do a ppgtt->switch_mm yourself, 46 * instead, the GPU will do it for you on the context switch. 47 * 48 * But, what about the ringbuffer control registers (head, tail, etc..)? 49 * shouldn't we just need a set of those per engine command streamer? This is 50 * where the name "Logical Rings" starts to make sense: by virtualizing the 51 * rings, the engine cs shifts to a new "ring buffer" with every context 52 * switch. When you want to submit a workload to the GPU you: A) choose your 53 * context, B) find its appropriate virtualized ring, C) write commands to it 54 * and then, finally, D) tell the GPU to switch to that context. 55 * 56 * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch 57 * to a contexts is via a context execution list, ergo "Execlists". 58 * 59 * LRC implementation: 60 * Regarding the creation of contexts, we have: 61 * 62 * - One global default context. 63 * - One local default context for each opened fd. 64 * - One local extra context for each context create ioctl call. 65 * 66 * Now that ringbuffers belong per-context (and not per-engine, like before) 67 * and that contexts are uniquely tied to a given engine (and not reusable, 68 * like before) we need: 69 * 70 * - One ringbuffer per-engine inside each context. 71 * - One backing object per-engine inside each context. 72 * 73 * The global default context starts its life with these new objects fully 74 * allocated and populated. The local default context for each opened fd is 75 * more complex, because we don't know at creation time which engine is going 76 * to use them. To handle this, we have implemented a deferred creation of LR 77 * contexts: 78 * 79 * The local context starts its life as a hollow or blank holder, that only 80 * gets populated for a given engine once we receive an execbuffer. If later 81 * on we receive another execbuffer ioctl for the same context but a different 82 * engine, we allocate/populate a new ringbuffer and context backing object and 83 * so on. 84 * 85 * Finally, regarding local contexts created using the ioctl call: as they are 86 * only allowed with the render ring, we can allocate & populate them right 87 * away (no need to defer anything, at least for now). 88 * 89 * Execlists implementation: 90 * Execlists are the new method by which, on gen8+ hardware, workloads are 91 * submitted for execution (as opposed to the legacy, ringbuffer-based, method). 92 * This method works as follows: 93 * 94 * When a request is committed, its commands (the BB start and any leading or 95 * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer 96 * for the appropriate context. The tail pointer in the hardware context is not 97 * updated at this time, but instead, kept by the driver in the ringbuffer 98 * structure. A structure representing this request is added to a request queue 99 * for the appropriate engine: this structure contains a copy of the context's 100 * tail after the request was written to the ring buffer and a pointer to the 101 * context itself. 102 * 103 * If the engine's request queue was empty before the request was added, the 104 * queue is processed immediately. Otherwise the queue will be processed during 105 * a context switch interrupt. In any case, elements on the queue will get sent 106 * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a 107 * globally unique 20-bits submission ID. 108 * 109 * When execution of a request completes, the GPU updates the context status 110 * buffer with a context complete event and generates a context switch interrupt. 111 * During the interrupt handling, the driver examines the events in the buffer: 112 * for each context complete event, if the announced ID matches that on the head 113 * of the request queue, then that request is retired and removed from the queue. 114 * 115 * After processing, if any requests were retired and the queue is not empty 116 * then a new execution list can be submitted. The two requests at the front of 117 * the queue are next to be submitted but since a context may not occur twice in 118 * an execution list, if subsequent requests have the same ID as the first then 119 * the two requests must be combined. This is done simply by discarding requests 120 * at the head of the queue until either only one requests is left (in which case 121 * we use a NULL second context) or the first two requests have unique IDs. 122 * 123 * By always executing the first two requests in the queue the driver ensures 124 * that the GPU is kept as busy as possible. In the case where a single context 125 * completes but a second context is still executing, the request for this second 126 * context will be at the head of the queue when we remove the first one. This 127 * request will then be resubmitted along with a new request for a different context, 128 * which will cause the hardware to continue executing the second request and queue 129 * the new request (the GPU detects the condition of a context getting preempted 130 * with the same context and optimizes the context switch flow by not doing 131 * preemption, but just sampling the new tail pointer). 132 * 133 */ 134 #include <linux/interrupt.h> 135 136 #include <drm/drmP.h> 137 #include <drm/i915_drm.h> 138 #include "i915_drv.h" 139 #include "intel_mocs.h" 140 141 #define RING_EXECLIST_QFULL (1 << 0x2) 142 #define RING_EXECLIST1_VALID (1 << 0x3) 143 #define RING_EXECLIST0_VALID (1 << 0x4) 144 #define RING_EXECLIST_ACTIVE_STATUS (3 << 0xE) 145 #define RING_EXECLIST1_ACTIVE (1 << 0x11) 146 #define RING_EXECLIST0_ACTIVE (1 << 0x12) 147 148 #define GEN8_CTX_STATUS_IDLE_ACTIVE (1 << 0) 149 #define GEN8_CTX_STATUS_PREEMPTED (1 << 1) 150 #define GEN8_CTX_STATUS_ELEMENT_SWITCH (1 << 2) 151 #define GEN8_CTX_STATUS_ACTIVE_IDLE (1 << 3) 152 #define GEN8_CTX_STATUS_COMPLETE (1 << 4) 153 #define GEN8_CTX_STATUS_LITE_RESTORE (1 << 15) 154 155 #define GEN8_CTX_STATUS_COMPLETED_MASK \ 156 (GEN8_CTX_STATUS_ACTIVE_IDLE | \ 157 GEN8_CTX_STATUS_PREEMPTED | \ 158 GEN8_CTX_STATUS_ELEMENT_SWITCH) 159 160 #define CTX_LRI_HEADER_0 0x01 161 #define CTX_CONTEXT_CONTROL 0x02 162 #define CTX_RING_HEAD 0x04 163 #define CTX_RING_TAIL 0x06 164 #define CTX_RING_BUFFER_START 0x08 165 #define CTX_RING_BUFFER_CONTROL 0x0a 166 #define CTX_BB_HEAD_U 0x0c 167 #define CTX_BB_HEAD_L 0x0e 168 #define CTX_BB_STATE 0x10 169 #define CTX_SECOND_BB_HEAD_U 0x12 170 #define CTX_SECOND_BB_HEAD_L 0x14 171 #define CTX_SECOND_BB_STATE 0x16 172 #define CTX_BB_PER_CTX_PTR 0x18 173 #define CTX_RCS_INDIRECT_CTX 0x1a 174 #define CTX_RCS_INDIRECT_CTX_OFFSET 0x1c 175 #define CTX_LRI_HEADER_1 0x21 176 #define CTX_CTX_TIMESTAMP 0x22 177 #define CTX_PDP3_UDW 0x24 178 #define CTX_PDP3_LDW 0x26 179 #define CTX_PDP2_UDW 0x28 180 #define CTX_PDP2_LDW 0x2a 181 #define CTX_PDP1_UDW 0x2c 182 #define CTX_PDP1_LDW 0x2e 183 #define CTX_PDP0_UDW 0x30 184 #define CTX_PDP0_LDW 0x32 185 #define CTX_LRI_HEADER_2 0x41 186 #define CTX_R_PWR_CLK_STATE 0x42 187 #define CTX_GPGPU_CSR_BASE_ADDRESS 0x44 188 189 #define CTX_REG(reg_state, pos, reg, val) do { \ 190 (reg_state)[(pos)+0] = i915_mmio_reg_offset(reg); \ 191 (reg_state)[(pos)+1] = (val); \ 192 } while (0) 193 194 #define ASSIGN_CTX_PDP(ppgtt, reg_state, n) do { \ 195 const u64 _addr = i915_page_dir_dma_addr((ppgtt), (n)); \ 196 reg_state[CTX_PDP ## n ## _UDW+1] = upper_32_bits(_addr); \ 197 reg_state[CTX_PDP ## n ## _LDW+1] = lower_32_bits(_addr); \ 198 } while (0) 199 200 #define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \ 201 reg_state[CTX_PDP0_UDW + 1] = upper_32_bits(px_dma(&ppgtt->pml4)); \ 202 reg_state[CTX_PDP0_LDW + 1] = lower_32_bits(px_dma(&ppgtt->pml4)); \ 203 } while (0) 204 205 #define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x17 206 #define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x26 207 #define GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT 0x19 208 209 /* Typical size of the average request (2 pipecontrols and a MI_BB) */ 210 #define EXECLISTS_REQUEST_SIZE 64 /* bytes */ 211 #define WA_TAIL_DWORDS 2 212 #define WA_TAIL_BYTES (sizeof(u32) * WA_TAIL_DWORDS) 213 #define PREEMPT_ID 0x1 214 215 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx, 216 struct intel_engine_cs *engine); 217 static void execlists_init_reg_state(u32 *reg_state, 218 struct i915_gem_context *ctx, 219 struct intel_engine_cs *engine, 220 struct intel_ring *ring); 221 222 /** 223 * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists 224 * @dev_priv: i915 device private 225 * @enable_execlists: value of i915.enable_execlists module parameter. 226 * 227 * Only certain platforms support Execlists (the prerequisites being 228 * support for Logical Ring Contexts and Aliasing PPGTT or better). 229 * 230 * Return: 1 if Execlists is supported and has to be enabled. 231 */ 232 int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, int enable_execlists) 233 { 234 /* On platforms with execlist available, vGPU will only 235 * support execlist mode, no ring buffer mode. 236 */ 237 if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && intel_vgpu_active(dev_priv)) 238 return 1; 239 240 if (INTEL_GEN(dev_priv) >= 9) 241 return 1; 242 243 if (enable_execlists == 0) 244 return 0; 245 246 if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && 247 USES_PPGTT(dev_priv)) 248 return 1; 249 250 return 0; 251 } 252 253 /** 254 * intel_lr_context_descriptor_update() - calculate & cache the descriptor 255 * descriptor for a pinned context 256 * @ctx: Context to work on 257 * @engine: Engine the descriptor will be used with 258 * 259 * The context descriptor encodes various attributes of a context, 260 * including its GTT address and some flags. Because it's fairly 261 * expensive to calculate, we'll just do it once and cache the result, 262 * which remains valid until the context is unpinned. 263 * 264 * This is what a descriptor looks like, from LSB to MSB:: 265 * 266 * bits 0-11: flags, GEN8_CTX_* (cached in ctx->desc_template) 267 * bits 12-31: LRCA, GTT address of (the HWSP of) this context 268 * bits 32-52: ctx ID, a globally unique tag 269 * bits 53-54: mbz, reserved for use by hardware 270 * bits 55-63: group ID, currently unused and set to 0 271 */ 272 static void 273 intel_lr_context_descriptor_update(struct i915_gem_context *ctx, 274 struct intel_engine_cs *engine) 275 { 276 struct intel_context *ce = &ctx->engine[engine->id]; 277 u64 desc; 278 279 BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (1<<GEN8_CTX_ID_WIDTH)); 280 281 desc = ctx->desc_template; /* bits 0-11 */ 282 desc |= i915_ggtt_offset(ce->state) + LRC_HEADER_PAGES * PAGE_SIZE; 283 /* bits 12-31 */ 284 desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT; /* bits 32-52 */ 285 286 ce->lrc_desc = desc; 287 } 288 289 static struct i915_priolist * 290 lookup_priolist(struct intel_engine_cs *engine, 291 struct i915_priotree *pt, 292 int prio) 293 { 294 struct intel_engine_execlists * const execlists = &engine->execlists; 295 struct i915_priolist *p; 296 struct rb_node **parent, *rb; 297 bool first = true; 298 299 if (unlikely(execlists->no_priolist)) 300 prio = I915_PRIORITY_NORMAL; 301 302 find_priolist: 303 /* most positive priority is scheduled first, equal priorities fifo */ 304 rb = NULL; 305 parent = &execlists->queue.rb_node; 306 while (*parent) { 307 rb = *parent; 308 p = rb_entry(rb, typeof(*p), node); 309 if (prio > p->priority) { 310 parent = &rb->rb_left; 311 } else if (prio < p->priority) { 312 parent = &rb->rb_right; 313 first = false; 314 } else { 315 return p; 316 } 317 } 318 319 if (prio == I915_PRIORITY_NORMAL) { 320 p = &execlists->default_priolist; 321 } else { 322 p = kmem_cache_alloc(engine->i915->priorities, GFP_ATOMIC); 323 /* Convert an allocation failure to a priority bump */ 324 if (unlikely(!p)) { 325 prio = I915_PRIORITY_NORMAL; /* recurses just once */ 326 327 /* To maintain ordering with all rendering, after an 328 * allocation failure we have to disable all scheduling. 329 * Requests will then be executed in fifo, and schedule 330 * will ensure that dependencies are emitted in fifo. 331 * There will be still some reordering with existing 332 * requests, so if userspace lied about their 333 * dependencies that reordering may be visible. 334 */ 335 execlists->no_priolist = true; 336 goto find_priolist; 337 } 338 } 339 340 p->priority = prio; 341 INIT_LIST_HEAD(&p->requests); 342 rb_link_node(&p->node, rb, parent); 343 rb_insert_color(&p->node, &execlists->queue); 344 345 if (first) 346 execlists->first = &p->node; 347 348 return ptr_pack_bits(p, first, 1); 349 } 350 351 static void unwind_wa_tail(struct drm_i915_gem_request *rq) 352 { 353 rq->tail = intel_ring_wrap(rq->ring, rq->wa_tail - WA_TAIL_BYTES); 354 assert_ring_tail_valid(rq->ring, rq->tail); 355 } 356 357 static void unwind_incomplete_requests(struct intel_engine_cs *engine) 358 { 359 struct drm_i915_gem_request *rq, *rn; 360 struct i915_priolist *p = NULL; 361 int last_prio = I915_PRIORITY_INVALID; 362 363 lockdep_assert_held(&engine->timeline->lock); 364 365 list_for_each_entry_safe_reverse(rq, rn, 366 &engine->timeline->requests, 367 link) { 368 if (i915_gem_request_completed(rq)) 369 return; 370 371 __i915_gem_request_unsubmit(rq); 372 unwind_wa_tail(rq); 373 374 GEM_BUG_ON(rq->priotree.priority == I915_PRIORITY_INVALID); 375 if (rq->priotree.priority != last_prio) { 376 p = lookup_priolist(engine, 377 &rq->priotree, 378 rq->priotree.priority); 379 p = ptr_mask_bits(p, 1); 380 381 last_prio = rq->priotree.priority; 382 } 383 384 list_add(&rq->priotree.link, &p->requests); 385 } 386 } 387 388 static inline void 389 execlists_context_status_change(struct drm_i915_gem_request *rq, 390 unsigned long status) 391 { 392 /* 393 * Only used when GVT-g is enabled now. When GVT-g is disabled, 394 * The compiler should eliminate this function as dead-code. 395 */ 396 if (!IS_ENABLED(CONFIG_DRM_I915_GVT)) 397 return; 398 399 atomic_notifier_call_chain(&rq->engine->context_status_notifier, 400 status, rq); 401 } 402 403 static void 404 execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state) 405 { 406 ASSIGN_CTX_PDP(ppgtt, reg_state, 3); 407 ASSIGN_CTX_PDP(ppgtt, reg_state, 2); 408 ASSIGN_CTX_PDP(ppgtt, reg_state, 1); 409 ASSIGN_CTX_PDP(ppgtt, reg_state, 0); 410 } 411 412 static u64 execlists_update_context(struct drm_i915_gem_request *rq) 413 { 414 struct intel_context *ce = &rq->ctx->engine[rq->engine->id]; 415 struct i915_hw_ppgtt *ppgtt = 416 rq->ctx->ppgtt ?: rq->i915->mm.aliasing_ppgtt; 417 u32 *reg_state = ce->lrc_reg_state; 418 419 reg_state[CTX_RING_TAIL+1] = intel_ring_set_tail(rq->ring, rq->tail); 420 421 /* True 32b PPGTT with dynamic page allocation: update PDP 422 * registers and point the unallocated PDPs to scratch page. 423 * PML4 is allocated during ppgtt init, so this is not needed 424 * in 48-bit mode. 425 */ 426 if (ppgtt && !i915_vm_is_48bit(&ppgtt->base)) 427 execlists_update_context_pdps(ppgtt, reg_state); 428 429 return ce->lrc_desc; 430 } 431 432 static inline void elsp_write(u64 desc, u32 __iomem *elsp) 433 { 434 writel(upper_32_bits(desc), elsp); 435 writel(lower_32_bits(desc), elsp); 436 } 437 438 static void execlists_submit_ports(struct intel_engine_cs *engine) 439 { 440 struct execlist_port *port = engine->execlists.port; 441 u32 __iomem *elsp = 442 engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine)); 443 unsigned int n; 444 445 for (n = execlists_num_ports(&engine->execlists); n--; ) { 446 struct drm_i915_gem_request *rq; 447 unsigned int count; 448 u64 desc; 449 450 rq = port_unpack(&port[n], &count); 451 if (rq) { 452 GEM_BUG_ON(count > !n); 453 if (!count++) 454 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN); 455 port_set(&port[n], port_pack(rq, count)); 456 desc = execlists_update_context(rq); 457 GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc)); 458 } else { 459 GEM_BUG_ON(!n); 460 desc = 0; 461 } 462 463 elsp_write(desc, elsp); 464 } 465 } 466 467 static bool ctx_single_port_submission(const struct i915_gem_context *ctx) 468 { 469 return (IS_ENABLED(CONFIG_DRM_I915_GVT) && 470 i915_gem_context_force_single_submission(ctx)); 471 } 472 473 static bool can_merge_ctx(const struct i915_gem_context *prev, 474 const struct i915_gem_context *next) 475 { 476 if (prev != next) 477 return false; 478 479 if (ctx_single_port_submission(prev)) 480 return false; 481 482 return true; 483 } 484 485 static void port_assign(struct execlist_port *port, 486 struct drm_i915_gem_request *rq) 487 { 488 GEM_BUG_ON(rq == port_request(port)); 489 490 if (port_isset(port)) 491 i915_gem_request_put(port_request(port)); 492 493 port_set(port, port_pack(i915_gem_request_get(rq), port_count(port))); 494 } 495 496 static void inject_preempt_context(struct intel_engine_cs *engine) 497 { 498 struct intel_context *ce = 499 &engine->i915->preempt_context->engine[engine->id]; 500 u32 __iomem *elsp = 501 engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine)); 502 unsigned int n; 503 504 GEM_BUG_ON(engine->i915->preempt_context->hw_id != PREEMPT_ID); 505 GEM_BUG_ON(!IS_ALIGNED(ce->ring->size, WA_TAIL_BYTES)); 506 507 memset(ce->ring->vaddr + ce->ring->tail, 0, WA_TAIL_BYTES); 508 ce->ring->tail += WA_TAIL_BYTES; 509 ce->ring->tail &= (ce->ring->size - 1); 510 ce->lrc_reg_state[CTX_RING_TAIL+1] = ce->ring->tail; 511 512 for (n = execlists_num_ports(&engine->execlists); --n; ) 513 elsp_write(0, elsp); 514 515 elsp_write(ce->lrc_desc, elsp); 516 } 517 518 static bool can_preempt(struct intel_engine_cs *engine) 519 { 520 return INTEL_INFO(engine->i915)->has_logical_ring_preemption; 521 } 522 523 static void execlists_dequeue(struct intel_engine_cs *engine) 524 { 525 struct intel_engine_execlists * const execlists = &engine->execlists; 526 struct execlist_port *port = execlists->port; 527 const struct execlist_port * const last_port = 528 &execlists->port[execlists->port_mask]; 529 struct drm_i915_gem_request *last = port_request(port); 530 struct rb_node *rb; 531 bool submit = false; 532 533 /* Hardware submission is through 2 ports. Conceptually each port 534 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is 535 * static for a context, and unique to each, so we only execute 536 * requests belonging to a single context from each ring. RING_HEAD 537 * is maintained by the CS in the context image, it marks the place 538 * where it got up to last time, and through RING_TAIL we tell the CS 539 * where we want to execute up to this time. 540 * 541 * In this list the requests are in order of execution. Consecutive 542 * requests from the same context are adjacent in the ringbuffer. We 543 * can combine these requests into a single RING_TAIL update: 544 * 545 * RING_HEAD...req1...req2 546 * ^- RING_TAIL 547 * since to execute req2 the CS must first execute req1. 548 * 549 * Our goal then is to point each port to the end of a consecutive 550 * sequence of requests as being the most optimal (fewest wake ups 551 * and context switches) submission. 552 */ 553 554 spin_lock_irq(&engine->timeline->lock); 555 rb = execlists->first; 556 GEM_BUG_ON(rb_first(&execlists->queue) != rb); 557 if (!rb) 558 goto unlock; 559 560 if (last) { 561 /* 562 * Don't resubmit or switch until all outstanding 563 * preemptions (lite-restore) are seen. Then we 564 * know the next preemption status we see corresponds 565 * to this ELSP update. 566 */ 567 if (port_count(&port[0]) > 1) 568 goto unlock; 569 570 if (can_preempt(engine) && 571 rb_entry(rb, struct i915_priolist, node)->priority > 572 max(last->priotree.priority, 0)) { 573 /* 574 * Switch to our empty preempt context so 575 * the state of the GPU is known (idle). 576 */ 577 inject_preempt_context(engine); 578 execlists_set_active(execlists, 579 EXECLISTS_ACTIVE_PREEMPT); 580 goto unlock; 581 } else { 582 /* 583 * In theory, we could coalesce more requests onto 584 * the second port (the first port is active, with 585 * no preemptions pending). However, that means we 586 * then have to deal with the possible lite-restore 587 * of the second port (as we submit the ELSP, there 588 * may be a context-switch) but also we may complete 589 * the resubmission before the context-switch. Ergo, 590 * coalescing onto the second port will cause a 591 * preemption event, but we cannot predict whether 592 * that will affect port[0] or port[1]. 593 * 594 * If the second port is already active, we can wait 595 * until the next context-switch before contemplating 596 * new requests. The GPU will be busy and we should be 597 * able to resubmit the new ELSP before it idles, 598 * avoiding pipeline bubbles (momentary pauses where 599 * the driver is unable to keep up the supply of new 600 * work). 601 */ 602 if (port_count(&port[1])) 603 goto unlock; 604 605 /* WaIdleLiteRestore:bdw,skl 606 * Apply the wa NOOPs to prevent 607 * ring:HEAD == req:TAIL as we resubmit the 608 * request. See gen8_emit_breadcrumb() for 609 * where we prepare the padding after the 610 * end of the request. 611 */ 612 last->tail = last->wa_tail; 613 } 614 } 615 616 do { 617 struct i915_priolist *p = rb_entry(rb, typeof(*p), node); 618 struct drm_i915_gem_request *rq, *rn; 619 620 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) { 621 /* 622 * Can we combine this request with the current port? 623 * It has to be the same context/ringbuffer and not 624 * have any exceptions (e.g. GVT saying never to 625 * combine contexts). 626 * 627 * If we can combine the requests, we can execute both 628 * by updating the RING_TAIL to point to the end of the 629 * second request, and so we never need to tell the 630 * hardware about the first. 631 */ 632 if (last && !can_merge_ctx(rq->ctx, last->ctx)) { 633 /* 634 * If we are on the second port and cannot 635 * combine this request with the last, then we 636 * are done. 637 */ 638 if (port == last_port) { 639 __list_del_many(&p->requests, 640 &rq->priotree.link); 641 goto done; 642 } 643 644 /* 645 * If GVT overrides us we only ever submit 646 * port[0], leaving port[1] empty. Note that we 647 * also have to be careful that we don't queue 648 * the same context (even though a different 649 * request) to the second port. 650 */ 651 if (ctx_single_port_submission(last->ctx) || 652 ctx_single_port_submission(rq->ctx)) { 653 __list_del_many(&p->requests, 654 &rq->priotree.link); 655 goto done; 656 } 657 658 GEM_BUG_ON(last->ctx == rq->ctx); 659 660 if (submit) 661 port_assign(port, last); 662 port++; 663 664 GEM_BUG_ON(port_isset(port)); 665 } 666 667 INIT_LIST_HEAD(&rq->priotree.link); 668 __i915_gem_request_submit(rq); 669 trace_i915_gem_request_in(rq, port_index(port, execlists)); 670 last = rq; 671 submit = true; 672 } 673 674 rb = rb_next(rb); 675 rb_erase(&p->node, &execlists->queue); 676 INIT_LIST_HEAD(&p->requests); 677 if (p->priority != I915_PRIORITY_NORMAL) 678 kmem_cache_free(engine->i915->priorities, p); 679 } while (rb); 680 done: 681 execlists->first = rb; 682 if (submit) 683 port_assign(port, last); 684 unlock: 685 spin_unlock_irq(&engine->timeline->lock); 686 687 if (submit) { 688 execlists_set_active(execlists, EXECLISTS_ACTIVE_USER); 689 execlists_submit_ports(engine); 690 } 691 } 692 693 static void 694 execlist_cancel_port_requests(struct intel_engine_execlists *execlists) 695 { 696 struct execlist_port *port = execlists->port; 697 unsigned int num_ports = execlists_num_ports(execlists); 698 699 while (num_ports-- && port_isset(port)) { 700 struct drm_i915_gem_request *rq = port_request(port); 701 702 GEM_BUG_ON(!execlists->active); 703 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_PREEMPTED); 704 i915_gem_request_put(rq); 705 706 memset(port, 0, sizeof(*port)); 707 port++; 708 } 709 } 710 711 static void execlists_cancel_requests(struct intel_engine_cs *engine) 712 { 713 struct intel_engine_execlists * const execlists = &engine->execlists; 714 struct drm_i915_gem_request *rq, *rn; 715 struct rb_node *rb; 716 unsigned long flags; 717 718 spin_lock_irqsave(&engine->timeline->lock, flags); 719 720 /* Cancel the requests on the HW and clear the ELSP tracker. */ 721 execlist_cancel_port_requests(execlists); 722 723 /* Mark all executing requests as skipped. */ 724 list_for_each_entry(rq, &engine->timeline->requests, link) { 725 GEM_BUG_ON(!rq->global_seqno); 726 if (!i915_gem_request_completed(rq)) 727 dma_fence_set_error(&rq->fence, -EIO); 728 } 729 730 /* Flush the queued requests to the timeline list (for retiring). */ 731 rb = execlists->first; 732 while (rb) { 733 struct i915_priolist *p = rb_entry(rb, typeof(*p), node); 734 735 list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) { 736 INIT_LIST_HEAD(&rq->priotree.link); 737 738 dma_fence_set_error(&rq->fence, -EIO); 739 __i915_gem_request_submit(rq); 740 } 741 742 rb = rb_next(rb); 743 rb_erase(&p->node, &execlists->queue); 744 INIT_LIST_HEAD(&p->requests); 745 if (p->priority != I915_PRIORITY_NORMAL) 746 kmem_cache_free(engine->i915->priorities, p); 747 } 748 749 /* Remaining _unready_ requests will be nop'ed when submitted */ 750 751 752 execlists->queue = LINUX_RB_ROOT; 753 execlists->first = NULL; 754 GEM_BUG_ON(port_isset(execlists->port)); 755 756 /* 757 * The port is checked prior to scheduling a tasklet, but 758 * just in case we have suspended the tasklet to do the 759 * wedging make sure that when it wakes, it decides there 760 * is no work to do by clearing the irq_posted bit. 761 */ 762 clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted); 763 764 spin_unlock_irqrestore(&engine->timeline->lock, flags); 765 } 766 767 /* 768 * Check the unread Context Status Buffers and manage the submission of new 769 * contexts to the ELSP accordingly. 770 */ 771 static void intel_lrc_irq_handler(unsigned long data) 772 { 773 struct intel_engine_cs * const engine = (struct intel_engine_cs *)data; 774 struct intel_engine_execlists * const execlists = &engine->execlists; 775 struct execlist_port * const port = execlists->port; 776 struct drm_i915_private *dev_priv = engine->i915; 777 778 /* We can skip acquiring intel_runtime_pm_get() here as it was taken 779 * on our behalf by the request (see i915_gem_mark_busy()) and it will 780 * not be relinquished until the device is idle (see 781 * i915_gem_idle_work_handler()). As a precaution, we make sure 782 * that all ELSP are drained i.e. we have processed the CSB, 783 * before allowing ourselves to idle and calling intel_runtime_pm_put(). 784 */ 785 GEM_BUG_ON(!dev_priv->gt.awake); 786 787 intel_uncore_forcewake_get(dev_priv, execlists->fw_domains); 788 789 /* Prefer doing test_and_clear_bit() as a two stage operation to avoid 790 * imposing the cost of a locked atomic transaction when submitting a 791 * new request (outside of the context-switch interrupt). 792 */ 793 while (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted)) { 794 /* The HWSP contains a (cacheable) mirror of the CSB */ 795 const u32 *buf = 796 &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX]; 797 unsigned int head, tail; 798 799 if (unlikely(execlists->csb_use_mmio)) { 800 buf = (u32 * __force) 801 (dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0))); 802 execlists->csb_head = -1; /* force mmio read of CSB ptrs */ 803 } 804 805 /* The write will be ordered by the uncached read (itself 806 * a memory barrier), so we do not need another in the form 807 * of a locked instruction. The race between the interrupt 808 * handler and the split test/clear is harmless as we order 809 * our clear before the CSB read. If the interrupt arrived 810 * first between the test and the clear, we read the updated 811 * CSB and clear the bit. If the interrupt arrives as we read 812 * the CSB or later (i.e. after we had cleared the bit) the bit 813 * is set and we do a new loop. 814 */ 815 __clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted); 816 if (unlikely(execlists->csb_head == -1)) { /* following a reset */ 817 head = readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine))); 818 tail = GEN8_CSB_WRITE_PTR(head); 819 head = GEN8_CSB_READ_PTR(head); 820 execlists->csb_head = head; 821 } else { 822 const int write_idx = 823 intel_hws_csb_write_index(dev_priv) - 824 I915_HWS_CSB_BUF0_INDEX; 825 826 head = execlists->csb_head; 827 tail = READ_ONCE(buf[write_idx]); 828 } 829 830 while (head != tail) { 831 struct drm_i915_gem_request *rq; 832 unsigned int status; 833 unsigned int count; 834 835 if (++head == GEN8_CSB_ENTRIES) 836 head = 0; 837 838 /* We are flying near dragons again. 839 * 840 * We hold a reference to the request in execlist_port[] 841 * but no more than that. We are operating in softirq 842 * context and so cannot hold any mutex or sleep. That 843 * prevents us stopping the requests we are processing 844 * in port[] from being retired simultaneously (the 845 * breadcrumb will be complete before we see the 846 * context-switch). As we only hold the reference to the 847 * request, any pointer chasing underneath the request 848 * is subject to a potential use-after-free. Thus we 849 * store all of the bookkeeping within port[] as 850 * required, and avoid using unguarded pointers beneath 851 * request itself. The same applies to the atomic 852 * status notifier. 853 */ 854 855 status = READ_ONCE(buf[2 * head]); /* maybe mmio! */ 856 if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK)) 857 continue; 858 859 if (status & GEN8_CTX_STATUS_ACTIVE_IDLE && 860 buf[2*head + 1] == PREEMPT_ID) { 861 execlist_cancel_port_requests(execlists); 862 863 spin_lock_irq(&engine->timeline->lock); 864 unwind_incomplete_requests(engine); 865 spin_unlock_irq(&engine->timeline->lock); 866 867 GEM_BUG_ON(!execlists_is_active(execlists, 868 EXECLISTS_ACTIVE_PREEMPT)); 869 execlists_clear_active(execlists, 870 EXECLISTS_ACTIVE_PREEMPT); 871 continue; 872 } 873 874 if (status & GEN8_CTX_STATUS_PREEMPTED && 875 execlists_is_active(execlists, 876 EXECLISTS_ACTIVE_PREEMPT)) 877 continue; 878 879 GEM_BUG_ON(!execlists_is_active(execlists, 880 EXECLISTS_ACTIVE_USER)); 881 882 /* Check the context/desc id for this event matches */ 883 GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id); 884 885 rq = port_unpack(port, &count); 886 GEM_BUG_ON(count == 0); 887 if (--count == 0) { 888 GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED); 889 GEM_BUG_ON(!i915_gem_request_completed(rq)); 890 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT); 891 892 trace_i915_gem_request_out(rq); 893 i915_gem_request_put(rq); 894 895 execlists_port_complete(execlists, port); 896 } else { 897 port_set(port, port_pack(rq, count)); 898 } 899 900 /* After the final element, the hw should be idle */ 901 GEM_BUG_ON(port_count(port) == 0 && 902 !(status & GEN8_CTX_STATUS_ACTIVE_IDLE)); 903 if (port_count(port) == 0) 904 execlists_clear_active(execlists, 905 EXECLISTS_ACTIVE_USER); 906 } 907 908 if (head != execlists->csb_head) { 909 execlists->csb_head = head; 910 writel(_MASKED_FIELD(GEN8_CSB_READ_PTR_MASK, head << 8), 911 dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine))); 912 } 913 } 914 915 if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT)) 916 execlists_dequeue(engine); 917 918 intel_uncore_forcewake_put(dev_priv, execlists->fw_domains); 919 } 920 921 static void insert_request(struct intel_engine_cs *engine, 922 struct i915_priotree *pt, 923 int prio) 924 { 925 struct i915_priolist *p = lookup_priolist(engine, pt, prio); 926 927 list_add_tail(&pt->link, &ptr_mask_bits(p, 1)->requests); 928 if (ptr_unmask_bits(p, 1)) 929 tasklet_hi_schedule(&engine->execlists.irq_tasklet); 930 } 931 932 static void execlists_submit_request(struct drm_i915_gem_request *request) 933 { 934 struct intel_engine_cs *engine = request->engine; 935 unsigned long flags; 936 937 /* Will be called from irq-context when using foreign fences. */ 938 spin_lock_irqsave(&engine->timeline->lock, flags); 939 940 insert_request(engine, &request->priotree, request->priotree.priority); 941 942 GEM_BUG_ON(!engine->execlists.first); 943 GEM_BUG_ON(list_empty(&request->priotree.link)); 944 945 spin_unlock_irqrestore(&engine->timeline->lock, flags); 946 } 947 948 static struct drm_i915_gem_request *pt_to_request(struct i915_priotree *pt) 949 { 950 return container_of(pt, struct drm_i915_gem_request, priotree); 951 } 952 953 static struct intel_engine_cs * 954 pt_lock_engine(struct i915_priotree *pt, struct intel_engine_cs *locked) 955 { 956 struct intel_engine_cs *engine = pt_to_request(pt)->engine; 957 958 GEM_BUG_ON(!locked); 959 960 if (engine != locked) { 961 lockmgr(&locked->timeline->lock, LK_RELEASE); 962 lockmgr(&engine->timeline->lock, LK_EXCLUSIVE); 963 } 964 965 return engine; 966 } 967 968 static void execlists_schedule(struct drm_i915_gem_request *request, int prio) 969 { 970 struct intel_engine_cs *engine; 971 struct i915_dependency *dep, *p; 972 struct i915_dependency stack; 973 LINUX_LIST_HEAD(dfs); 974 975 GEM_BUG_ON(prio == I915_PRIORITY_INVALID); 976 977 if (i915_gem_request_completed(request)) 978 return; 979 980 if (prio <= READ_ONCE(request->priotree.priority)) 981 return; 982 983 /* Need BKL in order to use the temporary link inside i915_dependency */ 984 lockdep_assert_held(&request->i915->drm.struct_mutex); 985 986 stack.signaler = &request->priotree; 987 list_add(&stack.dfs_link, &dfs); 988 989 /* Recursively bump all dependent priorities to match the new request. 990 * 991 * A naive approach would be to use recursion: 992 * static void update_priorities(struct i915_priotree *pt, prio) { 993 * list_for_each_entry(dep, &pt->signalers_list, signal_link) 994 * update_priorities(dep->signal, prio) 995 * insert_request(pt); 996 * } 997 * but that may have unlimited recursion depth and so runs a very 998 * real risk of overunning the kernel stack. Instead, we build 999 * a flat list of all dependencies starting with the current request. 1000 * As we walk the list of dependencies, we add all of its dependencies 1001 * to the end of the list (this may include an already visited 1002 * request) and continue to walk onwards onto the new dependencies. The 1003 * end result is a topological list of requests in reverse order, the 1004 * last element in the list is the request we must execute first. 1005 */ 1006 list_for_each_entry_safe(dep, p, &dfs, dfs_link) { 1007 struct i915_priotree *pt = dep->signaler; 1008 1009 /* Within an engine, there can be no cycle, but we may 1010 * refer to the same dependency chain multiple times 1011 * (redundant dependencies are not eliminated) and across 1012 * engines. 1013 */ 1014 list_for_each_entry(p, &pt->signalers_list, signal_link) { 1015 if (i915_gem_request_completed(pt_to_request(p->signaler))) 1016 continue; 1017 1018 GEM_BUG_ON(p->signaler->priority < pt->priority); 1019 if (prio > READ_ONCE(p->signaler->priority)) 1020 list_move_tail(&p->dfs_link, &dfs); 1021 } 1022 1023 list_safe_reset_next(dep, p, dfs_link); 1024 } 1025 1026 /* If we didn't need to bump any existing priorities, and we haven't 1027 * yet submitted this request (i.e. there is no potential race with 1028 * execlists_submit_request()), we can set our own priority and skip 1029 * acquiring the engine locks. 1030 */ 1031 if (request->priotree.priority == I915_PRIORITY_INVALID) { 1032 GEM_BUG_ON(!list_empty(&request->priotree.link)); 1033 request->priotree.priority = prio; 1034 if (stack.dfs_link.next == stack.dfs_link.prev) 1035 return; 1036 __list_del_entry(&stack.dfs_link); 1037 } 1038 1039 engine = request->engine; 1040 spin_lock_irq(&engine->timeline->lock); 1041 1042 /* Fifo and depth-first replacement ensure our deps execute before us */ 1043 list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) { 1044 struct i915_priotree *pt = dep->signaler; 1045 1046 INIT_LIST_HEAD(&dep->dfs_link); 1047 1048 engine = pt_lock_engine(pt, engine); 1049 1050 if (prio <= pt->priority) 1051 continue; 1052 1053 pt->priority = prio; 1054 if (!list_empty(&pt->link)) { 1055 __list_del_entry(&pt->link); 1056 insert_request(engine, pt, prio); 1057 } 1058 } 1059 1060 spin_unlock_irq(&engine->timeline->lock); 1061 } 1062 1063 static struct intel_ring * 1064 execlists_context_pin(struct intel_engine_cs *engine, 1065 struct i915_gem_context *ctx) 1066 { 1067 struct intel_context *ce = &ctx->engine[engine->id]; 1068 unsigned int flags; 1069 void *vaddr; 1070 int ret; 1071 1072 lockdep_assert_held(&ctx->i915->drm.struct_mutex); 1073 1074 if (likely(ce->pin_count++)) 1075 goto out; 1076 GEM_BUG_ON(!ce->pin_count); /* no overflow please! */ 1077 1078 if (!ce->state) { 1079 ret = execlists_context_deferred_alloc(ctx, engine); 1080 if (ret) 1081 goto err; 1082 } 1083 GEM_BUG_ON(!ce->state); 1084 1085 flags = PIN_GLOBAL | PIN_HIGH; 1086 if (ctx->ggtt_offset_bias) 1087 flags |= PIN_OFFSET_BIAS | ctx->ggtt_offset_bias; 1088 1089 ret = i915_vma_pin(ce->state, 0, GEN8_LR_CONTEXT_ALIGN, flags); 1090 if (ret) 1091 goto err; 1092 1093 vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB); 1094 if (IS_ERR(vaddr)) { 1095 ret = PTR_ERR(vaddr); 1096 goto unpin_vma; 1097 } 1098 1099 ret = intel_ring_pin(ce->ring, ctx->i915, ctx->ggtt_offset_bias); 1100 if (ret) 1101 goto unpin_map; 1102 1103 intel_lr_context_descriptor_update(ctx, engine); 1104 1105 ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE; 1106 ce->lrc_reg_state[CTX_RING_BUFFER_START+1] = 1107 i915_ggtt_offset(ce->ring->vma); 1108 1109 ce->state->obj->mm.dirty = true; 1110 ce->state->obj->pin_global++; 1111 1112 i915_gem_context_get(ctx); 1113 out: 1114 return ce->ring; 1115 1116 unpin_map: 1117 i915_gem_object_unpin_map(ce->state->obj); 1118 unpin_vma: 1119 __i915_vma_unpin(ce->state); 1120 err: 1121 ce->pin_count = 0; 1122 return ERR_PTR(ret); 1123 } 1124 1125 static void execlists_context_unpin(struct intel_engine_cs *engine, 1126 struct i915_gem_context *ctx) 1127 { 1128 struct intel_context *ce = &ctx->engine[engine->id]; 1129 1130 lockdep_assert_held(&ctx->i915->drm.struct_mutex); 1131 GEM_BUG_ON(ce->pin_count == 0); 1132 1133 if (--ce->pin_count) 1134 return; 1135 1136 intel_ring_unpin(ce->ring); 1137 1138 ce->state->obj->pin_global--; 1139 i915_gem_object_unpin_map(ce->state->obj); 1140 i915_vma_unpin(ce->state); 1141 1142 i915_gem_context_put(ctx); 1143 } 1144 1145 static int execlists_request_alloc(struct drm_i915_gem_request *request) 1146 { 1147 struct intel_engine_cs *engine = request->engine; 1148 struct intel_context *ce = &request->ctx->engine[engine->id]; 1149 u32 *cs; 1150 int ret; 1151 1152 GEM_BUG_ON(!ce->pin_count); 1153 1154 /* Flush enough space to reduce the likelihood of waiting after 1155 * we start building the request - in which case we will just 1156 * have to repeat work. 1157 */ 1158 request->reserved_space += EXECLISTS_REQUEST_SIZE; 1159 1160 cs = intel_ring_begin(request, 0); 1161 if (IS_ERR(cs)) 1162 return PTR_ERR(cs); 1163 1164 if (!ce->initialised) { 1165 ret = engine->init_context(request); 1166 if (ret) 1167 return ret; 1168 1169 ce->initialised = true; 1170 } 1171 1172 /* Note that after this point, we have committed to using 1173 * this request as it is being used to both track the 1174 * state of engine initialisation and liveness of the 1175 * golden renderstate above. Think twice before you try 1176 * to cancel/unwind this request now. 1177 */ 1178 1179 request->reserved_space -= EXECLISTS_REQUEST_SIZE; 1180 return 0; 1181 } 1182 1183 /* 1184 * In this WA we need to set GEN8_L3SQCREG4[21:21] and reset it after 1185 * PIPE_CONTROL instruction. This is required for the flush to happen correctly 1186 * but there is a slight complication as this is applied in WA batch where the 1187 * values are only initialized once so we cannot take register value at the 1188 * beginning and reuse it further; hence we save its value to memory, upload a 1189 * constant value with bit21 set and then we restore it back with the saved value. 1190 * To simplify the WA, a constant value is formed by using the default value 1191 * of this register. This shouldn't be a problem because we are only modifying 1192 * it for a short period and this batch in non-premptible. We can ofcourse 1193 * use additional instructions that read the actual value of the register 1194 * at that time and set our bit of interest but it makes the WA complicated. 1195 * 1196 * This WA is also required for Gen9 so extracting as a function avoids 1197 * code duplication. 1198 */ 1199 static u32 * 1200 gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, u32 *batch) 1201 { 1202 *batch++ = MI_STORE_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT; 1203 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4); 1204 *batch++ = i915_ggtt_offset(engine->scratch) + 256; 1205 *batch++ = 0; 1206 1207 *batch++ = MI_LOAD_REGISTER_IMM(1); 1208 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4); 1209 *batch++ = 0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES; 1210 1211 batch = gen8_emit_pipe_control(batch, 1212 PIPE_CONTROL_CS_STALL | 1213 PIPE_CONTROL_DC_FLUSH_ENABLE, 1214 0); 1215 1216 *batch++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT; 1217 *batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4); 1218 *batch++ = i915_ggtt_offset(engine->scratch) + 256; 1219 *batch++ = 0; 1220 1221 return batch; 1222 } 1223 1224 /* 1225 * Typically we only have one indirect_ctx and per_ctx batch buffer which are 1226 * initialized at the beginning and shared across all contexts but this field 1227 * helps us to have multiple batches at different offsets and select them based 1228 * on a criteria. At the moment this batch always start at the beginning of the page 1229 * and at this point we don't have multiple wa_ctx batch buffers. 1230 * 1231 * The number of WA applied are not known at the beginning; we use this field 1232 * to return the no of DWORDS written. 1233 * 1234 * It is to be noted that this batch does not contain MI_BATCH_BUFFER_END 1235 * so it adds NOOPs as padding to make it cacheline aligned. 1236 * MI_BATCH_BUFFER_END will be added to perctx batch and both of them together 1237 * makes a complete batch buffer. 1238 */ 1239 static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) 1240 { 1241 /* WaDisableCtxRestoreArbitration:bdw,chv */ 1242 *batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1243 1244 /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */ 1245 if (IS_BROADWELL(engine->i915)) 1246 batch = gen8_emit_flush_coherentl3_wa(engine, batch); 1247 1248 /* WaClearSlmSpaceAtContextSwitch:bdw,chv */ 1249 /* Actual scratch location is at 128 bytes offset */ 1250 batch = gen8_emit_pipe_control(batch, 1251 PIPE_CONTROL_FLUSH_L3 | 1252 PIPE_CONTROL_GLOBAL_GTT_IVB | 1253 PIPE_CONTROL_CS_STALL | 1254 PIPE_CONTROL_QW_WRITE, 1255 i915_ggtt_offset(engine->scratch) + 1256 2 * CACHELINE_BYTES); 1257 1258 *batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1259 1260 /* Pad to end of cacheline */ 1261 while ((unsigned long)batch % CACHELINE_BYTES) 1262 *batch++ = MI_NOOP; 1263 1264 /* 1265 * MI_BATCH_BUFFER_END is not required in Indirect ctx BB because 1266 * execution depends on the length specified in terms of cache lines 1267 * in the register CTX_RCS_INDIRECT_CTX 1268 */ 1269 1270 return batch; 1271 } 1272 1273 static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) 1274 { 1275 *batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1276 1277 /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt,glk */ 1278 batch = gen8_emit_flush_coherentl3_wa(engine, batch); 1279 1280 /* WaDisableGatherAtSetShaderCommonSlice:skl,bxt,kbl,glk */ 1281 *batch++ = MI_LOAD_REGISTER_IMM(1); 1282 *batch++ = i915_mmio_reg_offset(COMMON_SLICE_CHICKEN2); 1283 *batch++ = _MASKED_BIT_DISABLE( 1284 GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE); 1285 *batch++ = MI_NOOP; 1286 1287 /* WaClearSlmSpaceAtContextSwitch:kbl */ 1288 /* Actual scratch location is at 128 bytes offset */ 1289 if (IS_KBL_REVID(engine->i915, 0, KBL_REVID_A0)) { 1290 batch = gen8_emit_pipe_control(batch, 1291 PIPE_CONTROL_FLUSH_L3 | 1292 PIPE_CONTROL_GLOBAL_GTT_IVB | 1293 PIPE_CONTROL_CS_STALL | 1294 PIPE_CONTROL_QW_WRITE, 1295 i915_ggtt_offset(engine->scratch) 1296 + 2 * CACHELINE_BYTES); 1297 } 1298 1299 /* WaMediaPoolStateCmdInWABB:bxt,glk */ 1300 if (HAS_POOLED_EU(engine->i915)) { 1301 /* 1302 * EU pool configuration is setup along with golden context 1303 * during context initialization. This value depends on 1304 * device type (2x6 or 3x6) and needs to be updated based 1305 * on which subslice is disabled especially for 2x6 1306 * devices, however it is safe to load default 1307 * configuration of 3x6 device instead of masking off 1308 * corresponding bits because HW ignores bits of a disabled 1309 * subslice and drops down to appropriate config. Please 1310 * see render_state_setup() in i915_gem_render_state.c for 1311 * possible configurations, to avoid duplication they are 1312 * not shown here again. 1313 */ 1314 *batch++ = GEN9_MEDIA_POOL_STATE; 1315 *batch++ = GEN9_MEDIA_POOL_ENABLE; 1316 *batch++ = 0x00777000; 1317 *batch++ = 0; 1318 *batch++ = 0; 1319 *batch++ = 0; 1320 } 1321 1322 *batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1323 1324 /* Pad to end of cacheline */ 1325 while ((unsigned long)batch % CACHELINE_BYTES) 1326 *batch++ = MI_NOOP; 1327 1328 return batch; 1329 } 1330 1331 #define CTX_WA_BB_OBJ_SIZE (PAGE_SIZE) 1332 1333 static int lrc_setup_wa_ctx(struct intel_engine_cs *engine) 1334 { 1335 struct drm_i915_gem_object *obj; 1336 struct i915_vma *vma; 1337 int err; 1338 1339 obj = i915_gem_object_create(engine->i915, CTX_WA_BB_OBJ_SIZE); 1340 if (IS_ERR(obj)) 1341 return PTR_ERR(obj); 1342 1343 vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL); 1344 if (IS_ERR(vma)) { 1345 err = PTR_ERR(vma); 1346 goto err; 1347 } 1348 1349 err = i915_vma_pin(vma, 0, PAGE_SIZE, PIN_GLOBAL | PIN_HIGH); 1350 if (err) 1351 goto err; 1352 1353 engine->wa_ctx.vma = vma; 1354 return 0; 1355 1356 err: 1357 i915_gem_object_put(obj); 1358 return err; 1359 } 1360 1361 static void lrc_destroy_wa_ctx(struct intel_engine_cs *engine) 1362 { 1363 i915_vma_unpin_and_release(&engine->wa_ctx.vma); 1364 } 1365 1366 typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch); 1367 1368 static int intel_init_workaround_bb(struct intel_engine_cs *engine) 1369 { 1370 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx; 1371 struct i915_wa_ctx_bb *wa_bb[2] = { &wa_ctx->indirect_ctx, 1372 &wa_ctx->per_ctx }; 1373 wa_bb_func_t wa_bb_fn[2]; 1374 struct page *page; 1375 void *batch, *batch_ptr; 1376 unsigned int i; 1377 int ret; 1378 1379 if (WARN_ON(engine->id != RCS || !engine->scratch)) 1380 return -EINVAL; 1381 1382 switch (INTEL_GEN(engine->i915)) { 1383 case 10: 1384 return 0; 1385 case 9: 1386 wa_bb_fn[0] = gen9_init_indirectctx_bb; 1387 wa_bb_fn[1] = NULL; 1388 break; 1389 case 8: 1390 wa_bb_fn[0] = gen8_init_indirectctx_bb; 1391 wa_bb_fn[1] = NULL; 1392 break; 1393 default: 1394 MISSING_CASE(INTEL_GEN(engine->i915)); 1395 return 0; 1396 } 1397 1398 ret = lrc_setup_wa_ctx(engine); 1399 if (ret) { 1400 DRM_DEBUG_DRIVER("Failed to setup context WA page: %d\n", ret); 1401 return ret; 1402 } 1403 1404 page = i915_gem_object_get_dirty_page(wa_ctx->vma->obj, 0); 1405 batch = batch_ptr = kmap_atomic(page); 1406 1407 /* 1408 * Emit the two workaround batch buffers, recording the offset from the 1409 * start of the workaround batch buffer object for each and their 1410 * respective sizes. 1411 */ 1412 for (i = 0; i < ARRAY_SIZE(wa_bb_fn); i++) { 1413 wa_bb[i]->offset = batch_ptr - batch; 1414 if (WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, CACHELINE_BYTES))) { 1415 ret = -EINVAL; 1416 break; 1417 } 1418 if (wa_bb_fn[i]) 1419 batch_ptr = wa_bb_fn[i](engine, batch_ptr); 1420 wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset); 1421 } 1422 1423 BUG_ON(batch_ptr - batch > CTX_WA_BB_OBJ_SIZE); 1424 1425 kunmap_atomic(batch); 1426 if (ret) 1427 lrc_destroy_wa_ctx(engine); 1428 1429 return ret; 1430 } 1431 1432 static u8 gtiir[] = { 1433 [RCS] = 0, 1434 [BCS] = 0, 1435 [VCS] = 1, 1436 [VCS2] = 1, 1437 [VECS] = 3, 1438 }; 1439 1440 static int gen8_init_common_ring(struct intel_engine_cs *engine) 1441 { 1442 struct drm_i915_private *dev_priv = engine->i915; 1443 struct intel_engine_execlists * const execlists = &engine->execlists; 1444 int ret; 1445 1446 ret = intel_mocs_init_engine(engine); 1447 if (ret) 1448 return ret; 1449 1450 intel_engine_reset_breadcrumbs(engine); 1451 intel_engine_init_hangcheck(engine); 1452 1453 I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff); 1454 I915_WRITE(RING_MODE_GEN7(engine), 1455 _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE)); 1456 I915_WRITE(RING_HWS_PGA(engine->mmio_base), 1457 engine->status_page.ggtt_offset); 1458 POSTING_READ(RING_HWS_PGA(engine->mmio_base)); 1459 1460 DRM_DEBUG_DRIVER("Execlists enabled for %s\n", engine->name); 1461 1462 GEM_BUG_ON(engine->id >= ARRAY_SIZE(gtiir)); 1463 1464 /* 1465 * Clear any pending interrupt state. 1466 * 1467 * We do it twice out of paranoia that some of the IIR are double 1468 * buffered, and if we only reset it once there may still be 1469 * an interrupt pending. 1470 */ 1471 I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]), 1472 GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift); 1473 I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]), 1474 GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift); 1475 clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted); 1476 execlists->csb_head = -1; 1477 execlists->active = 0; 1478 1479 /* After a GPU reset, we may have requests to replay */ 1480 if (!i915_modparams.enable_guc_submission && execlists->first) 1481 tasklet_schedule(&execlists->irq_tasklet); 1482 1483 return 0; 1484 } 1485 1486 static int gen8_init_render_ring(struct intel_engine_cs *engine) 1487 { 1488 struct drm_i915_private *dev_priv = engine->i915; 1489 int ret; 1490 1491 ret = gen8_init_common_ring(engine); 1492 if (ret) 1493 return ret; 1494 1495 /* We need to disable the AsyncFlip performance optimisations in order 1496 * to use MI_WAIT_FOR_EVENT within the CS. It should already be 1497 * programmed to '1' on all products. 1498 * 1499 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv 1500 */ 1501 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); 1502 1503 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); 1504 1505 return init_workarounds_ring(engine); 1506 } 1507 1508 static int gen9_init_render_ring(struct intel_engine_cs *engine) 1509 { 1510 int ret; 1511 1512 ret = gen8_init_common_ring(engine); 1513 if (ret) 1514 return ret; 1515 1516 return init_workarounds_ring(engine); 1517 } 1518 1519 static void reset_common_ring(struct intel_engine_cs *engine, 1520 struct drm_i915_gem_request *request) 1521 { 1522 struct intel_engine_execlists * const execlists = &engine->execlists; 1523 struct intel_context *ce; 1524 unsigned long flags; 1525 1526 spin_lock_irqsave(&engine->timeline->lock, flags); 1527 1528 /* 1529 * Catch up with any missed context-switch interrupts. 1530 * 1531 * Ideally we would just read the remaining CSB entries now that we 1532 * know the gpu is idle. However, the CSB registers are sometimes^W 1533 * often trashed across a GPU reset! Instead we have to rely on 1534 * guessing the missed context-switch events by looking at what 1535 * requests were completed. 1536 */ 1537 execlist_cancel_port_requests(execlists); 1538 1539 /* Push back any incomplete requests for replay after the reset. */ 1540 unwind_incomplete_requests(engine); 1541 1542 spin_unlock_irqrestore(&engine->timeline->lock, flags); 1543 1544 /* If the request was innocent, we leave the request in the ELSP 1545 * and will try to replay it on restarting. The context image may 1546 * have been corrupted by the reset, in which case we may have 1547 * to service a new GPU hang, but more likely we can continue on 1548 * without impact. 1549 * 1550 * If the request was guilty, we presume the context is corrupt 1551 * and have to at least restore the RING register in the context 1552 * image back to the expected values to skip over the guilty request. 1553 */ 1554 if (!request || request->fence.error != -EIO) 1555 return; 1556 1557 /* We want a simple context + ring to execute the breadcrumb update. 1558 * We cannot rely on the context being intact across the GPU hang, 1559 * so clear it and rebuild just what we need for the breadcrumb. 1560 * All pending requests for this context will be zapped, and any 1561 * future request will be after userspace has had the opportunity 1562 * to recreate its own state. 1563 */ 1564 ce = &request->ctx->engine[engine->id]; 1565 execlists_init_reg_state(ce->lrc_reg_state, 1566 request->ctx, engine, ce->ring); 1567 1568 /* Move the RING_HEAD onto the breadcrumb, past the hanging batch */ 1569 ce->lrc_reg_state[CTX_RING_BUFFER_START+1] = 1570 i915_ggtt_offset(ce->ring->vma); 1571 ce->lrc_reg_state[CTX_RING_HEAD+1] = request->postfix; 1572 1573 request->ring->head = request->postfix; 1574 intel_ring_update_space(request->ring); 1575 1576 /* Reset WaIdleLiteRestore:bdw,skl as well */ 1577 unwind_wa_tail(request); 1578 } 1579 1580 static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req) 1581 { 1582 struct i915_hw_ppgtt *ppgtt = req->ctx->ppgtt; 1583 struct intel_engine_cs *engine = req->engine; 1584 const int num_lri_cmds = GEN8_3LVL_PDPES * 2; 1585 u32 *cs; 1586 int i; 1587 1588 cs = intel_ring_begin(req, num_lri_cmds * 2 + 2); 1589 if (IS_ERR(cs)) 1590 return PTR_ERR(cs); 1591 1592 *cs++ = MI_LOAD_REGISTER_IMM(num_lri_cmds); 1593 for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) { 1594 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 1595 1596 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, i)); 1597 *cs++ = upper_32_bits(pd_daddr); 1598 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, i)); 1599 *cs++ = lower_32_bits(pd_daddr); 1600 } 1601 1602 *cs++ = MI_NOOP; 1603 intel_ring_advance(req, cs); 1604 1605 return 0; 1606 } 1607 1608 static int gen8_emit_bb_start(struct drm_i915_gem_request *req, 1609 u64 offset, u32 len, 1610 const unsigned int flags) 1611 { 1612 u32 *cs; 1613 int ret; 1614 1615 /* Don't rely in hw updating PDPs, specially in lite-restore. 1616 * Ideally, we should set Force PD Restore in ctx descriptor, 1617 * but we can't. Force Restore would be a second option, but 1618 * it is unsafe in case of lite-restore (because the ctx is 1619 * not idle). PML4 is allocated during ppgtt init so this is 1620 * not needed in 48-bit.*/ 1621 if (req->ctx->ppgtt && 1622 (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings) && 1623 !i915_vm_is_48bit(&req->ctx->ppgtt->base) && 1624 !intel_vgpu_active(req->i915)) { 1625 ret = intel_logical_ring_emit_pdps(req); 1626 if (ret) 1627 return ret; 1628 1629 req->ctx->ppgtt->pd_dirty_rings &= ~intel_engine_flag(req->engine); 1630 } 1631 1632 cs = intel_ring_begin(req, 4); 1633 if (IS_ERR(cs)) 1634 return PTR_ERR(cs); 1635 1636 /* 1637 * WaDisableCtxRestoreArbitration:bdw,chv 1638 * 1639 * We don't need to perform MI_ARB_ENABLE as often as we do (in 1640 * particular all the gen that do not need the w/a at all!), if we 1641 * took care to make sure that on every switch into this context 1642 * (both ordinary and for preemption) that arbitrartion was enabled 1643 * we would be fine. However, there doesn't seem to be a downside to 1644 * being paranoid and making sure it is set before each batch and 1645 * every context-switch. 1646 * 1647 * Note that if we fail to enable arbitration before the request 1648 * is complete, then we do not see the context-switch interrupt and 1649 * the engine hangs (with RING_HEAD == RING_TAIL). 1650 * 1651 * That satisfies both the GPGPU w/a and our heavy-handed paranoia. 1652 */ 1653 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1654 1655 /* FIXME(BDW): Address space and security selectors. */ 1656 *cs++ = MI_BATCH_BUFFER_START_GEN8 | 1657 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8)) | 1658 (flags & I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0); 1659 *cs++ = lower_32_bits(offset); 1660 *cs++ = upper_32_bits(offset); 1661 intel_ring_advance(req, cs); 1662 1663 return 0; 1664 } 1665 1666 static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine) 1667 { 1668 struct drm_i915_private *dev_priv = engine->i915; 1669 I915_WRITE_IMR(engine, 1670 ~(engine->irq_enable_mask | engine->irq_keep_mask)); 1671 POSTING_READ_FW(RING_IMR(engine->mmio_base)); 1672 } 1673 1674 static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine) 1675 { 1676 struct drm_i915_private *dev_priv = engine->i915; 1677 I915_WRITE_IMR(engine, ~engine->irq_keep_mask); 1678 } 1679 1680 static int gen8_emit_flush(struct drm_i915_gem_request *request, u32 mode) 1681 { 1682 u32 cmd, *cs; 1683 1684 cs = intel_ring_begin(request, 4); 1685 if (IS_ERR(cs)) 1686 return PTR_ERR(cs); 1687 1688 cmd = MI_FLUSH_DW + 1; 1689 1690 /* We always require a command barrier so that subsequent 1691 * commands, such as breadcrumb interrupts, are strictly ordered 1692 * wrt the contents of the write cache being flushed to memory 1693 * (and thus being coherent from the CPU). 1694 */ 1695 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW; 1696 1697 if (mode & EMIT_INVALIDATE) { 1698 cmd |= MI_INVALIDATE_TLB; 1699 if (request->engine->id == VCS) 1700 cmd |= MI_INVALIDATE_BSD; 1701 } 1702 1703 *cs++ = cmd; 1704 *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; 1705 *cs++ = 0; /* upper addr */ 1706 *cs++ = 0; /* value */ 1707 intel_ring_advance(request, cs); 1708 1709 return 0; 1710 } 1711 1712 static int gen8_emit_flush_render(struct drm_i915_gem_request *request, 1713 u32 mode) 1714 { 1715 struct intel_engine_cs *engine = request->engine; 1716 u32 scratch_addr = 1717 i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES; 1718 bool vf_flush_wa = false, dc_flush_wa = false; 1719 u32 *cs, flags = 0; 1720 int len; 1721 1722 flags |= PIPE_CONTROL_CS_STALL; 1723 1724 if (mode & EMIT_FLUSH) { 1725 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; 1726 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; 1727 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; 1728 flags |= PIPE_CONTROL_FLUSH_ENABLE; 1729 } 1730 1731 if (mode & EMIT_INVALIDATE) { 1732 flags |= PIPE_CONTROL_TLB_INVALIDATE; 1733 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; 1734 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 1735 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; 1736 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; 1737 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; 1738 flags |= PIPE_CONTROL_QW_WRITE; 1739 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; 1740 1741 /* 1742 * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL 1743 * pipe control. 1744 */ 1745 if (IS_GEN9(request->i915)) 1746 vf_flush_wa = true; 1747 1748 /* WaForGAMHang:kbl */ 1749 if (IS_KBL_REVID(request->i915, 0, KBL_REVID_B0)) 1750 dc_flush_wa = true; 1751 } 1752 1753 len = 6; 1754 1755 if (vf_flush_wa) 1756 len += 6; 1757 1758 if (dc_flush_wa) 1759 len += 12; 1760 1761 cs = intel_ring_begin(request, len); 1762 if (IS_ERR(cs)) 1763 return PTR_ERR(cs); 1764 1765 if (vf_flush_wa) 1766 cs = gen8_emit_pipe_control(cs, 0, 0); 1767 1768 if (dc_flush_wa) 1769 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE, 1770 0); 1771 1772 cs = gen8_emit_pipe_control(cs, flags, scratch_addr); 1773 1774 if (dc_flush_wa) 1775 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0); 1776 1777 intel_ring_advance(request, cs); 1778 1779 return 0; 1780 } 1781 1782 /* 1783 * Reserve space for 2 NOOPs at the end of each request to be 1784 * used as a workaround for not being allowed to do lite 1785 * restore with HEAD==TAIL (WaIdleLiteRestore). 1786 */ 1787 static void gen8_emit_wa_tail(struct drm_i915_gem_request *request, u32 *cs) 1788 { 1789 /* Ensure there's always at least one preemption point per-request. */ 1790 *cs++ = MI_ARB_CHECK; 1791 *cs++ = MI_NOOP; 1792 request->wa_tail = intel_ring_offset(request, cs); 1793 } 1794 1795 static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, u32 *cs) 1796 { 1797 /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */ 1798 BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5)); 1799 1800 *cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW; 1801 *cs++ = intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT; 1802 *cs++ = 0; 1803 *cs++ = request->global_seqno; 1804 *cs++ = MI_USER_INTERRUPT; 1805 *cs++ = MI_NOOP; 1806 request->tail = intel_ring_offset(request, cs); 1807 assert_ring_tail_valid(request->ring, request->tail); 1808 1809 gen8_emit_wa_tail(request, cs); 1810 } 1811 static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS; 1812 1813 static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request, 1814 u32 *cs) 1815 { 1816 /* We're using qword write, seqno should be aligned to 8 bytes. */ 1817 BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1); 1818 1819 /* w/a for post sync ops following a GPGPU operation we 1820 * need a prior CS_STALL, which is emitted by the flush 1821 * following the batch. 1822 */ 1823 *cs++ = GFX_OP_PIPE_CONTROL(6); 1824 *cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL | 1825 PIPE_CONTROL_QW_WRITE; 1826 *cs++ = intel_hws_seqno_address(request->engine); 1827 *cs++ = 0; 1828 *cs++ = request->global_seqno; 1829 /* We're thrashing one dword of HWS. */ 1830 *cs++ = 0; 1831 *cs++ = MI_USER_INTERRUPT; 1832 *cs++ = MI_NOOP; 1833 request->tail = intel_ring_offset(request, cs); 1834 assert_ring_tail_valid(request->ring, request->tail); 1835 1836 gen8_emit_wa_tail(request, cs); 1837 } 1838 static const int gen8_emit_breadcrumb_render_sz = 8 + WA_TAIL_DWORDS; 1839 1840 static int gen8_init_rcs_context(struct drm_i915_gem_request *req) 1841 { 1842 int ret; 1843 1844 ret = intel_ring_workarounds_emit(req); 1845 if (ret) 1846 return ret; 1847 1848 ret = intel_rcs_context_init_mocs(req); 1849 /* 1850 * Failing to program the MOCS is non-fatal.The system will not 1851 * run at peak performance. So generate an error and carry on. 1852 */ 1853 if (ret) 1854 DRM_ERROR("MOCS failed to program: expect performance issues.\n"); 1855 1856 return i915_gem_render_state_emit(req); 1857 } 1858 1859 /** 1860 * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer 1861 * @engine: Engine Command Streamer. 1862 */ 1863 void intel_logical_ring_cleanup(struct intel_engine_cs *engine) 1864 { 1865 struct drm_i915_private *dev_priv; 1866 1867 /* 1868 * Tasklet cannot be active at this point due intel_mark_active/idle 1869 * so this is just for documentation. 1870 */ 1871 if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->execlists.irq_tasklet.state))) 1872 tasklet_kill(&engine->execlists.irq_tasklet); 1873 1874 dev_priv = engine->i915; 1875 1876 if (engine->buffer) { 1877 WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0); 1878 } 1879 1880 if (engine->cleanup) 1881 engine->cleanup(engine); 1882 1883 intel_engine_cleanup_common(engine); 1884 1885 lrc_destroy_wa_ctx(engine); 1886 engine->i915 = NULL; 1887 dev_priv->engine[engine->id] = NULL; 1888 kfree(engine); 1889 } 1890 1891 static void execlists_set_default_submission(struct intel_engine_cs *engine) 1892 { 1893 engine->submit_request = execlists_submit_request; 1894 engine->cancel_requests = execlists_cancel_requests; 1895 engine->schedule = execlists_schedule; 1896 engine->execlists.irq_tasklet.func = intel_lrc_irq_handler; 1897 } 1898 1899 static void 1900 logical_ring_default_vfuncs(struct intel_engine_cs *engine) 1901 { 1902 /* Default vfuncs which can be overriden by each engine. */ 1903 engine->init_hw = gen8_init_common_ring; 1904 engine->reset_hw = reset_common_ring; 1905 1906 engine->context_pin = execlists_context_pin; 1907 engine->context_unpin = execlists_context_unpin; 1908 1909 engine->request_alloc = execlists_request_alloc; 1910 1911 engine->emit_flush = gen8_emit_flush; 1912 engine->emit_breadcrumb = gen8_emit_breadcrumb; 1913 engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz; 1914 1915 engine->set_default_submission = execlists_set_default_submission; 1916 1917 engine->irq_enable = gen8_logical_ring_enable_irq; 1918 engine->irq_disable = gen8_logical_ring_disable_irq; 1919 engine->emit_bb_start = gen8_emit_bb_start; 1920 } 1921 1922 static inline void 1923 logical_ring_default_irqs(struct intel_engine_cs *engine) 1924 { 1925 unsigned shift = engine->irq_shift; 1926 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift; 1927 engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift; 1928 } 1929 1930 static void 1931 logical_ring_setup(struct intel_engine_cs *engine) 1932 { 1933 struct drm_i915_private *dev_priv = engine->i915; 1934 enum forcewake_domains fw_domains; 1935 1936 intel_engine_setup_common(engine); 1937 1938 /* Intentionally left blank. */ 1939 engine->buffer = NULL; 1940 1941 fw_domains = intel_uncore_forcewake_for_reg(dev_priv, 1942 RING_ELSP(engine), 1943 FW_REG_WRITE); 1944 1945 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, 1946 RING_CONTEXT_STATUS_PTR(engine), 1947 FW_REG_READ | FW_REG_WRITE); 1948 1949 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv, 1950 RING_CONTEXT_STATUS_BUF_BASE(engine), 1951 FW_REG_READ); 1952 1953 engine->execlists.fw_domains = fw_domains; 1954 1955 tasklet_init(&engine->execlists.irq_tasklet, 1956 intel_lrc_irq_handler, (unsigned long)engine); 1957 1958 logical_ring_default_vfuncs(engine); 1959 logical_ring_default_irqs(engine); 1960 } 1961 1962 static int logical_ring_init(struct intel_engine_cs *engine) 1963 { 1964 int ret; 1965 1966 ret = intel_engine_init_common(engine); 1967 if (ret) 1968 goto error; 1969 1970 return 0; 1971 1972 error: 1973 intel_logical_ring_cleanup(engine); 1974 return ret; 1975 } 1976 1977 int logical_render_ring_init(struct intel_engine_cs *engine) 1978 { 1979 struct drm_i915_private *dev_priv = engine->i915; 1980 int ret; 1981 1982 logical_ring_setup(engine); 1983 1984 if (HAS_L3_DPF(dev_priv)) 1985 engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; 1986 1987 /* Override some for render ring. */ 1988 if (INTEL_GEN(dev_priv) >= 9) 1989 engine->init_hw = gen9_init_render_ring; 1990 else 1991 engine->init_hw = gen8_init_render_ring; 1992 engine->init_context = gen8_init_rcs_context; 1993 engine->emit_flush = gen8_emit_flush_render; 1994 engine->emit_breadcrumb = gen8_emit_breadcrumb_render; 1995 engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_render_sz; 1996 1997 ret = intel_engine_create_scratch(engine, PAGE_SIZE); 1998 if (ret) 1999 return ret; 2000 2001 ret = intel_init_workaround_bb(engine); 2002 if (ret) { 2003 /* 2004 * We continue even if we fail to initialize WA batch 2005 * because we only expect rare glitches but nothing 2006 * critical to prevent us from using GPU 2007 */ 2008 DRM_ERROR("WA batch buffer initialization failed: %d\n", 2009 ret); 2010 } 2011 2012 return logical_ring_init(engine); 2013 } 2014 2015 int logical_xcs_ring_init(struct intel_engine_cs *engine) 2016 { 2017 logical_ring_setup(engine); 2018 2019 return logical_ring_init(engine); 2020 } 2021 2022 static u32 2023 make_rpcs(struct drm_i915_private *dev_priv) 2024 { 2025 u32 rpcs = 0; 2026 2027 /* 2028 * No explicit RPCS request is needed to ensure full 2029 * slice/subslice/EU enablement prior to Gen9. 2030 */ 2031 if (INTEL_GEN(dev_priv) < 9) 2032 return 0; 2033 2034 /* 2035 * Starting in Gen9, render power gating can leave 2036 * slice/subslice/EU in a partially enabled state. We 2037 * must make an explicit request through RPCS for full 2038 * enablement. 2039 */ 2040 if (INTEL_INFO(dev_priv)->sseu.has_slice_pg) { 2041 rpcs |= GEN8_RPCS_S_CNT_ENABLE; 2042 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.slice_mask) << 2043 GEN8_RPCS_S_CNT_SHIFT; 2044 rpcs |= GEN8_RPCS_ENABLE; 2045 } 2046 2047 if (INTEL_INFO(dev_priv)->sseu.has_subslice_pg) { 2048 rpcs |= GEN8_RPCS_SS_CNT_ENABLE; 2049 rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.subslice_mask) << 2050 GEN8_RPCS_SS_CNT_SHIFT; 2051 rpcs |= GEN8_RPCS_ENABLE; 2052 } 2053 2054 if (INTEL_INFO(dev_priv)->sseu.has_eu_pg) { 2055 rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice << 2056 GEN8_RPCS_EU_MIN_SHIFT; 2057 rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice << 2058 GEN8_RPCS_EU_MAX_SHIFT; 2059 rpcs |= GEN8_RPCS_ENABLE; 2060 } 2061 2062 return rpcs; 2063 } 2064 2065 static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine) 2066 { 2067 u32 indirect_ctx_offset; 2068 2069 switch (INTEL_GEN(engine->i915)) { 2070 default: 2071 MISSING_CASE(INTEL_GEN(engine->i915)); 2072 /* fall through */ 2073 case 10: 2074 indirect_ctx_offset = 2075 GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 2076 break; 2077 case 9: 2078 indirect_ctx_offset = 2079 GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 2080 break; 2081 case 8: 2082 indirect_ctx_offset = 2083 GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; 2084 break; 2085 } 2086 2087 return indirect_ctx_offset; 2088 } 2089 2090 static void execlists_init_reg_state(u32 *regs, 2091 struct i915_gem_context *ctx, 2092 struct intel_engine_cs *engine, 2093 struct intel_ring *ring) 2094 { 2095 struct drm_i915_private *dev_priv = engine->i915; 2096 struct i915_hw_ppgtt *ppgtt = ctx->ppgtt ?: dev_priv->mm.aliasing_ppgtt; 2097 u32 base = engine->mmio_base; 2098 bool rcs = engine->id == RCS; 2099 2100 /* A context is actually a big batch buffer with several 2101 * MI_LOAD_REGISTER_IMM commands followed by (reg, value) pairs. The 2102 * values we are setting here are only for the first context restore: 2103 * on a subsequent save, the GPU will recreate this batchbuffer with new 2104 * values (including all the missing MI_LOAD_REGISTER_IMM commands that 2105 * we are not initializing here). 2106 */ 2107 regs[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(rcs ? 14 : 11) | 2108 MI_LRI_FORCE_POSTED; 2109 2110 CTX_REG(regs, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(engine), 2111 _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | 2112 CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | 2113 (HAS_RESOURCE_STREAMER(dev_priv) ? 2114 CTX_CTRL_RS_CTX_ENABLE : 0))); 2115 CTX_REG(regs, CTX_RING_HEAD, RING_HEAD(base), 0); 2116 CTX_REG(regs, CTX_RING_TAIL, RING_TAIL(base), 0); 2117 CTX_REG(regs, CTX_RING_BUFFER_START, RING_START(base), 0); 2118 CTX_REG(regs, CTX_RING_BUFFER_CONTROL, RING_CTL(base), 2119 RING_CTL_SIZE(ring->size) | RING_VALID); 2120 CTX_REG(regs, CTX_BB_HEAD_U, RING_BBADDR_UDW(base), 0); 2121 CTX_REG(regs, CTX_BB_HEAD_L, RING_BBADDR(base), 0); 2122 CTX_REG(regs, CTX_BB_STATE, RING_BBSTATE(base), RING_BB_PPGTT); 2123 CTX_REG(regs, CTX_SECOND_BB_HEAD_U, RING_SBBADDR_UDW(base), 0); 2124 CTX_REG(regs, CTX_SECOND_BB_HEAD_L, RING_SBBADDR(base), 0); 2125 CTX_REG(regs, CTX_SECOND_BB_STATE, RING_SBBSTATE(base), 0); 2126 if (rcs) { 2127 struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx; 2128 2129 CTX_REG(regs, CTX_RCS_INDIRECT_CTX, RING_INDIRECT_CTX(base), 0); 2130 CTX_REG(regs, CTX_RCS_INDIRECT_CTX_OFFSET, 2131 RING_INDIRECT_CTX_OFFSET(base), 0); 2132 if (wa_ctx->indirect_ctx.size) { 2133 u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); 2134 2135 regs[CTX_RCS_INDIRECT_CTX + 1] = 2136 (ggtt_offset + wa_ctx->indirect_ctx.offset) | 2137 (wa_ctx->indirect_ctx.size / CACHELINE_BYTES); 2138 2139 regs[CTX_RCS_INDIRECT_CTX_OFFSET + 1] = 2140 intel_lr_indirect_ctx_offset(engine) << 6; 2141 } 2142 2143 CTX_REG(regs, CTX_BB_PER_CTX_PTR, RING_BB_PER_CTX_PTR(base), 0); 2144 if (wa_ctx->per_ctx.size) { 2145 u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); 2146 2147 regs[CTX_BB_PER_CTX_PTR + 1] = 2148 (ggtt_offset + wa_ctx->per_ctx.offset) | 0x01; 2149 } 2150 } 2151 2152 regs[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9) | MI_LRI_FORCE_POSTED; 2153 2154 CTX_REG(regs, CTX_CTX_TIMESTAMP, RING_CTX_TIMESTAMP(base), 0); 2155 /* PDP values well be assigned later if needed */ 2156 CTX_REG(regs, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(engine, 3), 0); 2157 CTX_REG(regs, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(engine, 3), 0); 2158 CTX_REG(regs, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(engine, 2), 0); 2159 CTX_REG(regs, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(engine, 2), 0); 2160 CTX_REG(regs, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(engine, 1), 0); 2161 CTX_REG(regs, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(engine, 1), 0); 2162 CTX_REG(regs, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(engine, 0), 0); 2163 CTX_REG(regs, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(engine, 0), 0); 2164 2165 if (ppgtt && i915_vm_is_48bit(&ppgtt->base)) { 2166 /* 64b PPGTT (48bit canonical) 2167 * PDP0_DESCRIPTOR contains the base address to PML4 and 2168 * other PDP Descriptors are ignored. 2169 */ 2170 ASSIGN_CTX_PML4(ppgtt, regs); 2171 } 2172 2173 if (rcs) { 2174 regs[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1); 2175 CTX_REG(regs, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, 2176 make_rpcs(dev_priv)); 2177 2178 i915_oa_init_reg_state(engine, ctx, regs); 2179 } 2180 } 2181 2182 static int 2183 populate_lr_context(struct i915_gem_context *ctx, 2184 struct drm_i915_gem_object *ctx_obj, 2185 struct intel_engine_cs *engine, 2186 struct intel_ring *ring) 2187 { 2188 void *vaddr; 2189 int ret; 2190 2191 ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true); 2192 if (ret) { 2193 DRM_DEBUG_DRIVER("Could not set to CPU domain\n"); 2194 return ret; 2195 } 2196 2197 vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB); 2198 if (IS_ERR(vaddr)) { 2199 ret = PTR_ERR(vaddr); 2200 DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret); 2201 return ret; 2202 } 2203 ctx_obj->mm.dirty = true; 2204 2205 /* The second page of the context object contains some fields which must 2206 * be set up prior to the first execution. */ 2207 2208 execlists_init_reg_state(vaddr + LRC_STATE_PN * PAGE_SIZE, 2209 ctx, engine, ring); 2210 2211 i915_gem_object_unpin_map(ctx_obj); 2212 2213 return 0; 2214 } 2215 2216 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx, 2217 struct intel_engine_cs *engine) 2218 { 2219 struct drm_i915_gem_object *ctx_obj; 2220 struct intel_context *ce = &ctx->engine[engine->id]; 2221 struct i915_vma *vma; 2222 uint32_t context_size; 2223 struct intel_ring *ring; 2224 int ret; 2225 2226 WARN_ON(ce->state); 2227 2228 context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE); 2229 2230 /* 2231 * Before the actual start of the context image, we insert a few pages 2232 * for our own use and for sharing with the GuC. 2233 */ 2234 context_size += LRC_HEADER_PAGES * PAGE_SIZE; 2235 2236 ctx_obj = i915_gem_object_create(ctx->i915, context_size); 2237 if (IS_ERR(ctx_obj)) { 2238 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n"); 2239 return PTR_ERR(ctx_obj); 2240 } 2241 2242 vma = i915_vma_instance(ctx_obj, &ctx->i915->ggtt.base, NULL); 2243 if (IS_ERR(vma)) { 2244 ret = PTR_ERR(vma); 2245 goto error_deref_obj; 2246 } 2247 2248 ring = intel_engine_create_ring(engine, ctx->ring_size); 2249 if (IS_ERR(ring)) { 2250 ret = PTR_ERR(ring); 2251 goto error_deref_obj; 2252 } 2253 2254 ret = populate_lr_context(ctx, ctx_obj, engine, ring); 2255 if (ret) { 2256 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret); 2257 goto error_ring_free; 2258 } 2259 2260 ce->ring = ring; 2261 ce->state = vma; 2262 ce->initialised |= engine->init_context == NULL; 2263 2264 return 0; 2265 2266 error_ring_free: 2267 intel_ring_free(ring); 2268 error_deref_obj: 2269 i915_gem_object_put(ctx_obj); 2270 return ret; 2271 } 2272 2273 void intel_lr_context_resume(struct drm_i915_private *dev_priv) 2274 { 2275 struct intel_engine_cs *engine; 2276 struct i915_gem_context *ctx; 2277 enum intel_engine_id id; 2278 2279 /* Because we emit WA_TAIL_DWORDS there may be a disparity 2280 * between our bookkeeping in ce->ring->head and ce->ring->tail and 2281 * that stored in context. As we only write new commands from 2282 * ce->ring->tail onwards, everything before that is junk. If the GPU 2283 * starts reading from its RING_HEAD from the context, it may try to 2284 * execute that junk and die. 2285 * 2286 * So to avoid that we reset the context images upon resume. For 2287 * simplicity, we just zero everything out. 2288 */ 2289 list_for_each_entry(ctx, &dev_priv->contexts.list, link) { 2290 for_each_engine(engine, dev_priv, id) { 2291 struct intel_context *ce = &ctx->engine[engine->id]; 2292 u32 *reg; 2293 2294 if (!ce->state) 2295 continue; 2296 2297 reg = i915_gem_object_pin_map(ce->state->obj, 2298 I915_MAP_WB); 2299 if (WARN_ON(IS_ERR(reg))) 2300 continue; 2301 2302 reg += LRC_STATE_PN * PAGE_SIZE / sizeof(*reg); 2303 reg[CTX_RING_HEAD+1] = 0; 2304 reg[CTX_RING_TAIL+1] = 0; 2305 2306 ce->state->obj->mm.dirty = true; 2307 i915_gem_object_unpin_map(ce->state->obj); 2308 2309 intel_ring_reset(ce->ring, 0); 2310 } 2311 } 2312 } 2313