1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2008-2021 Intel Corporation 4 */ 5 6 #include <drm/drm_cache.h> 7 8 #include "gem/i915_gem_internal.h" 9 10 #include "gen2_engine_cs.h" 11 #include "gen6_engine_cs.h" 12 #include "gen6_ppgtt.h" 13 #include "gen7_renderclear.h" 14 #include "i915_drv.h" 15 #include "i915_mitigations.h" 16 #include "intel_breadcrumbs.h" 17 #include "intel_context.h" 18 #include "intel_engine_regs.h" 19 #include "intel_gt.h" 20 #include "intel_gt_irq.h" 21 #include "intel_gt_regs.h" 22 #include "intel_reset.h" 23 #include "intel_ring.h" 24 #include "shmem_utils.h" 25 #include "intel_engine_heartbeat.h" 26 #include "intel_engine_pm.h" 27 28 /* Rough estimate of the typical request size, performing a flush, 29 * set-context and then emitting the batch. 30 */ 31 #define LEGACY_REQUEST_SIZE 200 32 33 static void set_hwstam(struct intel_engine_cs *engine, u32 mask) 34 { 35 /* 36 * Keep the render interrupt unmasked as this papers over 37 * lost interrupts following a reset. 38 */ 39 if (engine->class == RENDER_CLASS) { 40 if (GRAPHICS_VER(engine->i915) >= 6) 41 mask &= ~BIT(0); 42 else 43 mask &= ~I915_USER_INTERRUPT; 44 } 45 46 intel_engine_set_hwsp_writemask(engine, mask); 47 } 48 49 static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys) 50 { 51 u32 addr; 52 53 addr = lower_32_bits(phys); 54 if (GRAPHICS_VER(engine->i915) >= 4) 55 addr |= (phys >> 28) & 0xf0; 56 57 intel_uncore_write(engine->uncore, HWS_PGA, addr); 58 } 59 60 static struct page *status_page(struct intel_engine_cs *engine) 61 { 62 struct drm_i915_gem_object *obj = engine->status_page.vma->obj; 63 64 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 65 return sg_page(obj->mm.pages->sgl); 66 } 67 68 static void ring_setup_phys_status_page(struct intel_engine_cs *engine) 69 { 70 set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine)))); 71 set_hwstam(engine, ~0u); 72 } 73 74 static void set_hwsp(struct intel_engine_cs *engine, u32 offset) 75 { 76 i915_reg_t hwsp; 77 78 /* 79 * The ring status page addresses are no longer next to the rest of 80 * the ring registers as of gen7. 81 */ 82 if (GRAPHICS_VER(engine->i915) == 7) { 83 switch (engine->id) { 84 /* 85 * No more rings exist on Gen7. Default case is only to shut up 86 * gcc switch check warning. 87 */ 88 default: 89 GEM_BUG_ON(engine->id); 90 fallthrough; 91 case RCS0: 92 hwsp = RENDER_HWS_PGA_GEN7; 93 break; 94 case BCS0: 95 hwsp = BLT_HWS_PGA_GEN7; 96 break; 97 case VCS0: 98 hwsp = BSD_HWS_PGA_GEN7; 99 break; 100 case VECS0: 101 hwsp = VEBOX_HWS_PGA_GEN7; 102 break; 103 } 104 } else if (GRAPHICS_VER(engine->i915) == 6) { 105 hwsp = RING_HWS_PGA_GEN6(engine->mmio_base); 106 } else { 107 hwsp = RING_HWS_PGA(engine->mmio_base); 108 } 109 110 intel_uncore_write_fw(engine->uncore, hwsp, offset); 111 intel_uncore_posting_read_fw(engine->uncore, hwsp); 112 } 113 114 static void flush_cs_tlb(struct intel_engine_cs *engine) 115 { 116 if (!IS_GRAPHICS_VER(engine->i915, 6, 7)) 117 return; 118 119 /* ring should be idle before issuing a sync flush*/ 120 if ((ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0) 121 drm_warn(&engine->i915->drm, "%s not idle before sync flush!\n", 122 engine->name); 123 124 ENGINE_WRITE_FW(engine, RING_INSTPM, 125 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | 126 INSTPM_SYNC_FLUSH)); 127 if (__intel_wait_for_register_fw(engine->uncore, 128 RING_INSTPM(engine->mmio_base), 129 INSTPM_SYNC_FLUSH, 0, 130 2000, 0, NULL)) 131 ENGINE_TRACE(engine, 132 "wait for SyncFlush to complete for TLB invalidation timed out\n"); 133 } 134 135 static void ring_setup_status_page(struct intel_engine_cs *engine) 136 { 137 set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma)); 138 set_hwstam(engine, ~0u); 139 140 flush_cs_tlb(engine); 141 } 142 143 static struct i915_address_space *vm_alias(struct i915_address_space *vm) 144 { 145 if (i915_is_ggtt(vm)) 146 vm = &i915_vm_to_ggtt(vm)->alias->vm; 147 148 return vm; 149 } 150 151 static u32 pp_dir(struct i915_address_space *vm) 152 { 153 return to_gen6_ppgtt(i915_vm_to_ppgtt(vm))->pp_dir; 154 } 155 156 static void set_pp_dir(struct intel_engine_cs *engine) 157 { 158 struct i915_address_space *vm = vm_alias(engine->gt->vm); 159 160 if (!vm) 161 return; 162 163 ENGINE_WRITE_FW(engine, RING_PP_DIR_DCLV, PP_DIR_DCLV_2G); 164 ENGINE_WRITE_FW(engine, RING_PP_DIR_BASE, pp_dir(vm)); 165 166 if (GRAPHICS_VER(engine->i915) >= 7) { 167 ENGINE_WRITE_FW(engine, 168 RING_MODE_GEN7, 169 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 170 } 171 } 172 173 static bool stop_ring(struct intel_engine_cs *engine) 174 { 175 /* Empty the ring by skipping to the end */ 176 ENGINE_WRITE_FW(engine, RING_HEAD, ENGINE_READ_FW(engine, RING_TAIL)); 177 ENGINE_POSTING_READ(engine, RING_HEAD); 178 179 /* The ring must be empty before it is disabled */ 180 ENGINE_WRITE_FW(engine, RING_CTL, 0); 181 ENGINE_POSTING_READ(engine, RING_CTL); 182 183 /* Then reset the disabled ring */ 184 ENGINE_WRITE_FW(engine, RING_HEAD, 0); 185 ENGINE_WRITE_FW(engine, RING_TAIL, 0); 186 187 return (ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR) == 0; 188 } 189 190 static int xcs_resume(struct intel_engine_cs *engine) 191 { 192 struct intel_ring *ring = engine->legacy.ring; 193 194 ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n", 195 ring->head, ring->tail); 196 197 /* 198 * Double check the ring is empty & disabled before we resume. Called 199 * from atomic context during PCI probe, so _hardirq(). 200 */ 201 intel_synchronize_hardirq(engine->i915); 202 if (!stop_ring(engine)) 203 goto err; 204 205 if (HWS_NEEDS_PHYSICAL(engine->i915)) 206 ring_setup_phys_status_page(engine); 207 else 208 ring_setup_status_page(engine); 209 210 intel_breadcrumbs_reset(engine->breadcrumbs); 211 212 /* Enforce ordering by reading HEAD register back */ 213 ENGINE_POSTING_READ(engine, RING_HEAD); 214 215 /* 216 * Initialize the ring. This must happen _after_ we've cleared the ring 217 * registers with the above sequence (the readback of the HEAD registers 218 * also enforces ordering), otherwise the hw might lose the new ring 219 * register values. 220 */ 221 ENGINE_WRITE_FW(engine, RING_START, i915_ggtt_offset(ring->vma)); 222 223 /* Check that the ring offsets point within the ring! */ 224 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head)); 225 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); 226 intel_ring_update_space(ring); 227 228 set_pp_dir(engine); 229 230 /* First wake the ring up to an empty/idle ring */ 231 ENGINE_WRITE_FW(engine, RING_HEAD, ring->head); 232 ENGINE_WRITE_FW(engine, RING_TAIL, ring->head); 233 ENGINE_POSTING_READ(engine, RING_TAIL); 234 235 ENGINE_WRITE_FW(engine, RING_CTL, 236 RING_CTL_SIZE(ring->size) | RING_VALID); 237 238 /* If the head is still not zero, the ring is dead */ 239 if (__intel_wait_for_register_fw(engine->uncore, 240 RING_CTL(engine->mmio_base), 241 RING_VALID, RING_VALID, 242 5000, 0, NULL)) 243 goto err; 244 245 if (GRAPHICS_VER(engine->i915) > 2) 246 ENGINE_WRITE_FW(engine, 247 RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); 248 249 /* Now awake, let it get started */ 250 if (ring->tail != ring->head) { 251 ENGINE_WRITE_FW(engine, RING_TAIL, ring->tail); 252 ENGINE_POSTING_READ(engine, RING_TAIL); 253 } 254 255 /* Papering over lost _interrupts_ immediately following the restart */ 256 intel_engine_signal_breadcrumbs(engine); 257 return 0; 258 259 err: 260 drm_err(&engine->i915->drm, 261 "%s initialization failed; " 262 "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n", 263 engine->name, 264 ENGINE_READ(engine, RING_CTL), 265 ENGINE_READ(engine, RING_CTL) & RING_VALID, 266 ENGINE_READ(engine, RING_HEAD), ring->head, 267 ENGINE_READ(engine, RING_TAIL), ring->tail, 268 ENGINE_READ(engine, RING_START), 269 i915_ggtt_offset(ring->vma)); 270 return -EIO; 271 } 272 273 static void sanitize_hwsp(struct intel_engine_cs *engine) 274 { 275 struct intel_timeline *tl; 276 277 list_for_each_entry(tl, &engine->status_page.timelines, engine_link) 278 intel_timeline_reset_seqno(tl); 279 } 280 281 static void xcs_sanitize(struct intel_engine_cs *engine) 282 { 283 /* 284 * Poison residual state on resume, in case the suspend didn't! 285 * 286 * We have to assume that across suspend/resume (or other loss 287 * of control) that the contents of our pinned buffers has been 288 * lost, replaced by garbage. Since this doesn't always happen, 289 * let's poison such state so that we more quickly spot when 290 * we falsely assume it has been preserved. 291 */ 292 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 293 memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE); 294 295 /* 296 * The kernel_context HWSP is stored in the status_page. As above, 297 * that may be lost on resume/initialisation, and so we need to 298 * reset the value in the HWSP. 299 */ 300 sanitize_hwsp(engine); 301 302 /* And scrub the dirty cachelines for the HWSP */ 303 drm_clflush_virt_range(engine->status_page.addr, PAGE_SIZE); 304 305 intel_engine_reset_pinned_contexts(engine); 306 } 307 308 static void reset_prepare(struct intel_engine_cs *engine) 309 { 310 /* 311 * We stop engines, otherwise we might get failed reset and a 312 * dead gpu (on elk). Also as modern gpu as kbl can suffer 313 * from system hang if batchbuffer is progressing when 314 * the reset is issued, regardless of READY_TO_RESET ack. 315 * Thus assume it is best to stop engines on all gens 316 * where we have a gpu reset. 317 * 318 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES) 319 * 320 * WaMediaResetMainRingCleanup:ctg,elk (presumably) 321 * WaClearRingBufHeadRegAtInit:ctg,elk 322 * 323 * FIXME: Wa for more modern gens needs to be validated 324 */ 325 ENGINE_TRACE(engine, "\n"); 326 intel_engine_stop_cs(engine); 327 328 if (!stop_ring(engine)) { 329 /* G45 ring initialization often fails to reset head to zero */ 330 ENGINE_TRACE(engine, 331 "HEAD not reset to zero, " 332 "{ CTL:%08x, HEAD:%08x, TAIL:%08x, START:%08x }\n", 333 ENGINE_READ_FW(engine, RING_CTL), 334 ENGINE_READ_FW(engine, RING_HEAD), 335 ENGINE_READ_FW(engine, RING_TAIL), 336 ENGINE_READ_FW(engine, RING_START)); 337 if (!stop_ring(engine)) { 338 drm_err(&engine->i915->drm, 339 "failed to set %s head to zero " 340 "ctl %08x head %08x tail %08x start %08x\n", 341 engine->name, 342 ENGINE_READ_FW(engine, RING_CTL), 343 ENGINE_READ_FW(engine, RING_HEAD), 344 ENGINE_READ_FW(engine, RING_TAIL), 345 ENGINE_READ_FW(engine, RING_START)); 346 } 347 } 348 } 349 350 static void reset_rewind(struct intel_engine_cs *engine, bool stalled) 351 { 352 struct i915_request *pos, *rq; 353 unsigned long flags; 354 u32 head; 355 356 rq = NULL; 357 spin_lock_irqsave(&engine->sched_engine->lock, flags); 358 rcu_read_lock(); 359 list_for_each_entry(pos, &engine->sched_engine->requests, sched.link) { 360 if (!__i915_request_is_complete(pos)) { 361 rq = pos; 362 break; 363 } 364 } 365 rcu_read_unlock(); 366 367 /* 368 * The guilty request will get skipped on a hung engine. 369 * 370 * Users of client default contexts do not rely on logical 371 * state preserved between batches so it is safe to execute 372 * queued requests following the hang. Non default contexts 373 * rely on preserved state, so skipping a batch loses the 374 * evolution of the state and it needs to be considered corrupted. 375 * Executing more queued batches on top of corrupted state is 376 * risky. But we take the risk by trying to advance through 377 * the queued requests in order to make the client behaviour 378 * more predictable around resets, by not throwing away random 379 * amount of batches it has prepared for execution. Sophisticated 380 * clients can use gem_reset_stats_ioctl and dma fence status 381 * (exported via sync_file info ioctl on explicit fences) to observe 382 * when it loses the context state and should rebuild accordingly. 383 * 384 * The context ban, and ultimately the client ban, mechanism are safety 385 * valves if client submission ends up resulting in nothing more than 386 * subsequent hangs. 387 */ 388 389 if (rq) { 390 /* 391 * Try to restore the logical GPU state to match the 392 * continuation of the request queue. If we skip the 393 * context/PD restore, then the next request may try to execute 394 * assuming that its context is valid and loaded on the GPU and 395 * so may try to access invalid memory, prompting repeated GPU 396 * hangs. 397 * 398 * If the request was guilty, we still restore the logical 399 * state in case the next request requires it (e.g. the 400 * aliasing ppgtt), but skip over the hung batch. 401 * 402 * If the request was innocent, we try to replay the request 403 * with the restored context. 404 */ 405 __i915_request_reset(rq, stalled); 406 407 GEM_BUG_ON(rq->ring != engine->legacy.ring); 408 head = rq->head; 409 } else { 410 head = engine->legacy.ring->tail; 411 } 412 engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head); 413 414 spin_unlock_irqrestore(&engine->sched_engine->lock, flags); 415 } 416 417 static void reset_finish(struct intel_engine_cs *engine) 418 { 419 } 420 421 static void reset_cancel(struct intel_engine_cs *engine) 422 { 423 struct i915_request *request; 424 unsigned long flags; 425 426 spin_lock_irqsave(&engine->sched_engine->lock, flags); 427 428 /* Mark all submitted requests as skipped. */ 429 list_for_each_entry(request, &engine->sched_engine->requests, sched.link) 430 i915_request_put(i915_request_mark_eio(request)); 431 intel_engine_signal_breadcrumbs(engine); 432 433 /* Remaining _unready_ requests will be nop'ed when submitted */ 434 435 spin_unlock_irqrestore(&engine->sched_engine->lock, flags); 436 } 437 438 static void i9xx_submit_request(struct i915_request *request) 439 { 440 i915_request_submit(request); 441 wmb(); /* paranoid flush writes out of the WCB before mmio */ 442 443 ENGINE_WRITE(request->engine, RING_TAIL, 444 intel_ring_set_tail(request->ring, request->tail)); 445 } 446 447 static void __ring_context_fini(struct intel_context *ce) 448 { 449 i915_vma_put(ce->state); 450 } 451 452 static void ring_context_destroy(struct kref *ref) 453 { 454 struct intel_context *ce = container_of(ref, typeof(*ce), ref); 455 456 GEM_BUG_ON(intel_context_is_pinned(ce)); 457 458 if (ce->state) 459 __ring_context_fini(ce); 460 461 intel_context_fini(ce); 462 intel_context_free(ce); 463 } 464 465 static int ring_context_init_default_state(struct intel_context *ce, 466 struct i915_gem_ww_ctx *ww) 467 { 468 struct drm_i915_gem_object *obj = ce->state->obj; 469 void *vaddr; 470 471 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 472 if (IS_ERR(vaddr)) 473 return PTR_ERR(vaddr); 474 475 shmem_read(ce->engine->default_state, 0, 476 vaddr, ce->engine->context_size); 477 478 i915_gem_object_flush_map(obj); 479 __i915_gem_object_release_map(obj); 480 481 __set_bit(CONTEXT_VALID_BIT, &ce->flags); 482 return 0; 483 } 484 485 static int ring_context_pre_pin(struct intel_context *ce, 486 struct i915_gem_ww_ctx *ww, 487 void **unused) 488 { 489 struct i915_address_space *vm; 490 int err = 0; 491 492 if (ce->engine->default_state && 493 !test_bit(CONTEXT_VALID_BIT, &ce->flags)) { 494 err = ring_context_init_default_state(ce, ww); 495 if (err) 496 return err; 497 } 498 499 vm = vm_alias(ce->vm); 500 if (vm) 501 err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww); 502 503 return err; 504 } 505 506 static void __context_unpin_ppgtt(struct intel_context *ce) 507 { 508 struct i915_address_space *vm; 509 510 vm = vm_alias(ce->vm); 511 if (vm) 512 gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm)); 513 } 514 515 static void ring_context_unpin(struct intel_context *ce) 516 { 517 } 518 519 static void ring_context_post_unpin(struct intel_context *ce) 520 { 521 __context_unpin_ppgtt(ce); 522 } 523 524 static struct i915_vma * 525 alloc_context_vma(struct intel_engine_cs *engine) 526 { 527 struct drm_i915_private *i915 = engine->i915; 528 struct drm_i915_gem_object *obj; 529 struct i915_vma *vma; 530 int err; 531 532 obj = i915_gem_object_create_shmem(i915, engine->context_size); 533 if (IS_ERR(obj)) 534 return ERR_CAST(obj); 535 536 /* 537 * Try to make the context utilize L3 as well as LLC. 538 * 539 * On VLV we don't have L3 controls in the PTEs so we 540 * shouldn't touch the cache level, especially as that 541 * would make the object snooped which might have a 542 * negative performance impact. 543 * 544 * Snooping is required on non-llc platforms in execlist 545 * mode, but since all GGTT accesses use PAT entry 0 we 546 * get snooping anyway regardless of cache_level. 547 * 548 * This is only applicable for Ivy Bridge devices since 549 * later platforms don't have L3 control bits in the PTE. 550 */ 551 if (IS_IVYBRIDGE(i915)) 552 i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC); 553 554 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 555 if (IS_ERR(vma)) { 556 err = PTR_ERR(vma); 557 goto err_obj; 558 } 559 560 return vma; 561 562 err_obj: 563 i915_gem_object_put(obj); 564 return ERR_PTR(err); 565 } 566 567 static int ring_context_alloc(struct intel_context *ce) 568 { 569 struct intel_engine_cs *engine = ce->engine; 570 571 /* One ringbuffer to rule them all */ 572 GEM_BUG_ON(!engine->legacy.ring); 573 ce->ring = engine->legacy.ring; 574 ce->timeline = intel_timeline_get(engine->legacy.timeline); 575 576 GEM_BUG_ON(ce->state); 577 if (engine->context_size) { 578 struct i915_vma *vma; 579 580 vma = alloc_context_vma(engine); 581 if (IS_ERR(vma)) 582 return PTR_ERR(vma); 583 584 ce->state = vma; 585 } 586 587 return 0; 588 } 589 590 static int ring_context_pin(struct intel_context *ce, void *unused) 591 { 592 return 0; 593 } 594 595 static void ring_context_reset(struct intel_context *ce) 596 { 597 intel_ring_reset(ce->ring, ce->ring->emit); 598 clear_bit(CONTEXT_VALID_BIT, &ce->flags); 599 } 600 601 static void ring_context_revoke(struct intel_context *ce, 602 struct i915_request *rq, 603 unsigned int preempt_timeout_ms) 604 { 605 struct intel_engine_cs *engine; 606 607 if (!rq || !i915_request_is_active(rq)) 608 return; 609 610 engine = rq->engine; 611 lockdep_assert_held(&engine->sched_engine->lock); 612 list_for_each_entry_continue(rq, &engine->sched_engine->requests, 613 sched.link) 614 if (rq->context == ce) { 615 i915_request_set_error_once(rq, -EIO); 616 __i915_request_skip(rq); 617 } 618 } 619 620 static void ring_context_cancel_request(struct intel_context *ce, 621 struct i915_request *rq) 622 { 623 struct intel_engine_cs *engine = NULL; 624 625 i915_request_active_engine(rq, &engine); 626 627 if (engine && intel_engine_pulse(engine)) 628 intel_gt_handle_error(engine->gt, engine->mask, 0, 629 "request cancellation by %s", 630 current->comm); 631 } 632 633 static const struct intel_context_ops ring_context_ops = { 634 .alloc = ring_context_alloc, 635 636 .cancel_request = ring_context_cancel_request, 637 638 .revoke = ring_context_revoke, 639 640 .pre_pin = ring_context_pre_pin, 641 .pin = ring_context_pin, 642 .unpin = ring_context_unpin, 643 .post_unpin = ring_context_post_unpin, 644 645 .enter = intel_context_enter_engine, 646 .exit = intel_context_exit_engine, 647 648 .reset = ring_context_reset, 649 .destroy = ring_context_destroy, 650 }; 651 652 static int load_pd_dir(struct i915_request *rq, 653 struct i915_address_space *vm, 654 u32 valid) 655 { 656 const struct intel_engine_cs * const engine = rq->engine; 657 u32 *cs; 658 659 cs = intel_ring_begin(rq, 12); 660 if (IS_ERR(cs)) 661 return PTR_ERR(cs); 662 663 *cs++ = MI_LOAD_REGISTER_IMM(1); 664 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base)); 665 *cs++ = valid; 666 667 *cs++ = MI_LOAD_REGISTER_IMM(1); 668 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 669 *cs++ = pp_dir(vm); 670 671 /* Stall until the page table load is complete? */ 672 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 673 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 674 *cs++ = intel_gt_scratch_offset(engine->gt, 675 INTEL_GT_SCRATCH_FIELD_DEFAULT); 676 677 *cs++ = MI_LOAD_REGISTER_IMM(1); 678 *cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base)); 679 *cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE); 680 681 intel_ring_advance(rq, cs); 682 683 return rq->engine->emit_flush(rq, EMIT_FLUSH); 684 } 685 686 static int mi_set_context(struct i915_request *rq, 687 struct intel_context *ce, 688 u32 flags) 689 { 690 struct intel_engine_cs *engine = rq->engine; 691 struct drm_i915_private *i915 = engine->i915; 692 enum intel_engine_id id; 693 const int num_engines = 694 IS_HASWELL(i915) ? engine->gt->info.num_engines - 1 : 0; 695 bool force_restore = false; 696 int len; 697 u32 *cs; 698 699 len = 4; 700 if (GRAPHICS_VER(i915) == 7) 701 len += 2 + (num_engines ? 4 * num_engines + 6 : 0); 702 else if (GRAPHICS_VER(i915) == 5) 703 len += 2; 704 if (flags & MI_FORCE_RESTORE) { 705 GEM_BUG_ON(flags & MI_RESTORE_INHIBIT); 706 flags &= ~MI_FORCE_RESTORE; 707 force_restore = true; 708 len += 2; 709 } 710 711 cs = intel_ring_begin(rq, len); 712 if (IS_ERR(cs)) 713 return PTR_ERR(cs); 714 715 /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ 716 if (GRAPHICS_VER(i915) == 7) { 717 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 718 if (num_engines) { 719 struct intel_engine_cs *signaller; 720 721 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 722 for_each_engine(signaller, engine->gt, id) { 723 if (signaller == engine) 724 continue; 725 726 *cs++ = i915_mmio_reg_offset( 727 RING_PSMI_CTL(signaller->mmio_base)); 728 *cs++ = _MASKED_BIT_ENABLE( 729 GEN6_PSMI_SLEEP_MSG_DISABLE); 730 } 731 } 732 } else if (GRAPHICS_VER(i915) == 5) { 733 /* 734 * This w/a is only listed for pre-production ilk a/b steppings, 735 * but is also mentioned for programming the powerctx. To be 736 * safe, just apply the workaround; we do not use SyncFlush so 737 * this should never take effect and so be a no-op! 738 */ 739 *cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN; 740 } 741 742 if (force_restore) { 743 /* 744 * The HW doesn't handle being told to restore the current 745 * context very well. Quite often it likes goes to go off and 746 * sulk, especially when it is meant to be reloading PP_DIR. 747 * A very simple fix to force the reload is to simply switch 748 * away from the current context and back again. 749 * 750 * Note that the kernel_context will contain random state 751 * following the INHIBIT_RESTORE. We accept this since we 752 * never use the kernel_context state; it is merely a 753 * placeholder we use to flush other contexts. 754 */ 755 *cs++ = MI_SET_CONTEXT; 756 *cs++ = i915_ggtt_offset(engine->kernel_context->state) | 757 MI_MM_SPACE_GTT | 758 MI_RESTORE_INHIBIT; 759 } 760 761 *cs++ = MI_NOOP; 762 *cs++ = MI_SET_CONTEXT; 763 *cs++ = i915_ggtt_offset(ce->state) | flags; 764 /* 765 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP 766 * WaMiSetContext_Hang:snb,ivb,vlv 767 */ 768 *cs++ = MI_NOOP; 769 770 if (GRAPHICS_VER(i915) == 7) { 771 if (num_engines) { 772 struct intel_engine_cs *signaller; 773 i915_reg_t last_reg = INVALID_MMIO_REG; /* keep gcc quiet */ 774 775 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 776 for_each_engine(signaller, engine->gt, id) { 777 if (signaller == engine) 778 continue; 779 780 last_reg = RING_PSMI_CTL(signaller->mmio_base); 781 *cs++ = i915_mmio_reg_offset(last_reg); 782 *cs++ = _MASKED_BIT_DISABLE( 783 GEN6_PSMI_SLEEP_MSG_DISABLE); 784 } 785 786 /* Insert a delay before the next switch! */ 787 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 788 *cs++ = i915_mmio_reg_offset(last_reg); 789 *cs++ = intel_gt_scratch_offset(engine->gt, 790 INTEL_GT_SCRATCH_FIELD_DEFAULT); 791 *cs++ = MI_NOOP; 792 } 793 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 794 } else if (GRAPHICS_VER(i915) == 5) { 795 *cs++ = MI_SUSPEND_FLUSH; 796 } 797 798 intel_ring_advance(rq, cs); 799 800 return 0; 801 } 802 803 static int remap_l3_slice(struct i915_request *rq, int slice) 804 { 805 #define L3LOG_DW (GEN7_L3LOG_SIZE / sizeof(u32)) 806 u32 *cs, *remap_info = rq->engine->i915->l3_parity.remap_info[slice]; 807 int i; 808 809 if (!remap_info) 810 return 0; 811 812 cs = intel_ring_begin(rq, L3LOG_DW * 2 + 2); 813 if (IS_ERR(cs)) 814 return PTR_ERR(cs); 815 816 /* 817 * Note: We do not worry about the concurrent register cacheline hang 818 * here because no other code should access these registers other than 819 * at initialization time. 820 */ 821 *cs++ = MI_LOAD_REGISTER_IMM(L3LOG_DW); 822 for (i = 0; i < L3LOG_DW; i++) { 823 *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i)); 824 *cs++ = remap_info[i]; 825 } 826 *cs++ = MI_NOOP; 827 intel_ring_advance(rq, cs); 828 829 return 0; 830 #undef L3LOG_DW 831 } 832 833 static int remap_l3(struct i915_request *rq) 834 { 835 struct i915_gem_context *ctx = i915_request_gem_context(rq); 836 int i, err; 837 838 if (!ctx || !ctx->remap_slice) 839 return 0; 840 841 for (i = 0; i < MAX_L3_SLICES; i++) { 842 if (!(ctx->remap_slice & BIT(i))) 843 continue; 844 845 err = remap_l3_slice(rq, i); 846 if (err) 847 return err; 848 } 849 850 ctx->remap_slice = 0; 851 return 0; 852 } 853 854 static int switch_mm(struct i915_request *rq, struct i915_address_space *vm) 855 { 856 int ret; 857 858 if (!vm) 859 return 0; 860 861 ret = rq->engine->emit_flush(rq, EMIT_FLUSH); 862 if (ret) 863 return ret; 864 865 /* 866 * Not only do we need a full barrier (post-sync write) after 867 * invalidating the TLBs, but we need to wait a little bit 868 * longer. Whether this is merely delaying us, or the 869 * subsequent flush is a key part of serialising with the 870 * post-sync op, this extra pass appears vital before a 871 * mm switch! 872 */ 873 ret = load_pd_dir(rq, vm, PP_DIR_DCLV_2G); 874 if (ret) 875 return ret; 876 877 return rq->engine->emit_flush(rq, EMIT_INVALIDATE); 878 } 879 880 static int clear_residuals(struct i915_request *rq) 881 { 882 struct intel_engine_cs *engine = rq->engine; 883 int ret; 884 885 ret = switch_mm(rq, vm_alias(engine->kernel_context->vm)); 886 if (ret) 887 return ret; 888 889 if (engine->kernel_context->state) { 890 ret = mi_set_context(rq, 891 engine->kernel_context, 892 MI_MM_SPACE_GTT | MI_RESTORE_INHIBIT); 893 if (ret) 894 return ret; 895 } 896 897 ret = engine->emit_bb_start(rq, 898 engine->wa_ctx.vma->node.start, 0, 899 0); 900 if (ret) 901 return ret; 902 903 ret = engine->emit_flush(rq, EMIT_FLUSH); 904 if (ret) 905 return ret; 906 907 /* Always invalidate before the next switch_mm() */ 908 return engine->emit_flush(rq, EMIT_INVALIDATE); 909 } 910 911 static int switch_context(struct i915_request *rq) 912 { 913 struct intel_engine_cs *engine = rq->engine; 914 struct intel_context *ce = rq->context; 915 void **residuals = NULL; 916 int ret; 917 918 GEM_BUG_ON(HAS_EXECLISTS(engine->i915)); 919 920 if (engine->wa_ctx.vma && ce != engine->kernel_context) { 921 if (engine->wa_ctx.vma->private != ce && 922 i915_mitigate_clear_residuals()) { 923 ret = clear_residuals(rq); 924 if (ret) 925 return ret; 926 927 residuals = &engine->wa_ctx.vma->private; 928 } 929 } 930 931 ret = switch_mm(rq, vm_alias(ce->vm)); 932 if (ret) 933 return ret; 934 935 if (ce->state) { 936 u32 flags; 937 938 GEM_BUG_ON(engine->id != RCS0); 939 940 /* For resource streamer on HSW+ and power context elsewhere */ 941 BUILD_BUG_ON(HSW_MI_RS_SAVE_STATE_EN != MI_SAVE_EXT_STATE_EN); 942 BUILD_BUG_ON(HSW_MI_RS_RESTORE_STATE_EN != MI_RESTORE_EXT_STATE_EN); 943 944 flags = MI_SAVE_EXT_STATE_EN | MI_MM_SPACE_GTT; 945 if (test_bit(CONTEXT_VALID_BIT, &ce->flags)) 946 flags |= MI_RESTORE_EXT_STATE_EN; 947 else 948 flags |= MI_RESTORE_INHIBIT; 949 950 ret = mi_set_context(rq, ce, flags); 951 if (ret) 952 return ret; 953 } 954 955 ret = remap_l3(rq); 956 if (ret) 957 return ret; 958 959 /* 960 * Now past the point of no return, this request _will_ be emitted. 961 * 962 * Or at least this preamble will be emitted, the request may be 963 * interrupted prior to submitting the user payload. If so, we 964 * still submit the "empty" request in order to preserve global 965 * state tracking such as this, our tracking of the current 966 * dirty context. 967 */ 968 if (residuals) { 969 intel_context_put(*residuals); 970 *residuals = intel_context_get(ce); 971 } 972 973 return 0; 974 } 975 976 static int ring_request_alloc(struct i915_request *request) 977 { 978 int ret; 979 980 GEM_BUG_ON(!intel_context_is_pinned(request->context)); 981 GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb); 982 983 /* 984 * Flush enough space to reduce the likelihood of waiting after 985 * we start building the request - in which case we will just 986 * have to repeat work. 987 */ 988 request->reserved_space += LEGACY_REQUEST_SIZE; 989 990 /* Unconditionally invalidate GPU caches and TLBs. */ 991 ret = request->engine->emit_flush(request, EMIT_INVALIDATE); 992 if (ret) 993 return ret; 994 995 ret = switch_context(request); 996 if (ret) 997 return ret; 998 999 request->reserved_space -= LEGACY_REQUEST_SIZE; 1000 return 0; 1001 } 1002 1003 static void gen6_bsd_submit_request(struct i915_request *request) 1004 { 1005 struct intel_uncore *uncore = request->engine->uncore; 1006 1007 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1008 1009 /* Every tail move must follow the sequence below */ 1010 1011 /* Disable notification that the ring is IDLE. The GT 1012 * will then assume that it is busy and bring it out of rc6. 1013 */ 1014 intel_uncore_write_fw(uncore, RING_PSMI_CTL(GEN6_BSD_RING_BASE), 1015 _MASKED_BIT_ENABLE(GEN6_PSMI_SLEEP_MSG_DISABLE)); 1016 1017 /* Clear the context id. Here be magic! */ 1018 intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0); 1019 1020 /* Wait for the ring not to be idle, i.e. for it to wake up. */ 1021 if (__intel_wait_for_register_fw(uncore, 1022 RING_PSMI_CTL(GEN6_BSD_RING_BASE), 1023 GEN6_BSD_SLEEP_INDICATOR, 1024 0, 1025 1000, 0, NULL)) 1026 drm_err(&uncore->i915->drm, 1027 "timed out waiting for the BSD ring to wake up\n"); 1028 1029 /* Now that the ring is fully powered up, update the tail */ 1030 i9xx_submit_request(request); 1031 1032 /* Let the ring send IDLE messages to the GT again, 1033 * and so let it sleep to conserve power when idle. 1034 */ 1035 intel_uncore_write_fw(uncore, RING_PSMI_CTL(GEN6_BSD_RING_BASE), 1036 _MASKED_BIT_DISABLE(GEN6_PSMI_SLEEP_MSG_DISABLE)); 1037 1038 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1039 } 1040 1041 static void i9xx_set_default_submission(struct intel_engine_cs *engine) 1042 { 1043 engine->submit_request = i9xx_submit_request; 1044 } 1045 1046 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine) 1047 { 1048 engine->submit_request = gen6_bsd_submit_request; 1049 } 1050 1051 static void ring_release(struct intel_engine_cs *engine) 1052 { 1053 struct drm_i915_private *dev_priv = engine->i915; 1054 1055 drm_WARN_ON(&dev_priv->drm, GRAPHICS_VER(dev_priv) > 2 && 1056 (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0); 1057 1058 intel_engine_cleanup_common(engine); 1059 1060 if (engine->wa_ctx.vma) { 1061 intel_context_put(engine->wa_ctx.vma->private); 1062 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); 1063 } 1064 1065 intel_ring_unpin(engine->legacy.ring); 1066 intel_ring_put(engine->legacy.ring); 1067 1068 intel_timeline_unpin(engine->legacy.timeline); 1069 intel_timeline_put(engine->legacy.timeline); 1070 } 1071 1072 static void irq_handler(struct intel_engine_cs *engine, u16 iir) 1073 { 1074 intel_engine_signal_breadcrumbs(engine); 1075 } 1076 1077 static void setup_irq(struct intel_engine_cs *engine) 1078 { 1079 struct drm_i915_private *i915 = engine->i915; 1080 1081 intel_engine_set_irq_handler(engine, irq_handler); 1082 1083 if (GRAPHICS_VER(i915) >= 6) { 1084 engine->irq_enable = gen6_irq_enable; 1085 engine->irq_disable = gen6_irq_disable; 1086 } else if (GRAPHICS_VER(i915) >= 5) { 1087 engine->irq_enable = gen5_irq_enable; 1088 engine->irq_disable = gen5_irq_disable; 1089 } else if (GRAPHICS_VER(i915) >= 3) { 1090 engine->irq_enable = gen3_irq_enable; 1091 engine->irq_disable = gen3_irq_disable; 1092 } else { 1093 engine->irq_enable = gen2_irq_enable; 1094 engine->irq_disable = gen2_irq_disable; 1095 } 1096 } 1097 1098 static void add_to_engine(struct i915_request *rq) 1099 { 1100 lockdep_assert_held(&rq->engine->sched_engine->lock); 1101 list_move_tail(&rq->sched.link, &rq->engine->sched_engine->requests); 1102 } 1103 1104 static void remove_from_engine(struct i915_request *rq) 1105 { 1106 spin_lock_irq(&rq->engine->sched_engine->lock); 1107 list_del_init(&rq->sched.link); 1108 1109 /* Prevent further __await_execution() registering a cb, then flush */ 1110 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); 1111 1112 spin_unlock_irq(&rq->engine->sched_engine->lock); 1113 1114 i915_request_notify_execute_cb_imm(rq); 1115 } 1116 1117 static void setup_common(struct intel_engine_cs *engine) 1118 { 1119 struct drm_i915_private *i915 = engine->i915; 1120 1121 /* gen8+ are only supported with execlists */ 1122 GEM_BUG_ON(GRAPHICS_VER(i915) >= 8); 1123 1124 setup_irq(engine); 1125 1126 engine->resume = xcs_resume; 1127 engine->sanitize = xcs_sanitize; 1128 1129 engine->reset.prepare = reset_prepare; 1130 engine->reset.rewind = reset_rewind; 1131 engine->reset.cancel = reset_cancel; 1132 engine->reset.finish = reset_finish; 1133 1134 engine->add_active_request = add_to_engine; 1135 engine->remove_active_request = remove_from_engine; 1136 1137 engine->cops = &ring_context_ops; 1138 engine->request_alloc = ring_request_alloc; 1139 1140 /* 1141 * Using a global execution timeline; the previous final breadcrumb is 1142 * equivalent to our next initial bread so we can elide 1143 * engine->emit_init_breadcrumb(). 1144 */ 1145 engine->emit_fini_breadcrumb = gen3_emit_breadcrumb; 1146 if (GRAPHICS_VER(i915) == 5) 1147 engine->emit_fini_breadcrumb = gen5_emit_breadcrumb; 1148 1149 engine->set_default_submission = i9xx_set_default_submission; 1150 1151 if (GRAPHICS_VER(i915) >= 6) 1152 engine->emit_bb_start = gen6_emit_bb_start; 1153 else if (GRAPHICS_VER(i915) >= 4) 1154 engine->emit_bb_start = gen4_emit_bb_start; 1155 else if (IS_I830(i915) || IS_I845G(i915)) 1156 engine->emit_bb_start = i830_emit_bb_start; 1157 else 1158 engine->emit_bb_start = gen3_emit_bb_start; 1159 } 1160 1161 static void setup_rcs(struct intel_engine_cs *engine) 1162 { 1163 struct drm_i915_private *i915 = engine->i915; 1164 1165 if (HAS_L3_DPF(i915)) 1166 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT; 1167 1168 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; 1169 1170 if (GRAPHICS_VER(i915) >= 7) { 1171 engine->emit_flush = gen7_emit_flush_rcs; 1172 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_rcs; 1173 } else if (GRAPHICS_VER(i915) == 6) { 1174 engine->emit_flush = gen6_emit_flush_rcs; 1175 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_rcs; 1176 } else if (GRAPHICS_VER(i915) == 5) { 1177 engine->emit_flush = gen4_emit_flush_rcs; 1178 } else { 1179 if (GRAPHICS_VER(i915) < 4) 1180 engine->emit_flush = gen2_emit_flush; 1181 else 1182 engine->emit_flush = gen4_emit_flush_rcs; 1183 engine->irq_enable_mask = I915_USER_INTERRUPT; 1184 } 1185 1186 if (IS_HASWELL(i915)) 1187 engine->emit_bb_start = hsw_emit_bb_start; 1188 } 1189 1190 static void setup_vcs(struct intel_engine_cs *engine) 1191 { 1192 struct drm_i915_private *i915 = engine->i915; 1193 1194 if (GRAPHICS_VER(i915) >= 6) { 1195 /* gen6 bsd needs a special wa for tail updates */ 1196 if (GRAPHICS_VER(i915) == 6) 1197 engine->set_default_submission = gen6_bsd_set_default_submission; 1198 engine->emit_flush = gen6_emit_flush_vcs; 1199 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT; 1200 1201 if (GRAPHICS_VER(i915) == 6) 1202 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs; 1203 else 1204 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; 1205 } else { 1206 engine->emit_flush = gen4_emit_flush_vcs; 1207 if (GRAPHICS_VER(i915) == 5) 1208 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT; 1209 else 1210 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT; 1211 } 1212 } 1213 1214 static void setup_bcs(struct intel_engine_cs *engine) 1215 { 1216 struct drm_i915_private *i915 = engine->i915; 1217 1218 engine->emit_flush = gen6_emit_flush_xcs; 1219 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT; 1220 1221 if (GRAPHICS_VER(i915) == 6) 1222 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs; 1223 else 1224 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; 1225 } 1226 1227 static void setup_vecs(struct intel_engine_cs *engine) 1228 { 1229 struct drm_i915_private *i915 = engine->i915; 1230 1231 GEM_BUG_ON(GRAPHICS_VER(i915) < 7); 1232 1233 engine->emit_flush = gen6_emit_flush_xcs; 1234 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; 1235 engine->irq_enable = hsw_irq_enable_vecs; 1236 engine->irq_disable = hsw_irq_disable_vecs; 1237 1238 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs; 1239 } 1240 1241 static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine, 1242 struct i915_vma * const vma) 1243 { 1244 return gen7_setup_clear_gpr_bb(engine, vma); 1245 } 1246 1247 static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine, 1248 struct i915_gem_ww_ctx *ww, 1249 struct i915_vma *vma) 1250 { 1251 int err; 1252 1253 err = i915_vma_pin_ww(vma, ww, 0, 0, PIN_USER | PIN_HIGH); 1254 if (err) 1255 return err; 1256 1257 err = i915_vma_sync(vma); 1258 if (err) 1259 goto err_unpin; 1260 1261 err = gen7_ctx_switch_bb_setup(engine, vma); 1262 if (err) 1263 goto err_unpin; 1264 1265 engine->wa_ctx.vma = vma; 1266 return 0; 1267 1268 err_unpin: 1269 i915_vma_unpin(vma); 1270 return err; 1271 } 1272 1273 static struct i915_vma *gen7_ctx_vma(struct intel_engine_cs *engine) 1274 { 1275 struct drm_i915_gem_object *obj; 1276 struct i915_vma *vma; 1277 int size, err; 1278 1279 if (GRAPHICS_VER(engine->i915) != 7 || engine->class != RENDER_CLASS) 1280 return NULL; 1281 1282 err = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */); 1283 if (err < 0) 1284 return ERR_PTR(err); 1285 if (!err) 1286 return NULL; 1287 1288 size = ALIGN(err, PAGE_SIZE); 1289 1290 obj = i915_gem_object_create_internal(engine->i915, size); 1291 if (IS_ERR(obj)) 1292 return ERR_CAST(obj); 1293 1294 vma = i915_vma_instance(obj, engine->gt->vm, NULL); 1295 if (IS_ERR(vma)) { 1296 i915_gem_object_put(obj); 1297 return ERR_CAST(vma); 1298 } 1299 1300 vma->private = intel_context_create(engine); /* dummy residuals */ 1301 if (IS_ERR(vma->private)) { 1302 err = PTR_ERR(vma->private); 1303 vma->private = NULL; 1304 i915_gem_object_put(obj); 1305 return ERR_PTR(err); 1306 } 1307 1308 return vma; 1309 } 1310 1311 int intel_ring_submission_setup(struct intel_engine_cs *engine) 1312 { 1313 struct i915_gem_ww_ctx ww; 1314 struct intel_timeline *timeline; 1315 struct intel_ring *ring; 1316 struct i915_vma *gen7_wa_vma; 1317 int err; 1318 1319 setup_common(engine); 1320 1321 switch (engine->class) { 1322 case RENDER_CLASS: 1323 setup_rcs(engine); 1324 break; 1325 case VIDEO_DECODE_CLASS: 1326 setup_vcs(engine); 1327 break; 1328 case COPY_ENGINE_CLASS: 1329 setup_bcs(engine); 1330 break; 1331 case VIDEO_ENHANCEMENT_CLASS: 1332 setup_vecs(engine); 1333 break; 1334 default: 1335 MISSING_CASE(engine->class); 1336 return -ENODEV; 1337 } 1338 1339 timeline = intel_timeline_create_from_engine(engine, 1340 I915_GEM_HWS_SEQNO_ADDR); 1341 if (IS_ERR(timeline)) { 1342 err = PTR_ERR(timeline); 1343 goto err; 1344 } 1345 GEM_BUG_ON(timeline->has_initial_breadcrumb); 1346 1347 ring = intel_engine_create_ring(engine, SZ_16K); 1348 if (IS_ERR(ring)) { 1349 err = PTR_ERR(ring); 1350 goto err_timeline; 1351 } 1352 1353 GEM_BUG_ON(engine->legacy.ring); 1354 engine->legacy.ring = ring; 1355 engine->legacy.timeline = timeline; 1356 1357 gen7_wa_vma = gen7_ctx_vma(engine); 1358 if (IS_ERR(gen7_wa_vma)) { 1359 err = PTR_ERR(gen7_wa_vma); 1360 goto err_ring; 1361 } 1362 1363 i915_gem_ww_ctx_init(&ww, false); 1364 1365 retry: 1366 err = i915_gem_object_lock(timeline->hwsp_ggtt->obj, &ww); 1367 if (!err && gen7_wa_vma) 1368 err = i915_gem_object_lock(gen7_wa_vma->obj, &ww); 1369 if (!err) 1370 err = i915_gem_object_lock(engine->legacy.ring->vma->obj, &ww); 1371 if (!err) 1372 err = intel_timeline_pin(timeline, &ww); 1373 if (!err) { 1374 err = intel_ring_pin(ring, &ww); 1375 if (err) 1376 intel_timeline_unpin(timeline); 1377 } 1378 if (err) 1379 goto out; 1380 1381 GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma); 1382 1383 if (gen7_wa_vma) { 1384 err = gen7_ctx_switch_bb_init(engine, &ww, gen7_wa_vma); 1385 if (err) { 1386 intel_ring_unpin(ring); 1387 intel_timeline_unpin(timeline); 1388 } 1389 } 1390 1391 out: 1392 if (err == -EDEADLK) { 1393 err = i915_gem_ww_ctx_backoff(&ww); 1394 if (!err) 1395 goto retry; 1396 } 1397 i915_gem_ww_ctx_fini(&ww); 1398 if (err) 1399 goto err_gen7_put; 1400 1401 /* Finally, take ownership and responsibility for cleanup! */ 1402 engine->release = ring_release; 1403 1404 return 0; 1405 1406 err_gen7_put: 1407 if (gen7_wa_vma) { 1408 intel_context_put(gen7_wa_vma->private); 1409 i915_gem_object_put(gen7_wa_vma->obj); 1410 } 1411 err_ring: 1412 intel_ring_put(ring); 1413 err_timeline: 1414 intel_timeline_put(timeline); 1415 err: 1416 intel_engine_cleanup_common(engine); 1417 return err; 1418 } 1419 1420 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1421 #include "selftest_ring_submission.c" 1422 #endif 1423