1 /* 2 * Copyright © 2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Ben Widawsky <ben@bwidawsk.net> 25 * Michel Thierry <michel.thierry@intel.com> 26 * Thomas Daniel <thomas.daniel@intel.com> 27 * Oscar Mateo <oscar.mateo@intel.com> 28 * 29 */ 30 31 /** 32 * DOC: Logical Rings, Logical Ring Contexts and Execlists 33 * 34 * Motivation: 35 * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts". 36 * These expanded contexts enable a number of new abilities, especially 37 * "Execlists" (also implemented in this file). 38 * 39 * One of the main differences with the legacy HW contexts is that logical 40 * ring contexts incorporate many more things to the context's state, like 41 * PDPs or ringbuffer control registers: 42 * 43 * The reason why PDPs are included in the context is straightforward: as 44 * PPGTTs (per-process GTTs) are actually per-context, having the PDPs 45 * contained there mean you don't need to do a ppgtt->switch_mm yourself, 46 * instead, the GPU will do it for you on the context switch. 47 * 48 * But, what about the ringbuffer control registers (head, tail, etc..)? 49 * shouldn't we just need a set of those per engine command streamer? This is 50 * where the name "Logical Rings" starts to make sense: by virtualizing the 51 * rings, the engine cs shifts to a new "ring buffer" with every context 52 * switch. When you want to submit a workload to the GPU you: A) choose your 53 * context, B) find its appropriate virtualized ring, C) write commands to it 54 * and then, finally, D) tell the GPU to switch to that context. 55 * 56 * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch 57 * to a contexts is via a context execution list, ergo "Execlists". 58 * 59 * LRC implementation: 60 * Regarding the creation of contexts, we have: 61 * 62 * - One global default context. 63 * - One local default context for each opened fd. 64 * - One local extra context for each context create ioctl call. 65 * 66 * Now that ringbuffers belong per-context (and not per-engine, like before) 67 * and that contexts are uniquely tied to a given engine (and not reusable, 68 * like before) we need: 69 * 70 * - One ringbuffer per-engine inside each context. 71 * - One backing object per-engine inside each context. 72 * 73 * The global default context starts its life with these new objects fully 74 * allocated and populated. The local default context for each opened fd is 75 * more complex, because we don't know at creation time which engine is going 76 * to use them. To handle this, we have implemented a deferred creation of LR 77 * contexts: 78 * 79 * The local context starts its life as a hollow or blank holder, that only 80 * gets populated for a given engine once we receive an execbuffer. If later 81 * on we receive another execbuffer ioctl for the same context but a different 82 * engine, we allocate/populate a new ringbuffer and context backing object and 83 * so on. 84 * 85 * Finally, regarding local contexts created using the ioctl call: as they are 86 * only allowed with the render ring, we can allocate & populate them right 87 * away (no need to defer anything, at least for now). 88 * 89 * Execlists implementation: 90 * Execlists are the new method by which, on gen8+ hardware, workloads are 91 * submitted for execution (as opposed to the legacy, ringbuffer-based, method). 92 * This method works as follows: 93 * 94 * When a request is committed, its commands (the BB start and any leading or 95 * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer 96 * for the appropriate context. The tail pointer in the hardware context is not 97 * updated at this time, but instead, kept by the driver in the ringbuffer 98 * structure. A structure representing this request is added to a request queue 99 * for the appropriate engine: this structure contains a copy of the context's 100 * tail after the request was written to the ring buffer and a pointer to the 101 * context itself. 102 * 103 * If the engine's request queue was empty before the request was added, the 104 * queue is processed immediately. Otherwise the queue will be processed during 105 * a context switch interrupt. In any case, elements on the queue will get sent 106 * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a 107 * globally unique 20-bits submission ID. 108 * 109 * When execution of a request completes, the GPU updates the context status 110 * buffer with a context complete event and generates a context switch interrupt. 111 * During the interrupt handling, the driver examines the events in the buffer: 112 * for each context complete event, if the announced ID matches that on the head 113 * of the request queue, then that request is retired and removed from the queue. 114 * 115 * After processing, if any requests were retired and the queue is not empty 116 * then a new execution list can be submitted. The two requests at the front of 117 * the queue are next to be submitted but since a context may not occur twice in 118 * an execution list, if subsequent requests have the same ID as the first then 119 * the two requests must be combined. This is done simply by discarding requests 120 * at the head of the queue until either only one requests is left (in which case 121 * we use a NULL second context) or the first two requests have unique IDs. 122 * 123 * By always executing the first two requests in the queue the driver ensures 124 * that the GPU is kept as busy as possible. In the case where a single context 125 * completes but a second context is still executing, the request for this second 126 * context will be at the head of the queue when we remove the first one. This 127 * request will then be resubmitted along with a new request for a different context, 128 * which will cause the hardware to continue executing the second request and queue 129 * the new request (the GPU detects the condition of a context getting preempted 130 * with the same context and optimizes the context switch flow by not doing 131 * preemption, but just sampling the new tail pointer). 132 * 133 */ 134 135 #include <drm/drmP.h> 136 #include <drm/i915_drm.h> 137 #include "i915_drv.h" 138 #include "intel_drv.h" 139 140 #define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE) 141 #define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE) 142 #define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE) 143 144 #define RING_EXECLIST_QFULL (1 << 0x2) 145 #define RING_EXECLIST1_VALID (1 << 0x3) 146 #define RING_EXECLIST0_VALID (1 << 0x4) 147 #define RING_EXECLIST_ACTIVE_STATUS (3 << 0xE) 148 #define RING_EXECLIST1_ACTIVE (1 << 0x11) 149 #define RING_EXECLIST0_ACTIVE (1 << 0x12) 150 151 #define GEN8_CTX_STATUS_IDLE_ACTIVE (1 << 0) 152 #define GEN8_CTX_STATUS_PREEMPTED (1 << 1) 153 #define GEN8_CTX_STATUS_ELEMENT_SWITCH (1 << 2) 154 #define GEN8_CTX_STATUS_ACTIVE_IDLE (1 << 3) 155 #define GEN8_CTX_STATUS_COMPLETE (1 << 4) 156 #define GEN8_CTX_STATUS_LITE_RESTORE (1 << 15) 157 158 #define CTX_LRI_HEADER_0 0x01 159 #define CTX_CONTEXT_CONTROL 0x02 160 #define CTX_RING_HEAD 0x04 161 #define CTX_RING_TAIL 0x06 162 #define CTX_RING_BUFFER_START 0x08 163 #define CTX_RING_BUFFER_CONTROL 0x0a 164 #define CTX_BB_HEAD_U 0x0c 165 #define CTX_BB_HEAD_L 0x0e 166 #define CTX_BB_STATE 0x10 167 #define CTX_SECOND_BB_HEAD_U 0x12 168 #define CTX_SECOND_BB_HEAD_L 0x14 169 #define CTX_SECOND_BB_STATE 0x16 170 #define CTX_BB_PER_CTX_PTR 0x18 171 #define CTX_RCS_INDIRECT_CTX 0x1a 172 #define CTX_RCS_INDIRECT_CTX_OFFSET 0x1c 173 #define CTX_LRI_HEADER_1 0x21 174 #define CTX_CTX_TIMESTAMP 0x22 175 #define CTX_PDP3_UDW 0x24 176 #define CTX_PDP3_LDW 0x26 177 #define CTX_PDP2_UDW 0x28 178 #define CTX_PDP2_LDW 0x2a 179 #define CTX_PDP1_UDW 0x2c 180 #define CTX_PDP1_LDW 0x2e 181 #define CTX_PDP0_UDW 0x30 182 #define CTX_PDP0_LDW 0x32 183 #define CTX_LRI_HEADER_2 0x41 184 #define CTX_R_PWR_CLK_STATE 0x42 185 #define CTX_GPGPU_CSR_BASE_ADDRESS 0x44 186 187 #define GEN8_CTX_VALID (1<<0) 188 #define GEN8_CTX_FORCE_PD_RESTORE (1<<1) 189 #define GEN8_CTX_FORCE_RESTORE (1<<2) 190 #define GEN8_CTX_L3LLC_COHERENT (1<<5) 191 #define GEN8_CTX_PRIVILEGE (1<<8) 192 enum { 193 ADVANCED_CONTEXT = 0, 194 LEGACY_CONTEXT, 195 ADVANCED_AD_CONTEXT, 196 LEGACY_64B_CONTEXT 197 }; 198 #define GEN8_CTX_MODE_SHIFT 3 199 enum { 200 FAULT_AND_HANG = 0, 201 FAULT_AND_HALT, /* Debug only */ 202 FAULT_AND_STREAM, 203 FAULT_AND_CONTINUE /* Unsupported */ 204 }; 205 #define GEN8_CTX_ID_SHIFT 32 206 207 static int intel_lr_context_pin(struct intel_engine_cs *ring, 208 struct intel_context *ctx); 209 210 /** 211 * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists 212 * @dev: DRM device. 213 * @enable_execlists: value of i915.enable_execlists module parameter. 214 * 215 * Only certain platforms support Execlists (the prerequisites being 216 * support for Logical Ring Contexts and Aliasing PPGTT or better). 217 * 218 * Return: 1 if Execlists is supported and has to be enabled. 219 */ 220 int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists) 221 { 222 WARN_ON(i915.enable_ppgtt == -1); 223 224 if (INTEL_INFO(dev)->gen >= 9) 225 return 1; 226 227 if (enable_execlists == 0) 228 return 0; 229 230 if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev) && 231 i915.use_mmio_flip >= 0) 232 return 1; 233 234 return 0; 235 } 236 237 /** 238 * intel_execlists_ctx_id() - get the Execlists Context ID 239 * @ctx_obj: Logical Ring Context backing object. 240 * 241 * Do not confuse with ctx->id! Unfortunately we have a name overload 242 * here: the old context ID we pass to userspace as a handler so that 243 * they can refer to a context, and the new context ID we pass to the 244 * ELSP so that the GPU can inform us of the context status via 245 * interrupts. 246 * 247 * Return: 20-bits globally unique context ID. 248 */ 249 u32 intel_execlists_ctx_id(struct drm_i915_gem_object *ctx_obj) 250 { 251 u32 lrca = i915_gem_obj_ggtt_offset(ctx_obj); 252 253 /* LRCA is required to be 4K aligned so the more significant 20 bits 254 * are globally unique */ 255 return lrca >> 12; 256 } 257 258 static uint64_t execlists_ctx_descriptor(struct drm_i915_gem_object *ctx_obj) 259 { 260 uint64_t desc; 261 uint64_t lrca = i915_gem_obj_ggtt_offset(ctx_obj); 262 263 WARN_ON(lrca & 0xFFFFFFFF00000FFFULL); 264 265 desc = GEN8_CTX_VALID; 266 desc |= LEGACY_CONTEXT << GEN8_CTX_MODE_SHIFT; 267 desc |= GEN8_CTX_L3LLC_COHERENT; 268 desc |= GEN8_CTX_PRIVILEGE; 269 desc |= lrca; 270 desc |= (u64)intel_execlists_ctx_id(ctx_obj) << GEN8_CTX_ID_SHIFT; 271 272 /* TODO: WaDisableLiteRestore when we start using semaphore 273 * signalling between Command Streamers */ 274 /* desc |= GEN8_CTX_FORCE_RESTORE; */ 275 276 return desc; 277 } 278 279 static void execlists_elsp_write(struct intel_engine_cs *ring, 280 struct drm_i915_gem_object *ctx_obj0, 281 struct drm_i915_gem_object *ctx_obj1) 282 { 283 struct drm_device *dev = ring->dev; 284 struct drm_i915_private *dev_priv = dev->dev_private; 285 uint64_t temp = 0; 286 uint32_t desc[4]; 287 288 /* XXX: You must always write both descriptors in the order below. */ 289 if (ctx_obj1) 290 temp = execlists_ctx_descriptor(ctx_obj1); 291 else 292 temp = 0; 293 desc[1] = (u32)(temp >> 32); 294 desc[0] = (u32)temp; 295 296 temp = execlists_ctx_descriptor(ctx_obj0); 297 desc[3] = (u32)(temp >> 32); 298 desc[2] = (u32)temp; 299 300 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 301 I915_WRITE(RING_ELSP(ring), desc[1]); 302 I915_WRITE(RING_ELSP(ring), desc[0]); 303 I915_WRITE(RING_ELSP(ring), desc[3]); 304 305 /* The context is automatically loaded after the following */ 306 I915_WRITE(RING_ELSP(ring), desc[2]); 307 308 /* ELSP is a wo register, so use another nearby reg for posting instead */ 309 POSTING_READ(RING_EXECLIST_STATUS(ring)); 310 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 311 } 312 313 static int execlists_update_context(struct drm_i915_gem_object *ctx_obj, 314 struct drm_i915_gem_object *ring_obj, 315 u32 tail) 316 { 317 struct vm_page *page; 318 uint32_t *reg_state; 319 320 page = i915_gem_object_get_page(ctx_obj, 1); 321 reg_state = kmap_atomic(page); 322 323 reg_state[CTX_RING_TAIL+1] = tail; 324 reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(ring_obj); 325 326 kunmap_atomic(reg_state); 327 328 return 0; 329 } 330 331 static void execlists_submit_contexts(struct intel_engine_cs *ring, 332 struct intel_context *to0, u32 tail0, 333 struct intel_context *to1, u32 tail1) 334 { 335 struct drm_i915_gem_object *ctx_obj0 = to0->engine[ring->id].state; 336 struct intel_ringbuffer *ringbuf0 = to0->engine[ring->id].ringbuf; 337 struct drm_i915_gem_object *ctx_obj1 = NULL; 338 struct intel_ringbuffer *ringbuf1 = NULL; 339 340 BUG_ON(!ctx_obj0); 341 WARN_ON(!i915_gem_obj_is_pinned(ctx_obj0)); 342 WARN_ON(!i915_gem_obj_is_pinned(ringbuf0->obj)); 343 344 execlists_update_context(ctx_obj0, ringbuf0->obj, tail0); 345 346 if (to1) { 347 ringbuf1 = to1->engine[ring->id].ringbuf; 348 ctx_obj1 = to1->engine[ring->id].state; 349 BUG_ON(!ctx_obj1); 350 WARN_ON(!i915_gem_obj_is_pinned(ctx_obj1)); 351 WARN_ON(!i915_gem_obj_is_pinned(ringbuf1->obj)); 352 353 execlists_update_context(ctx_obj1, ringbuf1->obj, tail1); 354 } 355 356 execlists_elsp_write(ring, ctx_obj0, ctx_obj1); 357 } 358 359 static void execlists_context_unqueue(struct intel_engine_cs *ring) 360 { 361 struct drm_i915_gem_request *req0 = NULL, *req1 = NULL; 362 struct drm_i915_gem_request *cursor = NULL, *tmp = NULL; 363 364 assert_spin_locked(&ring->execlist_lock); 365 366 if (list_empty(&ring->execlist_queue)) 367 return; 368 369 /* Try to read in pairs */ 370 list_for_each_entry_safe(cursor, tmp, &ring->execlist_queue, 371 execlist_link) { 372 if (!req0) { 373 req0 = cursor; 374 } else if (req0->ctx == cursor->ctx) { 375 /* Same ctx: ignore first request, as second request 376 * will update tail past first request's workload */ 377 cursor->elsp_submitted = req0->elsp_submitted; 378 list_del(&req0->execlist_link); 379 list_add_tail(&req0->execlist_link, 380 &ring->execlist_retired_req_list); 381 req0 = cursor; 382 } else { 383 req1 = cursor; 384 break; 385 } 386 } 387 388 WARN_ON(req1 && req1->elsp_submitted); 389 390 execlists_submit_contexts(ring, req0->ctx, req0->tail, 391 req1 ? req1->ctx : NULL, 392 req1 ? req1->tail : 0); 393 394 req0->elsp_submitted++; 395 if (req1) 396 req1->elsp_submitted++; 397 } 398 399 static bool execlists_check_remove_request(struct intel_engine_cs *ring, 400 u32 request_id) 401 { 402 struct drm_i915_gem_request *head_req; 403 404 assert_spin_locked(&ring->execlist_lock); 405 406 head_req = list_first_entry_or_null(&ring->execlist_queue, 407 struct drm_i915_gem_request, 408 execlist_link); 409 410 if (head_req != NULL) { 411 struct drm_i915_gem_object *ctx_obj = 412 head_req->ctx->engine[ring->id].state; 413 if (intel_execlists_ctx_id(ctx_obj) == request_id) { 414 WARN(head_req->elsp_submitted == 0, 415 "Never submitted head request\n"); 416 417 if (--head_req->elsp_submitted <= 0) { 418 list_del(&head_req->execlist_link); 419 list_add_tail(&head_req->execlist_link, 420 &ring->execlist_retired_req_list); 421 return true; 422 } 423 } 424 } 425 426 return false; 427 } 428 429 /** 430 * intel_lrc_irq_handler() - handle Context Switch interrupts 431 * @ring: Engine Command Streamer to handle. 432 * 433 * Check the unread Context Status Buffers and manage the submission of new 434 * contexts to the ELSP accordingly. 435 */ 436 void intel_lrc_irq_handler(struct intel_engine_cs *ring) 437 { 438 struct drm_i915_private *dev_priv = ring->dev->dev_private; 439 u32 status_pointer; 440 u8 read_pointer; 441 u8 write_pointer; 442 u32 status; 443 u32 status_id; 444 u32 submit_contexts = 0; 445 446 status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(ring)); 447 448 read_pointer = ring->next_context_status_buffer; 449 write_pointer = status_pointer & 0x07; 450 if (read_pointer > write_pointer) 451 write_pointer += 6; 452 453 lockmgr(&ring->execlist_lock, LK_EXCLUSIVE); 454 455 while (read_pointer < write_pointer) { 456 read_pointer++; 457 status = I915_READ(RING_CONTEXT_STATUS_BUF(ring) + 458 (read_pointer % 6) * 8); 459 status_id = I915_READ(RING_CONTEXT_STATUS_BUF(ring) + 460 (read_pointer % 6) * 8 + 4); 461 462 if (status & GEN8_CTX_STATUS_PREEMPTED) { 463 if (status & GEN8_CTX_STATUS_LITE_RESTORE) { 464 if (execlists_check_remove_request(ring, status_id)) 465 WARN(1, "Lite Restored request removed from queue\n"); 466 } else 467 WARN(1, "Preemption without Lite Restore\n"); 468 } 469 470 if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) || 471 (status & GEN8_CTX_STATUS_ELEMENT_SWITCH)) { 472 if (execlists_check_remove_request(ring, status_id)) 473 submit_contexts++; 474 } 475 } 476 477 if (submit_contexts != 0) 478 execlists_context_unqueue(ring); 479 480 lockmgr(&ring->execlist_lock, LK_RELEASE); 481 482 WARN(submit_contexts > 2, "More than two context complete events?\n"); 483 ring->next_context_status_buffer = write_pointer % 6; 484 485 I915_WRITE(RING_CONTEXT_STATUS_PTR(ring), 486 ((u32)ring->next_context_status_buffer & 0x07) << 8); 487 } 488 489 static int execlists_context_queue(struct intel_engine_cs *ring, 490 struct intel_context *to, 491 u32 tail, 492 struct drm_i915_gem_request *request) 493 { 494 struct drm_i915_gem_request *cursor; 495 struct drm_i915_private *dev_priv = ring->dev->dev_private; 496 int num_elements = 0; 497 498 if (to != ring->default_context) 499 intel_lr_context_pin(ring, to); 500 501 if (!request) { 502 /* 503 * If there isn't a request associated with this submission, 504 * create one as a temporary holder. 505 */ 506 request = kzalloc(sizeof(*request), GFP_KERNEL); 507 if (request == NULL) 508 return -ENOMEM; 509 request->ring = ring; 510 request->ctx = to; 511 kref_init(&request->ref); 512 request->uniq = dev_priv->request_uniq++; 513 i915_gem_context_reference(request->ctx); 514 } else { 515 i915_gem_request_reference(request); 516 WARN_ON(to != request->ctx); 517 } 518 request->tail = tail; 519 520 intel_runtime_pm_get(dev_priv); 521 522 lockmgr(&ring->execlist_lock, LK_EXCLUSIVE); 523 524 list_for_each_entry(cursor, &ring->execlist_queue, execlist_link) 525 if (++num_elements > 2) 526 break; 527 528 if (num_elements > 2) { 529 struct drm_i915_gem_request *tail_req; 530 531 tail_req = list_last_entry(&ring->execlist_queue, 532 struct drm_i915_gem_request, 533 execlist_link); 534 535 if (to == tail_req->ctx) { 536 WARN(tail_req->elsp_submitted != 0, 537 "More than 2 already-submitted reqs queued\n"); 538 list_del(&tail_req->execlist_link); 539 list_add_tail(&tail_req->execlist_link, 540 &ring->execlist_retired_req_list); 541 } 542 } 543 544 list_add_tail(&request->execlist_link, &ring->execlist_queue); 545 if (num_elements == 0) 546 execlists_context_unqueue(ring); 547 548 lockmgr(&ring->execlist_lock, LK_RELEASE); 549 550 return 0; 551 } 552 553 static int logical_ring_invalidate_all_caches(struct intel_ringbuffer *ringbuf, 554 struct intel_context *ctx) 555 { 556 struct intel_engine_cs *ring = ringbuf->ring; 557 uint32_t flush_domains; 558 int ret; 559 560 flush_domains = 0; 561 if (ring->gpu_caches_dirty) 562 flush_domains = I915_GEM_GPU_DOMAINS; 563 564 ret = ring->emit_flush(ringbuf, ctx, 565 I915_GEM_GPU_DOMAINS, flush_domains); 566 if (ret) 567 return ret; 568 569 ring->gpu_caches_dirty = false; 570 return 0; 571 } 572 573 static int execlists_move_to_gpu(struct intel_ringbuffer *ringbuf, 574 struct intel_context *ctx, 575 struct list_head *vmas) 576 { 577 struct intel_engine_cs *ring = ringbuf->ring; 578 struct i915_vma *vma; 579 uint32_t flush_domains = 0; 580 bool flush_chipset = false; 581 int ret; 582 583 list_for_each_entry(vma, vmas, exec_list) { 584 struct drm_i915_gem_object *obj = vma->obj; 585 586 ret = i915_gem_object_sync(obj, ring); 587 if (ret) 588 return ret; 589 590 if (obj->base.write_domain & I915_GEM_DOMAIN_CPU) 591 flush_chipset |= i915_gem_clflush_object(obj, false); 592 593 flush_domains |= obj->base.write_domain; 594 } 595 596 if (flush_domains & I915_GEM_DOMAIN_GTT) 597 wmb(); 598 599 /* Unconditionally invalidate gpu caches and ensure that we do flush 600 * any residual writes from the previous batch. 601 */ 602 return logical_ring_invalidate_all_caches(ringbuf, ctx); 603 } 604 605 /** 606 * execlists_submission() - submit a batchbuffer for execution, Execlists style 607 * @dev: DRM device. 608 * @file: DRM file. 609 * @ring: Engine Command Streamer to submit to. 610 * @ctx: Context to employ for this submission. 611 * @args: execbuffer call arguments. 612 * @vmas: list of vmas. 613 * @batch_obj: the batchbuffer to submit. 614 * @exec_start: batchbuffer start virtual address pointer. 615 * @flags: translated execbuffer call flags. 616 * 617 * This is the evil twin version of i915_gem_ringbuffer_submission. It abstracts 618 * away the submission details of the execbuffer ioctl call. 619 * 620 * Return: non-zero if the submission fails. 621 */ 622 int intel_execlists_submission(struct drm_device *dev, struct drm_file *file, 623 struct intel_engine_cs *ring, 624 struct intel_context *ctx, 625 struct drm_i915_gem_execbuffer2 *args, 626 struct list_head *vmas, 627 struct drm_i915_gem_object *batch_obj, 628 u64 exec_start, u32 flags) 629 { 630 struct drm_i915_private *dev_priv = dev->dev_private; 631 struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf; 632 int instp_mode; 633 u32 instp_mask; 634 int ret; 635 636 instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK; 637 instp_mask = I915_EXEC_CONSTANTS_MASK; 638 switch (instp_mode) { 639 case I915_EXEC_CONSTANTS_REL_GENERAL: 640 case I915_EXEC_CONSTANTS_ABSOLUTE: 641 case I915_EXEC_CONSTANTS_REL_SURFACE: 642 if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) { 643 DRM_DEBUG("non-0 rel constants mode on non-RCS\n"); 644 return -EINVAL; 645 } 646 647 if (instp_mode != dev_priv->relative_constants_mode) { 648 if (instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) { 649 DRM_DEBUG("rel surface constants mode invalid on gen5+\n"); 650 return -EINVAL; 651 } 652 653 /* The HW changed the meaning on this bit on gen6 */ 654 instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE; 655 } 656 break; 657 default: 658 DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode); 659 return -EINVAL; 660 } 661 662 if (args->num_cliprects != 0) { 663 DRM_DEBUG("clip rectangles are only valid on pre-gen5\n"); 664 return -EINVAL; 665 } else { 666 if (args->DR4 == 0xffffffff) { 667 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n"); 668 args->DR4 = 0; 669 } 670 671 if (args->DR1 || args->DR4 || args->cliprects_ptr) { 672 DRM_DEBUG("0 cliprects but dirt in cliprects fields\n"); 673 return -EINVAL; 674 } 675 } 676 677 if (args->flags & I915_EXEC_GEN7_SOL_RESET) { 678 DRM_DEBUG("sol reset is gen7 only\n"); 679 return -EINVAL; 680 } 681 682 ret = execlists_move_to_gpu(ringbuf, ctx, vmas); 683 if (ret) 684 return ret; 685 686 if (ring == &dev_priv->ring[RCS] && 687 instp_mode != dev_priv->relative_constants_mode) { 688 ret = intel_logical_ring_begin(ringbuf, ctx, 4); 689 if (ret) 690 return ret; 691 692 intel_logical_ring_emit(ringbuf, MI_NOOP); 693 intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(1)); 694 intel_logical_ring_emit(ringbuf, INSTPM); 695 intel_logical_ring_emit(ringbuf, instp_mask << 16 | instp_mode); 696 intel_logical_ring_advance(ringbuf); 697 698 dev_priv->relative_constants_mode = instp_mode; 699 } 700 701 ret = ring->emit_bb_start(ringbuf, ctx, exec_start, flags); 702 if (ret) 703 return ret; 704 705 i915_gem_execbuffer_move_to_active(vmas, ring); 706 i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj); 707 708 return 0; 709 } 710 711 void intel_execlists_retire_requests(struct intel_engine_cs *ring) 712 { 713 struct drm_i915_gem_request *req, *tmp; 714 struct drm_i915_private *dev_priv = ring->dev->dev_private; 715 struct list_head retired_list; 716 717 WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex)); 718 if (list_empty(&ring->execlist_retired_req_list)) 719 return; 720 721 INIT_LIST_HEAD(&retired_list); 722 lockmgr(&ring->execlist_lock, LK_EXCLUSIVE); 723 list_replace_init(&ring->execlist_retired_req_list, &retired_list); 724 lockmgr(&ring->execlist_lock, LK_RELEASE); 725 726 list_for_each_entry_safe(req, tmp, &retired_list, execlist_link) { 727 struct intel_context *ctx = req->ctx; 728 struct drm_i915_gem_object *ctx_obj = 729 ctx->engine[ring->id].state; 730 731 if (ctx_obj && (ctx != ring->default_context)) 732 intel_lr_context_unpin(ring, ctx); 733 intel_runtime_pm_put(dev_priv); 734 list_del(&req->execlist_link); 735 i915_gem_request_unreference(req); 736 } 737 } 738 739 void intel_logical_ring_stop(struct intel_engine_cs *ring) 740 { 741 struct drm_i915_private *dev_priv = ring->dev->dev_private; 742 int ret; 743 744 if (!intel_ring_initialized(ring)) 745 return; 746 747 ret = intel_ring_idle(ring); 748 if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error)) 749 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n", 750 ring->name, ret); 751 752 /* TODO: Is this correct with Execlists enabled? */ 753 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING)); 754 if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) { 755 DRM_ERROR("%s :timed out trying to stop ring\n", ring->name); 756 return; 757 } 758 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING)); 759 } 760 761 int logical_ring_flush_all_caches(struct intel_ringbuffer *ringbuf, 762 struct intel_context *ctx) 763 { 764 struct intel_engine_cs *ring = ringbuf->ring; 765 int ret; 766 767 if (!ring->gpu_caches_dirty) 768 return 0; 769 770 ret = ring->emit_flush(ringbuf, ctx, 0, I915_GEM_GPU_DOMAINS); 771 if (ret) 772 return ret; 773 774 ring->gpu_caches_dirty = false; 775 return 0; 776 } 777 778 /** 779 * intel_logical_ring_advance_and_submit() - advance the tail and submit the workload 780 * @ringbuf: Logical Ringbuffer to advance. 781 * 782 * The tail is updated in our logical ringbuffer struct, not in the actual context. What 783 * really happens during submission is that the context and current tail will be placed 784 * on a queue waiting for the ELSP to be ready to accept a new context submission. At that 785 * point, the tail *inside* the context is updated and the ELSP written to. 786 */ 787 void intel_logical_ring_advance_and_submit(struct intel_ringbuffer *ringbuf, 788 struct intel_context *ctx, 789 struct drm_i915_gem_request *request) 790 { 791 struct intel_engine_cs *ring = ringbuf->ring; 792 793 intel_logical_ring_advance(ringbuf); 794 795 if (intel_ring_stopped(ring)) 796 return; 797 798 execlists_context_queue(ring, ctx, ringbuf->tail, request); 799 } 800 801 static int intel_lr_context_pin(struct intel_engine_cs *ring, 802 struct intel_context *ctx) 803 { 804 struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state; 805 struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf; 806 int ret = 0; 807 808 WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex)); 809 if (ctx->engine[ring->id].pin_count++ == 0) { 810 ret = i915_gem_obj_ggtt_pin(ctx_obj, 811 GEN8_LR_CONTEXT_ALIGN, 0); 812 if (ret) 813 goto reset_pin_count; 814 815 ret = intel_pin_and_map_ringbuffer_obj(ring->dev, ringbuf); 816 if (ret) 817 goto unpin_ctx_obj; 818 } 819 820 return ret; 821 822 unpin_ctx_obj: 823 i915_gem_object_ggtt_unpin(ctx_obj); 824 reset_pin_count: 825 ctx->engine[ring->id].pin_count = 0; 826 827 return ret; 828 } 829 830 void intel_lr_context_unpin(struct intel_engine_cs *ring, 831 struct intel_context *ctx) 832 { 833 struct drm_i915_gem_object *ctx_obj = ctx->engine[ring->id].state; 834 struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf; 835 836 if (ctx_obj) { 837 WARN_ON(!mutex_is_locked(&ring->dev->struct_mutex)); 838 if (--ctx->engine[ring->id].pin_count == 0) { 839 intel_unpin_ringbuffer_obj(ringbuf); 840 i915_gem_object_ggtt_unpin(ctx_obj); 841 } 842 } 843 } 844 845 static int logical_ring_alloc_request(struct intel_engine_cs *ring, 846 struct intel_context *ctx) 847 { 848 struct drm_i915_gem_request *request; 849 struct drm_i915_private *dev_private = ring->dev->dev_private; 850 int ret; 851 852 if (ring->outstanding_lazy_request) 853 return 0; 854 855 request = kzalloc(sizeof(*request), GFP_KERNEL); 856 if (request == NULL) 857 return -ENOMEM; 858 859 if (ctx != ring->default_context) { 860 ret = intel_lr_context_pin(ring, ctx); 861 if (ret) { 862 kfree(request); 863 return ret; 864 } 865 } 866 867 kref_init(&request->ref); 868 request->ring = ring; 869 request->uniq = dev_private->request_uniq++; 870 871 ret = i915_gem_get_seqno(ring->dev, &request->seqno); 872 if (ret) { 873 intel_lr_context_unpin(ring, ctx); 874 kfree(request); 875 return ret; 876 } 877 878 /* Hold a reference to the context this request belongs to 879 * (we will need it when the time comes to emit/retire the 880 * request). 881 */ 882 request->ctx = ctx; 883 i915_gem_context_reference(request->ctx); 884 885 ring->outstanding_lazy_request = request; 886 return 0; 887 } 888 889 static int logical_ring_wait_request(struct intel_ringbuffer *ringbuf, 890 int bytes) 891 { 892 struct intel_engine_cs *ring = ringbuf->ring; 893 struct drm_i915_gem_request *request; 894 int ret; 895 896 if (intel_ring_space(ringbuf) >= bytes) 897 return 0; 898 899 list_for_each_entry(request, &ring->request_list, list) { 900 /* 901 * The request queue is per-engine, so can contain requests 902 * from multiple ringbuffers. Here, we must ignore any that 903 * aren't from the ringbuffer we're considering. 904 */ 905 struct intel_context *ctx = request->ctx; 906 if (ctx->engine[ring->id].ringbuf != ringbuf) 907 continue; 908 909 /* Would completion of this request free enough space? */ 910 if (__intel_ring_space(request->tail, ringbuf->tail, 911 ringbuf->size) >= bytes) { 912 break; 913 } 914 } 915 916 if (&request->list == &ring->request_list) 917 return -ENOSPC; 918 919 ret = i915_wait_request(request); 920 if (ret) 921 return ret; 922 923 i915_gem_retire_requests_ring(ring); 924 925 return intel_ring_space(ringbuf) >= bytes ? 0 : -ENOSPC; 926 } 927 928 static int logical_ring_wait_for_space(struct intel_ringbuffer *ringbuf, 929 struct intel_context *ctx, 930 int bytes) 931 { 932 struct intel_engine_cs *ring = ringbuf->ring; 933 struct drm_device *dev = ring->dev; 934 struct drm_i915_private *dev_priv = dev->dev_private; 935 unsigned long end; 936 int ret; 937 938 ret = logical_ring_wait_request(ringbuf, bytes); 939 if (ret != -ENOSPC) 940 return ret; 941 942 /* Force the context submission in case we have been skipping it */ 943 intel_logical_ring_advance_and_submit(ringbuf, ctx, NULL); 944 945 /* With GEM the hangcheck timer should kick us out of the loop, 946 * leaving it early runs the risk of corrupting GEM state (due 947 * to running on almost untested codepaths). But on resume 948 * timers don't work yet, so prevent a complete hang in that 949 * case by choosing an insanely large timeout. */ 950 end = jiffies + 60 * HZ; 951 952 ret = 0; 953 do { 954 if (intel_ring_space(ringbuf) >= bytes) 955 break; 956 957 msleep(1); 958 959 if (dev_priv->mm.interruptible && signal_pending(curthread->td_lwp)) { 960 ret = -ERESTARTSYS; 961 break; 962 } 963 964 ret = i915_gem_check_wedge(&dev_priv->gpu_error, 965 dev_priv->mm.interruptible); 966 if (ret) 967 break; 968 969 if (time_after(jiffies, end)) { 970 ret = -EBUSY; 971 break; 972 } 973 } while (1); 974 975 return ret; 976 } 977 978 static int logical_ring_wrap_buffer(struct intel_ringbuffer *ringbuf, 979 struct intel_context *ctx) 980 { 981 uint32_t __iomem *virt; 982 int rem = ringbuf->size - ringbuf->tail; 983 984 if (ringbuf->space < rem) { 985 int ret = logical_ring_wait_for_space(ringbuf, ctx, rem); 986 987 if (ret) 988 return ret; 989 } 990 991 virt = (unsigned int *)((char *)ringbuf->virtual_start + ringbuf->tail); 992 rem /= 4; 993 while (rem--) 994 iowrite32(MI_NOOP, virt++); 995 996 ringbuf->tail = 0; 997 intel_ring_update_space(ringbuf); 998 999 return 0; 1000 } 1001 1002 static int logical_ring_prepare(struct intel_ringbuffer *ringbuf, 1003 struct intel_context *ctx, int bytes) 1004 { 1005 int ret; 1006 1007 if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) { 1008 ret = logical_ring_wrap_buffer(ringbuf, ctx); 1009 if (unlikely(ret)) 1010 return ret; 1011 } 1012 1013 if (unlikely(ringbuf->space < bytes)) { 1014 ret = logical_ring_wait_for_space(ringbuf, ctx, bytes); 1015 if (unlikely(ret)) 1016 return ret; 1017 } 1018 1019 return 0; 1020 } 1021 1022 /** 1023 * intel_logical_ring_begin() - prepare the logical ringbuffer to accept some commands 1024 * 1025 * @ringbuf: Logical ringbuffer. 1026 * @num_dwords: number of DWORDs that we plan to write to the ringbuffer. 1027 * 1028 * The ringbuffer might not be ready to accept the commands right away (maybe it needs to 1029 * be wrapped, or wait a bit for the tail to be updated). This function takes care of that 1030 * and also preallocates a request (every workload submission is still mediated through 1031 * requests, same as it did with legacy ringbuffer submission). 1032 * 1033 * Return: non-zero if the ringbuffer is not ready to be written to. 1034 */ 1035 int intel_logical_ring_begin(struct intel_ringbuffer *ringbuf, 1036 struct intel_context *ctx, int num_dwords) 1037 { 1038 struct intel_engine_cs *ring = ringbuf->ring; 1039 struct drm_device *dev = ring->dev; 1040 struct drm_i915_private *dev_priv = dev->dev_private; 1041 int ret; 1042 1043 ret = i915_gem_check_wedge(&dev_priv->gpu_error, 1044 dev_priv->mm.interruptible); 1045 if (ret) 1046 return ret; 1047 1048 ret = logical_ring_prepare(ringbuf, ctx, num_dwords * sizeof(uint32_t)); 1049 if (ret) 1050 return ret; 1051 1052 /* Preallocate the olr before touching the ring */ 1053 ret = logical_ring_alloc_request(ring, ctx); 1054 if (ret) 1055 return ret; 1056 1057 ringbuf->space -= num_dwords * sizeof(uint32_t); 1058 return 0; 1059 } 1060 1061 static int intel_logical_ring_workarounds_emit(struct intel_engine_cs *ring, 1062 struct intel_context *ctx) 1063 { 1064 int ret, i; 1065 struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf; 1066 struct drm_device *dev = ring->dev; 1067 struct drm_i915_private *dev_priv = dev->dev_private; 1068 struct i915_workarounds *w = &dev_priv->workarounds; 1069 1070 if (WARN_ON_ONCE(w->count == 0)) 1071 return 0; 1072 1073 ring->gpu_caches_dirty = true; 1074 ret = logical_ring_flush_all_caches(ringbuf, ctx); 1075 if (ret) 1076 return ret; 1077 1078 ret = intel_logical_ring_begin(ringbuf, ctx, w->count * 2 + 2); 1079 if (ret) 1080 return ret; 1081 1082 intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(w->count)); 1083 for (i = 0; i < w->count; i++) { 1084 intel_logical_ring_emit(ringbuf, w->reg[i].addr); 1085 intel_logical_ring_emit(ringbuf, w->reg[i].value); 1086 } 1087 intel_logical_ring_emit(ringbuf, MI_NOOP); 1088 1089 intel_logical_ring_advance(ringbuf); 1090 1091 ring->gpu_caches_dirty = true; 1092 ret = logical_ring_flush_all_caches(ringbuf, ctx); 1093 if (ret) 1094 return ret; 1095 1096 return 0; 1097 } 1098 1099 static int gen8_init_common_ring(struct intel_engine_cs *ring) 1100 { 1101 struct drm_device *dev = ring->dev; 1102 struct drm_i915_private *dev_priv = dev->dev_private; 1103 1104 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask)); 1105 I915_WRITE(RING_HWSTAM(ring->mmio_base), 0xffffffff); 1106 1107 I915_WRITE(RING_MODE_GEN7(ring), 1108 _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) | 1109 _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE)); 1110 POSTING_READ(RING_MODE_GEN7(ring)); 1111 ring->next_context_status_buffer = 0; 1112 DRM_DEBUG_DRIVER("Execlists enabled for %s\n", ring->name); 1113 1114 memset(&ring->hangcheck, 0, sizeof(ring->hangcheck)); 1115 1116 return 0; 1117 } 1118 1119 static int gen8_init_render_ring(struct intel_engine_cs *ring) 1120 { 1121 struct drm_device *dev = ring->dev; 1122 struct drm_i915_private *dev_priv = dev->dev_private; 1123 int ret; 1124 1125 ret = gen8_init_common_ring(ring); 1126 if (ret) 1127 return ret; 1128 1129 /* We need to disable the AsyncFlip performance optimisations in order 1130 * to use MI_WAIT_FOR_EVENT within the CS. It should already be 1131 * programmed to '1' on all products. 1132 * 1133 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv 1134 */ 1135 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE)); 1136 1137 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); 1138 1139 return init_workarounds_ring(ring); 1140 } 1141 1142 static int gen8_emit_bb_start(struct intel_ringbuffer *ringbuf, 1143 struct intel_context *ctx, 1144 u64 offset, unsigned flags) 1145 { 1146 bool ppgtt = !(flags & I915_DISPATCH_SECURE); 1147 int ret; 1148 1149 ret = intel_logical_ring_begin(ringbuf, ctx, 4); 1150 if (ret) 1151 return ret; 1152 1153 /* FIXME(BDW): Address space and security selectors. */ 1154 intel_logical_ring_emit(ringbuf, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8)); 1155 intel_logical_ring_emit(ringbuf, lower_32_bits(offset)); 1156 intel_logical_ring_emit(ringbuf, upper_32_bits(offset)); 1157 intel_logical_ring_emit(ringbuf, MI_NOOP); 1158 intel_logical_ring_advance(ringbuf); 1159 1160 return 0; 1161 } 1162 1163 static bool gen8_logical_ring_get_irq(struct intel_engine_cs *ring) 1164 { 1165 struct drm_device *dev = ring->dev; 1166 struct drm_i915_private *dev_priv = dev->dev_private; 1167 1168 if (WARN_ON(!intel_irqs_enabled(dev_priv))) 1169 return false; 1170 1171 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 1172 if (ring->irq_refcount++ == 0) { 1173 I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask)); 1174 POSTING_READ(RING_IMR(ring->mmio_base)); 1175 } 1176 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 1177 1178 return true; 1179 } 1180 1181 static void gen8_logical_ring_put_irq(struct intel_engine_cs *ring) 1182 { 1183 struct drm_device *dev = ring->dev; 1184 struct drm_i915_private *dev_priv = dev->dev_private; 1185 1186 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE); 1187 if (--ring->irq_refcount == 0) { 1188 I915_WRITE_IMR(ring, ~ring->irq_keep_mask); 1189 POSTING_READ(RING_IMR(ring->mmio_base)); 1190 } 1191 lockmgr(&dev_priv->irq_lock, LK_RELEASE); 1192 } 1193 1194 static int gen8_emit_flush(struct intel_ringbuffer *ringbuf, 1195 struct intel_context *ctx, 1196 u32 invalidate_domains, 1197 u32 unused) 1198 { 1199 struct intel_engine_cs *ring = ringbuf->ring; 1200 struct drm_device *dev = ring->dev; 1201 struct drm_i915_private *dev_priv = dev->dev_private; 1202 uint32_t cmd; 1203 int ret; 1204 1205 ret = intel_logical_ring_begin(ringbuf, ctx, 4); 1206 if (ret) 1207 return ret; 1208 1209 cmd = MI_FLUSH_DW + 1; 1210 1211 /* We always require a command barrier so that subsequent 1212 * commands, such as breadcrumb interrupts, are strictly ordered 1213 * wrt the contents of the write cache being flushed to memory 1214 * (and thus being coherent from the CPU). 1215 */ 1216 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW; 1217 1218 if (invalidate_domains & I915_GEM_GPU_DOMAINS) { 1219 cmd |= MI_INVALIDATE_TLB; 1220 if (ring == &dev_priv->ring[VCS]) 1221 cmd |= MI_INVALIDATE_BSD; 1222 } 1223 1224 intel_logical_ring_emit(ringbuf, cmd); 1225 intel_logical_ring_emit(ringbuf, 1226 I915_GEM_HWS_SCRATCH_ADDR | 1227 MI_FLUSH_DW_USE_GTT); 1228 intel_logical_ring_emit(ringbuf, 0); /* upper addr */ 1229 intel_logical_ring_emit(ringbuf, 0); /* value */ 1230 intel_logical_ring_advance(ringbuf); 1231 1232 return 0; 1233 } 1234 1235 static int gen8_emit_flush_render(struct intel_ringbuffer *ringbuf, 1236 struct intel_context *ctx, 1237 u32 invalidate_domains, 1238 u32 flush_domains) 1239 { 1240 struct intel_engine_cs *ring = ringbuf->ring; 1241 u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES; 1242 u32 flags = 0; 1243 int ret; 1244 1245 flags |= PIPE_CONTROL_CS_STALL; 1246 1247 if (flush_domains) { 1248 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; 1249 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; 1250 } 1251 1252 if (invalidate_domains) { 1253 flags |= PIPE_CONTROL_TLB_INVALIDATE; 1254 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; 1255 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 1256 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; 1257 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; 1258 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; 1259 flags |= PIPE_CONTROL_QW_WRITE; 1260 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; 1261 } 1262 1263 ret = intel_logical_ring_begin(ringbuf, ctx, 6); 1264 if (ret) 1265 return ret; 1266 1267 intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6)); 1268 intel_logical_ring_emit(ringbuf, flags); 1269 intel_logical_ring_emit(ringbuf, scratch_addr); 1270 intel_logical_ring_emit(ringbuf, 0); 1271 intel_logical_ring_emit(ringbuf, 0); 1272 intel_logical_ring_emit(ringbuf, 0); 1273 intel_logical_ring_advance(ringbuf); 1274 1275 return 0; 1276 } 1277 1278 static u32 gen8_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency) 1279 { 1280 return intel_read_status_page(ring, I915_GEM_HWS_INDEX); 1281 } 1282 1283 static void gen8_set_seqno(struct intel_engine_cs *ring, u32 seqno) 1284 { 1285 intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno); 1286 } 1287 1288 static int gen8_emit_request(struct intel_ringbuffer *ringbuf, 1289 struct drm_i915_gem_request *request) 1290 { 1291 struct intel_engine_cs *ring = ringbuf->ring; 1292 u32 cmd; 1293 int ret; 1294 1295 ret = intel_logical_ring_begin(ringbuf, request->ctx, 6); 1296 if (ret) 1297 return ret; 1298 1299 cmd = MI_STORE_DWORD_IMM_GEN4; 1300 cmd |= MI_GLOBAL_GTT; 1301 1302 intel_logical_ring_emit(ringbuf, cmd); 1303 intel_logical_ring_emit(ringbuf, 1304 (ring->status_page.gfx_addr + 1305 (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT))); 1306 intel_logical_ring_emit(ringbuf, 0); 1307 intel_logical_ring_emit(ringbuf, 1308 i915_gem_request_get_seqno(ring->outstanding_lazy_request)); 1309 intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT); 1310 intel_logical_ring_emit(ringbuf, MI_NOOP); 1311 intel_logical_ring_advance_and_submit(ringbuf, request->ctx, request); 1312 1313 return 0; 1314 } 1315 1316 static int gen8_init_rcs_context(struct intel_engine_cs *ring, 1317 struct intel_context *ctx) 1318 { 1319 int ret; 1320 1321 ret = intel_logical_ring_workarounds_emit(ring, ctx); 1322 if (ret) 1323 return ret; 1324 1325 return intel_lr_context_render_state_init(ring, ctx); 1326 } 1327 1328 /** 1329 * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer 1330 * 1331 * @ring: Engine Command Streamer. 1332 * 1333 */ 1334 void intel_logical_ring_cleanup(struct intel_engine_cs *ring) 1335 { 1336 struct drm_i915_private *dev_priv; 1337 1338 if (!intel_ring_initialized(ring)) 1339 return; 1340 1341 dev_priv = ring->dev->dev_private; 1342 1343 intel_logical_ring_stop(ring); 1344 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0); 1345 i915_gem_request_assign(&ring->outstanding_lazy_request, NULL); 1346 1347 if (ring->cleanup) 1348 ring->cleanup(ring); 1349 1350 i915_cmd_parser_fini_ring(ring); 1351 1352 if (ring->status_page.obj) { 1353 kunmap(ring->status_page.obj->pages[0]); 1354 ring->status_page.obj = NULL; 1355 } 1356 } 1357 1358 static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring) 1359 { 1360 int ret; 1361 1362 /* Intentionally left blank. */ 1363 ring->buffer = NULL; 1364 1365 ring->dev = dev; 1366 INIT_LIST_HEAD(&ring->active_list); 1367 INIT_LIST_HEAD(&ring->request_list); 1368 init_waitqueue_head(&ring->irq_queue); 1369 1370 INIT_LIST_HEAD(&ring->execlist_queue); 1371 INIT_LIST_HEAD(&ring->execlist_retired_req_list); 1372 lockinit(&ring->execlist_lock, "i915el", 0, LK_CANRECURSE); 1373 1374 ret = i915_cmd_parser_init_ring(ring); 1375 if (ret) 1376 return ret; 1377 1378 ret = intel_lr_context_deferred_create(ring->default_context, ring); 1379 1380 return ret; 1381 } 1382 1383 static int logical_render_ring_init(struct drm_device *dev) 1384 { 1385 struct drm_i915_private *dev_priv = dev->dev_private; 1386 struct intel_engine_cs *ring = &dev_priv->ring[RCS]; 1387 int ret; 1388 1389 ring->name = "render ring"; 1390 ring->id = RCS; 1391 ring->mmio_base = RENDER_RING_BASE; 1392 ring->irq_enable_mask = 1393 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT; 1394 ring->irq_keep_mask = 1395 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT; 1396 if (HAS_L3_DPF(dev)) 1397 ring->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; 1398 1399 ring->init_hw = gen8_init_render_ring; 1400 ring->init_context = gen8_init_rcs_context; 1401 ring->cleanup = intel_fini_pipe_control; 1402 ring->get_seqno = gen8_get_seqno; 1403 ring->set_seqno = gen8_set_seqno; 1404 ring->emit_request = gen8_emit_request; 1405 ring->emit_flush = gen8_emit_flush_render; 1406 ring->irq_get = gen8_logical_ring_get_irq; 1407 ring->irq_put = gen8_logical_ring_put_irq; 1408 ring->emit_bb_start = gen8_emit_bb_start; 1409 1410 ring->dev = dev; 1411 ret = logical_ring_init(dev, ring); 1412 if (ret) 1413 return ret; 1414 1415 return intel_init_pipe_control(ring); 1416 } 1417 1418 static int logical_bsd_ring_init(struct drm_device *dev) 1419 { 1420 struct drm_i915_private *dev_priv = dev->dev_private; 1421 struct intel_engine_cs *ring = &dev_priv->ring[VCS]; 1422 1423 ring->name = "bsd ring"; 1424 ring->id = VCS; 1425 ring->mmio_base = GEN6_BSD_RING_BASE; 1426 ring->irq_enable_mask = 1427 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT; 1428 ring->irq_keep_mask = 1429 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT; 1430 1431 ring->init_hw = gen8_init_common_ring; 1432 ring->get_seqno = gen8_get_seqno; 1433 ring->set_seqno = gen8_set_seqno; 1434 ring->emit_request = gen8_emit_request; 1435 ring->emit_flush = gen8_emit_flush; 1436 ring->irq_get = gen8_logical_ring_get_irq; 1437 ring->irq_put = gen8_logical_ring_put_irq; 1438 ring->emit_bb_start = gen8_emit_bb_start; 1439 1440 return logical_ring_init(dev, ring); 1441 } 1442 1443 static int logical_bsd2_ring_init(struct drm_device *dev) 1444 { 1445 struct drm_i915_private *dev_priv = dev->dev_private; 1446 struct intel_engine_cs *ring = &dev_priv->ring[VCS2]; 1447 1448 ring->name = "bds2 ring"; 1449 ring->id = VCS2; 1450 ring->mmio_base = GEN8_BSD2_RING_BASE; 1451 ring->irq_enable_mask = 1452 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT; 1453 ring->irq_keep_mask = 1454 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT; 1455 1456 ring->init_hw = gen8_init_common_ring; 1457 ring->get_seqno = gen8_get_seqno; 1458 ring->set_seqno = gen8_set_seqno; 1459 ring->emit_request = gen8_emit_request; 1460 ring->emit_flush = gen8_emit_flush; 1461 ring->irq_get = gen8_logical_ring_get_irq; 1462 ring->irq_put = gen8_logical_ring_put_irq; 1463 ring->emit_bb_start = gen8_emit_bb_start; 1464 1465 return logical_ring_init(dev, ring); 1466 } 1467 1468 static int logical_blt_ring_init(struct drm_device *dev) 1469 { 1470 struct drm_i915_private *dev_priv = dev->dev_private; 1471 struct intel_engine_cs *ring = &dev_priv->ring[BCS]; 1472 1473 ring->name = "blitter ring"; 1474 ring->id = BCS; 1475 ring->mmio_base = BLT_RING_BASE; 1476 ring->irq_enable_mask = 1477 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT; 1478 ring->irq_keep_mask = 1479 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT; 1480 1481 ring->init_hw = gen8_init_common_ring; 1482 ring->get_seqno = gen8_get_seqno; 1483 ring->set_seqno = gen8_set_seqno; 1484 ring->emit_request = gen8_emit_request; 1485 ring->emit_flush = gen8_emit_flush; 1486 ring->irq_get = gen8_logical_ring_get_irq; 1487 ring->irq_put = gen8_logical_ring_put_irq; 1488 ring->emit_bb_start = gen8_emit_bb_start; 1489 1490 return logical_ring_init(dev, ring); 1491 } 1492 1493 static int logical_vebox_ring_init(struct drm_device *dev) 1494 { 1495 struct drm_i915_private *dev_priv = dev->dev_private; 1496 struct intel_engine_cs *ring = &dev_priv->ring[VECS]; 1497 1498 ring->name = "video enhancement ring"; 1499 ring->id = VECS; 1500 ring->mmio_base = VEBOX_RING_BASE; 1501 ring->irq_enable_mask = 1502 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT; 1503 ring->irq_keep_mask = 1504 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT; 1505 1506 ring->init_hw = gen8_init_common_ring; 1507 ring->get_seqno = gen8_get_seqno; 1508 ring->set_seqno = gen8_set_seqno; 1509 ring->emit_request = gen8_emit_request; 1510 ring->emit_flush = gen8_emit_flush; 1511 ring->irq_get = gen8_logical_ring_get_irq; 1512 ring->irq_put = gen8_logical_ring_put_irq; 1513 ring->emit_bb_start = gen8_emit_bb_start; 1514 1515 return logical_ring_init(dev, ring); 1516 } 1517 1518 /** 1519 * intel_logical_rings_init() - allocate, populate and init the Engine Command Streamers 1520 * @dev: DRM device. 1521 * 1522 * This function inits the engines for an Execlists submission style (the equivalent in the 1523 * legacy ringbuffer submission world would be i915_gem_init_rings). It does it only for 1524 * those engines that are present in the hardware. 1525 * 1526 * Return: non-zero if the initialization failed. 1527 */ 1528 int intel_logical_rings_init(struct drm_device *dev) 1529 { 1530 struct drm_i915_private *dev_priv = dev->dev_private; 1531 int ret; 1532 1533 ret = logical_render_ring_init(dev); 1534 if (ret) 1535 return ret; 1536 1537 if (HAS_BSD(dev)) { 1538 ret = logical_bsd_ring_init(dev); 1539 if (ret) 1540 goto cleanup_render_ring; 1541 } 1542 1543 if (HAS_BLT(dev)) { 1544 ret = logical_blt_ring_init(dev); 1545 if (ret) 1546 goto cleanup_bsd_ring; 1547 } 1548 1549 if (HAS_VEBOX(dev)) { 1550 ret = logical_vebox_ring_init(dev); 1551 if (ret) 1552 goto cleanup_blt_ring; 1553 } 1554 1555 if (HAS_BSD2(dev)) { 1556 ret = logical_bsd2_ring_init(dev); 1557 if (ret) 1558 goto cleanup_vebox_ring; 1559 } 1560 1561 ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000)); 1562 if (ret) 1563 goto cleanup_bsd2_ring; 1564 1565 return 0; 1566 1567 cleanup_bsd2_ring: 1568 intel_logical_ring_cleanup(&dev_priv->ring[VCS2]); 1569 cleanup_vebox_ring: 1570 intel_logical_ring_cleanup(&dev_priv->ring[VECS]); 1571 cleanup_blt_ring: 1572 intel_logical_ring_cleanup(&dev_priv->ring[BCS]); 1573 cleanup_bsd_ring: 1574 intel_logical_ring_cleanup(&dev_priv->ring[VCS]); 1575 cleanup_render_ring: 1576 intel_logical_ring_cleanup(&dev_priv->ring[RCS]); 1577 1578 return ret; 1579 } 1580 1581 int intel_lr_context_render_state_init(struct intel_engine_cs *ring, 1582 struct intel_context *ctx) 1583 { 1584 struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf; 1585 struct render_state so; 1586 struct drm_i915_file_private *file_priv = ctx->file_priv; 1587 struct drm_file *file = file_priv ? file_priv->file : NULL; 1588 int ret; 1589 1590 ret = i915_gem_render_state_prepare(ring, &so); 1591 if (ret) 1592 return ret; 1593 1594 if (so.rodata == NULL) 1595 return 0; 1596 1597 ret = ring->emit_bb_start(ringbuf, 1598 ctx, 1599 so.ggtt_offset, 1600 I915_DISPATCH_SECURE); 1601 if (ret) 1602 goto out; 1603 1604 i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), ring); 1605 1606 ret = __i915_add_request(ring, file, so.obj); 1607 /* intel_logical_ring_add_request moves object to inactive if it 1608 * fails */ 1609 out: 1610 i915_gem_render_state_fini(&so); 1611 return ret; 1612 } 1613 1614 static int 1615 populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_obj, 1616 struct intel_engine_cs *ring, struct intel_ringbuffer *ringbuf) 1617 { 1618 struct drm_device *dev = ring->dev; 1619 struct drm_i915_private *dev_priv = dev->dev_private; 1620 struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; 1621 struct vm_page *page; 1622 uint32_t *reg_state; 1623 int ret; 1624 1625 if (!ppgtt) 1626 ppgtt = dev_priv->mm.aliasing_ppgtt; 1627 1628 ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true); 1629 if (ret) { 1630 DRM_DEBUG_DRIVER("Could not set to CPU domain\n"); 1631 return ret; 1632 } 1633 1634 ret = i915_gem_object_get_pages(ctx_obj); 1635 if (ret) { 1636 DRM_DEBUG_DRIVER("Could not get object pages\n"); 1637 return ret; 1638 } 1639 1640 i915_gem_object_pin_pages(ctx_obj); 1641 1642 /* The second page of the context object contains some fields which must 1643 * be set up prior to the first execution. */ 1644 page = i915_gem_object_get_page(ctx_obj, 1); 1645 reg_state = kmap_atomic(page); 1646 1647 /* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM 1648 * commands followed by (reg, value) pairs. The values we are setting here are 1649 * only for the first context restore: on a subsequent save, the GPU will 1650 * recreate this batchbuffer with new values (including all the missing 1651 * MI_LOAD_REGISTER_IMM commands that we are not initializing here). */ 1652 if (ring->id == RCS) 1653 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(14); 1654 else 1655 reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(11); 1656 reg_state[CTX_LRI_HEADER_0] |= MI_LRI_FORCE_POSTED; 1657 reg_state[CTX_CONTEXT_CONTROL] = RING_CONTEXT_CONTROL(ring); 1658 reg_state[CTX_CONTEXT_CONTROL+1] = 1659 _MASKED_BIT_ENABLE((1<<3) | MI_RESTORE_INHIBIT); 1660 reg_state[CTX_RING_HEAD] = RING_HEAD(ring->mmio_base); 1661 reg_state[CTX_RING_HEAD+1] = 0; 1662 reg_state[CTX_RING_TAIL] = RING_TAIL(ring->mmio_base); 1663 reg_state[CTX_RING_TAIL+1] = 0; 1664 reg_state[CTX_RING_BUFFER_START] = RING_START(ring->mmio_base); 1665 /* Ring buffer start address is not known until the buffer is pinned. 1666 * It is written to the context image in execlists_update_context() 1667 */ 1668 reg_state[CTX_RING_BUFFER_CONTROL] = RING_CTL(ring->mmio_base); 1669 reg_state[CTX_RING_BUFFER_CONTROL+1] = 1670 ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID; 1671 reg_state[CTX_BB_HEAD_U] = ring->mmio_base + 0x168; 1672 reg_state[CTX_BB_HEAD_U+1] = 0; 1673 reg_state[CTX_BB_HEAD_L] = ring->mmio_base + 0x140; 1674 reg_state[CTX_BB_HEAD_L+1] = 0; 1675 reg_state[CTX_BB_STATE] = ring->mmio_base + 0x110; 1676 reg_state[CTX_BB_STATE+1] = (1<<5); 1677 reg_state[CTX_SECOND_BB_HEAD_U] = ring->mmio_base + 0x11c; 1678 reg_state[CTX_SECOND_BB_HEAD_U+1] = 0; 1679 reg_state[CTX_SECOND_BB_HEAD_L] = ring->mmio_base + 0x114; 1680 reg_state[CTX_SECOND_BB_HEAD_L+1] = 0; 1681 reg_state[CTX_SECOND_BB_STATE] = ring->mmio_base + 0x118; 1682 reg_state[CTX_SECOND_BB_STATE+1] = 0; 1683 if (ring->id == RCS) { 1684 /* TODO: according to BSpec, the register state context 1685 * for CHV does not have these. OTOH, these registers do 1686 * exist in CHV. I'm waiting for a clarification */ 1687 reg_state[CTX_BB_PER_CTX_PTR] = ring->mmio_base + 0x1c0; 1688 reg_state[CTX_BB_PER_CTX_PTR+1] = 0; 1689 reg_state[CTX_RCS_INDIRECT_CTX] = ring->mmio_base + 0x1c4; 1690 reg_state[CTX_RCS_INDIRECT_CTX+1] = 0; 1691 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET] = ring->mmio_base + 0x1c8; 1692 reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] = 0; 1693 } 1694 reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9); 1695 reg_state[CTX_LRI_HEADER_1] |= MI_LRI_FORCE_POSTED; 1696 reg_state[CTX_CTX_TIMESTAMP] = ring->mmio_base + 0x3a8; 1697 reg_state[CTX_CTX_TIMESTAMP+1] = 0; 1698 reg_state[CTX_PDP3_UDW] = GEN8_RING_PDP_UDW(ring, 3); 1699 reg_state[CTX_PDP3_LDW] = GEN8_RING_PDP_LDW(ring, 3); 1700 reg_state[CTX_PDP2_UDW] = GEN8_RING_PDP_UDW(ring, 2); 1701 reg_state[CTX_PDP2_LDW] = GEN8_RING_PDP_LDW(ring, 2); 1702 reg_state[CTX_PDP1_UDW] = GEN8_RING_PDP_UDW(ring, 1); 1703 reg_state[CTX_PDP1_LDW] = GEN8_RING_PDP_LDW(ring, 1); 1704 reg_state[CTX_PDP0_UDW] = GEN8_RING_PDP_UDW(ring, 0); 1705 reg_state[CTX_PDP0_LDW] = GEN8_RING_PDP_LDW(ring, 0); 1706 reg_state[CTX_PDP3_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[3]); 1707 reg_state[CTX_PDP3_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[3]); 1708 reg_state[CTX_PDP2_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[2]); 1709 reg_state[CTX_PDP2_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[2]); 1710 reg_state[CTX_PDP1_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[1]); 1711 reg_state[CTX_PDP1_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[1]); 1712 reg_state[CTX_PDP0_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[0]); 1713 reg_state[CTX_PDP0_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[0]); 1714 if (ring->id == RCS) { 1715 reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1); 1716 reg_state[CTX_R_PWR_CLK_STATE] = 0x20c8; 1717 reg_state[CTX_R_PWR_CLK_STATE+1] = 0; 1718 } 1719 1720 kunmap_atomic(reg_state); 1721 1722 ctx_obj->dirty = 1; 1723 set_page_dirty(page); 1724 i915_gem_object_unpin_pages(ctx_obj); 1725 1726 return 0; 1727 } 1728 1729 /** 1730 * intel_lr_context_free() - free the LRC specific bits of a context 1731 * @ctx: the LR context to free. 1732 * 1733 * The real context freeing is done in i915_gem_context_free: this only 1734 * takes care of the bits that are LRC related: the per-engine backing 1735 * objects and the logical ringbuffer. 1736 */ 1737 void intel_lr_context_free(struct intel_context *ctx) 1738 { 1739 int i; 1740 1741 for (i = 0; i < I915_NUM_RINGS; i++) { 1742 struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state; 1743 1744 if (ctx_obj) { 1745 struct intel_ringbuffer *ringbuf = 1746 ctx->engine[i].ringbuf; 1747 struct intel_engine_cs *ring = ringbuf->ring; 1748 1749 if (ctx == ring->default_context) { 1750 intel_unpin_ringbuffer_obj(ringbuf); 1751 i915_gem_object_ggtt_unpin(ctx_obj); 1752 } 1753 WARN_ON(ctx->engine[ring->id].pin_count); 1754 intel_destroy_ringbuffer_obj(ringbuf); 1755 kfree(ringbuf); 1756 drm_gem_object_unreference(&ctx_obj->base); 1757 } 1758 } 1759 } 1760 1761 static uint32_t get_lr_context_size(struct intel_engine_cs *ring) 1762 { 1763 int ret = 0; 1764 1765 WARN_ON(INTEL_INFO(ring->dev)->gen < 8); 1766 1767 switch (ring->id) { 1768 case RCS: 1769 if (INTEL_INFO(ring->dev)->gen >= 9) 1770 ret = GEN9_LR_CONTEXT_RENDER_SIZE; 1771 else 1772 ret = GEN8_LR_CONTEXT_RENDER_SIZE; 1773 break; 1774 case VCS: 1775 case BCS: 1776 case VECS: 1777 case VCS2: 1778 ret = GEN8_LR_CONTEXT_OTHER_SIZE; 1779 break; 1780 } 1781 1782 return ret; 1783 } 1784 1785 static void lrc_setup_hardware_status_page(struct intel_engine_cs *ring, 1786 struct drm_i915_gem_object *default_ctx_obj) 1787 { 1788 struct drm_i915_private *dev_priv = ring->dev->dev_private; 1789 1790 /* The status page is offset 0 from the default context object 1791 * in LRC mode. */ 1792 ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(default_ctx_obj); 1793 ring->status_page.page_addr = 1794 kmap(default_ctx_obj->pages[0]); 1795 ring->status_page.obj = default_ctx_obj; 1796 1797 I915_WRITE(RING_HWS_PGA(ring->mmio_base), 1798 (u32)ring->status_page.gfx_addr); 1799 POSTING_READ(RING_HWS_PGA(ring->mmio_base)); 1800 } 1801 1802 /** 1803 * intel_lr_context_deferred_create() - create the LRC specific bits of a context 1804 * @ctx: LR context to create. 1805 * @ring: engine to be used with the context. 1806 * 1807 * This function can be called more than once, with different engines, if we plan 1808 * to use the context with them. The context backing objects and the ringbuffers 1809 * (specially the ringbuffer backing objects) suck a lot of memory up, and that's why 1810 * the creation is a deferred call: it's better to make sure first that we need to use 1811 * a given ring with the context. 1812 * 1813 * Return: non-zero on error. 1814 */ 1815 int intel_lr_context_deferred_create(struct intel_context *ctx, 1816 struct intel_engine_cs *ring) 1817 { 1818 const bool is_global_default_ctx = (ctx == ring->default_context); 1819 struct drm_device *dev = ring->dev; 1820 struct drm_i915_gem_object *ctx_obj; 1821 uint32_t context_size; 1822 struct intel_ringbuffer *ringbuf; 1823 int ret; 1824 1825 WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL); 1826 WARN_ON(ctx->engine[ring->id].state); 1827 1828 context_size = round_up(get_lr_context_size(ring), 4096); 1829 1830 ctx_obj = i915_gem_alloc_context_obj(dev, context_size); 1831 if (IS_ERR(ctx_obj)) { 1832 ret = PTR_ERR(ctx_obj); 1833 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed: %d\n", ret); 1834 return ret; 1835 } 1836 1837 if (is_global_default_ctx) { 1838 ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, 0); 1839 if (ret) { 1840 DRM_DEBUG_DRIVER("Pin LRC backing obj failed: %d\n", 1841 ret); 1842 drm_gem_object_unreference(&ctx_obj->base); 1843 return ret; 1844 } 1845 } 1846 1847 ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL); 1848 if (!ringbuf) { 1849 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n", 1850 ring->name); 1851 ret = -ENOMEM; 1852 goto error_unpin_ctx; 1853 } 1854 1855 ringbuf->ring = ring; 1856 1857 ringbuf->size = 32 * PAGE_SIZE; 1858 ringbuf->effective_size = ringbuf->size; 1859 ringbuf->head = 0; 1860 ringbuf->tail = 0; 1861 ringbuf->last_retired_head = -1; 1862 intel_ring_update_space(ringbuf); 1863 1864 if (ringbuf->obj == NULL) { 1865 ret = intel_alloc_ringbuffer_obj(dev, ringbuf); 1866 if (ret) { 1867 DRM_DEBUG_DRIVER( 1868 "Failed to allocate ringbuffer obj %s: %d\n", 1869 ring->name, ret); 1870 goto error_free_rbuf; 1871 } 1872 1873 if (is_global_default_ctx) { 1874 ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf); 1875 if (ret) { 1876 DRM_ERROR( 1877 "Failed to pin and map ringbuffer %s: %d\n", 1878 ring->name, ret); 1879 goto error_destroy_rbuf; 1880 } 1881 } 1882 1883 } 1884 1885 ret = populate_lr_context(ctx, ctx_obj, ring, ringbuf); 1886 if (ret) { 1887 DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret); 1888 goto error; 1889 } 1890 1891 ctx->engine[ring->id].ringbuf = ringbuf; 1892 ctx->engine[ring->id].state = ctx_obj; 1893 1894 if (ctx == ring->default_context) 1895 lrc_setup_hardware_status_page(ring, ctx_obj); 1896 else if (ring->id == RCS && !ctx->rcs_initialized) { 1897 if (ring->init_context) { 1898 ret = ring->init_context(ring, ctx); 1899 if (ret) { 1900 DRM_ERROR("ring init context: %d\n", ret); 1901 ctx->engine[ring->id].ringbuf = NULL; 1902 ctx->engine[ring->id].state = NULL; 1903 goto error; 1904 } 1905 } 1906 1907 ctx->rcs_initialized = true; 1908 } 1909 1910 return 0; 1911 1912 error: 1913 if (is_global_default_ctx) 1914 intel_unpin_ringbuffer_obj(ringbuf); 1915 error_destroy_rbuf: 1916 intel_destroy_ringbuffer_obj(ringbuf); 1917 error_free_rbuf: 1918 kfree(ringbuf); 1919 error_unpin_ctx: 1920 if (is_global_default_ctx) 1921 i915_gem_object_ggtt_unpin(ctx_obj); 1922 drm_gem_object_unreference(&ctx_obj->base); 1923 return ret; 1924 } 1925