1 /* 2 * Copyright © 2011-2012 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Ben Widawsky <ben@bwidawsk.net> 25 * 26 */ 27 28 /* 29 * This file implements HW context support. On gen5+ a HW context consists of an 30 * opaque GPU object which is referenced at times of context saves and restores. 31 * With RC6 enabled, the context is also referenced as the GPU enters and exists 32 * from RC6 (GPU has it's own internal power context, except on gen5). Though 33 * something like a context does exist for the media ring, the code only 34 * supports contexts for the render ring. 35 * 36 * In software, there is a distinction between contexts created by the user, 37 * and the default HW context. The default HW context is used by GPU clients 38 * that do not request setup of their own hardware context. The default 39 * context's state is never restored to help prevent programming errors. This 40 * would happen if a client ran and piggy-backed off another clients GPU state. 41 * The default context only exists to give the GPU some offset to load as the 42 * current to invoke a save of the context we actually care about. In fact, the 43 * code could likely be constructed, albeit in a more complicated fashion, to 44 * never use the default context, though that limits the driver's ability to 45 * swap out, and/or destroy other contexts. 46 * 47 * All other contexts are created as a request by the GPU client. These contexts 48 * store GPU state, and thus allow GPU clients to not re-emit state (and 49 * potentially query certain state) at any time. The kernel driver makes 50 * certain that the appropriate commands are inserted. 51 * 52 * The context life cycle is semi-complicated in that context BOs may live 53 * longer than the context itself because of the way the hardware, and object 54 * tracking works. Below is a very crude representation of the state machine 55 * describing the context life. 56 * refcount pincount active 57 * S0: initial state 0 0 0 58 * S1: context created 1 0 0 59 * S2: context is currently running 2 1 X 60 * S3: GPU referenced, but not current 2 0 1 61 * S4: context is current, but destroyed 1 1 0 62 * S5: like S3, but destroyed 1 0 1 63 * 64 * The most common (but not all) transitions: 65 * S0->S1: client creates a context 66 * S1->S2: client submits execbuf with context 67 * S2->S3: other clients submits execbuf with context 68 * S3->S1: context object was retired 69 * S3->S2: clients submits another execbuf 70 * S2->S4: context destroy called with current context 71 * S3->S5->S0: destroy path 72 * S4->S5->S0: destroy path on current context 73 * 74 * There are two confusing terms used above: 75 * The "current context" means the context which is currently running on the 76 * GPU. The GPU has loaded its state already and has stored away the gtt 77 * offset of the BO. The GPU is not actively referencing the data at this 78 * offset, but it will on the next context switch. The only way to avoid this 79 * is to do a GPU reset. 80 * 81 * An "active context' is one which was previously the "current context" and is 82 * on the active list waiting for the next context switch to occur. Until this 83 * happens, the object must remain at the same gtt offset. It is therefore 84 * possible to destroy a context, but it is still active. 85 * 86 */ 87 88 #include <drm/drmP.h> 89 #include <drm/i915_drm.h> 90 #include "i915_drv.h" 91 #include "i915_trace.h" 92 93 #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 94 95 static int get_context_size(struct drm_i915_private *dev_priv) 96 { 97 int ret; 98 u32 reg; 99 100 switch (INTEL_GEN(dev_priv)) { 101 case 6: 102 reg = I915_READ(CXT_SIZE); 103 ret = GEN6_CXT_TOTAL_SIZE(reg) * 64; 104 break; 105 case 7: 106 reg = I915_READ(GEN7_CXT_SIZE); 107 if (IS_HASWELL(dev_priv)) 108 ret = HSW_CXT_TOTAL_SIZE; 109 else 110 ret = GEN7_CXT_TOTAL_SIZE(reg) * 64; 111 break; 112 case 8: 113 ret = GEN8_CXT_TOTAL_SIZE; 114 break; 115 default: 116 BUG(); 117 } 118 119 return ret; 120 } 121 122 void i915_gem_context_free(struct kref *ctx_ref) 123 { 124 struct i915_gem_context *ctx = container_of(ctx_ref, typeof(*ctx), ref); 125 int i; 126 127 lockdep_assert_held(&ctx->i915->drm.struct_mutex); 128 trace_i915_context_free(ctx); 129 GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 130 131 i915_ppgtt_put(ctx->ppgtt); 132 133 for (i = 0; i < I915_NUM_ENGINES; i++) { 134 struct intel_context *ce = &ctx->engine[i]; 135 136 if (!ce->state) 137 continue; 138 139 WARN_ON(ce->pin_count); 140 if (ce->ring) 141 intel_ring_free(ce->ring); 142 143 __i915_gem_object_release_unless_active(ce->state->obj); 144 } 145 146 kfree(ctx->name); 147 put_pid(ctx->pid); 148 list_del(&ctx->link); 149 150 ida_simple_remove(&ctx->i915->context_hw_ida, ctx->hw_id); 151 kfree(ctx); 152 } 153 154 static struct drm_i915_gem_object * 155 alloc_context_obj(struct drm_i915_private *dev_priv, u64 size) 156 { 157 struct drm_i915_gem_object *obj; 158 int ret; 159 160 lockdep_assert_held(&dev_priv->drm.struct_mutex); 161 162 obj = i915_gem_object_create(dev_priv, size); 163 if (IS_ERR(obj)) 164 return obj; 165 166 /* 167 * Try to make the context utilize L3 as well as LLC. 168 * 169 * On VLV we don't have L3 controls in the PTEs so we 170 * shouldn't touch the cache level, especially as that 171 * would make the object snooped which might have a 172 * negative performance impact. 173 * 174 * Snooping is required on non-llc platforms in execlist 175 * mode, but since all GGTT accesses use PAT entry 0 we 176 * get snooping anyway regardless of cache_level. 177 * 178 * This is only applicable for Ivy Bridge devices since 179 * later platforms don't have L3 control bits in the PTE. 180 */ 181 if (IS_IVYBRIDGE(dev_priv)) { 182 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC); 183 /* Failure shouldn't ever happen this early */ 184 if (WARN_ON(ret)) { 185 i915_gem_object_put(obj); 186 return ERR_PTR(ret); 187 } 188 } 189 190 return obj; 191 } 192 193 static void context_close(struct i915_gem_context *ctx) 194 { 195 i915_gem_context_set_closed(ctx); 196 if (ctx->ppgtt) 197 i915_ppgtt_close(&ctx->ppgtt->base); 198 ctx->file_priv = ERR_PTR(-EBADF); 199 i915_gem_context_put(ctx); 200 } 201 202 static int assign_hw_id(struct drm_i915_private *dev_priv, unsigned *out) 203 { 204 int ret; 205 206 ret = ida_simple_get(&dev_priv->context_hw_ida, 207 0, MAX_CONTEXT_HW_ID, GFP_KERNEL); 208 if (ret < 0) { 209 /* Contexts are only released when no longer active. 210 * Flush any pending retires to hopefully release some 211 * stale contexts and try again. 212 */ 213 i915_gem_retire_requests(dev_priv); 214 ret = ida_simple_get(&dev_priv->context_hw_ida, 215 0, MAX_CONTEXT_HW_ID, GFP_KERNEL); 216 if (ret < 0) 217 return ret; 218 } 219 220 *out = ret; 221 return 0; 222 } 223 224 static u32 default_desc_template(const struct drm_i915_private *i915, 225 const struct i915_hw_ppgtt *ppgtt) 226 { 227 u32 address_mode; 228 u32 desc; 229 230 desc = GEN8_CTX_VALID | GEN8_CTX_PRIVILEGE; 231 232 address_mode = INTEL_LEGACY_32B_CONTEXT; 233 if (ppgtt && i915_vm_is_48bit(&ppgtt->base)) 234 address_mode = INTEL_LEGACY_64B_CONTEXT; 235 desc |= address_mode << GEN8_CTX_ADDRESSING_MODE_SHIFT; 236 237 if (IS_GEN8(i915)) 238 desc |= GEN8_CTX_L3LLC_COHERENT; 239 240 /* TODO: WaDisableLiteRestore when we start using semaphore 241 * signalling between Command Streamers 242 * ring->ctx_desc_template |= GEN8_CTX_FORCE_RESTORE; 243 */ 244 245 return desc; 246 } 247 248 static struct i915_gem_context * 249 __create_hw_context(struct drm_i915_private *dev_priv, 250 struct drm_i915_file_private *file_priv) 251 { 252 struct i915_gem_context *ctx; 253 int ret; 254 255 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 256 if (ctx == NULL) 257 return ERR_PTR(-ENOMEM); 258 259 ret = assign_hw_id(dev_priv, &ctx->hw_id); 260 if (ret) { 261 kfree(ctx); 262 return ERR_PTR(ret); 263 } 264 265 kref_init(&ctx->ref); 266 list_add_tail(&ctx->link, &dev_priv->context_list); 267 ctx->i915 = dev_priv; 268 269 if (dev_priv->hw_context_size) { 270 struct drm_i915_gem_object *obj; 271 struct i915_vma *vma; 272 273 obj = alloc_context_obj(dev_priv, dev_priv->hw_context_size); 274 if (IS_ERR(obj)) { 275 ret = PTR_ERR(obj); 276 goto err_out; 277 } 278 279 vma = i915_vma_instance(obj, &dev_priv->ggtt.base, NULL); 280 if (IS_ERR(vma)) { 281 i915_gem_object_put(obj); 282 ret = PTR_ERR(vma); 283 goto err_out; 284 } 285 286 ctx->engine[RCS].state = vma; 287 } 288 289 /* Default context will never have a file_priv */ 290 ret = DEFAULT_CONTEXT_HANDLE; 291 if (file_priv) { 292 ret = idr_alloc(&file_priv->context_idr, ctx, 293 DEFAULT_CONTEXT_HANDLE, 0, GFP_KERNEL); 294 if (ret < 0) 295 goto err_out; 296 } 297 ctx->user_handle = ret; 298 299 ctx->file_priv = file_priv; 300 if (file_priv) { 301 #ifndef __DragonFly__ 302 ctx->pid = get_task_pid(current, PIDTYPE_PID); 303 ctx->name = kasprintf(GFP_KERNEL, "%s[%d]/%x", 304 current->comm, 305 pid_nr(ctx->pid), 306 ctx->user_handle); 307 #else 308 ctx->pid = DRM_CURRENTPID; 309 ctx->name = kasprintf(GFP_KERNEL, "%s[%d]/%x", 310 "current->comm", 311 pid_nr(ctx->pid), 312 ctx->user_handle); 313 #endif 314 if (!ctx->name) { 315 ret = -ENOMEM; 316 goto err_pid; 317 } 318 } 319 320 /* NB: Mark all slices as needing a remap so that when the context first 321 * loads it will restore whatever remap state already exists. If there 322 * is no remap info, it will be a NOP. */ 323 ctx->remap_slice = ALL_L3_SLICES(dev_priv); 324 325 i915_gem_context_set_bannable(ctx); 326 ctx->ring_size = 4 * PAGE_SIZE; 327 ctx->desc_template = 328 default_desc_template(dev_priv, dev_priv->mm.aliasing_ppgtt); 329 330 /* GuC requires the ring to be placed above GUC_WOPCM_TOP. If GuC is not 331 * present or not in use we still need a small bias as ring wraparound 332 * at offset 0 sometimes hangs. No idea why. 333 */ 334 if (HAS_GUC(dev_priv) && i915.enable_guc_loading) 335 ctx->ggtt_offset_bias = GUC_WOPCM_TOP; 336 else 337 ctx->ggtt_offset_bias = I915_GTT_PAGE_SIZE; 338 339 return ctx; 340 341 err_pid: 342 put_pid(ctx->pid); 343 idr_remove(&file_priv->context_idr, ctx->user_handle); 344 err_out: 345 context_close(ctx); 346 return ERR_PTR(ret); 347 } 348 349 static void __destroy_hw_context(struct i915_gem_context *ctx, 350 struct drm_i915_file_private *file_priv) 351 { 352 idr_remove(&file_priv->context_idr, ctx->user_handle); 353 context_close(ctx); 354 } 355 356 /** 357 * The default context needs to exist per ring that uses contexts. It stores the 358 * context state of the GPU for applications that don't utilize HW contexts, as 359 * well as an idle case. 360 */ 361 static struct i915_gem_context * 362 i915_gem_create_context(struct drm_i915_private *dev_priv, 363 struct drm_i915_file_private *file_priv) 364 { 365 struct i915_gem_context *ctx; 366 367 lockdep_assert_held(&dev_priv->drm.struct_mutex); 368 369 ctx = __create_hw_context(dev_priv, file_priv); 370 if (IS_ERR(ctx)) 371 return ctx; 372 373 if (USES_FULL_PPGTT(dev_priv)) { 374 struct i915_hw_ppgtt *ppgtt; 375 376 ppgtt = i915_ppgtt_create(dev_priv, file_priv, ctx->name); 377 if (IS_ERR(ppgtt)) { 378 DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n", 379 PTR_ERR(ppgtt)); 380 __destroy_hw_context(ctx, file_priv); 381 return ERR_CAST(ppgtt); 382 } 383 384 ctx->ppgtt = ppgtt; 385 ctx->desc_template = default_desc_template(dev_priv, ppgtt); 386 } 387 388 trace_i915_context_create(ctx); 389 390 return ctx; 391 } 392 393 /** 394 * i915_gem_context_create_gvt - create a GVT GEM context 395 * @dev: drm device * 396 * 397 * This function is used to create a GVT specific GEM context. 398 * 399 * Returns: 400 * pointer to i915_gem_context on success, error pointer if failed 401 * 402 */ 403 struct i915_gem_context * 404 i915_gem_context_create_gvt(struct drm_device *dev) 405 { 406 struct i915_gem_context *ctx; 407 int ret; 408 409 if (!IS_ENABLED(CONFIG_DRM_I915_GVT)) 410 return ERR_PTR(-ENODEV); 411 412 ret = i915_mutex_lock_interruptible(dev); 413 if (ret) 414 return ERR_PTR(ret); 415 416 ctx = __create_hw_context(to_i915(dev), NULL); 417 if (IS_ERR(ctx)) 418 goto out; 419 420 ctx->file_priv = ERR_PTR(-EBADF); 421 i915_gem_context_set_closed(ctx); /* not user accessible */ 422 i915_gem_context_clear_bannable(ctx); 423 i915_gem_context_set_force_single_submission(ctx); 424 if (!i915.enable_guc_submission) 425 ctx->ring_size = 512 * PAGE_SIZE; /* Max ring buffer size */ 426 427 GEM_BUG_ON(i915_gem_context_is_kernel(ctx)); 428 out: 429 mutex_unlock(&dev->struct_mutex); 430 return ctx; 431 } 432 433 int i915_gem_context_init(struct drm_i915_private *dev_priv) 434 { 435 struct i915_gem_context *ctx; 436 437 /* Init should only be called once per module load. Eventually the 438 * restriction on the context_disabled check can be loosened. */ 439 if (WARN_ON(dev_priv->kernel_context)) 440 return 0; 441 442 if (intel_vgpu_active(dev_priv) && 443 HAS_LOGICAL_RING_CONTEXTS(dev_priv)) { 444 if (!i915.enable_execlists) { 445 DRM_INFO("Only EXECLIST mode is supported in vgpu.\n"); 446 return -EINVAL; 447 } 448 } 449 450 /* Using the simple ida interface, the max is limited by sizeof(int) */ 451 BUILD_BUG_ON(MAX_CONTEXT_HW_ID > INT_MAX); 452 ida_init(&dev_priv->context_hw_ida); 453 454 if (i915.enable_execlists) { 455 /* NB: intentionally left blank. We will allocate our own 456 * backing objects as we need them, thank you very much */ 457 dev_priv->hw_context_size = 0; 458 } else if (HAS_HW_CONTEXTS(dev_priv)) { 459 dev_priv->hw_context_size = 460 round_up(get_context_size(dev_priv), 461 I915_GTT_PAGE_SIZE); 462 if (dev_priv->hw_context_size > (1<<20)) { 463 DRM_DEBUG_DRIVER("Disabling HW Contexts; invalid size %d\n", 464 dev_priv->hw_context_size); 465 dev_priv->hw_context_size = 0; 466 } 467 } 468 469 ctx = i915_gem_create_context(dev_priv, NULL); 470 if (IS_ERR(ctx)) { 471 DRM_ERROR("Failed to create default global context (error %ld)\n", 472 PTR_ERR(ctx)); 473 return PTR_ERR(ctx); 474 } 475 476 /* For easy recognisablity, we want the kernel context to be 0 and then 477 * all user contexts will have non-zero hw_id. 478 */ 479 GEM_BUG_ON(ctx->hw_id); 480 481 i915_gem_context_clear_bannable(ctx); 482 ctx->priority = I915_PRIORITY_MIN; /* lowest priority; idle task */ 483 dev_priv->kernel_context = ctx; 484 485 GEM_BUG_ON(!i915_gem_context_is_kernel(ctx)); 486 487 DRM_DEBUG_DRIVER("%s context support initialized\n", 488 i915.enable_execlists ? "LR" : 489 dev_priv->hw_context_size ? "HW" : "fake"); 490 return 0; 491 } 492 493 void i915_gem_context_lost(struct drm_i915_private *dev_priv) 494 { 495 struct intel_engine_cs *engine; 496 enum intel_engine_id id; 497 498 lockdep_assert_held(&dev_priv->drm.struct_mutex); 499 500 for_each_engine(engine, dev_priv, id) { 501 engine->legacy_active_context = NULL; 502 503 if (!engine->last_retired_context) 504 continue; 505 506 engine->context_unpin(engine, engine->last_retired_context); 507 engine->last_retired_context = NULL; 508 } 509 510 /* Force the GPU state to be restored on enabling */ 511 if (!i915.enable_execlists) { 512 struct i915_gem_context *ctx; 513 514 list_for_each_entry(ctx, &dev_priv->context_list, link) { 515 if (!i915_gem_context_is_default(ctx)) 516 continue; 517 518 for_each_engine(engine, dev_priv, id) 519 ctx->engine[engine->id].initialised = false; 520 521 ctx->remap_slice = ALL_L3_SLICES(dev_priv); 522 } 523 524 for_each_engine(engine, dev_priv, id) { 525 struct intel_context *kce = 526 &dev_priv->kernel_context->engine[engine->id]; 527 528 kce->initialised = true; 529 } 530 } 531 } 532 533 void i915_gem_context_fini(struct drm_i915_private *dev_priv) 534 { 535 struct i915_gem_context *dctx = dev_priv->kernel_context; 536 537 lockdep_assert_held(&dev_priv->drm.struct_mutex); 538 539 GEM_BUG_ON(!i915_gem_context_is_kernel(dctx)); 540 541 context_close(dctx); 542 dev_priv->kernel_context = NULL; 543 544 ida_destroy(&dev_priv->context_hw_ida); 545 } 546 547 static int context_idr_cleanup(int id, void *p, void *data) 548 { 549 struct i915_gem_context *ctx = p; 550 551 context_close(ctx); 552 return 0; 553 } 554 555 int i915_gem_context_open(struct drm_device *dev, struct drm_file *file) 556 { 557 struct drm_i915_file_private *file_priv = file->driver_priv; 558 struct i915_gem_context *ctx; 559 560 idr_init(&file_priv->context_idr); 561 562 mutex_lock(&dev->struct_mutex); 563 ctx = i915_gem_create_context(to_i915(dev), file_priv); 564 mutex_unlock(&dev->struct_mutex); 565 566 GEM_BUG_ON(i915_gem_context_is_kernel(ctx)); 567 568 if (IS_ERR(ctx)) { 569 idr_destroy(&file_priv->context_idr); 570 return PTR_ERR(ctx); 571 } 572 573 return 0; 574 } 575 576 void i915_gem_context_close(struct drm_device *dev, struct drm_file *file) 577 { 578 struct drm_i915_file_private *file_priv = file->driver_priv; 579 580 lockdep_assert_held(&dev->struct_mutex); 581 582 idr_for_each(&file_priv->context_idr, context_idr_cleanup, NULL); 583 idr_destroy(&file_priv->context_idr); 584 } 585 586 static inline int 587 mi_set_context(struct drm_i915_gem_request *req, u32 flags) 588 { 589 struct drm_i915_private *dev_priv = req->i915; 590 struct intel_engine_cs *engine = req->engine; 591 enum intel_engine_id id; 592 const int num_rings = 593 /* Use an extended w/a on gen7 if signalling from other rings */ 594 (i915.semaphores && INTEL_GEN(dev_priv) == 7) ? 595 INTEL_INFO(dev_priv)->num_rings - 1 : 596 0; 597 int len; 598 u32 *cs; 599 600 flags |= MI_MM_SPACE_GTT; 601 if (IS_HASWELL(dev_priv) || INTEL_GEN(dev_priv) >= 8) 602 /* These flags are for resource streamer on HSW+ */ 603 flags |= HSW_MI_RS_SAVE_STATE_EN | HSW_MI_RS_RESTORE_STATE_EN; 604 else 605 flags |= MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN; 606 607 len = 4; 608 if (INTEL_GEN(dev_priv) >= 7) 609 len += 2 + (num_rings ? 4*num_rings + 6 : 0); 610 611 cs = intel_ring_begin(req, len); 612 if (IS_ERR(cs)) 613 return PTR_ERR(cs); 614 615 /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ 616 if (INTEL_GEN(dev_priv) >= 7) { 617 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 618 if (num_rings) { 619 struct intel_engine_cs *signaller; 620 621 *cs++ = MI_LOAD_REGISTER_IMM(num_rings); 622 for_each_engine(signaller, dev_priv, id) { 623 if (signaller == engine) 624 continue; 625 626 *cs++ = i915_mmio_reg_offset( 627 RING_PSMI_CTL(signaller->mmio_base)); 628 *cs++ = _MASKED_BIT_ENABLE( 629 GEN6_PSMI_SLEEP_MSG_DISABLE); 630 } 631 } 632 } 633 634 *cs++ = MI_NOOP; 635 *cs++ = MI_SET_CONTEXT; 636 *cs++ = i915_ggtt_offset(req->ctx->engine[RCS].state) | flags; 637 /* 638 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP 639 * WaMiSetContext_Hang:snb,ivb,vlv 640 */ 641 *cs++ = MI_NOOP; 642 643 if (INTEL_GEN(dev_priv) >= 7) { 644 if (num_rings) { 645 struct intel_engine_cs *signaller; 646 i915_reg_t last_reg = {}; /* keep gcc quiet */ 647 648 *cs++ = MI_LOAD_REGISTER_IMM(num_rings); 649 for_each_engine(signaller, dev_priv, id) { 650 if (signaller == engine) 651 continue; 652 653 last_reg = RING_PSMI_CTL(signaller->mmio_base); 654 *cs++ = i915_mmio_reg_offset(last_reg); 655 *cs++ = _MASKED_BIT_DISABLE( 656 GEN6_PSMI_SLEEP_MSG_DISABLE); 657 } 658 659 /* Insert a delay before the next switch! */ 660 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 661 *cs++ = i915_mmio_reg_offset(last_reg); 662 *cs++ = i915_ggtt_offset(engine->scratch); 663 *cs++ = MI_NOOP; 664 } 665 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 666 } 667 668 intel_ring_advance(req, cs); 669 670 return 0; 671 } 672 673 static int remap_l3(struct drm_i915_gem_request *req, int slice) 674 { 675 u32 *cs, *remap_info = req->i915->l3_parity.remap_info[slice]; 676 int i; 677 678 if (!remap_info) 679 return 0; 680 681 cs = intel_ring_begin(req, GEN7_L3LOG_SIZE/4 * 2 + 2); 682 if (IS_ERR(cs)) 683 return PTR_ERR(cs); 684 685 /* 686 * Note: We do not worry about the concurrent register cacheline hang 687 * here because no other code should access these registers other than 688 * at initialization time. 689 */ 690 *cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4); 691 for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) { 692 *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i)); 693 *cs++ = remap_info[i]; 694 } 695 *cs++ = MI_NOOP; 696 intel_ring_advance(req, cs); 697 698 return 0; 699 } 700 701 static inline bool skip_rcs_switch(struct i915_hw_ppgtt *ppgtt, 702 struct intel_engine_cs *engine, 703 struct i915_gem_context *to) 704 { 705 if (to->remap_slice) 706 return false; 707 708 if (!to->engine[RCS].initialised) 709 return false; 710 711 if (ppgtt && (intel_engine_flag(engine) & ppgtt->pd_dirty_rings)) 712 return false; 713 714 return to == engine->legacy_active_context; 715 } 716 717 static bool 718 needs_pd_load_pre(struct i915_hw_ppgtt *ppgtt, 719 struct intel_engine_cs *engine, 720 struct i915_gem_context *to) 721 { 722 if (!ppgtt) 723 return false; 724 725 /* Always load the ppgtt on first use */ 726 if (!engine->legacy_active_context) 727 return true; 728 729 /* Same context without new entries, skip */ 730 if (engine->legacy_active_context == to && 731 !(intel_engine_flag(engine) & ppgtt->pd_dirty_rings)) 732 return false; 733 734 if (engine->id != RCS) 735 return true; 736 737 if (INTEL_GEN(engine->i915) < 8) 738 return true; 739 740 return false; 741 } 742 743 static bool 744 needs_pd_load_post(struct i915_hw_ppgtt *ppgtt, 745 struct i915_gem_context *to, 746 u32 hw_flags) 747 { 748 if (!ppgtt) 749 return false; 750 751 if (!IS_GEN8(to->i915)) 752 return false; 753 754 if (hw_flags & MI_RESTORE_INHIBIT) 755 return true; 756 757 return false; 758 } 759 760 static int do_rcs_switch(struct drm_i915_gem_request *req) 761 { 762 struct i915_gem_context *to = req->ctx; 763 struct intel_engine_cs *engine = req->engine; 764 struct i915_hw_ppgtt *ppgtt = to->ppgtt ?: req->i915->mm.aliasing_ppgtt; 765 struct i915_gem_context *from = engine->legacy_active_context; 766 u32 hw_flags; 767 int ret, i; 768 769 GEM_BUG_ON(engine->id != RCS); 770 771 if (skip_rcs_switch(ppgtt, engine, to)) 772 return 0; 773 774 if (needs_pd_load_pre(ppgtt, engine, to)) { 775 /* Older GENs and non render rings still want the load first, 776 * "PP_DCLV followed by PP_DIR_BASE register through Load 777 * Register Immediate commands in Ring Buffer before submitting 778 * a context."*/ 779 trace_switch_mm(engine, to); 780 ret = ppgtt->switch_mm(ppgtt, req); 781 if (ret) 782 return ret; 783 } 784 785 if (!to->engine[RCS].initialised || i915_gem_context_is_default(to)) 786 /* NB: If we inhibit the restore, the context is not allowed to 787 * die because future work may end up depending on valid address 788 * space. This means we must enforce that a page table load 789 * occur when this occurs. */ 790 hw_flags = MI_RESTORE_INHIBIT; 791 else if (ppgtt && intel_engine_flag(engine) & ppgtt->pd_dirty_rings) 792 hw_flags = MI_FORCE_RESTORE; 793 else 794 hw_flags = 0; 795 796 if (to != from || (hw_flags & MI_FORCE_RESTORE)) { 797 ret = mi_set_context(req, hw_flags); 798 if (ret) 799 return ret; 800 801 engine->legacy_active_context = to; 802 } 803 804 /* GEN8 does *not* require an explicit reload if the PDPs have been 805 * setup, and we do not wish to move them. 806 */ 807 if (needs_pd_load_post(ppgtt, to, hw_flags)) { 808 trace_switch_mm(engine, to); 809 ret = ppgtt->switch_mm(ppgtt, req); 810 /* The hardware context switch is emitted, but we haven't 811 * actually changed the state - so it's probably safe to bail 812 * here. Still, let the user know something dangerous has 813 * happened. 814 */ 815 if (ret) 816 return ret; 817 } 818 819 if (ppgtt) 820 ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine); 821 822 for (i = 0; i < MAX_L3_SLICES; i++) { 823 if (!(to->remap_slice & (1<<i))) 824 continue; 825 826 ret = remap_l3(req, i); 827 if (ret) 828 return ret; 829 830 to->remap_slice &= ~(1<<i); 831 } 832 833 if (!to->engine[RCS].initialised) { 834 if (engine->init_context) { 835 ret = engine->init_context(req); 836 if (ret) 837 return ret; 838 } 839 to->engine[RCS].initialised = true; 840 } 841 842 return 0; 843 } 844 845 /** 846 * i915_switch_context() - perform a GPU context switch. 847 * @req: request for which we'll execute the context switch 848 * 849 * The context life cycle is simple. The context refcount is incremented and 850 * decremented by 1 and create and destroy. If the context is in use by the GPU, 851 * it will have a refcount > 1. This allows us to destroy the context abstract 852 * object while letting the normal object tracking destroy the backing BO. 853 * 854 * This function should not be used in execlists mode. Instead the context is 855 * switched by writing to the ELSP and requests keep a reference to their 856 * context. 857 */ 858 int i915_switch_context(struct drm_i915_gem_request *req) 859 { 860 struct intel_engine_cs *engine = req->engine; 861 862 lockdep_assert_held(&req->i915->drm.struct_mutex); 863 if (i915.enable_execlists) 864 return 0; 865 866 if (!req->ctx->engine[engine->id].state) { 867 struct i915_gem_context *to = req->ctx; 868 struct i915_hw_ppgtt *ppgtt = 869 to->ppgtt ?: req->i915->mm.aliasing_ppgtt; 870 871 if (needs_pd_load_pre(ppgtt, engine, to)) { 872 int ret; 873 874 trace_switch_mm(engine, to); 875 ret = ppgtt->switch_mm(ppgtt, req); 876 if (ret) 877 return ret; 878 879 ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine); 880 } 881 882 return 0; 883 } 884 885 return do_rcs_switch(req); 886 } 887 888 static bool engine_has_kernel_context(struct intel_engine_cs *engine) 889 { 890 struct i915_gem_timeline *timeline; 891 892 list_for_each_entry(timeline, &engine->i915->gt.timelines, link) { 893 struct intel_timeline *tl; 894 895 if (timeline == &engine->i915->gt.global_timeline) 896 continue; 897 898 tl = &timeline->engine[engine->id]; 899 if (i915_gem_active_peek(&tl->last_request, 900 &engine->i915->drm.struct_mutex)) 901 return false; 902 } 903 904 return (!engine->last_retired_context || 905 i915_gem_context_is_kernel(engine->last_retired_context)); 906 } 907 908 int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv) 909 { 910 struct intel_engine_cs *engine; 911 struct i915_gem_timeline *timeline; 912 enum intel_engine_id id; 913 914 lockdep_assert_held(&dev_priv->drm.struct_mutex); 915 916 i915_gem_retire_requests(dev_priv); 917 918 for_each_engine(engine, dev_priv, id) { 919 struct drm_i915_gem_request *req; 920 int ret; 921 922 if (engine_has_kernel_context(engine)) 923 continue; 924 925 req = i915_gem_request_alloc(engine, dev_priv->kernel_context); 926 if (IS_ERR(req)) 927 return PTR_ERR(req); 928 929 /* Queue this switch after all other activity */ 930 list_for_each_entry(timeline, &dev_priv->gt.timelines, link) { 931 struct drm_i915_gem_request *prev; 932 struct intel_timeline *tl; 933 934 tl = &timeline->engine[engine->id]; 935 prev = i915_gem_active_raw(&tl->last_request, 936 &dev_priv->drm.struct_mutex); 937 if (prev) 938 i915_sw_fence_await_sw_fence_gfp(&req->submit, 939 &prev->submit, 940 GFP_KERNEL); 941 } 942 943 ret = i915_switch_context(req); 944 i915_add_request(req); 945 if (ret) 946 return ret; 947 } 948 949 return 0; 950 } 951 952 static bool contexts_enabled(struct drm_device *dev) 953 { 954 return i915.enable_execlists || to_i915(dev)->hw_context_size; 955 } 956 957 static bool client_is_banned(struct drm_i915_file_private *file_priv) 958 { 959 return file_priv->context_bans > I915_MAX_CLIENT_CONTEXT_BANS; 960 } 961 962 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, 963 struct drm_file *file) 964 { 965 struct drm_i915_gem_context_create *args = data; 966 struct drm_i915_file_private *file_priv = file->driver_priv; 967 struct i915_gem_context *ctx; 968 int ret; 969 970 if (!contexts_enabled(dev)) 971 return -ENODEV; 972 973 if (args->pad != 0) 974 return -EINVAL; 975 976 if (client_is_banned(file_priv)) { 977 DRM_DEBUG("client %s[%d] banned from creating ctx\n", 978 current->comm, 979 pid_nr(get_task_pid(current, PIDTYPE_PID))); 980 981 return -EIO; 982 } 983 984 ret = i915_mutex_lock_interruptible(dev); 985 if (ret) 986 return ret; 987 988 ctx = i915_gem_create_context(to_i915(dev), file_priv); 989 mutex_unlock(&dev->struct_mutex); 990 if (IS_ERR(ctx)) 991 return PTR_ERR(ctx); 992 993 GEM_BUG_ON(i915_gem_context_is_kernel(ctx)); 994 995 args->ctx_id = ctx->user_handle; 996 DRM_DEBUG("HW context %d created\n", args->ctx_id); 997 998 return 0; 999 } 1000 1001 int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, 1002 struct drm_file *file) 1003 { 1004 struct drm_i915_gem_context_destroy *args = data; 1005 struct drm_i915_file_private *file_priv = file->driver_priv; 1006 struct i915_gem_context *ctx; 1007 int ret; 1008 1009 if (args->pad != 0) 1010 return -EINVAL; 1011 1012 if (args->ctx_id == DEFAULT_CONTEXT_HANDLE) 1013 return -ENOENT; 1014 1015 ret = i915_mutex_lock_interruptible(dev); 1016 if (ret) 1017 return ret; 1018 1019 ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 1020 if (IS_ERR(ctx)) { 1021 mutex_unlock(&dev->struct_mutex); 1022 return PTR_ERR(ctx); 1023 } 1024 1025 __destroy_hw_context(ctx, file_priv); 1026 mutex_unlock(&dev->struct_mutex); 1027 1028 DRM_DEBUG("HW context %d destroyed\n", args->ctx_id); 1029 return 0; 1030 } 1031 1032 int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, 1033 struct drm_file *file) 1034 { 1035 struct drm_i915_file_private *file_priv = file->driver_priv; 1036 struct drm_i915_gem_context_param *args = data; 1037 struct i915_gem_context *ctx; 1038 int ret; 1039 1040 ret = i915_mutex_lock_interruptible(dev); 1041 if (ret) 1042 return ret; 1043 1044 ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 1045 if (IS_ERR(ctx)) { 1046 mutex_unlock(&dev->struct_mutex); 1047 return PTR_ERR(ctx); 1048 } 1049 1050 args->size = 0; 1051 switch (args->param) { 1052 case I915_CONTEXT_PARAM_BAN_PERIOD: 1053 ret = -EINVAL; 1054 break; 1055 case I915_CONTEXT_PARAM_NO_ZEROMAP: 1056 args->value = ctx->flags & CONTEXT_NO_ZEROMAP; 1057 break; 1058 case I915_CONTEXT_PARAM_GTT_SIZE: 1059 if (ctx->ppgtt) 1060 args->value = ctx->ppgtt->base.total; 1061 else if (to_i915(dev)->mm.aliasing_ppgtt) 1062 args->value = to_i915(dev)->mm.aliasing_ppgtt->base.total; 1063 else 1064 args->value = to_i915(dev)->ggtt.base.total; 1065 break; 1066 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 1067 args->value = i915_gem_context_no_error_capture(ctx); 1068 break; 1069 case I915_CONTEXT_PARAM_BANNABLE: 1070 args->value = i915_gem_context_is_bannable(ctx); 1071 break; 1072 default: 1073 ret = -EINVAL; 1074 break; 1075 } 1076 mutex_unlock(&dev->struct_mutex); 1077 1078 return ret; 1079 } 1080 1081 int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, 1082 struct drm_file *file) 1083 { 1084 struct drm_i915_file_private *file_priv = file->driver_priv; 1085 struct drm_i915_gem_context_param *args = data; 1086 struct i915_gem_context *ctx; 1087 int ret; 1088 1089 ret = i915_mutex_lock_interruptible(dev); 1090 if (ret) 1091 return ret; 1092 1093 ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 1094 if (IS_ERR(ctx)) { 1095 mutex_unlock(&dev->struct_mutex); 1096 return PTR_ERR(ctx); 1097 } 1098 1099 switch (args->param) { 1100 case I915_CONTEXT_PARAM_BAN_PERIOD: 1101 ret = -EINVAL; 1102 break; 1103 case I915_CONTEXT_PARAM_NO_ZEROMAP: 1104 if (args->size) { 1105 ret = -EINVAL; 1106 } else { 1107 ctx->flags &= ~CONTEXT_NO_ZEROMAP; 1108 ctx->flags |= args->value ? CONTEXT_NO_ZEROMAP : 0; 1109 } 1110 break; 1111 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 1112 if (args->size) 1113 ret = -EINVAL; 1114 else if (args->value) 1115 i915_gem_context_set_no_error_capture(ctx); 1116 else 1117 i915_gem_context_clear_no_error_capture(ctx); 1118 break; 1119 case I915_CONTEXT_PARAM_BANNABLE: 1120 if (args->size) 1121 ret = -EINVAL; 1122 else if (!capable(CAP_SYS_ADMIN) && !args->value) 1123 ret = -EPERM; 1124 else if (args->value) 1125 i915_gem_context_set_bannable(ctx); 1126 else 1127 i915_gem_context_clear_bannable(ctx); 1128 break; 1129 default: 1130 ret = -EINVAL; 1131 break; 1132 } 1133 mutex_unlock(&dev->struct_mutex); 1134 1135 return ret; 1136 } 1137 1138 int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, 1139 void *data, struct drm_file *file) 1140 { 1141 struct drm_i915_private *dev_priv = to_i915(dev); 1142 struct drm_i915_reset_stats *args = data; 1143 struct i915_gem_context *ctx; 1144 int ret; 1145 1146 if (args->flags || args->pad) 1147 return -EINVAL; 1148 1149 if (args->ctx_id == DEFAULT_CONTEXT_HANDLE && !capable(CAP_SYS_ADMIN)) 1150 return -EPERM; 1151 1152 ret = i915_mutex_lock_interruptible(dev); 1153 if (ret) 1154 return ret; 1155 1156 ctx = i915_gem_context_lookup(file->driver_priv, args->ctx_id); 1157 if (IS_ERR(ctx)) { 1158 mutex_unlock(&dev->struct_mutex); 1159 return PTR_ERR(ctx); 1160 } 1161 1162 if (capable(CAP_SYS_ADMIN)) 1163 args->reset_count = i915_reset_count(&dev_priv->gpu_error); 1164 else 1165 args->reset_count = 0; 1166 1167 args->batch_active = ctx->guilty_count; 1168 args->batch_pending = ctx->active_count; 1169 1170 mutex_unlock(&dev->struct_mutex); 1171 1172 return 0; 1173 } 1174 1175 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 1176 #include "selftests/mock_context.c" 1177 #include "selftests/i915_gem_context.c" 1178 #endif 1179