1 /* 2 * Copyright © 2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 #include <linux/firmware.h> 25 #include <linux/circ_buf.h> 26 #include "i915_drv.h" 27 #include "intel_guc.h" 28 29 /** 30 * DOC: GuC-based command submission 31 * 32 * i915_guc_client: 33 * We use the term client to avoid confusion with contexts. A i915_guc_client is 34 * equivalent to GuC object guc_context_desc. This context descriptor is 35 * allocated from a pool of 1024 entries. Kernel driver will allocate doorbell 36 * and workqueue for it. Also the process descriptor (guc_process_desc), which 37 * is mapped to client space. So the client can write Work Item then ring the 38 * doorbell. 39 * 40 * To simplify the implementation, we allocate one gem object that contains all 41 * pages for doorbell, process descriptor and workqueue. 42 * 43 * The Scratch registers: 44 * There are 16 MMIO-based registers start from 0xC180. The kernel driver writes 45 * a value to the action register (SOFT_SCRATCH_0) along with any data. It then 46 * triggers an interrupt on the GuC via another register write (0xC4C8). 47 * Firmware writes a success/fail code back to the action register after 48 * processes the request. The kernel driver polls waiting for this update and 49 * then proceeds. 50 * See host2guc_action() 51 * 52 * Doorbells: 53 * Doorbells are interrupts to uKernel. A doorbell is a single cache line (QW) 54 * mapped into process space. 55 * 56 * Work Items: 57 * There are several types of work items that the host may place into a 58 * workqueue, each with its own requirements and limitations. Currently only 59 * WQ_TYPE_INORDER is needed to support legacy submission via GuC, which 60 * represents in-order queue. The kernel driver packs ring tail pointer and an 61 * ELSP context descriptor dword into Work Item. 62 * See guc_add_workqueue_item() 63 * 64 */ 65 66 /* 67 * Read GuC command/status register (SOFT_SCRATCH_0) 68 * Return true if it contains a response rather than a command 69 */ 70 static inline bool host2guc_action_response(struct drm_i915_private *dev_priv, 71 u32 *status) 72 { 73 u32 val = I915_READ(SOFT_SCRATCH(0)); 74 *status = val; 75 return GUC2HOST_IS_RESPONSE(val); 76 } 77 78 static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len) 79 { 80 struct drm_i915_private *dev_priv = guc_to_i915(guc); 81 u32 status; 82 int i; 83 int ret; 84 85 if (WARN_ON(len < 1 || len > 15)) 86 return -EINVAL; 87 88 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 89 90 dev_priv->guc.action_count += 1; 91 dev_priv->guc.action_cmd = data[0]; 92 93 for (i = 0; i < len; i++) 94 I915_WRITE(SOFT_SCRATCH(i), data[i]); 95 96 POSTING_READ(SOFT_SCRATCH(i - 1)); 97 98 I915_WRITE(HOST2GUC_INTERRUPT, HOST2GUC_TRIGGER); 99 100 /* No HOST2GUC command should take longer than 10ms */ 101 ret = wait_for_atomic(host2guc_action_response(dev_priv, &status), 10); 102 if (status != GUC2HOST_STATUS_SUCCESS) { 103 /* 104 * Either the GuC explicitly returned an error (which 105 * we convert to -EIO here) or no response at all was 106 * received within the timeout limit (-ETIMEDOUT) 107 */ 108 if (ret != -ETIMEDOUT) 109 ret = -EIO; 110 111 DRM_ERROR("GUC: host2guc action 0x%X failed. ret=%d " 112 "status=0x%08X response=0x%08X\n", 113 data[0], ret, status, 114 I915_READ(SOFT_SCRATCH(15))); 115 116 dev_priv->guc.action_fail += 1; 117 dev_priv->guc.action_err = ret; 118 } 119 dev_priv->guc.action_status = status; 120 121 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 122 123 return ret; 124 } 125 126 /* 127 * Tell the GuC to allocate or deallocate a specific doorbell 128 */ 129 130 static int host2guc_allocate_doorbell(struct intel_guc *guc, 131 struct i915_guc_client *client) 132 { 133 u32 data[2]; 134 135 data[0] = HOST2GUC_ACTION_ALLOCATE_DOORBELL; 136 data[1] = client->ctx_index; 137 138 return host2guc_action(guc, data, 2); 139 } 140 141 static int host2guc_release_doorbell(struct intel_guc *guc, 142 struct i915_guc_client *client) 143 { 144 u32 data[2]; 145 146 data[0] = HOST2GUC_ACTION_DEALLOCATE_DOORBELL; 147 data[1] = client->ctx_index; 148 149 return host2guc_action(guc, data, 2); 150 } 151 152 static int host2guc_sample_forcewake(struct intel_guc *guc, 153 struct i915_guc_client *client) 154 { 155 struct drm_i915_private *dev_priv = guc_to_i915(guc); 156 struct drm_device *dev = dev_priv->dev; 157 u32 data[2]; 158 159 data[0] = HOST2GUC_ACTION_SAMPLE_FORCEWAKE; 160 /* WaRsDisableCoarsePowerGating:skl,bxt */ 161 if (!intel_enable_rc6(dev) || 162 NEEDS_WaRsDisableCoarsePowerGating(dev)) 163 data[1] = 0; 164 else 165 /* bit 0 and 1 are for Render and Media domain separately */ 166 data[1] = GUC_FORCEWAKE_RENDER | GUC_FORCEWAKE_MEDIA; 167 168 return host2guc_action(guc, data, ARRAY_SIZE(data)); 169 } 170 171 /* 172 * Initialise, update, or clear doorbell data shared with the GuC 173 * 174 * These functions modify shared data and so need access to the mapped 175 * client object which contains the page being used for the doorbell 176 */ 177 178 static void guc_init_doorbell(struct intel_guc *guc, 179 struct i915_guc_client *client) 180 { 181 struct guc_doorbell_info *doorbell; 182 183 doorbell = (struct guc_doorbell_info *)((char *)client->client_base + client->doorbell_offset); 184 185 doorbell->db_status = GUC_DOORBELL_ENABLED; 186 doorbell->cookie = 0; 187 } 188 189 static int guc_ring_doorbell(struct i915_guc_client *gc) 190 { 191 struct guc_process_desc *desc; 192 union guc_doorbell_qw db_cmp, db_exc, db_ret; 193 union guc_doorbell_qw *db; 194 int attempt = 2, ret = -EAGAIN; 195 196 desc = (struct guc_process_desc *)((char *)gc->client_base + gc->proc_desc_offset); 197 198 /* Update the tail so it is visible to GuC */ 199 desc->tail = gc->wq_tail; 200 201 /* current cookie */ 202 db_cmp.db_status = GUC_DOORBELL_ENABLED; 203 db_cmp.cookie = gc->cookie; 204 205 /* cookie to be updated */ 206 db_exc.db_status = GUC_DOORBELL_ENABLED; 207 db_exc.cookie = gc->cookie + 1; 208 if (db_exc.cookie == 0) 209 db_exc.cookie = 1; 210 211 /* pointer of current doorbell cacheline */ 212 db = (union guc_doorbell_qw *)((char *)gc->client_base + gc->doorbell_offset); 213 214 while (attempt--) { 215 /* lets ring the doorbell */ 216 db_ret.value_qw = atomic64_cmpxchg((atomic64_t *)db, 217 db_cmp.value_qw, db_exc.value_qw); 218 219 /* if the exchange was successfully executed */ 220 if (db_ret.value_qw == db_cmp.value_qw) { 221 /* db was successfully rung */ 222 gc->cookie = db_exc.cookie; 223 ret = 0; 224 break; 225 } 226 227 /* XXX: doorbell was lost and need to acquire it again */ 228 if (db_ret.db_status == GUC_DOORBELL_DISABLED) 229 break; 230 231 DRM_ERROR("Cookie mismatch. Expected %d, returned %d\n", 232 db_cmp.cookie, db_ret.cookie); 233 234 /* update the cookie to newly read cookie from GuC */ 235 db_cmp.cookie = db_ret.cookie; 236 db_exc.cookie = db_ret.cookie + 1; 237 if (db_exc.cookie == 0) 238 db_exc.cookie = 1; 239 } 240 241 return ret; 242 } 243 244 static void guc_disable_doorbell(struct intel_guc *guc, 245 struct i915_guc_client *client) 246 { 247 struct drm_i915_private *dev_priv = guc_to_i915(guc); 248 struct guc_doorbell_info *doorbell; 249 i915_reg_t drbreg = GEN8_DRBREGL(client->doorbell_id); 250 int value; 251 252 doorbell = (struct guc_doorbell_info*)((char *)client->client_base + client->doorbell_offset); 253 254 doorbell->db_status = GUC_DOORBELL_DISABLED; 255 256 I915_WRITE(drbreg, I915_READ(drbreg) & ~GEN8_DRB_VALID); 257 258 value = I915_READ(drbreg); 259 WARN_ON((value & GEN8_DRB_VALID) != 0); 260 261 I915_WRITE(GEN8_DRBREGU(client->doorbell_id), 0); 262 I915_WRITE(drbreg, 0); 263 264 /* XXX: wait for any interrupts */ 265 /* XXX: wait for workqueue to drain */ 266 } 267 268 /* 269 * Select, assign and relase doorbell cachelines 270 * 271 * These functions track which doorbell cachelines are in use. 272 * The data they manipulate is protected by the host2guc lock. 273 */ 274 275 static uint32_t select_doorbell_cacheline(struct intel_guc *guc) 276 { 277 const uint32_t cacheline_size = cache_line_size(); 278 uint32_t offset; 279 280 /* Doorbell uses a single cache line within a page */ 281 offset = offset_in_page(guc->db_cacheline); 282 283 /* Moving to next cache line to reduce contention */ 284 guc->db_cacheline += cacheline_size; 285 286 DRM_DEBUG_DRIVER("selected doorbell cacheline 0x%x, next 0x%x, linesize %u\n", 287 offset, guc->db_cacheline, cacheline_size); 288 289 return offset; 290 } 291 292 static uint16_t assign_doorbell(struct intel_guc *guc, uint32_t priority) 293 { 294 /* 295 * The bitmap is split into two halves; the first half is used for 296 * normal priority contexts, the second half for high-priority ones. 297 * Note that logically higher priorities are numerically less than 298 * normal ones, so the test below means "is it high-priority?" 299 */ 300 const bool hi_pri = (priority <= GUC_CTX_PRIORITY_HIGH); 301 const uint16_t half = GUC_MAX_DOORBELLS / 2; 302 const uint16_t start = hi_pri ? half : 0; 303 const uint16_t end = start + half; 304 uint16_t id; 305 306 id = find_next_zero_bit(guc->doorbell_bitmap, end, start); 307 if (id == end) 308 id = GUC_INVALID_DOORBELL_ID; 309 else 310 bitmap_set(guc->doorbell_bitmap, id, 1); 311 312 DRM_DEBUG_DRIVER("assigned %s priority doorbell id 0x%x\n", 313 hi_pri ? "high" : "normal", id); 314 315 return id; 316 } 317 318 static void release_doorbell(struct intel_guc *guc, uint16_t id) 319 { 320 bitmap_clear(guc->doorbell_bitmap, id, 1); 321 } 322 323 /* 324 * Initialise the process descriptor shared with the GuC firmware. 325 */ 326 static void guc_init_proc_desc(struct intel_guc *guc, 327 struct i915_guc_client *client) 328 { 329 struct guc_process_desc *desc; 330 331 desc = (struct guc_process_desc *)((char *)client->client_base + client->proc_desc_offset); 332 333 memset(desc, 0, sizeof(*desc)); 334 335 /* 336 * XXX: pDoorbell and WQVBaseAddress are pointers in process address 337 * space for ring3 clients (set them as in mmap_ioctl) or kernel 338 * space for kernel clients (map on demand instead? May make debug 339 * easier to have it mapped). 340 */ 341 desc->wq_base_addr = 0; 342 desc->db_base_addr = 0; 343 344 desc->context_id = client->ctx_index; 345 desc->wq_size_bytes = client->wq_size; 346 desc->wq_status = WQ_STATUS_ACTIVE; 347 desc->priority = client->priority; 348 } 349 350 /* 351 * Initialise/clear the context descriptor shared with the GuC firmware. 352 * 353 * This descriptor tells the GuC where (in GGTT space) to find the important 354 * data structures relating to this client (doorbell, process descriptor, 355 * write queue, etc). 356 */ 357 358 static void guc_init_ctx_desc(struct intel_guc *guc, 359 struct i915_guc_client *client) 360 { 361 struct drm_i915_gem_object *client_obj = client->client_obj; 362 struct drm_i915_private *dev_priv = guc_to_i915(guc); 363 struct intel_engine_cs *engine; 364 struct intel_context *ctx = client->owner; 365 struct guc_context_desc desc; 366 struct sg_table *sg; 367 enum intel_engine_id id; 368 u32 gfx_addr; 369 370 memset(&desc, 0, sizeof(desc)); 371 372 desc.attribute = GUC_CTX_DESC_ATTR_ACTIVE | GUC_CTX_DESC_ATTR_KERNEL; 373 desc.context_id = client->ctx_index; 374 desc.priority = client->priority; 375 desc.db_id = client->doorbell_id; 376 377 for_each_engine_id(engine, dev_priv, id) { 378 struct guc_execlist_context *lrc = &desc.lrc[engine->guc_id]; 379 struct drm_i915_gem_object *obj; 380 uint64_t ctx_desc; 381 382 /* TODO: We have a design issue to be solved here. Only when we 383 * receive the first batch, we know which engine is used by the 384 * user. But here GuC expects the lrc and ring to be pinned. It 385 * is not an issue for default context, which is the only one 386 * for now who owns a GuC client. But for future owner of GuC 387 * client, need to make sure lrc is pinned prior to enter here. 388 */ 389 obj = ctx->engine[id].state; 390 if (!obj) 391 break; /* XXX: continue? */ 392 393 ctx_desc = intel_lr_context_descriptor(ctx, engine); 394 lrc->context_desc = (u32)ctx_desc; 395 396 /* The state page is after PPHWSP */ 397 gfx_addr = i915_gem_obj_ggtt_offset(obj); 398 lrc->ring_lcra = gfx_addr + LRC_STATE_PN * PAGE_SIZE; 399 lrc->context_id = (client->ctx_index << GUC_ELC_CTXID_OFFSET) | 400 (engine->guc_id << GUC_ELC_ENGINE_OFFSET); 401 402 obj = ctx->engine[id].ringbuf->obj; 403 gfx_addr = i915_gem_obj_ggtt_offset(obj); 404 405 lrc->ring_begin = gfx_addr; 406 lrc->ring_end = gfx_addr + obj->base.size - 1; 407 lrc->ring_next_free_location = gfx_addr; 408 lrc->ring_current_tail_pointer_value = 0; 409 410 desc.engines_used |= (1 << engine->guc_id); 411 } 412 413 WARN_ON(desc.engines_used == 0); 414 415 /* 416 * The doorbell, process descriptor, and workqueue are all parts 417 * of the client object, which the GuC will reference via the GGTT 418 */ 419 gfx_addr = i915_gem_obj_ggtt_offset(client_obj); 420 desc.db_trigger_phy = sg_dma_address(client_obj->pages->sgl) + 421 client->doorbell_offset; 422 desc.db_trigger_cpu = (uintptr_t)client->client_base + 423 client->doorbell_offset; 424 desc.db_trigger_uk = gfx_addr + client->doorbell_offset; 425 desc.process_desc = gfx_addr + client->proc_desc_offset; 426 desc.wq_addr = gfx_addr + client->wq_offset; 427 desc.wq_size = client->wq_size; 428 429 /* 430 * XXX: Take LRCs from an existing intel_context if this is not an 431 * IsKMDCreatedContext client 432 */ 433 desc.desc_private = (uintptr_t)client; 434 435 /* Pool context is pinned already */ 436 sg = guc->ctx_pool_obj->pages; 437 sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc), 438 sizeof(desc) * client->ctx_index); 439 } 440 441 static void guc_fini_ctx_desc(struct intel_guc *guc, 442 struct i915_guc_client *client) 443 { 444 struct guc_context_desc desc; 445 struct sg_table *sg; 446 447 memset(&desc, 0, sizeof(desc)); 448 449 sg = guc->ctx_pool_obj->pages; 450 sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc), 451 sizeof(desc) * client->ctx_index); 452 } 453 454 int i915_guc_wq_check_space(struct i915_guc_client *gc) 455 { 456 struct guc_process_desc *desc; 457 u32 size = sizeof(struct guc_wq_item); 458 int ret = -ETIMEDOUT, timeout_counter = 200; 459 460 if (!gc) 461 return 0; 462 463 desc = (struct guc_process_desc *)((char *)gc->client_base + gc->proc_desc_offset); 464 465 while (timeout_counter-- > 0) { 466 if (CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size) >= size) { 467 ret = 0; 468 break; 469 } 470 471 if (timeout_counter) 472 usleep_range(1000, 2000); 473 }; 474 475 return ret; 476 } 477 478 static int guc_add_workqueue_item(struct i915_guc_client *gc, 479 struct drm_i915_gem_request *rq) 480 { 481 struct guc_process_desc *desc; 482 struct guc_wq_item *wqi; 483 void *base; 484 u32 tail, wq_len, wq_off, space; 485 486 desc = (struct guc_process_desc *)((char *)gc->client_base + gc->proc_desc_offset); 487 space = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size); 488 if (WARN_ON(space < sizeof(struct guc_wq_item))) 489 return -ENOSPC; /* shouldn't happen */ 490 491 /* postincrement WQ tail for next time */ 492 wq_off = gc->wq_tail; 493 gc->wq_tail += sizeof(struct guc_wq_item); 494 gc->wq_tail &= gc->wq_size - 1; 495 496 /* For now workqueue item is 4 DWs; workqueue buffer is 2 pages. So we 497 * should not have the case where structure wqi is across page, neither 498 * wrapped to the beginning. This simplifies the implementation below. 499 * 500 * XXX: if not the case, we need save data to a temp wqi and copy it to 501 * workqueue buffer dw by dw. 502 */ 503 WARN_ON(sizeof(struct guc_wq_item) != 16); 504 WARN_ON(wq_off & 3); 505 506 /* wq starts from the page after doorbell / process_desc */ 507 base = kmap_atomic(i915_gem_object_get_page(gc->client_obj, 508 (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT)); 509 wq_off &= PAGE_SIZE - 1; 510 wqi = (struct guc_wq_item *)((char *)base + wq_off); 511 512 /* len does not include the header */ 513 wq_len = sizeof(struct guc_wq_item) / sizeof(u32) - 1; 514 wqi->header = WQ_TYPE_INORDER | 515 (wq_len << WQ_LEN_SHIFT) | 516 (rq->engine->guc_id << WQ_TARGET_SHIFT) | 517 WQ_NO_WCFLUSH_WAIT; 518 519 /* The GuC wants only the low-order word of the context descriptor */ 520 wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, 521 rq->engine); 522 523 /* The GuC firmware wants the tail index in QWords, not bytes */ 524 tail = rq->ringbuf->tail >> 3; 525 wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT; 526 wqi->fence_id = 0; /*XXX: what fence to be here */ 527 528 kunmap_atomic(base); 529 530 return 0; 531 } 532 533 /** 534 * i915_guc_submit() - Submit commands through GuC 535 * @client: the guc client where commands will go through 536 * @rq: request associated with the commands 537 * 538 * Return: 0 if succeed 539 */ 540 int i915_guc_submit(struct i915_guc_client *client, 541 struct drm_i915_gem_request *rq) 542 { 543 struct intel_guc *guc = client->guc; 544 unsigned int engine_id = rq->engine->guc_id; 545 int q_ret, b_ret; 546 547 q_ret = guc_add_workqueue_item(client, rq); 548 if (q_ret == 0) 549 b_ret = guc_ring_doorbell(client); 550 551 client->submissions[engine_id] += 1; 552 if (q_ret) { 553 client->q_fail += 1; 554 client->retcode = q_ret; 555 } else if (b_ret) { 556 client->b_fail += 1; 557 client->retcode = q_ret = b_ret; 558 } else { 559 client->retcode = 0; 560 } 561 guc->submissions[engine_id] += 1; 562 guc->last_seqno[engine_id] = rq->seqno; 563 564 return q_ret; 565 } 566 567 /* 568 * Everything below here is concerned with setup & teardown, and is 569 * therefore not part of the somewhat time-critical batch-submission 570 * path of i915_guc_submit() above. 571 */ 572 573 /** 574 * gem_allocate_guc_obj() - Allocate gem object for GuC usage 575 * @dev: drm device 576 * @size: size of object 577 * 578 * This is a wrapper to create a gem obj. In order to use it inside GuC, the 579 * object needs to be pinned lifetime. Also we must pin it to gtt space other 580 * than [0, GUC_WOPCM_TOP) because this range is reserved inside GuC. 581 * 582 * Return: A drm_i915_gem_object if successful, otherwise NULL. 583 */ 584 static struct drm_i915_gem_object *gem_allocate_guc_obj(struct drm_device *dev, 585 u32 size) 586 { 587 struct drm_i915_private *dev_priv = dev->dev_private; 588 struct drm_i915_gem_object *obj; 589 590 obj = i915_gem_alloc_object(dev, size); 591 if (!obj) 592 return NULL; 593 594 if (i915_gem_object_get_pages(obj)) { 595 drm_gem_object_unreference(&obj->base); 596 return NULL; 597 } 598 599 if (i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, 600 PIN_OFFSET_BIAS | GUC_WOPCM_TOP)) { 601 drm_gem_object_unreference(&obj->base); 602 return NULL; 603 } 604 605 /* Invalidate GuC TLB to let GuC take the latest updates to GTT. */ 606 I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE); 607 608 return obj; 609 } 610 611 /** 612 * gem_release_guc_obj() - Release gem object allocated for GuC usage 613 * @obj: gem obj to be released 614 */ 615 static void gem_release_guc_obj(struct drm_i915_gem_object *obj) 616 { 617 if (!obj) 618 return; 619 620 if (i915_gem_obj_is_pinned(obj)) 621 i915_gem_object_ggtt_unpin(obj); 622 623 drm_gem_object_unreference(&obj->base); 624 } 625 626 static void guc_client_free(struct drm_device *dev, 627 struct i915_guc_client *client) 628 { 629 struct drm_i915_private *dev_priv = dev->dev_private; 630 struct intel_guc *guc = &dev_priv->guc; 631 632 if (!client) 633 return; 634 635 /* 636 * XXX: wait for any outstanding submissions before freeing memory. 637 * Be sure to drop any locks 638 */ 639 640 if (client->client_base) { 641 /* 642 * If we got as far as setting up a doorbell, make sure 643 * we shut it down before unmapping & deallocating the 644 * memory. So first disable the doorbell, then tell the 645 * GuC that we've finished with it, finally deallocate 646 * it in our bitmap 647 */ 648 if (client->doorbell_id != GUC_INVALID_DOORBELL_ID) { 649 guc_disable_doorbell(guc, client); 650 host2guc_release_doorbell(guc, client); 651 release_doorbell(guc, client->doorbell_id); 652 } 653 654 kunmap(kmap_to_page(client->client_base)); 655 } 656 657 gem_release_guc_obj(client->client_obj); 658 659 if (client->ctx_index != GUC_INVALID_CTX_ID) { 660 guc_fini_ctx_desc(guc, client); 661 ida_simple_remove(&guc->ctx_ids, client->ctx_index); 662 } 663 664 kfree(client); 665 } 666 667 /** 668 * guc_client_alloc() - Allocate an i915_guc_client 669 * @dev: drm device 670 * @priority: four levels priority _CRITICAL, _HIGH, _NORMAL and _LOW 671 * The kernel client to replace ExecList submission is created with 672 * NORMAL priority. Priority of a client for scheduler can be HIGH, 673 * while a preemption context can use CRITICAL. 674 * @ctx: the context that owns the client (we use the default render 675 * context) 676 * 677 * Return: An i915_guc_client object if success, else NULL. 678 */ 679 static struct i915_guc_client *guc_client_alloc(struct drm_device *dev, 680 uint32_t priority, 681 struct intel_context *ctx) 682 { 683 struct i915_guc_client *client; 684 struct drm_i915_private *dev_priv = dev->dev_private; 685 struct intel_guc *guc = &dev_priv->guc; 686 struct drm_i915_gem_object *obj; 687 688 client = kzalloc(sizeof(*client), GFP_KERNEL); 689 if (!client) 690 return NULL; 691 692 client->doorbell_id = GUC_INVALID_DOORBELL_ID; 693 client->priority = priority; 694 client->owner = ctx; 695 client->guc = guc; 696 697 client->ctx_index = (uint32_t)ida_simple_get(&guc->ctx_ids, 0, 698 GUC_MAX_GPU_CONTEXTS, GFP_KERNEL); 699 if (client->ctx_index >= GUC_MAX_GPU_CONTEXTS) { 700 client->ctx_index = GUC_INVALID_CTX_ID; 701 goto err; 702 } 703 704 /* The first page is doorbell/proc_desc. Two followed pages are wq. */ 705 obj = gem_allocate_guc_obj(dev, GUC_DB_SIZE + GUC_WQ_SIZE); 706 if (!obj) 707 goto err; 708 709 /* We'll keep just the first (doorbell/proc) page permanently kmap'd. */ 710 client->client_obj = obj; 711 client->client_base = kmap(i915_gem_object_get_page(obj, 0)); 712 client->wq_offset = GUC_DB_SIZE; 713 client->wq_size = GUC_WQ_SIZE; 714 715 client->doorbell_offset = select_doorbell_cacheline(guc); 716 717 /* 718 * Since the doorbell only requires a single cacheline, we can save 719 * space by putting the application process descriptor in the same 720 * page. Use the half of the page that doesn't include the doorbell. 721 */ 722 if (client->doorbell_offset >= (GUC_DB_SIZE / 2)) 723 client->proc_desc_offset = 0; 724 else 725 client->proc_desc_offset = (GUC_DB_SIZE / 2); 726 727 client->doorbell_id = assign_doorbell(guc, client->priority); 728 if (client->doorbell_id == GUC_INVALID_DOORBELL_ID) 729 /* XXX: evict a doorbell instead */ 730 goto err; 731 732 guc_init_proc_desc(guc, client); 733 guc_init_ctx_desc(guc, client); 734 guc_init_doorbell(guc, client); 735 736 /* XXX: Any cache flushes needed? General domain mgmt calls? */ 737 738 if (host2guc_allocate_doorbell(guc, client)) 739 goto err; 740 741 DRM_DEBUG_DRIVER("new priority %u client %p: ctx_index %u db_id %u\n", 742 priority, client, client->ctx_index, client->doorbell_id); 743 744 return client; 745 746 err: 747 DRM_ERROR("FAILED to create priority %u GuC client!\n", priority); 748 749 guc_client_free(dev, client); 750 return NULL; 751 } 752 753 static void guc_create_log(struct intel_guc *guc) 754 { 755 struct drm_i915_private *dev_priv = guc_to_i915(guc); 756 struct drm_i915_gem_object *obj; 757 unsigned long offset; 758 uint32_t size, flags; 759 760 if (i915.guc_log_level < GUC_LOG_VERBOSITY_MIN) 761 return; 762 763 if (i915.guc_log_level > GUC_LOG_VERBOSITY_MAX) 764 i915.guc_log_level = GUC_LOG_VERBOSITY_MAX; 765 766 /* The first page is to save log buffer state. Allocate one 767 * extra page for others in case for overlap */ 768 size = (1 + GUC_LOG_DPC_PAGES + 1 + 769 GUC_LOG_ISR_PAGES + 1 + 770 GUC_LOG_CRASH_PAGES + 1) << PAGE_SHIFT; 771 772 obj = guc->log_obj; 773 if (!obj) { 774 obj = gem_allocate_guc_obj(dev_priv->dev, size); 775 if (!obj) { 776 /* logging will be off */ 777 i915.guc_log_level = -1; 778 return; 779 } 780 781 guc->log_obj = obj; 782 } 783 784 /* each allocated unit is a page */ 785 flags = GUC_LOG_VALID | GUC_LOG_NOTIFY_ON_HALF_FULL | 786 (GUC_LOG_DPC_PAGES << GUC_LOG_DPC_SHIFT) | 787 (GUC_LOG_ISR_PAGES << GUC_LOG_ISR_SHIFT) | 788 (GUC_LOG_CRASH_PAGES << GUC_LOG_CRASH_SHIFT); 789 790 offset = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT; /* in pages */ 791 guc->log_flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags; 792 } 793 794 static void init_guc_policies(struct guc_policies *policies) 795 { 796 struct guc_policy *policy; 797 u32 p, i; 798 799 policies->dpc_promote_time = 500000; 800 policies->max_num_work_items = POLICY_MAX_NUM_WI; 801 802 for (p = 0; p < GUC_CTX_PRIORITY_NUM; p++) { 803 for (i = GUC_RENDER_ENGINE; i < GUC_MAX_ENGINES_NUM; i++) { 804 policy = &policies->policy[p][i]; 805 806 policy->execution_quantum = 1000000; 807 policy->preemption_time = 500000; 808 policy->fault_time = 250000; 809 policy->policy_flags = 0; 810 } 811 } 812 813 policies->is_valid = 1; 814 } 815 816 static void guc_create_ads(struct intel_guc *guc) 817 { 818 struct drm_i915_private *dev_priv = guc_to_i915(guc); 819 struct drm_i915_gem_object *obj; 820 struct guc_ads *ads; 821 struct guc_policies *policies; 822 struct guc_mmio_reg_state *reg_state; 823 struct intel_engine_cs *engine; 824 struct vm_page *page; 825 u32 size; 826 827 /* The ads obj includes the struct itself and buffers passed to GuC */ 828 size = sizeof(struct guc_ads) + sizeof(struct guc_policies) + 829 sizeof(struct guc_mmio_reg_state) + 830 GUC_S3_SAVE_SPACE_PAGES * PAGE_SIZE; 831 832 obj = guc->ads_obj; 833 if (!obj) { 834 obj = gem_allocate_guc_obj(dev_priv->dev, PAGE_ALIGN(size)); 835 if (!obj) 836 return; 837 838 guc->ads_obj = obj; 839 } 840 841 page = i915_gem_object_get_page(obj, 0); 842 ads = kmap(page); 843 844 /* 845 * The GuC requires a "Golden Context" when it reinitialises 846 * engines after a reset. Here we use the Render ring default 847 * context, which must already exist and be pinned in the GGTT, 848 * so its address won't change after we've told the GuC where 849 * to find it. 850 */ 851 engine = &dev_priv->engine[RCS]; 852 ads->golden_context_lrca = engine->status_page.gfx_addr; 853 854 for_each_engine(engine, dev_priv) 855 ads->eng_state_size[engine->guc_id] = intel_lr_context_size(engine); 856 857 /* GuC scheduling policies */ 858 policies = (struct guc_policies *)((char *)ads + sizeof(struct guc_ads)); 859 init_guc_policies(policies); 860 861 ads->scheduler_policies = i915_gem_obj_ggtt_offset(obj) + 862 sizeof(struct guc_ads); 863 864 /* MMIO reg state */ 865 reg_state = (struct guc_mmio_reg_state *)((char *)policies + sizeof(struct guc_policies)); 866 867 for_each_engine(engine, dev_priv) { 868 reg_state->mmio_white_list[engine->guc_id].mmio_start = 869 engine->mmio_base + GUC_MMIO_WHITE_LIST_START; 870 871 /* Nothing to be saved or restored for now. */ 872 reg_state->mmio_white_list[engine->guc_id].count = 0; 873 } 874 875 ads->reg_state_addr = ads->scheduler_policies + 876 sizeof(struct guc_policies); 877 878 ads->reg_state_buffer = ads->reg_state_addr + 879 sizeof(struct guc_mmio_reg_state); 880 881 kunmap(page); 882 } 883 884 /* 885 * Set up the memory resources to be shared with the GuC. At this point, 886 * we require just one object that can be mapped through the GGTT. 887 */ 888 int i915_guc_submission_init(struct drm_device *dev) 889 { 890 struct drm_i915_private *dev_priv = dev->dev_private; 891 const size_t ctxsize = sizeof(struct guc_context_desc); 892 const size_t poolsize = GUC_MAX_GPU_CONTEXTS * ctxsize; 893 const size_t gemsize = round_up(poolsize, PAGE_SIZE); 894 struct intel_guc *guc = &dev_priv->guc; 895 896 if (!i915.enable_guc_submission) 897 return 0; /* not enabled */ 898 899 if (guc->ctx_pool_obj) 900 return 0; /* already allocated */ 901 902 guc->ctx_pool_obj = gem_allocate_guc_obj(dev_priv->dev, gemsize); 903 if (!guc->ctx_pool_obj) 904 return -ENOMEM; 905 906 ida_init(&guc->ctx_ids); 907 908 guc_create_log(guc); 909 910 guc_create_ads(guc); 911 912 return 0; 913 } 914 915 int i915_guc_submission_enable(struct drm_device *dev) 916 { 917 struct drm_i915_private *dev_priv = dev->dev_private; 918 struct intel_guc *guc = &dev_priv->guc; 919 struct intel_context *ctx = dev_priv->kernel_context; 920 struct i915_guc_client *client; 921 922 /* client for execbuf submission */ 923 client = guc_client_alloc(dev, GUC_CTX_PRIORITY_KMD_NORMAL, ctx); 924 if (!client) { 925 DRM_ERROR("Failed to create execbuf guc_client\n"); 926 return -ENOMEM; 927 } 928 929 guc->execbuf_client = client; 930 931 host2guc_sample_forcewake(guc, client); 932 933 return 0; 934 } 935 936 void i915_guc_submission_disable(struct drm_device *dev) 937 { 938 struct drm_i915_private *dev_priv = dev->dev_private; 939 struct intel_guc *guc = &dev_priv->guc; 940 941 guc_client_free(dev, guc->execbuf_client); 942 guc->execbuf_client = NULL; 943 } 944 945 void i915_guc_submission_fini(struct drm_device *dev) 946 { 947 struct drm_i915_private *dev_priv = dev->dev_private; 948 struct intel_guc *guc = &dev_priv->guc; 949 950 gem_release_guc_obj(dev_priv->guc.ads_obj); 951 guc->ads_obj = NULL; 952 953 gem_release_guc_obj(dev_priv->guc.log_obj); 954 guc->log_obj = NULL; 955 956 if (guc->ctx_pool_obj) 957 ida_destroy(&guc->ctx_ids); 958 gem_release_guc_obj(guc->ctx_pool_obj); 959 guc->ctx_pool_obj = NULL; 960 } 961 962 /** 963 * intel_guc_suspend() - notify GuC entering suspend state 964 * @dev: drm device 965 */ 966 int intel_guc_suspend(struct drm_device *dev) 967 { 968 struct drm_i915_private *dev_priv = dev->dev_private; 969 struct intel_guc *guc = &dev_priv->guc; 970 struct intel_context *ctx; 971 u32 data[3]; 972 973 if (!i915.enable_guc_submission) 974 return 0; 975 976 ctx = dev_priv->kernel_context; 977 978 data[0] = HOST2GUC_ACTION_ENTER_S_STATE; 979 /* any value greater than GUC_POWER_D0 */ 980 data[1] = GUC_POWER_D1; 981 /* first page is shared data with GuC */ 982 data[2] = i915_gem_obj_ggtt_offset(ctx->engine[RCS].state); 983 984 return host2guc_action(guc, data, ARRAY_SIZE(data)); 985 } 986 987 988 /** 989 * intel_guc_resume() - notify GuC resuming from suspend state 990 * @dev: drm device 991 */ 992 int intel_guc_resume(struct drm_device *dev) 993 { 994 struct drm_i915_private *dev_priv = dev->dev_private; 995 struct intel_guc *guc = &dev_priv->guc; 996 struct intel_context *ctx; 997 u32 data[3]; 998 999 if (!i915.enable_guc_submission) 1000 return 0; 1001 1002 ctx = dev_priv->kernel_context; 1003 1004 data[0] = HOST2GUC_ACTION_EXIT_S_STATE; 1005 data[1] = GUC_POWER_D0; 1006 /* first page is shared data with GuC */ 1007 data[2] = i915_gem_obj_ggtt_offset(ctx->engine[RCS].state); 1008 1009 return host2guc_action(guc, data, ARRAY_SIZE(data)); 1010 } 1011