1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Dave Airlie 30 */ 31 #include <drm/drmP.h> 32 #include "radeon_reg.h" 33 #include "radeon.h" 34 #ifdef TRACE_TODO 35 #include "radeon_trace.h" 36 #endif 37 38 /* 39 * Fences 40 * Fences mark an event in the GPUs pipeline and are used 41 * for GPU/CPU synchronization. When the fence is written, 42 * it is expected that all buffers associated with that fence 43 * are no longer in use by the associated ring on the GPU and 44 * that the the relevant GPU caches have been flushed. Whether 45 * we use a scratch register or memory location depends on the asic 46 * and whether writeback is enabled. 47 */ 48 49 /** 50 * radeon_fence_write - write a fence value 51 * 52 * @rdev: radeon_device pointer 53 * @seq: sequence number to write 54 * @ring: ring index the fence is associated with 55 * 56 * Writes a fence value to memory or a scratch register (all asics). 57 */ 58 static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring) 59 { 60 struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; 61 if (likely(rdev->wb.enabled || !drv->scratch_reg)) { 62 if (drv->cpu_addr) { 63 *drv->cpu_addr = cpu_to_le32(seq); 64 } 65 } else { 66 WREG32(drv->scratch_reg, seq); 67 } 68 } 69 70 /** 71 * radeon_fence_read - read a fence value 72 * 73 * @rdev: radeon_device pointer 74 * @ring: ring index the fence is associated with 75 * 76 * Reads a fence value from memory or a scratch register (all asics). 77 * Returns the value of the fence read from memory or register. 78 */ 79 static u32 radeon_fence_read(struct radeon_device *rdev, int ring) 80 { 81 struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; 82 u32 seq = 0; 83 84 if (likely(rdev->wb.enabled || !drv->scratch_reg)) { 85 if (drv->cpu_addr) { 86 seq = le32_to_cpu(*drv->cpu_addr); 87 } else { 88 seq = lower_32_bits(atomic64_read(&drv->last_seq)); 89 } 90 } else { 91 seq = RREG32(drv->scratch_reg); 92 } 93 return seq; 94 } 95 96 /** 97 * radeon_fence_schedule_check - schedule lockup check 98 * 99 * @rdev: radeon_device pointer 100 * @ring: ring index we should work with 101 * 102 * Queues a delayed work item to check for lockups. 103 */ 104 static void radeon_fence_schedule_check(struct radeon_device *rdev, int ring) 105 { 106 /* 107 * Do not reset the timer here with mod_delayed_work, 108 * this can livelock in an interaction with TTM delayed destroy. 109 */ 110 queue_delayed_work(system_power_efficient_wq, 111 &rdev->fence_drv[ring].lockup_work, 112 RADEON_FENCE_JIFFIES_TIMEOUT); 113 } 114 115 /** 116 * radeon_fence_emit - emit a fence on the requested ring 117 * 118 * @rdev: radeon_device pointer 119 * @fence: radeon fence object 120 * @ring: ring index the fence is associated with 121 * 122 * Emits a fence command on the requested ring (all asics). 123 * Returns 0 on success, -ENOMEM on failure. 124 */ 125 int radeon_fence_emit(struct radeon_device *rdev, 126 struct radeon_fence **fence, 127 int ring) 128 { 129 /* we are protected by the ring emission mutex */ 130 *fence = kmalloc(sizeof(struct radeon_fence), M_DRM, 131 M_WAITOK); 132 if ((*fence) == NULL) { 133 return -ENOMEM; 134 } 135 refcount_init(&((*fence)->kref), 1); 136 (*fence)->rdev = rdev; 137 (*fence)->seq = ++rdev->fence_drv[ring].sync_seq[ring]; 138 (*fence)->ring = ring; 139 radeon_fence_ring_emit(rdev, ring, *fence); 140 #ifdef TRACE_TODO 141 trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq); 142 #endif 143 radeon_fence_schedule_check(rdev, ring); 144 return 0; 145 } 146 147 /** 148 * radeon_fence_activity - check for fence activity 149 * 150 * @rdev: radeon_device pointer 151 * @ring: ring index the fence is associated with 152 * 153 * Checks the current fence value and calculates the last 154 * signalled fence value. Returns true if activity occured 155 * on the ring, and the fence_queue should be waken up. 156 */ 157 static bool radeon_fence_activity(struct radeon_device *rdev, int ring) 158 { 159 uint64_t seq, last_seq, last_emitted; 160 unsigned count_loop = 0; 161 bool wake = false; 162 163 /* Note there is a scenario here for an infinite loop but it's 164 * very unlikely to happen. For it to happen, the current polling 165 * process need to be interrupted by another process and another 166 * process needs to update the last_seq btw the atomic read and 167 * xchg of the current process. 168 * 169 * More over for this to go in infinite loop there need to be 170 * continuously new fence signaled ie radeon_fence_read needs 171 * to return a different value each time for both the currently 172 * polling process and the other process that xchg the last_seq 173 * btw atomic read and xchg of the current process. And the 174 * value the other process set as last seq must be higher than 175 * the seq value we just read. Which means that current process 176 * need to be interrupted after radeon_fence_read and before 177 * atomic xchg. 178 * 179 * To be even more safe we count the number of time we loop and 180 * we bail after 10 loop just accepting the fact that we might 181 * have temporarly set the last_seq not to the true real last 182 * seq but to an older one. 183 */ 184 last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq); 185 do { 186 last_emitted = rdev->fence_drv[ring].sync_seq[ring]; 187 seq = radeon_fence_read(rdev, ring); 188 seq |= last_seq & 0xffffffff00000000LL; 189 if (seq < last_seq) { 190 seq &= 0xffffffff; 191 seq |= last_emitted & 0xffffffff00000000LL; 192 } 193 194 if (seq <= last_seq || seq > last_emitted) { 195 break; 196 } 197 /* If we loop over we don't want to return without 198 * checking if a fence is signaled as it means that the 199 * seq we just read is different from the previous on. 200 */ 201 wake = true; 202 last_seq = seq; 203 if ((count_loop++) > 10) { 204 /* We looped over too many time leave with the 205 * fact that we might have set an older fence 206 * seq then the current real last seq as signaled 207 * by the hw. 208 */ 209 break; 210 } 211 } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); 212 213 if (seq < last_emitted) 214 radeon_fence_schedule_check(rdev, ring); 215 216 return wake; 217 } 218 219 /** 220 * radeon_fence_check_lockup - check for hardware lockup 221 * 222 * @work: delayed work item 223 * 224 * Checks for fence activity and if there is none probe 225 * the hardware if a lockup occured. 226 */ 227 static void radeon_fence_check_lockup(struct work_struct *work) 228 { 229 struct radeon_fence_driver *fence_drv; 230 struct radeon_device *rdev; 231 int ring; 232 233 fence_drv = container_of(work, struct radeon_fence_driver, 234 lockup_work.work); 235 rdev = fence_drv->rdev; 236 ring = fence_drv - &rdev->fence_drv[0]; 237 238 if (lockmgr(&rdev->exclusive_lock, LK_EXCLUSIVE|LK_NOWAIT)) { 239 /* just reschedule the check if a reset is going on */ 240 radeon_fence_schedule_check(rdev, ring); 241 return; 242 } 243 244 if (radeon_fence_activity(rdev, ring)) 245 wake_up_all(&rdev->fence_queue); 246 247 else if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { 248 249 /* good news we believe it's a lockup */ 250 dev_warn(rdev->dev, "GPU lockup (current fence id " 251 "0x%016lx last fence id 0x%016lx on ring %d)\n", 252 (uint64_t)atomic64_read(&fence_drv->last_seq), 253 fence_drv->sync_seq[ring], ring); 254 255 /* remember that we need an reset */ 256 rdev->needs_reset = true; 257 wake_up_all(&rdev->fence_queue); 258 } 259 lockmgr(&rdev->exclusive_lock, LK_RELEASE); 260 } 261 262 /** 263 * radeon_fence_process - process a fence 264 * 265 * @rdev: radeon_device pointer 266 * @ring: ring index the fence is associated with 267 * 268 * Checks the current fence value and wakes the fence queue 269 * if the sequence number has increased (all asics). 270 */ 271 void radeon_fence_process(struct radeon_device *rdev, int ring) 272 { 273 if (radeon_fence_activity(rdev, ring)) 274 wake_up_all(&rdev->fence_queue); 275 } 276 277 /** 278 * radeon_fence_destroy - destroy a fence 279 * 280 * @kref: fence kref 281 * 282 * Frees the fence object (all asics). 283 */ 284 static void radeon_fence_destroy(struct radeon_fence *fence) 285 { 286 287 kfree(fence); 288 } 289 290 /** 291 * radeon_fence_seq_signaled - check if a fence sequence number has signaled 292 * 293 * @rdev: radeon device pointer 294 * @seq: sequence number 295 * @ring: ring index the fence is associated with 296 * 297 * Check if the last signaled fence sequnce number is >= the requested 298 * sequence number (all asics). 299 * Returns true if the fence has signaled (current fence value 300 * is >= requested value) or false if it has not (current fence 301 * value is < the requested value. Helper function for 302 * radeon_fence_signaled(). 303 */ 304 static bool radeon_fence_seq_signaled(struct radeon_device *rdev, 305 u64 seq, unsigned ring) 306 { 307 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { 308 return true; 309 } 310 /* poll new last sequence at least once */ 311 radeon_fence_process(rdev, ring); 312 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { 313 return true; 314 } 315 return false; 316 } 317 318 /** 319 * radeon_fence_signaled - check if a fence has signaled 320 * 321 * @fence: radeon fence object 322 * 323 * Check if the requested fence has signaled (all asics). 324 * Returns true if the fence has signaled or false if it has not. 325 */ 326 bool radeon_fence_signaled(struct radeon_fence *fence) 327 { 328 if (!fence) 329 return true; 330 if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) 331 return true; 332 return false; 333 } 334 335 /** 336 * radeon_fence_any_seq_signaled - check if any sequence number is signaled 337 * 338 * @rdev: radeon device pointer 339 * @seq: sequence numbers 340 * 341 * Check if the last signaled fence sequnce number is >= the requested 342 * sequence number (all asics). 343 * Returns true if any has signaled (current value is >= requested value) 344 * or false if it has not. Helper function for radeon_fence_wait_seq. 345 */ 346 static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq) 347 { 348 unsigned i; 349 350 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 351 if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) 352 return true; 353 } 354 return false; 355 } 356 357 /** 358 * radeon_fence_wait_seq_timeout - wait for a specific sequence numbers 359 * 360 * @rdev: radeon device pointer 361 * @target_seq: sequence number(s) we want to wait for 362 * @intr: use interruptable sleep 363 * @timeout: maximum time to wait, or MAX_SCHEDULE_TIMEOUT for infinite wait 364 * 365 * Wait for the requested sequence number(s) to be written by any ring 366 * (all asics). Sequnce number array is indexed by ring id. 367 * @intr selects whether to use interruptable (true) or non-interruptable 368 * (false) sleep when waiting for the sequence number. Helper function 369 * for radeon_fence_wait_*(). 370 * Returns remaining time if the sequence number has passed, 0 when 371 * the wait timeout, or an error for all other cases. 372 * -EDEADLK is returned when a GPU lockup has been detected. 373 */ 374 static int radeon_fence_wait_seq_timeout(struct radeon_device *rdev, 375 u64 *target_seq, bool intr, 376 int timeout) 377 { 378 long r; 379 int i; 380 381 if (radeon_fence_any_seq_signaled(rdev, target_seq)) 382 return timeout; 383 384 /* enable IRQs and tracing */ 385 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 386 if (!target_seq[i]) 387 continue; 388 389 #ifdef TRACE_TODO 390 trace_radeon_fence_wait_begin(rdev->ddev, i, target_seq[i]); 391 #endif 392 radeon_irq_kms_sw_irq_get(rdev, i); 393 } 394 395 if (intr) { 396 r = wait_event_interruptible_timeout(rdev->fence_queue, ( 397 radeon_fence_any_seq_signaled(rdev, target_seq) 398 || rdev->needs_reset), timeout); 399 } else { 400 r = wait_event_timeout(rdev->fence_queue, ( 401 radeon_fence_any_seq_signaled(rdev, target_seq) 402 || rdev->needs_reset), timeout); 403 } 404 405 if (rdev->needs_reset) 406 r = -EDEADLK; 407 408 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 409 if (!target_seq[i]) 410 continue; 411 412 radeon_irq_kms_sw_irq_put(rdev, i); 413 #if TRACE_TODO 414 trace_radeon_fence_wait_end(rdev->ddev, i, target_seq[i]); 415 #endif 416 } 417 418 return r; 419 } 420 421 /** 422 * radeon_fence_wait - wait for a fence to signal 423 * 424 * @fence: radeon fence object 425 * @intr: use interruptible sleep 426 * 427 * Wait for the requested fence to signal (all asics). 428 * @intr selects whether to use interruptable (true) or non-interruptable 429 * (false) sleep when waiting for the fence. 430 * Returns 0 if the fence has passed, error for all other cases. 431 */ 432 int radeon_fence_wait(struct radeon_fence *fence, bool intr) 433 { 434 uint64_t seq[RADEON_NUM_RINGS] = {}; 435 int r; 436 437 if (fence == NULL) { 438 WARN(1, "Querying an invalid fence : %p !\n", fence); 439 return -EINVAL; 440 } 441 442 seq[fence->ring] = fence->seq; 443 r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, INT_MAX); 444 if (r < 0) { 445 return r; 446 } 447 448 return 0; 449 } 450 451 /** 452 * radeon_fence_wait_any - wait for a fence to signal on any ring 453 * 454 * @rdev: radeon device pointer 455 * @fences: radeon fence object(s) 456 * @intr: use interruptable sleep 457 * 458 * Wait for any requested fence to signal (all asics). Fence 459 * array is indexed by ring id. @intr selects whether to use 460 * interruptable (true) or non-interruptable (false) sleep when 461 * waiting for the fences. Used by the suballocator. 462 * Returns 0 if any fence has passed, error for all other cases. 463 */ 464 int radeon_fence_wait_any(struct radeon_device *rdev, 465 struct radeon_fence **fences, 466 bool intr) 467 { 468 uint64_t seq[RADEON_NUM_RINGS]; 469 unsigned i, num_rings = 0; 470 int r; 471 472 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 473 seq[i] = 0; 474 475 if (!fences[i]) { 476 continue; 477 } 478 479 seq[i] = fences[i]->seq; 480 ++num_rings; 481 } 482 483 /* nothing to wait for ? */ 484 if (num_rings == 0) 485 return -ENOENT; 486 487 r = radeon_fence_wait_seq_timeout(rdev, seq, intr, INT_MAX); 488 if (r < 0) { 489 return r; 490 } 491 return 0; 492 } 493 494 /** 495 * radeon_fence_wait_next - wait for the next fence to signal 496 * 497 * @rdev: radeon device pointer 498 * @ring: ring index the fence is associated with 499 * 500 * Wait for the next fence on the requested ring to signal (all asics). 501 * Returns 0 if the next fence has passed, error for all other cases. 502 * Caller must hold ring lock. 503 */ 504 int radeon_fence_wait_next(struct radeon_device *rdev, int ring) 505 { 506 uint64_t seq[RADEON_NUM_RINGS] = {}; 507 int r; 508 509 seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; 510 if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) { 511 /* nothing to wait for, last_seq is 512 already the last emited fence */ 513 return -ENOENT; 514 } 515 r = radeon_fence_wait_seq_timeout(rdev, seq, false, INT_MAX); 516 if (r < 0) 517 return r; 518 return 0; 519 } 520 521 /** 522 * radeon_fence_wait_empty - wait for all fences to signal 523 * 524 * @rdev: radeon device pointer 525 * @ring: ring index the fence is associated with 526 * 527 * Wait for all fences on the requested ring to signal (all asics). 528 * Returns 0 if the fences have passed, error for all other cases. 529 * Caller must hold ring lock. 530 */ 531 int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) 532 { 533 uint64_t seq[RADEON_NUM_RINGS] = {}; 534 int r; 535 536 seq[ring] = rdev->fence_drv[ring].sync_seq[ring]; 537 if (!seq[ring]) 538 return 0; 539 540 r = radeon_fence_wait_seq_timeout(rdev, seq, false, INT_MAX); 541 if (r < 0) { 542 if (r == -EDEADLK) 543 return -EDEADLK; 544 545 dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%d)\n", 546 ring, r); 547 } 548 return 0; 549 } 550 551 /** 552 * radeon_fence_ref - take a ref on a fence 553 * 554 * @fence: radeon fence object 555 * 556 * Take a reference on a fence (all asics). 557 * Returns the fence. 558 */ 559 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) 560 { 561 refcount_acquire(&fence->kref); 562 return fence; 563 } 564 565 /** 566 * radeon_fence_unref - remove a ref on a fence 567 * 568 * @fence: radeon fence object 569 * 570 * Remove a reference on a fence (all asics). 571 */ 572 void radeon_fence_unref(struct radeon_fence **fence) 573 { 574 struct radeon_fence *tmp = *fence; 575 576 *fence = NULL; 577 if (tmp) { 578 if (refcount_release(&tmp->kref)) { 579 radeon_fence_destroy(tmp); 580 } 581 } 582 } 583 584 /** 585 * radeon_fence_count_emitted - get the count of emitted fences 586 * 587 * @rdev: radeon device pointer 588 * @ring: ring index the fence is associated with 589 * 590 * Get the number of fences emitted on the requested ring (all asics). 591 * Returns the number of emitted fences on the ring. Used by the 592 * dynpm code to ring track activity. 593 */ 594 unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring) 595 { 596 uint64_t emitted; 597 598 /* We are not protected by ring lock when reading the last sequence 599 * but it's ok to report slightly wrong fence count here. 600 */ 601 radeon_fence_process(rdev, ring); 602 emitted = rdev->fence_drv[ring].sync_seq[ring] 603 - atomic64_read(&rdev->fence_drv[ring].last_seq); 604 /* to avoid 32bits warp around */ 605 if (emitted > 0x10000000) { 606 emitted = 0x10000000; 607 } 608 return (unsigned)emitted; 609 } 610 611 /** 612 * radeon_fence_need_sync - do we need a semaphore 613 * 614 * @fence: radeon fence object 615 * @dst_ring: which ring to check against 616 * 617 * Check if the fence needs to be synced against another ring 618 * (all asics). If so, we need to emit a semaphore. 619 * Returns true if we need to sync with another ring, false if 620 * not. 621 */ 622 bool radeon_fence_need_sync(struct radeon_fence *fence, int dst_ring) 623 { 624 struct radeon_fence_driver *fdrv; 625 626 if (!fence) { 627 return false; 628 } 629 630 if (fence->ring == dst_ring) { 631 return false; 632 } 633 634 /* we are protected by the ring mutex */ 635 fdrv = &fence->rdev->fence_drv[dst_ring]; 636 if (fence->seq <= fdrv->sync_seq[fence->ring]) { 637 return false; 638 } 639 640 return true; 641 } 642 643 /** 644 * radeon_fence_note_sync - record the sync point 645 * 646 * @fence: radeon fence object 647 * @dst_ring: which ring to check against 648 * 649 * Note the sequence number at which point the fence will 650 * be synced with the requested ring (all asics). 651 */ 652 void radeon_fence_note_sync(struct radeon_fence *fence, int dst_ring) 653 { 654 struct radeon_fence_driver *dst, *src; 655 unsigned i; 656 657 if (!fence) { 658 return; 659 } 660 661 if (fence->ring == dst_ring) { 662 return; 663 } 664 665 /* we are protected by the ring mutex */ 666 src = &fence->rdev->fence_drv[fence->ring]; 667 dst = &fence->rdev->fence_drv[dst_ring]; 668 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 669 if (i == dst_ring) { 670 continue; 671 } 672 dst->sync_seq[i] = max(dst->sync_seq[i], src->sync_seq[i]); 673 } 674 } 675 676 /** 677 * radeon_fence_driver_start_ring - make the fence driver 678 * ready for use on the requested ring. 679 * 680 * @rdev: radeon device pointer 681 * @ring: ring index to start the fence driver on 682 * 683 * Make the fence driver ready for processing (all asics). 684 * Not all asics have all rings, so each asic will only 685 * start the fence driver on the rings it has. 686 * Returns 0 for success, errors for failure. 687 */ 688 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) 689 { 690 uint64_t index; 691 int r; 692 693 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); 694 if (rdev->wb.use_event || !radeon_ring_supports_scratch_reg(rdev, &rdev->ring[ring])) { 695 rdev->fence_drv[ring].scratch_reg = 0; 696 if (ring != R600_RING_TYPE_UVD_INDEX) { 697 index = R600_WB_EVENT_OFFSET + ring * 4; 698 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; 699 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + 700 index; 701 702 } else { 703 /* put fence directly behind firmware */ 704 index = ALIGN(rdev->uvd_fw->datasize, 8); 705 rdev->fence_drv[ring].cpu_addr = (void*)((uint8_t*)rdev->uvd.cpu_addr + index); 706 rdev->fence_drv[ring].gpu_addr = rdev->uvd.gpu_addr + index; 707 } 708 709 } else { 710 r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg); 711 if (r) { 712 dev_err(rdev->dev, "fence failed to get scratch register\n"); 713 return r; 714 } 715 index = RADEON_WB_SCRATCH_OFFSET + 716 rdev->fence_drv[ring].scratch_reg - 717 rdev->scratch.reg_base; 718 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; 719 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; 720 } 721 radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring); 722 rdev->fence_drv[ring].initialized = true; 723 dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016jx and cpu addr 0x%p\n", 724 ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr); 725 return 0; 726 } 727 728 /** 729 * radeon_fence_driver_init_ring - init the fence driver 730 * for the requested ring. 731 * 732 * @rdev: radeon device pointer 733 * @ring: ring index to start the fence driver on 734 * 735 * Init the fence driver for the requested ring (all asics). 736 * Helper function for radeon_fence_driver_init(). 737 */ 738 static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring) 739 { 740 int i; 741 742 rdev->fence_drv[ring].scratch_reg = -1; 743 rdev->fence_drv[ring].cpu_addr = NULL; 744 rdev->fence_drv[ring].gpu_addr = 0; 745 for (i = 0; i < RADEON_NUM_RINGS; ++i) 746 rdev->fence_drv[ring].sync_seq[i] = 0; 747 atomic64_set(&rdev->fence_drv[ring].last_seq, 0); 748 rdev->fence_drv[ring].initialized = false; 749 INIT_DELAYED_WORK(&rdev->fence_drv[ring].lockup_work, 750 radeon_fence_check_lockup); 751 rdev->fence_drv[ring].rdev = rdev; 752 } 753 754 /** 755 * radeon_fence_driver_init - init the fence driver 756 * for all possible rings. 757 * 758 * @rdev: radeon device pointer 759 * 760 * Init the fence driver for all possible rings (all asics). 761 * Not all asics have all rings, so each asic will only 762 * start the fence driver on the rings it has using 763 * radeon_fence_driver_start_ring(). 764 * Returns 0 for success. 765 */ 766 int radeon_fence_driver_init(struct radeon_device *rdev) 767 { 768 int ring; 769 770 init_waitqueue_head(&rdev->fence_queue); 771 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { 772 radeon_fence_driver_init_ring(rdev, ring); 773 } 774 if (radeon_debugfs_fence_init(rdev)) { 775 dev_err(rdev->dev, "fence debugfs file creation failed\n"); 776 } 777 return 0; 778 } 779 780 /** 781 * radeon_fence_driver_fini - tear down the fence driver 782 * for all possible rings. 783 * 784 * @rdev: radeon device pointer 785 * 786 * Tear down the fence driver for all possible rings (all asics). 787 */ 788 void radeon_fence_driver_fini(struct radeon_device *rdev) 789 { 790 int ring, r; 791 792 lockmgr(&rdev->ring_lock, LK_EXCLUSIVE); 793 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { 794 if (!rdev->fence_drv[ring].initialized) 795 continue; 796 r = radeon_fence_wait_empty(rdev, ring); 797 if (r) { 798 /* no need to trigger GPU reset as we are unloading */ 799 radeon_fence_driver_force_completion(rdev, ring); 800 } 801 cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); 802 wake_up_all(&rdev->fence_queue); 803 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); 804 rdev->fence_drv[ring].initialized = false; 805 } 806 lockmgr(&rdev->ring_lock, LK_RELEASE); 807 } 808 809 /** 810 * radeon_fence_driver_force_completion - force all fence waiter to complete 811 * 812 * @rdev: radeon device pointer 813 * @ring: the ring to complete 814 * 815 * In case of GPU reset failure make sure no process keep waiting on fence 816 * that will never complete. 817 */ 818 void radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring) 819 { 820 if (rdev->fence_drv[ring].initialized) { 821 radeon_fence_write(rdev, rdev->fence_drv[ring].sync_seq[ring], ring); 822 cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); 823 } 824 } 825 826 827 /* 828 * Fence debugfs 829 */ 830 #if defined(CONFIG_DEBUG_FS) 831 static int radeon_debugfs_fence_info(struct seq_file *m, void *data) 832 { 833 struct drm_info_node *node = (struct drm_info_node *)m->private; 834 struct drm_device *dev = node->minor->dev; 835 struct radeon_device *rdev = dev->dev_private; 836 int i, j; 837 838 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 839 if (!rdev->fence_drv[i].initialized) 840 continue; 841 842 radeon_fence_process(rdev, i); 843 844 seq_printf(m, "--- ring %d ---\n", i); 845 seq_printf(m, "Last signaled fence 0x%016llx\n", 846 (unsigned long long)atomic_load_acq_64(&rdev->fence_drv[i].last_seq)); 847 seq_printf(m, "Last emitted 0x%016llx\n", 848 rdev->fence_drv[i].sync_seq[i]); 849 850 for (j = 0; j < RADEON_NUM_RINGS; ++j) { 851 if (i != j && rdev->fence_drv[j].initialized) 852 seq_printf(m, "Last sync to ring %d 0x%016llx\n", 853 j, rdev->fence_drv[i].sync_seq[j]); 854 } 855 } 856 return 0; 857 } 858 859 /** 860 * radeon_debugfs_gpu_reset - manually trigger a gpu reset 861 * 862 * Manually trigger a gpu reset at the next fence wait. 863 */ 864 static int radeon_debugfs_gpu_reset(struct seq_file *m, void *data) 865 { 866 struct drm_info_node *node = (struct drm_info_node *) m->private; 867 struct drm_device *dev = node->minor->dev; 868 struct radeon_device *rdev = dev->dev_private; 869 870 down_read(&rdev->exclusive_lock); 871 seq_printf(m, "%d\n", rdev->needs_reset); 872 rdev->needs_reset = true; 873 wake_up_all(&rdev->fence_queue); 874 up_read(&rdev->exclusive_lock); 875 876 return 0; 877 } 878 879 static struct drm_info_list radeon_debugfs_fence_list[] = { 880 {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL}, 881 {"radeon_gpu_reset", &radeon_debugfs_gpu_reset, 0, NULL} 882 }; 883 #endif 884 885 int radeon_debugfs_fence_init(struct radeon_device *rdev) 886 { 887 #if defined(CONFIG_DEBUG_FS) 888 return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 2); 889 #else 890 return 0; 891 #endif 892 } 893