1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Dave Airlie 30 * 31 * $FreeBSD: head/sys/dev/drm2/radeon/radeon_fence.c 254885 2013-08-25 19:37:15Z dumbbell $ 32 */ 33 #include <drm/drmP.h> 34 #include "radeon_reg.h" 35 #include "radeon.h" 36 #ifdef DUMBBELL_WIP 37 #include "radeon_trace.h" 38 #endif /* DUMBBELL_WIP */ 39 40 /* 41 * Fences 42 * Fences mark an event in the GPUs pipeline and are used 43 * for GPU/CPU synchronization. When the fence is written, 44 * it is expected that all buffers associated with that fence 45 * are no longer in use by the associated ring on the GPU and 46 * that the the relevant GPU caches have been flushed. Whether 47 * we use a scratch register or memory location depends on the asic 48 * and whether writeback is enabled. 49 */ 50 51 /** 52 * radeon_fence_write - write a fence value 53 * 54 * @rdev: radeon_device pointer 55 * @seq: sequence number to write 56 * @ring: ring index the fence is associated with 57 * 58 * Writes a fence value to memory or a scratch register (all asics). 59 */ 60 static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring) 61 { 62 struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; 63 if (likely(rdev->wb.enabled || !drv->scratch_reg)) { 64 if (drv->cpu_addr) { 65 *drv->cpu_addr = cpu_to_le32(seq); 66 } 67 } else { 68 WREG32(drv->scratch_reg, seq); 69 } 70 } 71 72 /** 73 * radeon_fence_read - read a fence value 74 * 75 * @rdev: radeon_device pointer 76 * @ring: ring index the fence is associated with 77 * 78 * Reads a fence value from memory or a scratch register (all asics). 79 * Returns the value of the fence read from memory or register. 80 */ 81 static u32 radeon_fence_read(struct radeon_device *rdev, int ring) 82 { 83 struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; 84 u32 seq = 0; 85 86 if (likely(rdev->wb.enabled || !drv->scratch_reg)) { 87 if (drv->cpu_addr) { 88 seq = le32_to_cpu(*drv->cpu_addr); 89 } else { 90 seq = lower_32_bits(atomic64_read(&drv->last_seq)); 91 } 92 } else { 93 seq = RREG32(drv->scratch_reg); 94 } 95 return seq; 96 } 97 98 /** 99 * radeon_fence_emit - emit a fence on the requested ring 100 * 101 * @rdev: radeon_device pointer 102 * @fence: radeon fence object 103 * @ring: ring index the fence is associated with 104 * 105 * Emits a fence command on the requested ring (all asics). 106 * Returns 0 on success, -ENOMEM on failure. 107 */ 108 int radeon_fence_emit(struct radeon_device *rdev, 109 struct radeon_fence **fence, 110 int ring) 111 { 112 /* we are protected by the ring emission mutex */ 113 *fence = kmalloc(sizeof(struct radeon_fence), M_DRM, 114 M_WAITOK); 115 if ((*fence) == NULL) { 116 return -ENOMEM; 117 } 118 refcount_init(&((*fence)->kref), 1); 119 (*fence)->rdev = rdev; 120 (*fence)->seq = ++rdev->fence_drv[ring].sync_seq[ring]; 121 (*fence)->ring = ring; 122 radeon_fence_ring_emit(rdev, ring, *fence); 123 #ifdef TRACE_TODO 124 trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq); 125 #endif 126 return 0; 127 } 128 129 /** 130 * radeon_fence_process - process a fence 131 * 132 * @rdev: radeon_device pointer 133 * @ring: ring index the fence is associated with 134 * 135 * Checks the current fence value and wakes the fence queue 136 * if the sequence number has increased (all asics). 137 */ 138 void radeon_fence_process(struct radeon_device *rdev, int ring) 139 { 140 uint64_t seq, last_seq, last_emitted; 141 unsigned count_loop = 0; 142 bool wake = false; 143 144 /* Note there is a scenario here for an infinite loop but it's 145 * very unlikely to happen. For it to happen, the current polling 146 * process need to be interrupted by another process and another 147 * process needs to update the last_seq btw the atomic read and 148 * xchg of the current process. 149 * 150 * More over for this to go in infinite loop there need to be 151 * continuously new fence signaled ie radeon_fence_read needs 152 * to return a different value each time for both the currently 153 * polling process and the other process that xchg the last_seq 154 * btw atomic read and xchg of the current process. And the 155 * value the other process set as last seq must be higher than 156 * the seq value we just read. Which means that current process 157 * need to be interrupted after radeon_fence_read and before 158 * atomic xchg. 159 * 160 * To be even more safe we count the number of time we loop and 161 * we bail after 10 loop just accepting the fact that we might 162 * have temporarly set the last_seq not to the true real last 163 * seq but to an older one. 164 */ 165 last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq); 166 do { 167 last_emitted = rdev->fence_drv[ring].sync_seq[ring]; 168 seq = radeon_fence_read(rdev, ring); 169 seq |= last_seq & 0xffffffff00000000LL; 170 if (seq < last_seq) { 171 seq &= 0xffffffff; 172 seq |= last_emitted & 0xffffffff00000000LL; 173 } 174 175 if (seq <= last_seq || seq > last_emitted) { 176 break; 177 } 178 /* If we loop over we don't want to return without 179 * checking if a fence is signaled as it means that the 180 * seq we just read is different from the previous on. 181 */ 182 wake = true; 183 last_seq = seq; 184 if ((count_loop++) > 10) { 185 /* We looped over too many time leave with the 186 * fact that we might have set an older fence 187 * seq then the current real last seq as signaled 188 * by the hw. 189 */ 190 break; 191 } 192 } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); 193 194 if (wake) 195 wake_up_all(&rdev->fence_queue); 196 } 197 198 /** 199 * radeon_fence_destroy - destroy a fence 200 * 201 * @kref: fence kref 202 * 203 * Frees the fence object (all asics). 204 */ 205 static void radeon_fence_destroy(struct radeon_fence *fence) 206 { 207 208 kfree(fence); 209 } 210 211 /** 212 * radeon_fence_seq_signaled - check if a fence sequence number has signaled 213 * 214 * @rdev: radeon device pointer 215 * @seq: sequence number 216 * @ring: ring index the fence is associated with 217 * 218 * Check if the last signaled fence sequnce number is >= the requested 219 * sequence number (all asics). 220 * Returns true if the fence has signaled (current fence value 221 * is >= requested value) or false if it has not (current fence 222 * value is < the requested value. Helper function for 223 * radeon_fence_signaled(). 224 */ 225 static bool radeon_fence_seq_signaled(struct radeon_device *rdev, 226 u64 seq, unsigned ring) 227 { 228 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { 229 return true; 230 } 231 /* poll new last sequence at least once */ 232 radeon_fence_process(rdev, ring); 233 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { 234 return true; 235 } 236 return false; 237 } 238 239 /** 240 * radeon_fence_signaled - check if a fence has signaled 241 * 242 * @fence: radeon fence object 243 * 244 * Check if the requested fence has signaled (all asics). 245 * Returns true if the fence has signaled or false if it has not. 246 */ 247 bool radeon_fence_signaled(struct radeon_fence *fence) 248 { 249 if (!fence) { 250 return true; 251 } 252 if (fence->seq == RADEON_FENCE_SIGNALED_SEQ) { 253 return true; 254 } 255 if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { 256 fence->seq = RADEON_FENCE_SIGNALED_SEQ; 257 return true; 258 } 259 return false; 260 } 261 262 /** 263 * radeon_fence_any_seq_signaled - check if any sequence number is signaled 264 * 265 * @rdev: radeon device pointer 266 * @seq: sequence numbers 267 * 268 * Check if the last signaled fence sequnce number is >= the requested 269 * sequence number (all asics). 270 * Returns true if any has signaled (current value is >= requested value) 271 * or false if it has not. Helper function for radeon_fence_wait_seq. 272 */ 273 static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq) 274 { 275 unsigned i; 276 277 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 278 if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) 279 return true; 280 } 281 return false; 282 } 283 284 /** 285 * radeon_fence_wait_seq - wait for a specific sequence numbers 286 * 287 * @rdev: radeon device pointer 288 * @target_seq: sequence number(s) we want to wait for 289 * @intr: use interruptable sleep 290 * 291 * Wait for the requested sequence number(s) to be written by any ring 292 * (all asics). Sequnce number array is indexed by ring id. 293 * @intr selects whether to use interruptable (true) or non-interruptable 294 * (false) sleep when waiting for the sequence number. Helper function 295 * for radeon_fence_wait_*(). 296 * Returns 0 if the sequence number has passed, error for all other cases. 297 * -EDEADLK is returned when a GPU lockup has been detected. 298 */ 299 static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 *target_seq, 300 bool intr) 301 { 302 uint64_t last_seq[RADEON_NUM_RINGS]; 303 bool signaled; 304 int i, r; 305 306 while (!radeon_fence_any_seq_signaled(rdev, target_seq)) { 307 308 /* Save current sequence values, used to check for GPU lockups */ 309 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 310 if (!target_seq[i]) 311 continue; 312 313 last_seq[i] = atomic64_read(&rdev->fence_drv[i].last_seq); 314 #if 0 315 trace_radeon_fence_wait_begin(rdev->ddev, i, target_seq[i]); 316 #endif 317 radeon_irq_kms_sw_irq_get(rdev, i); 318 } 319 320 if (intr) { 321 r = wait_event_interruptible_timeout(rdev->fence_queue, ( 322 (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)) 323 || rdev->needs_reset), RADEON_FENCE_JIFFIES_TIMEOUT); 324 } else { 325 r = wait_event_timeout(rdev->fence_queue, ( 326 (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)) 327 || rdev->needs_reset), RADEON_FENCE_JIFFIES_TIMEOUT); 328 } 329 330 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 331 if (!target_seq[i]) 332 continue; 333 334 radeon_irq_kms_sw_irq_put(rdev, i); 335 #if 0 336 trace_radeon_fence_wait_end(rdev->ddev, i, target_seq[i]); 337 #endif 338 } 339 340 if (unlikely(r < 0)) 341 return r; 342 343 if (unlikely(!signaled)) { 344 if (rdev->needs_reset) 345 return -EDEADLK; 346 347 /* we were interrupted for some reason and fence 348 * isn't signaled yet, resume waiting */ 349 if (r) 350 continue; 351 352 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 353 if (!target_seq[i]) 354 continue; 355 356 if (last_seq[i] != atomic64_read(&rdev->fence_drv[i].last_seq)) 357 break; 358 } 359 360 if (i != RADEON_NUM_RINGS) 361 continue; 362 363 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 364 if (!target_seq[i]) 365 continue; 366 367 if (radeon_ring_is_lockup(rdev, i, &rdev->ring[i])) 368 break; 369 } 370 371 if (i < RADEON_NUM_RINGS) { 372 /* good news we believe it's a lockup */ 373 dev_warn(rdev->dev, "GPU lockup (waiting for " 374 "0x%016lx last fence id 0x%016lx on" 375 " ring %d)\n", 376 target_seq[i], last_seq[i], i); 377 378 /* remember that we need an reset */ 379 rdev->needs_reset = true; 380 wake_up_all(&rdev->fence_queue); 381 return -EDEADLK; 382 } 383 } 384 } 385 return 0; 386 } 387 388 /** 389 * radeon_fence_wait - wait for a fence to signal 390 * 391 * @fence: radeon fence object 392 * @intr: use interruptable sleep 393 * 394 * Wait for the requested fence to signal (all asics). 395 * @intr selects whether to use interruptable (true) or non-interruptable 396 * (false) sleep when waiting for the fence. 397 * Returns 0 if the fence has passed, error for all other cases. 398 */ 399 int radeon_fence_wait(struct radeon_fence *fence, bool intr) 400 { 401 uint64_t seq[RADEON_NUM_RINGS] = {}; 402 int r; 403 404 if (fence == NULL) { 405 WARN(1, "Querying an invalid fence : %p !\n", fence); 406 return -EINVAL; 407 } 408 409 seq[fence->ring] = fence->seq; 410 if (seq[fence->ring] == RADEON_FENCE_SIGNALED_SEQ) 411 return 0; 412 413 r = radeon_fence_wait_seq(fence->rdev, seq, intr); 414 if (r) 415 return r; 416 417 fence->seq = RADEON_FENCE_SIGNALED_SEQ; 418 return 0; 419 } 420 421 /** 422 * radeon_fence_wait_any - wait for a fence to signal on any ring 423 * 424 * @rdev: radeon device pointer 425 * @fences: radeon fence object(s) 426 * @intr: use interruptable sleep 427 * 428 * Wait for any requested fence to signal (all asics). Fence 429 * array is indexed by ring id. @intr selects whether to use 430 * interruptable (true) or non-interruptable (false) sleep when 431 * waiting for the fences. Used by the suballocator. 432 * Returns 0 if any fence has passed, error for all other cases. 433 */ 434 int radeon_fence_wait_any(struct radeon_device *rdev, 435 struct radeon_fence **fences, 436 bool intr) 437 { 438 uint64_t seq[RADEON_NUM_RINGS]; 439 unsigned i, num_rings = 0; 440 int r; 441 442 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 443 seq[i] = 0; 444 445 if (!fences[i]) { 446 continue; 447 } 448 449 seq[i] = fences[i]->seq; 450 ++num_rings; 451 452 /* test if something was allready signaled */ 453 if (seq[i] == RADEON_FENCE_SIGNALED_SEQ) 454 return 0; 455 } 456 457 /* nothing to wait for ? */ 458 if (num_rings == 0) 459 return -ENOENT; 460 461 r = radeon_fence_wait_seq(rdev, seq, intr); 462 if (r) { 463 return r; 464 } 465 return 0; 466 } 467 468 /** 469 * radeon_fence_wait_next - wait for the next fence to signal 470 * 471 * @rdev: radeon device pointer 472 * @ring: ring index the fence is associated with 473 * 474 * Wait for the next fence on the requested ring to signal (all asics). 475 * Returns 0 if the next fence has passed, error for all other cases. 476 * Caller must hold ring lock. 477 */ 478 int radeon_fence_wait_next(struct radeon_device *rdev, int ring) 479 { 480 uint64_t seq[RADEON_NUM_RINGS] = {}; 481 482 seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; 483 if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) { 484 /* nothing to wait for, last_seq is 485 already the last emited fence */ 486 return -ENOENT; 487 } 488 return radeon_fence_wait_seq(rdev, seq, false); 489 } 490 491 /** 492 * radeon_fence_wait_empty - wait for all fences to signal 493 * 494 * @rdev: radeon device pointer 495 * @ring: ring index the fence is associated with 496 * 497 * Wait for all fences on the requested ring to signal (all asics). 498 * Returns 0 if the fences have passed, error for all other cases. 499 * Caller must hold ring lock. 500 */ 501 int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) 502 { 503 uint64_t seq[RADEON_NUM_RINGS] = {}; 504 int r; 505 506 seq[ring] = rdev->fence_drv[ring].sync_seq[ring]; 507 if (!seq[ring]) 508 return 0; 509 510 r = radeon_fence_wait_seq(rdev, seq, false); 511 if (r) { 512 if (r == -EDEADLK) 513 return -EDEADLK; 514 515 dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%d)\n", 516 ring, r); 517 } 518 return 0; 519 } 520 521 /** 522 * radeon_fence_ref - take a ref on a fence 523 * 524 * @fence: radeon fence object 525 * 526 * Take a reference on a fence (all asics). 527 * Returns the fence. 528 */ 529 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) 530 { 531 refcount_acquire(&fence->kref); 532 return fence; 533 } 534 535 /** 536 * radeon_fence_unref - remove a ref on a fence 537 * 538 * @fence: radeon fence object 539 * 540 * Remove a reference on a fence (all asics). 541 */ 542 void radeon_fence_unref(struct radeon_fence **fence) 543 { 544 struct radeon_fence *tmp = *fence; 545 546 *fence = NULL; 547 if (tmp) { 548 if (refcount_release(&tmp->kref)) { 549 radeon_fence_destroy(tmp); 550 } 551 } 552 } 553 554 /** 555 * radeon_fence_count_emitted - get the count of emitted fences 556 * 557 * @rdev: radeon device pointer 558 * @ring: ring index the fence is associated with 559 * 560 * Get the number of fences emitted on the requested ring (all asics). 561 * Returns the number of emitted fences on the ring. Used by the 562 * dynpm code to ring track activity. 563 */ 564 unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring) 565 { 566 uint64_t emitted; 567 568 /* We are not protected by ring lock when reading the last sequence 569 * but it's ok to report slightly wrong fence count here. 570 */ 571 radeon_fence_process(rdev, ring); 572 emitted = rdev->fence_drv[ring].sync_seq[ring] 573 - atomic64_read(&rdev->fence_drv[ring].last_seq); 574 /* to avoid 32bits warp around */ 575 if (emitted > 0x10000000) { 576 emitted = 0x10000000; 577 } 578 return (unsigned)emitted; 579 } 580 581 /** 582 * radeon_fence_need_sync - do we need a semaphore 583 * 584 * @fence: radeon fence object 585 * @dst_ring: which ring to check against 586 * 587 * Check if the fence needs to be synced against another ring 588 * (all asics). If so, we need to emit a semaphore. 589 * Returns true if we need to sync with another ring, false if 590 * not. 591 */ 592 bool radeon_fence_need_sync(struct radeon_fence *fence, int dst_ring) 593 { 594 struct radeon_fence_driver *fdrv; 595 596 if (!fence) { 597 return false; 598 } 599 600 if (fence->ring == dst_ring) { 601 return false; 602 } 603 604 /* we are protected by the ring mutex */ 605 fdrv = &fence->rdev->fence_drv[dst_ring]; 606 if (fence->seq <= fdrv->sync_seq[fence->ring]) { 607 return false; 608 } 609 610 return true; 611 } 612 613 /** 614 * radeon_fence_note_sync - record the sync point 615 * 616 * @fence: radeon fence object 617 * @dst_ring: which ring to check against 618 * 619 * Note the sequence number at which point the fence will 620 * be synced with the requested ring (all asics). 621 */ 622 void radeon_fence_note_sync(struct radeon_fence *fence, int dst_ring) 623 { 624 struct radeon_fence_driver *dst, *src; 625 unsigned i; 626 627 if (!fence) { 628 return; 629 } 630 631 if (fence->ring == dst_ring) { 632 return; 633 } 634 635 /* we are protected by the ring mutex */ 636 src = &fence->rdev->fence_drv[fence->ring]; 637 dst = &fence->rdev->fence_drv[dst_ring]; 638 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 639 if (i == dst_ring) { 640 continue; 641 } 642 dst->sync_seq[i] = max(dst->sync_seq[i], src->sync_seq[i]); 643 } 644 } 645 646 /** 647 * radeon_fence_driver_start_ring - make the fence driver 648 * ready for use on the requested ring. 649 * 650 * @rdev: radeon device pointer 651 * @ring: ring index to start the fence driver on 652 * 653 * Make the fence driver ready for processing (all asics). 654 * Not all asics have all rings, so each asic will only 655 * start the fence driver on the rings it has. 656 * Returns 0 for success, errors for failure. 657 */ 658 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) 659 { 660 uint64_t index; 661 int r; 662 663 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); 664 if (rdev->wb.use_event || !radeon_ring_supports_scratch_reg(rdev, &rdev->ring[ring])) { 665 rdev->fence_drv[ring].scratch_reg = 0; 666 if (ring != R600_RING_TYPE_UVD_INDEX) { 667 index = R600_WB_EVENT_OFFSET + ring * 4; 668 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; 669 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + 670 index; 671 672 } else { 673 /* put fence directly behind firmware */ 674 index = ALIGN(rdev->uvd_fw->datasize, 8); 675 rdev->fence_drv[ring].cpu_addr = (void*)((uint8_t*)rdev->uvd.cpu_addr + index); 676 rdev->fence_drv[ring].gpu_addr = rdev->uvd.gpu_addr + index; 677 } 678 679 } else { 680 r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg); 681 if (r) { 682 dev_err(rdev->dev, "fence failed to get scratch register\n"); 683 return r; 684 } 685 index = RADEON_WB_SCRATCH_OFFSET + 686 rdev->fence_drv[ring].scratch_reg - 687 rdev->scratch.reg_base; 688 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; 689 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; 690 } 691 radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring); 692 rdev->fence_drv[ring].initialized = true; 693 dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016jx and cpu addr 0x%p\n", 694 ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr); 695 return 0; 696 } 697 698 /** 699 * radeon_fence_driver_init_ring - init the fence driver 700 * for the requested ring. 701 * 702 * @rdev: radeon device pointer 703 * @ring: ring index to start the fence driver on 704 * 705 * Init the fence driver for the requested ring (all asics). 706 * Helper function for radeon_fence_driver_init(). 707 */ 708 static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring) 709 { 710 int i; 711 712 rdev->fence_drv[ring].scratch_reg = -1; 713 rdev->fence_drv[ring].cpu_addr = NULL; 714 rdev->fence_drv[ring].gpu_addr = 0; 715 for (i = 0; i < RADEON_NUM_RINGS; ++i) 716 rdev->fence_drv[ring].sync_seq[i] = 0; 717 atomic64_set(&rdev->fence_drv[ring].last_seq, 0); 718 rdev->fence_drv[ring].initialized = false; 719 } 720 721 /** 722 * radeon_fence_driver_init - init the fence driver 723 * for all possible rings. 724 * 725 * @rdev: radeon device pointer 726 * 727 * Init the fence driver for all possible rings (all asics). 728 * Not all asics have all rings, so each asic will only 729 * start the fence driver on the rings it has using 730 * radeon_fence_driver_start_ring(). 731 * Returns 0 for success. 732 */ 733 int radeon_fence_driver_init(struct radeon_device *rdev) 734 { 735 int ring; 736 737 init_waitqueue_head(&rdev->fence_queue); 738 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { 739 radeon_fence_driver_init_ring(rdev, ring); 740 } 741 if (radeon_debugfs_fence_init(rdev)) { 742 dev_err(rdev->dev, "fence debugfs file creation failed\n"); 743 } 744 return 0; 745 } 746 747 /** 748 * radeon_fence_driver_fini - tear down the fence driver 749 * for all possible rings. 750 * 751 * @rdev: radeon device pointer 752 * 753 * Tear down the fence driver for all possible rings (all asics). 754 */ 755 void radeon_fence_driver_fini(struct radeon_device *rdev) 756 { 757 int ring, r; 758 759 lockmgr(&rdev->ring_lock, LK_EXCLUSIVE); 760 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { 761 if (!rdev->fence_drv[ring].initialized) 762 continue; 763 r = radeon_fence_wait_empty(rdev, ring); 764 if (r) { 765 /* no need to trigger GPU reset as we are unloading */ 766 radeon_fence_driver_force_completion(rdev, ring); 767 } 768 wake_up_all(&rdev->fence_queue); 769 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); 770 rdev->fence_drv[ring].initialized = false; 771 } 772 lockmgr(&rdev->ring_lock, LK_RELEASE); 773 } 774 775 /** 776 * radeon_fence_driver_force_completion - force all fence waiter to complete 777 * 778 * @rdev: radeon device pointer 779 * @ring: the ring to complete 780 * 781 * In case of GPU reset failure make sure no process keep waiting on fence 782 * that will never complete. 783 */ 784 void radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring) 785 { 786 if (rdev->fence_drv[ring].initialized) 787 radeon_fence_write(rdev, rdev->fence_drv[ring].sync_seq[ring], ring); 788 } 789 790 791 /* 792 * Fence debugfs 793 */ 794 #if defined(CONFIG_DEBUG_FS) 795 static int radeon_debugfs_fence_info(struct seq_file *m, void *data) 796 { 797 struct drm_info_node *node = (struct drm_info_node *)m->private; 798 struct drm_device *dev = node->minor->dev; 799 struct radeon_device *rdev = dev->dev_private; 800 int i, j; 801 802 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 803 if (!rdev->fence_drv[i].initialized) 804 continue; 805 806 radeon_fence_process(rdev, i); 807 808 seq_printf(m, "--- ring %d ---\n", i); 809 seq_printf(m, "Last signaled fence 0x%016llx\n", 810 (unsigned long long)atomic_load_acq_64(&rdev->fence_drv[i].last_seq)); 811 seq_printf(m, "Last emitted 0x%016llx\n", 812 rdev->fence_drv[i].sync_seq[i]); 813 814 for (j = 0; j < RADEON_NUM_RINGS; ++j) { 815 if (i != j && rdev->fence_drv[j].initialized) 816 seq_printf(m, "Last sync to ring %d 0x%016llx\n", 817 j, rdev->fence_drv[i].sync_seq[j]); 818 } 819 } 820 return 0; 821 } 822 823 /** 824 * radeon_debugfs_gpu_reset - manually trigger a gpu reset 825 * 826 * Manually trigger a gpu reset at the next fence wait. 827 */ 828 static int radeon_debugfs_gpu_reset(struct seq_file *m, void *data) 829 { 830 struct drm_info_node *node = (struct drm_info_node *) m->private; 831 struct drm_device *dev = node->minor->dev; 832 struct radeon_device *rdev = dev->dev_private; 833 834 down_read(&rdev->exclusive_lock); 835 seq_printf(m, "%d\n", rdev->needs_reset); 836 rdev->needs_reset = true; 837 up_read(&rdev->exclusive_lock); 838 839 return 0; 840 } 841 842 static struct drm_info_list radeon_debugfs_fence_list[] = { 843 {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL}, 844 {"radeon_gpu_reset", &radeon_debugfs_gpu_reset, 0, NULL} 845 }; 846 #endif 847 848 int radeon_debugfs_fence_init(struct radeon_device *rdev) 849 { 850 #if defined(CONFIG_DEBUG_FS) 851 return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 2); 852 #else 853 return 0; 854 #endif 855 } 856