1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Copyright 2021 HabanaLabs, Ltd. 5 * All Rights Reserved. 6 */ 7 8 #include <linux/vmalloc.h> 9 #include <uapi/drm/habanalabs_accel.h> 10 #include "habanalabs.h" 11 12 /** 13 * hl_format_as_binary - helper function, format an integer as binary 14 * using supplied scratch buffer 15 * @buf: the buffer to use 16 * @buf_len: buffer capacity 17 * @n: number to format 18 * 19 * Returns pointer to buffer 20 */ 21 char *hl_format_as_binary(char *buf, size_t buf_len, u32 n) 22 { 23 int i; 24 u32 bit; 25 bool leading0 = true; 26 char *wrptr = buf; 27 28 if (buf_len > 0 && buf_len < 3) { 29 *wrptr = '\0'; 30 return buf; 31 } 32 33 wrptr[0] = '0'; 34 wrptr[1] = 'b'; 35 wrptr += 2; 36 /* Remove 3 characters from length for '0b' and '\0' termination */ 37 buf_len -= 3; 38 39 for (i = 0; i < sizeof(n) * BITS_PER_BYTE && buf_len; ++i, n <<= 1) { 40 /* Writing bit calculation in one line would cause a false 41 * positive static code analysis error, so splitting. 42 */ 43 bit = n & (1 << (sizeof(n) * BITS_PER_BYTE - 1)); 44 bit = !!bit; 45 leading0 &= !bit; 46 if (!leading0) { 47 *wrptr = '0' + bit; 48 ++wrptr; 49 } 50 } 51 52 *wrptr = '\0'; 53 54 return buf; 55 } 56 57 /** 58 * resize_to_fit - helper function, resize buffer to fit given amount of data 59 * @buf: destination buffer double pointer 60 * @size: pointer to the size container 61 * @desired_size: size the buffer must contain 62 * 63 * Returns 0 on success or error code on failure. 64 * On success, the size of buffer is at least desired_size. Buffer is allocated 65 * via vmalloc and must be freed with vfree. 66 */ 67 static int resize_to_fit(char **buf, size_t *size, size_t desired_size) 68 { 69 char *resized_buf; 70 size_t new_size; 71 72 if (*size >= desired_size) 73 return 0; 74 75 /* Not enough space to print all, have to resize */ 76 new_size = max_t(size_t, PAGE_SIZE, round_up(desired_size, PAGE_SIZE)); 77 resized_buf = vmalloc(new_size); 78 if (!resized_buf) 79 return -ENOMEM; 80 memcpy(resized_buf, *buf, *size); 81 vfree(*buf); 82 *buf = resized_buf; 83 *size = new_size; 84 85 return 1; 86 } 87 88 /** 89 * hl_snprintf_resize() - print formatted data to buffer, resize as needed 90 * @buf: buffer double pointer, to be written to and resized, must be either 91 * NULL or allocated with vmalloc. 92 * @size: current size of the buffer 93 * @offset: current offset to write to 94 * @format: format of the data 95 * 96 * This function will write formatted data into the buffer. If buffer is not 97 * large enough, it will be resized using vmalloc. Size may be modified if the 98 * buffer was resized, offset will be advanced by the number of bytes written 99 * not including the terminating character 100 * 101 * Returns 0 on success or error code on failure 102 * 103 * Note that the buffer has to be manually released using vfree. 104 */ 105 int hl_snprintf_resize(char **buf, size_t *size, size_t *offset, 106 const char *format, ...) 107 { 108 va_list args; 109 size_t length; 110 int rc; 111 112 if (*buf == NULL && (*size != 0 || *offset != 0)) 113 return -EINVAL; 114 115 va_start(args, format); 116 length = vsnprintf(*buf + *offset, *size - *offset, format, args); 117 va_end(args); 118 119 rc = resize_to_fit(buf, size, *offset + length + 1); 120 if (rc < 0) 121 return rc; 122 else if (rc > 0) { 123 /* Resize was needed, write again */ 124 va_start(args, format); 125 length = vsnprintf(*buf + *offset, *size - *offset, format, 126 args); 127 va_end(args); 128 } 129 130 *offset += length; 131 132 return 0; 133 } 134 135 /** 136 * hl_sync_engine_to_string - convert engine type enum to string literal 137 * @engine_type: engine type (TPC/MME/DMA) 138 * 139 * Return the resolved string literal 140 */ 141 const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type) 142 { 143 switch (engine_type) { 144 case ENGINE_DMA: 145 return "DMA"; 146 case ENGINE_MME: 147 return "MME"; 148 case ENGINE_TPC: 149 return "TPC"; 150 } 151 return "Invalid Engine Type"; 152 } 153 154 /** 155 * hl_print_resize_sync_engine - helper function, format engine name and ID 156 * using hl_snprintf_resize 157 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 158 * @size: pointer to the size container 159 * @offset: pointer to the offset container 160 * @engine_type: engine type (TPC/MME/DMA) 161 * @engine_id: engine numerical id 162 * 163 * Returns 0 on success or error code on failure 164 */ 165 static int hl_print_resize_sync_engine(char **buf, size_t *size, size_t *offset, 166 enum hl_sync_engine_type engine_type, 167 u32 engine_id) 168 { 169 return hl_snprintf_resize(buf, size, offset, "%s%u", 170 hl_sync_engine_to_string(engine_type), engine_id); 171 } 172 173 /** 174 * hl_state_dump_get_sync_name - transform sync object id to name if available 175 * @hdev: pointer to the device 176 * @sync_id: sync object id 177 * 178 * Returns a name literal or NULL if not resolved. 179 * Note: returning NULL shall not be considered as a failure, as not all 180 * sync objects are named. 181 */ 182 const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id) 183 { 184 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 185 struct hl_hw_obj_name_entry *entry; 186 187 hash_for_each_possible(sds->so_id_to_str_tb, entry, 188 node, sync_id) 189 if (sync_id == entry->id) 190 return entry->name; 191 192 return NULL; 193 } 194 195 /** 196 * hl_state_dump_get_monitor_name - transform monitor object dump to monitor 197 * name if available 198 * @hdev: pointer to the device 199 * @mon: monitor state dump 200 * 201 * Returns a name literal or NULL if not resolved. 202 * Note: returning NULL shall not be considered as a failure, as not all 203 * monitors are named. 204 */ 205 const char *hl_state_dump_get_monitor_name(struct hl_device *hdev, 206 struct hl_mon_state_dump *mon) 207 { 208 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 209 struct hl_hw_obj_name_entry *entry; 210 211 hash_for_each_possible(sds->monitor_id_to_str_tb, 212 entry, node, mon->id) 213 if (mon->id == entry->id) 214 return entry->name; 215 216 return NULL; 217 } 218 219 /** 220 * hl_state_dump_free_sync_to_engine_map - free sync object to engine map 221 * @map: sync object to engine map 222 * 223 * Note: generic free implementation, the allocation is implemented per ASIC. 224 */ 225 void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map) 226 { 227 struct hl_sync_to_engine_map_entry *entry; 228 struct hlist_node *tmp_node; 229 int i; 230 231 hash_for_each_safe(map->tb, i, tmp_node, entry, node) { 232 hash_del(&entry->node); 233 kfree(entry); 234 } 235 } 236 237 /** 238 * hl_state_dump_get_sync_to_engine - transform sync_id to 239 * hl_sync_to_engine_map_entry if available for current id 240 * @map: sync object to engine map 241 * @sync_id: sync object id 242 * 243 * Returns the translation entry if found or NULL if not. 244 * Note, returned NULL shall not be considered as a failure as the map 245 * does not cover all possible, it is a best effort sync ids. 246 */ 247 static struct hl_sync_to_engine_map_entry * 248 hl_state_dump_get_sync_to_engine(struct hl_sync_to_engine_map *map, u32 sync_id) 249 { 250 struct hl_sync_to_engine_map_entry *entry; 251 252 hash_for_each_possible(map->tb, entry, node, sync_id) 253 if (entry->sync_id == sync_id) 254 return entry; 255 return NULL; 256 } 257 258 /** 259 * hl_state_dump_read_sync_objects - read sync objects array 260 * @hdev: pointer to the device 261 * @index: sync manager block index starting with E_N 262 * 263 * Returns array of size SP_SYNC_OBJ_AMOUNT on success or NULL on failure 264 */ 265 static u32 *hl_state_dump_read_sync_objects(struct hl_device *hdev, u32 index) 266 { 267 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 268 u32 *sync_objects; 269 s64 base_addr; /* Base addr can be negative */ 270 int i; 271 272 base_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] + 273 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index; 274 275 sync_objects = vmalloc(sds->props[SP_SYNC_OBJ_AMOUNT] * sizeof(u32)); 276 if (!sync_objects) 277 return NULL; 278 279 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i) 280 sync_objects[i] = RREG32(base_addr + i * sizeof(u32)); 281 282 return sync_objects; 283 } 284 285 /** 286 * hl_state_dump_free_sync_objects - free sync objects array allocated by 287 * hl_state_dump_read_sync_objects 288 * @sync_objects: sync objects array 289 */ 290 static void hl_state_dump_free_sync_objects(u32 *sync_objects) 291 { 292 vfree(sync_objects); 293 } 294 295 296 /** 297 * hl_state_dump_print_syncs_single_block - print active sync objects on a 298 * single block 299 * @hdev: pointer to the device 300 * @index: sync manager block index starting with E_N 301 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 302 * @size: pointer to the size container 303 * @offset: pointer to the offset container 304 * @map: sync engines names map 305 * 306 * Returns 0 on success or error code on failure 307 */ 308 static int 309 hl_state_dump_print_syncs_single_block(struct hl_device *hdev, u32 index, 310 char **buf, size_t *size, size_t *offset, 311 struct hl_sync_to_engine_map *map) 312 { 313 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 314 const char *sync_name; 315 u32 *sync_objects = NULL; 316 int rc = 0, i; 317 318 if (sds->sync_namager_names) { 319 rc = hl_snprintf_resize( 320 buf, size, offset, "%s\n", 321 sds->sync_namager_names[index]); 322 if (rc) 323 goto out; 324 } 325 326 sync_objects = hl_state_dump_read_sync_objects(hdev, index); 327 if (!sync_objects) { 328 rc = -ENOMEM; 329 goto out; 330 } 331 332 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i) { 333 struct hl_sync_to_engine_map_entry *entry; 334 u64 sync_object_addr; 335 336 if (!sync_objects[i]) 337 continue; 338 339 sync_object_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] + 340 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index + 341 i * sizeof(u32); 342 343 rc = hl_snprintf_resize(buf, size, offset, "sync id: %u", i); 344 if (rc) 345 goto free_sync_objects; 346 sync_name = hl_state_dump_get_sync_name(hdev, i); 347 if (sync_name) { 348 rc = hl_snprintf_resize(buf, size, offset, " %s", 349 sync_name); 350 if (rc) 351 goto free_sync_objects; 352 } 353 rc = hl_snprintf_resize(buf, size, offset, ", value: %u", 354 sync_objects[i]); 355 if (rc) 356 goto free_sync_objects; 357 358 /* Append engine string */ 359 entry = hl_state_dump_get_sync_to_engine(map, 360 (u32)sync_object_addr); 361 if (entry) { 362 rc = hl_snprintf_resize(buf, size, offset, 363 ", Engine: "); 364 if (rc) 365 goto free_sync_objects; 366 rc = hl_print_resize_sync_engine(buf, size, offset, 367 entry->engine_type, 368 entry->engine_id); 369 if (rc) 370 goto free_sync_objects; 371 } 372 373 rc = hl_snprintf_resize(buf, size, offset, "\n"); 374 if (rc) 375 goto free_sync_objects; 376 } 377 378 free_sync_objects: 379 hl_state_dump_free_sync_objects(sync_objects); 380 out: 381 return rc; 382 } 383 384 /** 385 * hl_state_dump_print_syncs - print active sync objects 386 * @hdev: pointer to the device 387 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 388 * @size: pointer to the size container 389 * @offset: pointer to the offset container 390 * 391 * Returns 0 on success or error code on failure 392 */ 393 static int hl_state_dump_print_syncs(struct hl_device *hdev, 394 char **buf, size_t *size, 395 size_t *offset) 396 397 { 398 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 399 struct hl_sync_to_engine_map *map; 400 u32 index; 401 int rc = 0; 402 403 map = kzalloc(sizeof(*map), GFP_KERNEL); 404 if (!map) 405 return -ENOMEM; 406 407 rc = sds->funcs.gen_sync_to_engine_map(hdev, map); 408 if (rc) 409 goto free_map_mem; 410 411 rc = hl_snprintf_resize(buf, size, offset, "Non zero sync objects:\n"); 412 if (rc) 413 goto out; 414 415 if (sds->sync_namager_names) { 416 for (index = 0; sds->sync_namager_names[index]; ++index) { 417 rc = hl_state_dump_print_syncs_single_block( 418 hdev, index, buf, size, offset, map); 419 if (rc) 420 goto out; 421 } 422 } else { 423 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) { 424 rc = hl_state_dump_print_syncs_single_block( 425 hdev, index, buf, size, offset, map); 426 if (rc) 427 goto out; 428 } 429 } 430 431 out: 432 hl_state_dump_free_sync_to_engine_map(map); 433 free_map_mem: 434 kfree(map); 435 436 return rc; 437 } 438 439 /** 440 * hl_state_dump_alloc_read_sm_block_monitors - read monitors for a specific 441 * block 442 * @hdev: pointer to the device 443 * @index: sync manager block index starting with E_N 444 * 445 * Returns an array of monitor data of size SP_MONITORS_AMOUNT or NULL 446 * on error 447 */ 448 static struct hl_mon_state_dump * 449 hl_state_dump_alloc_read_sm_block_monitors(struct hl_device *hdev, u32 index) 450 { 451 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 452 struct hl_mon_state_dump *monitors; 453 s64 base_addr; /* Base addr can be negative */ 454 int i; 455 456 monitors = vmalloc(sds->props[SP_MONITORS_AMOUNT] * 457 sizeof(struct hl_mon_state_dump)); 458 if (!monitors) 459 return NULL; 460 461 base_addr = sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index; 462 463 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) { 464 monitors[i].id = i; 465 monitors[i].wr_addr_low = 466 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_LOW] + 467 i * sizeof(u32)); 468 469 monitors[i].wr_addr_high = 470 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_HIGH] + 471 i * sizeof(u32)); 472 473 monitors[i].wr_data = 474 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_DATA] + 475 i * sizeof(u32)); 476 477 monitors[i].arm_data = 478 RREG32(base_addr + sds->props[SP_MON_OBJ_ARM_DATA] + 479 i * sizeof(u32)); 480 481 monitors[i].status = 482 RREG32(base_addr + sds->props[SP_MON_OBJ_STATUS] + 483 i * sizeof(u32)); 484 } 485 486 return monitors; 487 } 488 489 /** 490 * hl_state_dump_free_monitors - free the monitors structure 491 * @monitors: monitors array created with 492 * hl_state_dump_alloc_read_sm_block_monitors 493 */ 494 static void hl_state_dump_free_monitors(struct hl_mon_state_dump *monitors) 495 { 496 vfree(monitors); 497 } 498 499 /** 500 * hl_state_dump_print_monitors_single_block - print active monitors on a 501 * single block 502 * @hdev: pointer to the device 503 * @index: sync manager block index starting with E_N 504 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 505 * @size: pointer to the size container 506 * @offset: pointer to the offset container 507 * 508 * Returns 0 on success or error code on failure 509 */ 510 static int hl_state_dump_print_monitors_single_block(struct hl_device *hdev, 511 u32 index, 512 char **buf, size_t *size, 513 size_t *offset) 514 { 515 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 516 struct hl_mon_state_dump *monitors = NULL; 517 int rc = 0, i; 518 519 if (sds->sync_namager_names) { 520 rc = hl_snprintf_resize( 521 buf, size, offset, "%s\n", 522 sds->sync_namager_names[index]); 523 if (rc) 524 goto out; 525 } 526 527 monitors = hl_state_dump_alloc_read_sm_block_monitors(hdev, index); 528 if (!monitors) { 529 rc = -ENOMEM; 530 goto out; 531 } 532 533 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) { 534 if (!(sds->funcs.monitor_valid(&monitors[i]))) 535 continue; 536 537 /* Monitor is valid, dump it */ 538 rc = sds->funcs.print_single_monitor(buf, size, offset, hdev, 539 &monitors[i]); 540 if (rc) 541 goto free_monitors; 542 543 hl_snprintf_resize(buf, size, offset, "\n"); 544 } 545 546 free_monitors: 547 hl_state_dump_free_monitors(monitors); 548 out: 549 return rc; 550 } 551 552 /** 553 * hl_state_dump_print_monitors - print active monitors 554 * @hdev: pointer to the device 555 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 556 * @size: pointer to the size container 557 * @offset: pointer to the offset container 558 * 559 * Returns 0 on success or error code on failure 560 */ 561 static int hl_state_dump_print_monitors(struct hl_device *hdev, 562 char **buf, size_t *size, 563 size_t *offset) 564 { 565 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 566 u32 index; 567 int rc = 0; 568 569 rc = hl_snprintf_resize(buf, size, offset, 570 "Valid (armed) monitor objects:\n"); 571 if (rc) 572 goto out; 573 574 if (sds->sync_namager_names) { 575 for (index = 0; sds->sync_namager_names[index]; ++index) { 576 rc = hl_state_dump_print_monitors_single_block( 577 hdev, index, buf, size, offset); 578 if (rc) 579 goto out; 580 } 581 } else { 582 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) { 583 rc = hl_state_dump_print_monitors_single_block( 584 hdev, index, buf, size, offset); 585 if (rc) 586 goto out; 587 } 588 } 589 590 out: 591 return rc; 592 } 593 594 /** 595 * hl_state_dump_print_engine_fences - print active fences for a specific 596 * engine 597 * @hdev: pointer to the device 598 * @engine_type: engine type to use 599 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 600 * @size: pointer to the size container 601 * @offset: pointer to the offset container 602 */ 603 static int 604 hl_state_dump_print_engine_fences(struct hl_device *hdev, 605 enum hl_sync_engine_type engine_type, 606 char **buf, size_t *size, size_t *offset) 607 { 608 struct hl_state_dump_specs *sds = &hdev->state_dump_specs; 609 int rc = 0, i, n_fences; 610 u64 base_addr, next_fence; 611 612 switch (engine_type) { 613 case ENGINE_TPC: 614 n_fences = sds->props[SP_NUM_OF_TPC_ENGINES]; 615 base_addr = sds->props[SP_TPC0_CMDQ]; 616 next_fence = sds->props[SP_NEXT_TPC]; 617 break; 618 case ENGINE_MME: 619 n_fences = sds->props[SP_NUM_OF_MME_ENGINES]; 620 base_addr = sds->props[SP_MME_CMDQ]; 621 next_fence = sds->props[SP_NEXT_MME]; 622 break; 623 case ENGINE_DMA: 624 n_fences = sds->props[SP_NUM_OF_DMA_ENGINES]; 625 base_addr = sds->props[SP_DMA_CMDQ]; 626 next_fence = sds->props[SP_DMA_QUEUES_OFFSET]; 627 break; 628 default: 629 return -EINVAL; 630 } 631 for (i = 0; i < n_fences; ++i) { 632 rc = sds->funcs.print_fences_single_engine( 633 hdev, 634 base_addr + next_fence * i + 635 sds->props[SP_FENCE0_CNT_OFFSET], 636 base_addr + next_fence * i + 637 sds->props[SP_CP_STS_OFFSET], 638 engine_type, i, buf, size, offset); 639 if (rc) 640 goto out; 641 } 642 out: 643 return rc; 644 } 645 646 /** 647 * hl_state_dump_print_fences - print active fences 648 * @hdev: pointer to the device 649 * @buf: destination buffer double pointer to be used with hl_snprintf_resize 650 * @size: pointer to the size container 651 * @offset: pointer to the offset container 652 */ 653 static int hl_state_dump_print_fences(struct hl_device *hdev, char **buf, 654 size_t *size, size_t *offset) 655 { 656 int rc = 0; 657 658 rc = hl_snprintf_resize(buf, size, offset, "Valid (armed) fences:\n"); 659 if (rc) 660 goto out; 661 662 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_TPC, buf, size, offset); 663 if (rc) 664 goto out; 665 666 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_MME, buf, size, offset); 667 if (rc) 668 goto out; 669 670 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_DMA, buf, size, offset); 671 if (rc) 672 goto out; 673 674 out: 675 return rc; 676 } 677 678 /** 679 * hl_state_dump() - dump system state 680 * @hdev: pointer to device structure 681 */ 682 int hl_state_dump(struct hl_device *hdev) 683 { 684 char *buf = NULL; 685 size_t offset = 0, size = 0; 686 int rc; 687 688 rc = hl_snprintf_resize(&buf, &size, &offset, 689 "Timestamp taken on: %llu\n\n", 690 ktime_to_ns(ktime_get())); 691 if (rc) 692 goto err; 693 694 rc = hl_state_dump_print_syncs(hdev, &buf, &size, &offset); 695 if (rc) 696 goto err; 697 698 hl_snprintf_resize(&buf, &size, &offset, "\n"); 699 700 rc = hl_state_dump_print_monitors(hdev, &buf, &size, &offset); 701 if (rc) 702 goto err; 703 704 hl_snprintf_resize(&buf, &size, &offset, "\n"); 705 706 rc = hl_state_dump_print_fences(hdev, &buf, &size, &offset); 707 if (rc) 708 goto err; 709 710 hl_snprintf_resize(&buf, &size, &offset, "\n"); 711 712 hl_debugfs_set_state_dump(hdev, buf, size); 713 714 return 0; 715 err: 716 vfree(buf); 717 return rc; 718 } 719