1 /* 2 * Copyright (c) 2018, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #define CREATE_TRACE_POINTS 33 #include "lib/eq.h" 34 #include "fw_tracer.h" 35 #include "fw_tracer_tracepoint.h" 36 37 static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer) 38 { 39 u32 *string_db_base_address_out = tracer->str_db.base_address_out; 40 u32 *string_db_size_out = tracer->str_db.size_out; 41 struct mlx5_core_dev *dev = tracer->dev; 42 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 43 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 44 void *mtrc_cap_sp; 45 int err, i; 46 47 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 48 MLX5_REG_MTRC_CAP, 0, 0); 49 if (err) { 50 mlx5_core_warn(dev, "FWTracer: Error reading tracer caps %d\n", 51 err); 52 return err; 53 } 54 55 if (!MLX5_GET(mtrc_cap, out, trace_to_memory)) { 56 mlx5_core_dbg(dev, "FWTracer: Device does not support logging traces to memory\n"); 57 return -ENOTSUPP; 58 } 59 60 tracer->trc_ver = MLX5_GET(mtrc_cap, out, trc_ver); 61 tracer->str_db.first_string_trace = 62 MLX5_GET(mtrc_cap, out, first_string_trace); 63 tracer->str_db.num_string_trace = 64 MLX5_GET(mtrc_cap, out, num_string_trace); 65 tracer->str_db.num_string_db = MLX5_GET(mtrc_cap, out, num_string_db); 66 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner); 67 68 for (i = 0; i < tracer->str_db.num_string_db; i++) { 69 mtrc_cap_sp = MLX5_ADDR_OF(mtrc_cap, out, string_db_param[i]); 70 string_db_base_address_out[i] = MLX5_GET(mtrc_string_db_param, 71 mtrc_cap_sp, 72 string_db_base_address); 73 string_db_size_out[i] = MLX5_GET(mtrc_string_db_param, 74 mtrc_cap_sp, string_db_size); 75 } 76 77 return err; 78 } 79 80 static int mlx5_set_mtrc_caps_trace_owner(struct mlx5_fw_tracer *tracer, 81 u32 *out, u32 out_size, 82 u8 trace_owner) 83 { 84 struct mlx5_core_dev *dev = tracer->dev; 85 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 86 87 MLX5_SET(mtrc_cap, in, trace_owner, trace_owner); 88 89 return mlx5_core_access_reg(dev, in, sizeof(in), out, out_size, 90 MLX5_REG_MTRC_CAP, 0, 1); 91 } 92 93 static int mlx5_fw_tracer_ownership_acquire(struct mlx5_fw_tracer *tracer) 94 { 95 struct mlx5_core_dev *dev = tracer->dev; 96 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 97 int err; 98 99 err = mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out), 100 MLX5_FW_TRACER_ACQUIRE_OWNERSHIP); 101 if (err) { 102 mlx5_core_warn(dev, "FWTracer: Acquire tracer ownership failed %d\n", 103 err); 104 return err; 105 } 106 107 tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner); 108 109 if (!tracer->owner) 110 return -EBUSY; 111 112 return 0; 113 } 114 115 static void mlx5_fw_tracer_ownership_release(struct mlx5_fw_tracer *tracer) 116 { 117 u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 118 119 mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out), 120 MLX5_FW_TRACER_RELEASE_OWNERSHIP); 121 tracer->owner = false; 122 } 123 124 static int mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer *tracer) 125 { 126 struct mlx5_core_dev *dev = tracer->dev; 127 struct device *ddev = &dev->pdev->dev; 128 dma_addr_t dma; 129 void *buff; 130 gfp_t gfp; 131 int err; 132 133 tracer->buff.size = TRACE_BUFFER_SIZE_BYTE; 134 135 gfp = GFP_KERNEL | __GFP_ZERO; 136 buff = (void *)__get_free_pages(gfp, 137 get_order(tracer->buff.size)); 138 if (!buff) { 139 err = -ENOMEM; 140 mlx5_core_warn(dev, "FWTracer: Failed to allocate pages, %d\n", err); 141 return err; 142 } 143 tracer->buff.log_buf = buff; 144 145 dma = dma_map_single(ddev, buff, tracer->buff.size, DMA_FROM_DEVICE); 146 if (dma_mapping_error(ddev, dma)) { 147 mlx5_core_warn(dev, "FWTracer: Unable to map DMA: %d\n", 148 dma_mapping_error(ddev, dma)); 149 err = -ENOMEM; 150 goto free_pages; 151 } 152 tracer->buff.dma = dma; 153 154 return 0; 155 156 free_pages: 157 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size)); 158 159 return err; 160 } 161 162 static void mlx5_fw_tracer_destroy_log_buf(struct mlx5_fw_tracer *tracer) 163 { 164 struct mlx5_core_dev *dev = tracer->dev; 165 struct device *ddev = &dev->pdev->dev; 166 167 if (!tracer->buff.log_buf) 168 return; 169 170 dma_unmap_single(ddev, tracer->buff.dma, tracer->buff.size, DMA_FROM_DEVICE); 171 free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size)); 172 } 173 174 static int mlx5_fw_tracer_create_mkey(struct mlx5_fw_tracer *tracer) 175 { 176 struct mlx5_core_dev *dev = tracer->dev; 177 int err, inlen, i; 178 __be64 *mtt; 179 void *mkc; 180 u32 *in; 181 182 inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + 183 sizeof(*mtt) * round_up(TRACER_BUFFER_PAGE_NUM, 2); 184 185 in = kvzalloc(inlen, GFP_KERNEL); 186 if (!in) 187 return -ENOMEM; 188 189 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 190 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2)); 191 mtt = (u64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 192 for (i = 0 ; i < TRACER_BUFFER_PAGE_NUM ; i++) 193 mtt[i] = cpu_to_be64(tracer->buff.dma + i * PAGE_SIZE); 194 195 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 196 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 197 MLX5_SET(mkc, mkc, lr, 1); 198 MLX5_SET(mkc, mkc, lw, 1); 199 MLX5_SET(mkc, mkc, pd, tracer->buff.pdn); 200 MLX5_SET(mkc, mkc, bsf_octword_size, 0); 201 MLX5_SET(mkc, mkc, qpn, 0xffffff); 202 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); 203 MLX5_SET(mkc, mkc, translations_octword_size, 204 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2)); 205 MLX5_SET64(mkc, mkc, start_addr, tracer->buff.dma); 206 MLX5_SET64(mkc, mkc, len, tracer->buff.size); 207 err = mlx5_core_create_mkey(dev, &tracer->buff.mkey, in, inlen); 208 if (err) 209 mlx5_core_warn(dev, "FWTracer: Failed to create mkey, %d\n", err); 210 211 kvfree(in); 212 213 return err; 214 } 215 216 static void mlx5_fw_tracer_free_strings_db(struct mlx5_fw_tracer *tracer) 217 { 218 u32 num_string_db = tracer->str_db.num_string_db; 219 int i; 220 221 for (i = 0; i < num_string_db; i++) { 222 kfree(tracer->str_db.buffer[i]); 223 tracer->str_db.buffer[i] = NULL; 224 } 225 } 226 227 static int mlx5_fw_tracer_allocate_strings_db(struct mlx5_fw_tracer *tracer) 228 { 229 u32 *string_db_size_out = tracer->str_db.size_out; 230 u32 num_string_db = tracer->str_db.num_string_db; 231 int i; 232 233 for (i = 0; i < num_string_db; i++) { 234 tracer->str_db.buffer[i] = kzalloc(string_db_size_out[i], GFP_KERNEL); 235 if (!tracer->str_db.buffer[i]) 236 goto free_strings_db; 237 } 238 239 return 0; 240 241 free_strings_db: 242 mlx5_fw_tracer_free_strings_db(tracer); 243 return -ENOMEM; 244 } 245 246 static void 247 mlx5_fw_tracer_init_saved_traces_array(struct mlx5_fw_tracer *tracer) 248 { 249 tracer->st_arr.saved_traces_index = 0; 250 mutex_init(&tracer->st_arr.lock); 251 } 252 253 static void 254 mlx5_fw_tracer_clean_saved_traces_array(struct mlx5_fw_tracer *tracer) 255 { 256 mutex_destroy(&tracer->st_arr.lock); 257 } 258 259 static void mlx5_tracer_read_strings_db(struct work_struct *work) 260 { 261 struct mlx5_fw_tracer *tracer = container_of(work, struct mlx5_fw_tracer, 262 read_fw_strings_work); 263 u32 num_of_reads, num_string_db = tracer->str_db.num_string_db; 264 struct mlx5_core_dev *dev = tracer->dev; 265 u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0}; 266 u32 leftovers, offset; 267 int err = 0, i, j; 268 u32 *out, outlen; 269 void *out_value; 270 271 outlen = MLX5_ST_SZ_BYTES(mtrc_stdb) + STRINGS_DB_READ_SIZE_BYTES; 272 out = kzalloc(outlen, GFP_KERNEL); 273 if (!out) { 274 err = -ENOMEM; 275 goto out; 276 } 277 278 for (i = 0; i < num_string_db; i++) { 279 offset = 0; 280 MLX5_SET(mtrc_stdb, in, string_db_index, i); 281 num_of_reads = tracer->str_db.size_out[i] / 282 STRINGS_DB_READ_SIZE_BYTES; 283 leftovers = (tracer->str_db.size_out[i] % 284 STRINGS_DB_READ_SIZE_BYTES) / 285 STRINGS_DB_LEFTOVER_SIZE_BYTES; 286 287 MLX5_SET(mtrc_stdb, in, read_size, STRINGS_DB_READ_SIZE_BYTES); 288 for (j = 0; j < num_of_reads; j++) { 289 MLX5_SET(mtrc_stdb, in, start_offset, offset); 290 291 err = mlx5_core_access_reg(dev, in, sizeof(in), out, 292 outlen, MLX5_REG_MTRC_STDB, 293 0, 1); 294 if (err) { 295 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n", 296 err); 297 goto out_free; 298 } 299 300 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data); 301 memcpy(tracer->str_db.buffer[i] + offset, out_value, 302 STRINGS_DB_READ_SIZE_BYTES); 303 offset += STRINGS_DB_READ_SIZE_BYTES; 304 } 305 306 /* Strings database is aligned to 64, need to read leftovers*/ 307 MLX5_SET(mtrc_stdb, in, read_size, 308 STRINGS_DB_LEFTOVER_SIZE_BYTES); 309 for (j = 0; j < leftovers; j++) { 310 MLX5_SET(mtrc_stdb, in, start_offset, offset); 311 312 err = mlx5_core_access_reg(dev, in, sizeof(in), out, 313 outlen, MLX5_REG_MTRC_STDB, 314 0, 1); 315 if (err) { 316 mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n", 317 err); 318 goto out_free; 319 } 320 321 out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data); 322 memcpy(tracer->str_db.buffer[i] + offset, out_value, 323 STRINGS_DB_LEFTOVER_SIZE_BYTES); 324 offset += STRINGS_DB_LEFTOVER_SIZE_BYTES; 325 } 326 } 327 328 tracer->str_db.loaded = true; 329 330 out_free: 331 kfree(out); 332 out: 333 return; 334 } 335 336 static void mlx5_fw_tracer_arm(struct mlx5_core_dev *dev) 337 { 338 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 339 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 340 int err; 341 342 MLX5_SET(mtrc_ctrl, in, arm_event, 1); 343 344 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 345 MLX5_REG_MTRC_CTRL, 0, 1); 346 if (err) 347 mlx5_core_warn(dev, "FWTracer: Failed to arm tracer event %d\n", err); 348 } 349 350 static const char *VAL_PARM = "%llx"; 351 static const char *REPLACE_64_VAL_PARM = "%x%x"; 352 static const char *PARAM_CHAR = "%"; 353 354 static int mlx5_tracer_message_hash(u32 message_id) 355 { 356 return jhash_1word(message_id, 0) & (MESSAGE_HASH_SIZE - 1); 357 } 358 359 static struct tracer_string_format *mlx5_tracer_message_insert(struct mlx5_fw_tracer *tracer, 360 struct tracer_event *tracer_event) 361 { 362 struct hlist_head *head = 363 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)]; 364 struct tracer_string_format *cur_string; 365 366 cur_string = kzalloc(sizeof(*cur_string), GFP_KERNEL); 367 if (!cur_string) 368 return NULL; 369 370 hlist_add_head(&cur_string->hlist, head); 371 372 return cur_string; 373 } 374 375 static struct tracer_string_format *mlx5_tracer_get_string(struct mlx5_fw_tracer *tracer, 376 struct tracer_event *tracer_event) 377 { 378 struct tracer_string_format *cur_string; 379 u32 str_ptr, offset; 380 int i; 381 382 str_ptr = tracer_event->string_event.string_param; 383 384 for (i = 0; i < tracer->str_db.num_string_db; i++) { 385 if (str_ptr > tracer->str_db.base_address_out[i] && 386 str_ptr < tracer->str_db.base_address_out[i] + 387 tracer->str_db.size_out[i]) { 388 offset = str_ptr - tracer->str_db.base_address_out[i]; 389 /* add it to the hash */ 390 cur_string = mlx5_tracer_message_insert(tracer, tracer_event); 391 if (!cur_string) 392 return NULL; 393 cur_string->string = (char *)(tracer->str_db.buffer[i] + 394 offset); 395 return cur_string; 396 } 397 } 398 399 return NULL; 400 } 401 402 static void mlx5_tracer_clean_message(struct tracer_string_format *str_frmt) 403 { 404 hlist_del(&str_frmt->hlist); 405 kfree(str_frmt); 406 } 407 408 static int mlx5_tracer_get_num_of_params(char *str) 409 { 410 char *substr, *pstr = str; 411 int num_of_params = 0; 412 413 /* replace %llx with %x%x */ 414 substr = strstr(pstr, VAL_PARM); 415 while (substr) { 416 memcpy(substr, REPLACE_64_VAL_PARM, 4); 417 pstr = substr; 418 substr = strstr(pstr, VAL_PARM); 419 } 420 421 /* count all the % characters */ 422 substr = strstr(str, PARAM_CHAR); 423 while (substr) { 424 num_of_params += 1; 425 str = substr + 1; 426 substr = strstr(str, PARAM_CHAR); 427 } 428 429 return num_of_params; 430 } 431 432 static struct tracer_string_format *mlx5_tracer_message_find(struct hlist_head *head, 433 u8 event_id, u32 tmsn) 434 { 435 struct tracer_string_format *message; 436 437 hlist_for_each_entry(message, head, hlist) 438 if (message->event_id == event_id && message->tmsn == tmsn) 439 return message; 440 441 return NULL; 442 } 443 444 static struct tracer_string_format *mlx5_tracer_message_get(struct mlx5_fw_tracer *tracer, 445 struct tracer_event *tracer_event) 446 { 447 struct hlist_head *head = 448 &tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)]; 449 450 return mlx5_tracer_message_find(head, tracer_event->event_id, tracer_event->string_event.tmsn); 451 } 452 453 static void poll_trace(struct mlx5_fw_tracer *tracer, 454 struct tracer_event *tracer_event, u64 *trace) 455 { 456 u32 timestamp_low, timestamp_mid, timestamp_high, urts; 457 458 tracer_event->event_id = MLX5_GET(tracer_event, trace, event_id); 459 tracer_event->lost_event = MLX5_GET(tracer_event, trace, lost); 460 461 switch (tracer_event->event_id) { 462 case TRACER_EVENT_TYPE_TIMESTAMP: 463 tracer_event->type = TRACER_EVENT_TYPE_TIMESTAMP; 464 urts = MLX5_GET(tracer_timestamp_event, trace, urts); 465 if (tracer->trc_ver == 0) 466 tracer_event->timestamp_event.unreliable = !!(urts >> 2); 467 else 468 tracer_event->timestamp_event.unreliable = !!(urts & 1); 469 470 timestamp_low = MLX5_GET(tracer_timestamp_event, 471 trace, timestamp7_0); 472 timestamp_mid = MLX5_GET(tracer_timestamp_event, 473 trace, timestamp39_8); 474 timestamp_high = MLX5_GET(tracer_timestamp_event, 475 trace, timestamp52_40); 476 477 tracer_event->timestamp_event.timestamp = 478 ((u64)timestamp_high << 40) | 479 ((u64)timestamp_mid << 8) | 480 (u64)timestamp_low; 481 break; 482 default: 483 if (tracer_event->event_id >= tracer->str_db.first_string_trace || 484 tracer_event->event_id <= tracer->str_db.first_string_trace + 485 tracer->str_db.num_string_trace) { 486 tracer_event->type = TRACER_EVENT_TYPE_STRING; 487 tracer_event->string_event.timestamp = 488 MLX5_GET(tracer_string_event, trace, timestamp); 489 tracer_event->string_event.string_param = 490 MLX5_GET(tracer_string_event, trace, string_param); 491 tracer_event->string_event.tmsn = 492 MLX5_GET(tracer_string_event, trace, tmsn); 493 tracer_event->string_event.tdsn = 494 MLX5_GET(tracer_string_event, trace, tdsn); 495 } else { 496 tracer_event->type = TRACER_EVENT_TYPE_UNRECOGNIZED; 497 } 498 break; 499 } 500 } 501 502 static u64 get_block_timestamp(struct mlx5_fw_tracer *tracer, u64 *ts_event) 503 { 504 struct tracer_event tracer_event; 505 u8 event_id; 506 507 event_id = MLX5_GET(tracer_event, ts_event, event_id); 508 509 if (event_id == TRACER_EVENT_TYPE_TIMESTAMP) 510 poll_trace(tracer, &tracer_event, ts_event); 511 else 512 tracer_event.timestamp_event.timestamp = 0; 513 514 return tracer_event.timestamp_event.timestamp; 515 } 516 517 static void mlx5_fw_tracer_clean_print_hash(struct mlx5_fw_tracer *tracer) 518 { 519 struct tracer_string_format *str_frmt; 520 struct hlist_node *n; 521 int i; 522 523 for (i = 0; i < MESSAGE_HASH_SIZE; i++) { 524 hlist_for_each_entry_safe(str_frmt, n, &tracer->hash[i], hlist) 525 mlx5_tracer_clean_message(str_frmt); 526 } 527 } 528 529 static void mlx5_fw_tracer_clean_ready_list(struct mlx5_fw_tracer *tracer) 530 { 531 struct tracer_string_format *str_frmt, *tmp_str; 532 533 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, 534 list) 535 list_del(&str_frmt->list); 536 } 537 538 static void mlx5_fw_tracer_save_trace(struct mlx5_fw_tracer *tracer, 539 u64 timestamp, bool lost, 540 u8 event_id, char *msg) 541 { 542 struct mlx5_fw_trace_data *trace_data; 543 544 mutex_lock(&tracer->st_arr.lock); 545 trace_data = &tracer->st_arr.straces[tracer->st_arr.saved_traces_index]; 546 trace_data->timestamp = timestamp; 547 trace_data->lost = lost; 548 trace_data->event_id = event_id; 549 strscpy_pad(trace_data->msg, msg, TRACE_STR_MSG); 550 551 tracer->st_arr.saved_traces_index = 552 (tracer->st_arr.saved_traces_index + 1) & (SAVED_TRACES_NUM - 1); 553 mutex_unlock(&tracer->st_arr.lock); 554 } 555 556 static noinline 557 void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt, 558 struct mlx5_core_dev *dev, 559 u64 trace_timestamp) 560 { 561 char tmp[512]; 562 563 snprintf(tmp, sizeof(tmp), str_frmt->string, 564 str_frmt->params[0], 565 str_frmt->params[1], 566 str_frmt->params[2], 567 str_frmt->params[3], 568 str_frmt->params[4], 569 str_frmt->params[5], 570 str_frmt->params[6]); 571 572 trace_mlx5_fw(dev->tracer, trace_timestamp, str_frmt->lost, 573 str_frmt->event_id, tmp); 574 575 mlx5_fw_tracer_save_trace(dev->tracer, trace_timestamp, 576 str_frmt->lost, str_frmt->event_id, tmp); 577 578 /* remove it from hash */ 579 mlx5_tracer_clean_message(str_frmt); 580 } 581 582 static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer, 583 struct tracer_event *tracer_event) 584 { 585 struct tracer_string_format *cur_string; 586 587 if (tracer_event->string_event.tdsn == 0) { 588 cur_string = mlx5_tracer_get_string(tracer, tracer_event); 589 if (!cur_string) 590 return -1; 591 592 cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string); 593 cur_string->last_param_num = 0; 594 cur_string->event_id = tracer_event->event_id; 595 cur_string->tmsn = tracer_event->string_event.tmsn; 596 cur_string->timestamp = tracer_event->string_event.timestamp; 597 cur_string->lost = tracer_event->lost_event; 598 if (cur_string->num_of_params == 0) /* trace with no params */ 599 list_add_tail(&cur_string->list, &tracer->ready_strings_list); 600 } else { 601 cur_string = mlx5_tracer_message_get(tracer, tracer_event); 602 if (!cur_string) { 603 pr_debug("%s Got string event for unknown string tdsm: %d\n", 604 __func__, tracer_event->string_event.tmsn); 605 return -1; 606 } 607 cur_string->last_param_num += 1; 608 if (cur_string->last_param_num > TRACER_MAX_PARAMS) { 609 pr_debug("%s Number of params exceeds the max (%d)\n", 610 __func__, TRACER_MAX_PARAMS); 611 list_add_tail(&cur_string->list, &tracer->ready_strings_list); 612 return 0; 613 } 614 /* keep the new parameter */ 615 cur_string->params[cur_string->last_param_num - 1] = 616 tracer_event->string_event.string_param; 617 if (cur_string->last_param_num == cur_string->num_of_params) 618 list_add_tail(&cur_string->list, &tracer->ready_strings_list); 619 } 620 621 return 0; 622 } 623 624 static void mlx5_tracer_handle_timestamp_trace(struct mlx5_fw_tracer *tracer, 625 struct tracer_event *tracer_event) 626 { 627 struct tracer_timestamp_event timestamp_event = 628 tracer_event->timestamp_event; 629 struct tracer_string_format *str_frmt, *tmp_str; 630 struct mlx5_core_dev *dev = tracer->dev; 631 u64 trace_timestamp; 632 633 list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, list) { 634 list_del(&str_frmt->list); 635 if (str_frmt->timestamp < (timestamp_event.timestamp & MASK_6_0)) 636 trace_timestamp = (timestamp_event.timestamp & MASK_52_7) | 637 (str_frmt->timestamp & MASK_6_0); 638 else 639 trace_timestamp = ((timestamp_event.timestamp & MASK_52_7) - 1) | 640 (str_frmt->timestamp & MASK_6_0); 641 642 mlx5_tracer_print_trace(str_frmt, dev, trace_timestamp); 643 } 644 } 645 646 static int mlx5_tracer_handle_trace(struct mlx5_fw_tracer *tracer, 647 struct tracer_event *tracer_event) 648 { 649 if (tracer_event->type == TRACER_EVENT_TYPE_STRING) { 650 mlx5_tracer_handle_string_trace(tracer, tracer_event); 651 } else if (tracer_event->type == TRACER_EVENT_TYPE_TIMESTAMP) { 652 if (!tracer_event->timestamp_event.unreliable) 653 mlx5_tracer_handle_timestamp_trace(tracer, tracer_event); 654 } else { 655 pr_debug("%s Got unrecognised type %d for parsing, exiting..\n", 656 __func__, tracer_event->type); 657 } 658 return 0; 659 } 660 661 static void mlx5_fw_tracer_handle_traces(struct work_struct *work) 662 { 663 struct mlx5_fw_tracer *tracer = 664 container_of(work, struct mlx5_fw_tracer, handle_traces_work); 665 u64 block_timestamp, last_block_timestamp, tmp_trace_block[TRACES_PER_BLOCK]; 666 u32 block_count, start_offset, prev_start_offset, prev_consumer_index; 667 u32 trace_event_size = MLX5_ST_SZ_BYTES(tracer_event); 668 struct mlx5_core_dev *dev = tracer->dev; 669 struct tracer_event tracer_event; 670 int i; 671 672 mlx5_core_dbg(dev, "FWTracer: Handle Trace event, owner=(%d)\n", tracer->owner); 673 if (!tracer->owner) 674 return; 675 676 block_count = tracer->buff.size / TRACER_BLOCK_SIZE_BYTE; 677 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE; 678 679 /* Copy the block to local buffer to avoid HW override while being processed*/ 680 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset, 681 TRACER_BLOCK_SIZE_BYTE); 682 683 block_timestamp = 684 get_block_timestamp(tracer, &tmp_trace_block[TRACES_PER_BLOCK - 1]); 685 686 while (block_timestamp > tracer->last_timestamp) { 687 /* Check block override if its not the first block */ 688 if (!tracer->last_timestamp) { 689 u64 *ts_event; 690 /* To avoid block override be the HW in case of buffer 691 * wraparound, the time stamp of the previous block 692 * should be compared to the last timestamp handled 693 * by the driver. 694 */ 695 prev_consumer_index = 696 (tracer->buff.consumer_index - 1) & (block_count - 1); 697 prev_start_offset = prev_consumer_index * TRACER_BLOCK_SIZE_BYTE; 698 699 ts_event = tracer->buff.log_buf + prev_start_offset + 700 (TRACES_PER_BLOCK - 1) * trace_event_size; 701 last_block_timestamp = get_block_timestamp(tracer, ts_event); 702 /* If previous timestamp different from last stored 703 * timestamp then there is a good chance that the 704 * current buffer is overwritten and therefore should 705 * not be parsed. 706 */ 707 if (tracer->last_timestamp != last_block_timestamp) { 708 mlx5_core_warn(dev, "FWTracer: Events were lost\n"); 709 tracer->last_timestamp = block_timestamp; 710 tracer->buff.consumer_index = 711 (tracer->buff.consumer_index + 1) & (block_count - 1); 712 break; 713 } 714 } 715 716 /* Parse events */ 717 for (i = 0; i < TRACES_PER_BLOCK ; i++) { 718 poll_trace(tracer, &tracer_event, &tmp_trace_block[i]); 719 mlx5_tracer_handle_trace(tracer, &tracer_event); 720 } 721 722 tracer->buff.consumer_index = 723 (tracer->buff.consumer_index + 1) & (block_count - 1); 724 725 tracer->last_timestamp = block_timestamp; 726 start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE; 727 memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset, 728 TRACER_BLOCK_SIZE_BYTE); 729 block_timestamp = get_block_timestamp(tracer, 730 &tmp_trace_block[TRACES_PER_BLOCK - 1]); 731 } 732 733 mlx5_fw_tracer_arm(dev); 734 } 735 736 static int mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer *tracer) 737 { 738 struct mlx5_core_dev *dev = tracer->dev; 739 u32 out[MLX5_ST_SZ_DW(mtrc_conf)] = {0}; 740 u32 in[MLX5_ST_SZ_DW(mtrc_conf)] = {0}; 741 int err; 742 743 MLX5_SET(mtrc_conf, in, trace_mode, TRACE_TO_MEMORY); 744 MLX5_SET(mtrc_conf, in, log_trace_buffer_size, 745 ilog2(TRACER_BUFFER_PAGE_NUM)); 746 MLX5_SET(mtrc_conf, in, trace_mkey, tracer->buff.mkey.key); 747 748 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 749 MLX5_REG_MTRC_CONF, 0, 1); 750 if (err) 751 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configurations %d\n", err); 752 753 return err; 754 } 755 756 static int mlx5_fw_tracer_set_mtrc_ctrl(struct mlx5_fw_tracer *tracer, u8 status, u8 arm) 757 { 758 struct mlx5_core_dev *dev = tracer->dev; 759 u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 760 u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0}; 761 int err; 762 763 MLX5_SET(mtrc_ctrl, in, modify_field_select, TRACE_STATUS); 764 MLX5_SET(mtrc_ctrl, in, trace_status, status); 765 MLX5_SET(mtrc_ctrl, in, arm_event, arm); 766 767 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), 768 MLX5_REG_MTRC_CTRL, 0, 1); 769 770 if (!err && status) 771 tracer->last_timestamp = 0; 772 773 return err; 774 } 775 776 static int mlx5_fw_tracer_start(struct mlx5_fw_tracer *tracer) 777 { 778 struct mlx5_core_dev *dev = tracer->dev; 779 int err; 780 781 err = mlx5_fw_tracer_ownership_acquire(tracer); 782 if (err) { 783 mlx5_core_dbg(dev, "FWTracer: Ownership was not granted %d\n", err); 784 /* Don't fail since ownership can be acquired on a later FW event */ 785 return 0; 786 } 787 788 err = mlx5_fw_tracer_set_mtrc_conf(tracer); 789 if (err) { 790 mlx5_core_warn(dev, "FWTracer: Failed to set tracer configuration %d\n", err); 791 goto release_ownership; 792 } 793 794 /* enable tracer & trace events */ 795 err = mlx5_fw_tracer_set_mtrc_ctrl(tracer, 1, 1); 796 if (err) { 797 mlx5_core_warn(dev, "FWTracer: Failed to enable tracer %d\n", err); 798 goto release_ownership; 799 } 800 801 mlx5_core_dbg(dev, "FWTracer: Ownership granted and active\n"); 802 return 0; 803 804 release_ownership: 805 mlx5_fw_tracer_ownership_release(tracer); 806 return err; 807 } 808 809 static void mlx5_fw_tracer_ownership_change(struct work_struct *work) 810 { 811 struct mlx5_fw_tracer *tracer = 812 container_of(work, struct mlx5_fw_tracer, ownership_change_work); 813 814 mlx5_core_dbg(tracer->dev, "FWTracer: ownership changed, current=(%d)\n", tracer->owner); 815 if (tracer->owner) { 816 tracer->owner = false; 817 tracer->buff.consumer_index = 0; 818 return; 819 } 820 821 mlx5_fw_tracer_start(tracer); 822 } 823 824 static int mlx5_fw_tracer_set_core_dump_reg(struct mlx5_core_dev *dev, 825 u32 *in, int size_in) 826 { 827 u32 out[MLX5_ST_SZ_DW(core_dump_reg)] = {}; 828 829 if (!MLX5_CAP_DEBUG(dev, core_dump_general) && 830 !MLX5_CAP_DEBUG(dev, core_dump_qp)) 831 return -EOPNOTSUPP; 832 833 return mlx5_core_access_reg(dev, in, size_in, out, sizeof(out), 834 MLX5_REG_CORE_DUMP, 0, 1); 835 } 836 837 int mlx5_fw_tracer_trigger_core_dump_general(struct mlx5_core_dev *dev) 838 { 839 struct mlx5_fw_tracer *tracer = dev->tracer; 840 u32 in[MLX5_ST_SZ_DW(core_dump_reg)] = {}; 841 int err; 842 843 if (!MLX5_CAP_DEBUG(dev, core_dump_general) || !tracer) 844 return -EOPNOTSUPP; 845 if (!tracer->owner) 846 return -EPERM; 847 848 MLX5_SET(core_dump_reg, in, core_dump_type, 0x0); 849 850 err = mlx5_fw_tracer_set_core_dump_reg(dev, in, sizeof(in)); 851 if (err) 852 return err; 853 queue_work(tracer->work_queue, &tracer->handle_traces_work); 854 flush_workqueue(tracer->work_queue); 855 return 0; 856 } 857 858 static int 859 mlx5_devlink_fmsg_fill_trace(struct devlink_fmsg *fmsg, 860 struct mlx5_fw_trace_data *trace_data) 861 { 862 int err; 863 864 err = devlink_fmsg_obj_nest_start(fmsg); 865 if (err) 866 return err; 867 868 err = devlink_fmsg_u64_pair_put(fmsg, "timestamp", trace_data->timestamp); 869 if (err) 870 return err; 871 872 err = devlink_fmsg_bool_pair_put(fmsg, "lost", trace_data->lost); 873 if (err) 874 return err; 875 876 err = devlink_fmsg_u8_pair_put(fmsg, "event_id", trace_data->event_id); 877 if (err) 878 return err; 879 880 err = devlink_fmsg_string_pair_put(fmsg, "msg", trace_data->msg); 881 if (err) 882 return err; 883 884 err = devlink_fmsg_obj_nest_end(fmsg); 885 if (err) 886 return err; 887 return 0; 888 } 889 890 int mlx5_fw_tracer_get_saved_traces_objects(struct mlx5_fw_tracer *tracer, 891 struct devlink_fmsg *fmsg) 892 { 893 struct mlx5_fw_trace_data *straces = tracer->st_arr.straces; 894 u32 index, start_index, end_index; 895 u32 saved_traces_index; 896 int err; 897 898 if (!straces[0].timestamp) 899 return -ENOMSG; 900 901 mutex_lock(&tracer->st_arr.lock); 902 saved_traces_index = tracer->st_arr.saved_traces_index; 903 if (straces[saved_traces_index].timestamp) 904 start_index = saved_traces_index; 905 else 906 start_index = 0; 907 end_index = (saved_traces_index - 1) & (SAVED_TRACES_NUM - 1); 908 909 err = devlink_fmsg_arr_pair_nest_start(fmsg, "dump fw traces"); 910 if (err) 911 goto unlock; 912 index = start_index; 913 while (index != end_index) { 914 err = mlx5_devlink_fmsg_fill_trace(fmsg, &straces[index]); 915 if (err) 916 goto unlock; 917 918 index = (index + 1) & (SAVED_TRACES_NUM - 1); 919 } 920 921 err = devlink_fmsg_arr_pair_nest_end(fmsg); 922 unlock: 923 mutex_unlock(&tracer->st_arr.lock); 924 return err; 925 } 926 927 /* Create software resources (Buffers, etc ..) */ 928 struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev) 929 { 930 struct mlx5_fw_tracer *tracer = NULL; 931 int err; 932 933 if (!MLX5_CAP_MCAM_REG(dev, tracer_registers)) { 934 mlx5_core_dbg(dev, "FWTracer: Tracer capability not present\n"); 935 return NULL; 936 } 937 938 tracer = kzalloc(sizeof(*tracer), GFP_KERNEL); 939 if (!tracer) 940 return ERR_PTR(-ENOMEM); 941 942 tracer->work_queue = create_singlethread_workqueue("mlx5_fw_tracer"); 943 if (!tracer->work_queue) { 944 err = -ENOMEM; 945 goto free_tracer; 946 } 947 948 tracer->dev = dev; 949 950 INIT_LIST_HEAD(&tracer->ready_strings_list); 951 INIT_WORK(&tracer->ownership_change_work, mlx5_fw_tracer_ownership_change); 952 INIT_WORK(&tracer->read_fw_strings_work, mlx5_tracer_read_strings_db); 953 INIT_WORK(&tracer->handle_traces_work, mlx5_fw_tracer_handle_traces); 954 955 956 err = mlx5_query_mtrc_caps(tracer); 957 if (err) { 958 mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err); 959 goto destroy_workqueue; 960 } 961 962 err = mlx5_fw_tracer_create_log_buf(tracer); 963 if (err) { 964 mlx5_core_warn(dev, "FWTracer: Create log buffer failed %d\n", err); 965 goto destroy_workqueue; 966 } 967 968 err = mlx5_fw_tracer_allocate_strings_db(tracer); 969 if (err) { 970 mlx5_core_warn(dev, "FWTracer: Allocate strings database failed %d\n", err); 971 goto free_log_buf; 972 } 973 974 mlx5_fw_tracer_init_saved_traces_array(tracer); 975 mlx5_core_dbg(dev, "FWTracer: Tracer created\n"); 976 977 return tracer; 978 979 free_log_buf: 980 mlx5_fw_tracer_destroy_log_buf(tracer); 981 destroy_workqueue: 982 tracer->dev = NULL; 983 destroy_workqueue(tracer->work_queue); 984 free_tracer: 985 kfree(tracer); 986 return ERR_PTR(err); 987 } 988 989 static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data); 990 991 /* Create HW resources + start tracer */ 992 int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer) 993 { 994 struct mlx5_core_dev *dev; 995 int err; 996 997 if (IS_ERR_OR_NULL(tracer)) 998 return 0; 999 1000 dev = tracer->dev; 1001 1002 if (!tracer->str_db.loaded) 1003 queue_work(tracer->work_queue, &tracer->read_fw_strings_work); 1004 1005 err = mlx5_core_alloc_pd(dev, &tracer->buff.pdn); 1006 if (err) { 1007 mlx5_core_warn(dev, "FWTracer: Failed to allocate PD %d\n", err); 1008 return err; 1009 } 1010 1011 err = mlx5_fw_tracer_create_mkey(tracer); 1012 if (err) { 1013 mlx5_core_warn(dev, "FWTracer: Failed to create mkey %d\n", err); 1014 goto err_dealloc_pd; 1015 } 1016 1017 MLX5_NB_INIT(&tracer->nb, fw_tracer_event, DEVICE_TRACER); 1018 mlx5_eq_notifier_register(dev, &tracer->nb); 1019 1020 mlx5_fw_tracer_start(tracer); 1021 1022 return 0; 1023 1024 err_dealloc_pd: 1025 mlx5_core_dealloc_pd(dev, tracer->buff.pdn); 1026 return err; 1027 } 1028 1029 /* Stop tracer + Cleanup HW resources */ 1030 void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer) 1031 { 1032 if (IS_ERR_OR_NULL(tracer)) 1033 return; 1034 1035 mlx5_core_dbg(tracer->dev, "FWTracer: Cleanup, is owner ? (%d)\n", 1036 tracer->owner); 1037 mlx5_eq_notifier_unregister(tracer->dev, &tracer->nb); 1038 cancel_work_sync(&tracer->ownership_change_work); 1039 cancel_work_sync(&tracer->handle_traces_work); 1040 1041 if (tracer->owner) 1042 mlx5_fw_tracer_ownership_release(tracer); 1043 1044 mlx5_core_destroy_mkey(tracer->dev, &tracer->buff.mkey); 1045 mlx5_core_dealloc_pd(tracer->dev, tracer->buff.pdn); 1046 } 1047 1048 /* Free software resources (Buffers, etc ..) */ 1049 void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer *tracer) 1050 { 1051 if (IS_ERR_OR_NULL(tracer)) 1052 return; 1053 1054 mlx5_core_dbg(tracer->dev, "FWTracer: Destroy\n"); 1055 1056 cancel_work_sync(&tracer->read_fw_strings_work); 1057 mlx5_fw_tracer_clean_ready_list(tracer); 1058 mlx5_fw_tracer_clean_print_hash(tracer); 1059 mlx5_fw_tracer_clean_saved_traces_array(tracer); 1060 mlx5_fw_tracer_free_strings_db(tracer); 1061 mlx5_fw_tracer_destroy_log_buf(tracer); 1062 flush_workqueue(tracer->work_queue); 1063 destroy_workqueue(tracer->work_queue); 1064 kfree(tracer); 1065 } 1066 1067 static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data) 1068 { 1069 struct mlx5_fw_tracer *tracer = mlx5_nb_cof(nb, struct mlx5_fw_tracer, nb); 1070 struct mlx5_core_dev *dev = tracer->dev; 1071 struct mlx5_eqe *eqe = data; 1072 1073 switch (eqe->sub_type) { 1074 case MLX5_TRACER_SUBTYPE_OWNERSHIP_CHANGE: 1075 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) 1076 queue_work(tracer->work_queue, &tracer->ownership_change_work); 1077 break; 1078 case MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE: 1079 if (likely(tracer->str_db.loaded)) 1080 queue_work(tracer->work_queue, &tracer->handle_traces_work); 1081 break; 1082 default: 1083 mlx5_core_dbg(dev, "FWTracer: Event with unrecognized subtype: sub_type %d\n", 1084 eqe->sub_type); 1085 } 1086 1087 return NOTIFY_OK; 1088 } 1089 1090 EXPORT_TRACEPOINT_SYMBOL(mlx5_fw); 1091