1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Loopback bridge driver for the Greybus loopback module. 4 * 5 * Copyright 2014 Google Inc. 6 * Copyright 2014 Linaro Ltd. 7 */ 8 9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 10 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/mutex.h> 14 #include <linux/slab.h> 15 #include <linux/kthread.h> 16 #include <linux/delay.h> 17 #include <linux/random.h> 18 #include <linux/sizes.h> 19 #include <linux/cdev.h> 20 #include <linux/fs.h> 21 #include <linux/kfifo.h> 22 #include <linux/debugfs.h> 23 #include <linux/list_sort.h> 24 #include <linux/spinlock.h> 25 #include <linux/workqueue.h> 26 #include <linux/atomic.h> 27 #include <linux/pm_runtime.h> 28 #include <linux/greybus.h> 29 #include <asm/div64.h> 30 31 #define NSEC_PER_DAY 86400000000000ULL 32 33 struct gb_loopback_stats { 34 u32 min; 35 u32 max; 36 u64 sum; 37 u32 count; 38 }; 39 40 struct gb_loopback_device { 41 struct dentry *root; 42 u32 count; 43 size_t size_max; 44 45 /* We need to take a lock in atomic context */ 46 spinlock_t lock; 47 wait_queue_head_t wq; 48 }; 49 50 static struct gb_loopback_device gb_dev; 51 52 struct gb_loopback_async_operation { 53 struct gb_loopback *gb; 54 struct gb_operation *operation; 55 ktime_t ts; 56 int (*completion)(struct gb_loopback_async_operation *op_async); 57 }; 58 59 struct gb_loopback { 60 struct gb_connection *connection; 61 62 struct dentry *file; 63 struct kfifo kfifo_lat; 64 struct mutex mutex; 65 struct task_struct *task; 66 struct device *dev; 67 wait_queue_head_t wq; 68 wait_queue_head_t wq_completion; 69 atomic_t outstanding_operations; 70 71 /* Per connection stats */ 72 ktime_t ts; 73 struct gb_loopback_stats latency; 74 struct gb_loopback_stats throughput; 75 struct gb_loopback_stats requests_per_second; 76 struct gb_loopback_stats apbridge_unipro_latency; 77 struct gb_loopback_stats gbphy_firmware_latency; 78 79 int type; 80 int async; 81 int id; 82 u32 size; 83 u32 iteration_max; 84 u32 iteration_count; 85 int us_wait; 86 u32 error; 87 u32 requests_completed; 88 u32 requests_timedout; 89 u32 timeout; 90 u32 jiffy_timeout; 91 u32 timeout_min; 92 u32 timeout_max; 93 u32 outstanding_operations_max; 94 u64 elapsed_nsecs; 95 u32 apbridge_latency_ts; 96 u32 gbphy_latency_ts; 97 98 u32 send_count; 99 }; 100 101 static struct class loopback_class = { 102 .name = "gb_loopback", 103 .owner = THIS_MODULE, 104 }; 105 static DEFINE_IDA(loopback_ida); 106 107 /* Min/max values in jiffies */ 108 #define GB_LOOPBACK_TIMEOUT_MIN 1 109 #define GB_LOOPBACK_TIMEOUT_MAX 10000 110 111 #define GB_LOOPBACK_FIFO_DEFAULT 8192 112 113 static unsigned int kfifo_depth = GB_LOOPBACK_FIFO_DEFAULT; 114 module_param(kfifo_depth, uint, 0444); 115 116 /* Maximum size of any one send data buffer we support */ 117 #define MAX_PACKET_SIZE (PAGE_SIZE * 2) 118 119 #define GB_LOOPBACK_US_WAIT_MAX 1000000 120 121 /* interface sysfs attributes */ 122 #define gb_loopback_ro_attr(field) \ 123 static ssize_t field##_show(struct device *dev, \ 124 struct device_attribute *attr, \ 125 char *buf) \ 126 { \ 127 struct gb_loopback *gb = dev_get_drvdata(dev); \ 128 return sprintf(buf, "%u\n", gb->field); \ 129 } \ 130 static DEVICE_ATTR_RO(field) 131 132 #define gb_loopback_ro_stats_attr(name, field, type) \ 133 static ssize_t name##_##field##_show(struct device *dev, \ 134 struct device_attribute *attr, \ 135 char *buf) \ 136 { \ 137 struct gb_loopback *gb = dev_get_drvdata(dev); \ 138 /* Report 0 for min and max if no transfer successed */ \ 139 if (!gb->requests_completed) \ 140 return sprintf(buf, "0\n"); \ 141 return sprintf(buf, "%" #type "\n", gb->name.field); \ 142 } \ 143 static DEVICE_ATTR_RO(name##_##field) 144 145 #define gb_loopback_ro_avg_attr(name) \ 146 static ssize_t name##_avg_show(struct device *dev, \ 147 struct device_attribute *attr, \ 148 char *buf) \ 149 { \ 150 struct gb_loopback_stats *stats; \ 151 struct gb_loopback *gb; \ 152 u64 avg, rem; \ 153 u32 count; \ 154 gb = dev_get_drvdata(dev); \ 155 stats = &gb->name; \ 156 count = stats->count ? stats->count : 1; \ 157 avg = stats->sum + count / 2000000; /* round closest */ \ 158 rem = do_div(avg, count); \ 159 rem *= 1000000; \ 160 do_div(rem, count); \ 161 return sprintf(buf, "%llu.%06u\n", avg, (u32)rem); \ 162 } \ 163 static DEVICE_ATTR_RO(name##_avg) 164 165 #define gb_loopback_stats_attrs(field) \ 166 gb_loopback_ro_stats_attr(field, min, u); \ 167 gb_loopback_ro_stats_attr(field, max, u); \ 168 gb_loopback_ro_avg_attr(field) 169 170 #define gb_loopback_attr(field, type) \ 171 static ssize_t field##_show(struct device *dev, \ 172 struct device_attribute *attr, \ 173 char *buf) \ 174 { \ 175 struct gb_loopback *gb = dev_get_drvdata(dev); \ 176 return sprintf(buf, "%" #type "\n", gb->field); \ 177 } \ 178 static ssize_t field##_store(struct device *dev, \ 179 struct device_attribute *attr, \ 180 const char *buf, \ 181 size_t len) \ 182 { \ 183 int ret; \ 184 struct gb_loopback *gb = dev_get_drvdata(dev); \ 185 mutex_lock(&gb->mutex); \ 186 ret = sscanf(buf, "%"#type, &gb->field); \ 187 if (ret != 1) \ 188 len = -EINVAL; \ 189 else \ 190 gb_loopback_check_attr(gb, bundle); \ 191 mutex_unlock(&gb->mutex); \ 192 return len; \ 193 } \ 194 static DEVICE_ATTR_RW(field) 195 196 #define gb_dev_loopback_ro_attr(field, conn) \ 197 static ssize_t field##_show(struct device *dev, \ 198 struct device_attribute *attr, \ 199 char *buf) \ 200 { \ 201 struct gb_loopback *gb = dev_get_drvdata(dev); \ 202 return sprintf(buf, "%u\n", gb->field); \ 203 } \ 204 static DEVICE_ATTR_RO(field) 205 206 #define gb_dev_loopback_rw_attr(field, type) \ 207 static ssize_t field##_show(struct device *dev, \ 208 struct device_attribute *attr, \ 209 char *buf) \ 210 { \ 211 struct gb_loopback *gb = dev_get_drvdata(dev); \ 212 return sprintf(buf, "%" #type "\n", gb->field); \ 213 } \ 214 static ssize_t field##_store(struct device *dev, \ 215 struct device_attribute *attr, \ 216 const char *buf, \ 217 size_t len) \ 218 { \ 219 int ret; \ 220 struct gb_loopback *gb = dev_get_drvdata(dev); \ 221 mutex_lock(&gb->mutex); \ 222 ret = sscanf(buf, "%"#type, &gb->field); \ 223 if (ret != 1) \ 224 len = -EINVAL; \ 225 else \ 226 gb_loopback_check_attr(gb); \ 227 mutex_unlock(&gb->mutex); \ 228 return len; \ 229 } \ 230 static DEVICE_ATTR_RW(field) 231 232 static void gb_loopback_reset_stats(struct gb_loopback *gb); 233 static void gb_loopback_check_attr(struct gb_loopback *gb) 234 { 235 if (gb->us_wait > GB_LOOPBACK_US_WAIT_MAX) 236 gb->us_wait = GB_LOOPBACK_US_WAIT_MAX; 237 if (gb->size > gb_dev.size_max) 238 gb->size = gb_dev.size_max; 239 gb->requests_timedout = 0; 240 gb->requests_completed = 0; 241 gb->iteration_count = 0; 242 gb->send_count = 0; 243 gb->error = 0; 244 245 if (kfifo_depth < gb->iteration_max) { 246 dev_warn(gb->dev, 247 "cannot log bytes %u kfifo_depth %u\n", 248 gb->iteration_max, kfifo_depth); 249 } 250 kfifo_reset_out(&gb->kfifo_lat); 251 252 switch (gb->type) { 253 case GB_LOOPBACK_TYPE_PING: 254 case GB_LOOPBACK_TYPE_TRANSFER: 255 case GB_LOOPBACK_TYPE_SINK: 256 gb->jiffy_timeout = usecs_to_jiffies(gb->timeout); 257 if (!gb->jiffy_timeout) 258 gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MIN; 259 else if (gb->jiffy_timeout > GB_LOOPBACK_TIMEOUT_MAX) 260 gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MAX; 261 gb_loopback_reset_stats(gb); 262 wake_up(&gb->wq); 263 break; 264 default: 265 gb->type = 0; 266 break; 267 } 268 } 269 270 /* Time to send and receive one message */ 271 gb_loopback_stats_attrs(latency); 272 /* Number of requests sent per second on this cport */ 273 gb_loopback_stats_attrs(requests_per_second); 274 /* Quantity of data sent and received on this cport */ 275 gb_loopback_stats_attrs(throughput); 276 /* Latency across the UniPro link from APBridge's perspective */ 277 gb_loopback_stats_attrs(apbridge_unipro_latency); 278 /* Firmware induced overhead in the GPBridge */ 279 gb_loopback_stats_attrs(gbphy_firmware_latency); 280 281 /* Number of errors encountered during loop */ 282 gb_loopback_ro_attr(error); 283 /* Number of requests successfully completed async */ 284 gb_loopback_ro_attr(requests_completed); 285 /* Number of requests timed out async */ 286 gb_loopback_ro_attr(requests_timedout); 287 /* Timeout minimum in useconds */ 288 gb_loopback_ro_attr(timeout_min); 289 /* Timeout minimum in useconds */ 290 gb_loopback_ro_attr(timeout_max); 291 292 /* 293 * Type of loopback message to send based on protocol type definitions 294 * 0 => Don't send message 295 * 2 => Send ping message continuously (message without payload) 296 * 3 => Send transfer message continuously (message with payload, 297 * payload returned in response) 298 * 4 => Send a sink message (message with payload, no payload in response) 299 */ 300 gb_dev_loopback_rw_attr(type, d); 301 /* Size of transfer message payload: 0-4096 bytes */ 302 gb_dev_loopback_rw_attr(size, u); 303 /* Time to wait between two messages: 0-1000 ms */ 304 gb_dev_loopback_rw_attr(us_wait, d); 305 /* Maximum iterations for a given operation: 1-(2^32-1), 0 implies infinite */ 306 gb_dev_loopback_rw_attr(iteration_max, u); 307 /* The current index of the for (i = 0; i < iteration_max; i++) loop */ 308 gb_dev_loopback_ro_attr(iteration_count, false); 309 /* A flag to indicate synchronous or asynchronous operations */ 310 gb_dev_loopback_rw_attr(async, u); 311 /* Timeout of an individual asynchronous request */ 312 gb_dev_loopback_rw_attr(timeout, u); 313 /* Maximum number of in-flight operations before back-off */ 314 gb_dev_loopback_rw_attr(outstanding_operations_max, u); 315 316 static struct attribute *loopback_attrs[] = { 317 &dev_attr_latency_min.attr, 318 &dev_attr_latency_max.attr, 319 &dev_attr_latency_avg.attr, 320 &dev_attr_requests_per_second_min.attr, 321 &dev_attr_requests_per_second_max.attr, 322 &dev_attr_requests_per_second_avg.attr, 323 &dev_attr_throughput_min.attr, 324 &dev_attr_throughput_max.attr, 325 &dev_attr_throughput_avg.attr, 326 &dev_attr_apbridge_unipro_latency_min.attr, 327 &dev_attr_apbridge_unipro_latency_max.attr, 328 &dev_attr_apbridge_unipro_latency_avg.attr, 329 &dev_attr_gbphy_firmware_latency_min.attr, 330 &dev_attr_gbphy_firmware_latency_max.attr, 331 &dev_attr_gbphy_firmware_latency_avg.attr, 332 &dev_attr_type.attr, 333 &dev_attr_size.attr, 334 &dev_attr_us_wait.attr, 335 &dev_attr_iteration_count.attr, 336 &dev_attr_iteration_max.attr, 337 &dev_attr_async.attr, 338 &dev_attr_error.attr, 339 &dev_attr_requests_completed.attr, 340 &dev_attr_requests_timedout.attr, 341 &dev_attr_timeout.attr, 342 &dev_attr_outstanding_operations_max.attr, 343 &dev_attr_timeout_min.attr, 344 &dev_attr_timeout_max.attr, 345 NULL, 346 }; 347 ATTRIBUTE_GROUPS(loopback); 348 349 static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error); 350 351 static u32 gb_loopback_nsec_to_usec_latency(u64 elapsed_nsecs) 352 { 353 do_div(elapsed_nsecs, NSEC_PER_USEC); 354 return elapsed_nsecs; 355 } 356 357 static u64 __gb_loopback_calc_latency(u64 t1, u64 t2) 358 { 359 if (t2 > t1) 360 return t2 - t1; 361 else 362 return NSEC_PER_DAY - t2 + t1; 363 } 364 365 static u64 gb_loopback_calc_latency(ktime_t ts, ktime_t te) 366 { 367 return __gb_loopback_calc_latency(ktime_to_ns(ts), ktime_to_ns(te)); 368 } 369 370 static int gb_loopback_operation_sync(struct gb_loopback *gb, int type, 371 void *request, int request_size, 372 void *response, int response_size) 373 { 374 struct gb_operation *operation; 375 ktime_t ts, te; 376 int ret; 377 378 ts = ktime_get(); 379 operation = gb_operation_create(gb->connection, type, request_size, 380 response_size, GFP_KERNEL); 381 if (!operation) 382 return -ENOMEM; 383 384 if (request_size) 385 memcpy(operation->request->payload, request, request_size); 386 387 ret = gb_operation_request_send_sync(operation); 388 if (ret) { 389 dev_err(&gb->connection->bundle->dev, 390 "synchronous operation failed: %d\n", ret); 391 goto out_put_operation; 392 } else { 393 if (response_size == operation->response->payload_size) { 394 memcpy(response, operation->response->payload, 395 response_size); 396 } else { 397 dev_err(&gb->connection->bundle->dev, 398 "response size %zu expected %d\n", 399 operation->response->payload_size, 400 response_size); 401 ret = -EINVAL; 402 goto out_put_operation; 403 } 404 } 405 406 te = ktime_get(); 407 408 /* Calculate the total time the message took */ 409 gb->elapsed_nsecs = gb_loopback_calc_latency(ts, te); 410 411 out_put_operation: 412 gb_operation_put(operation); 413 414 return ret; 415 } 416 417 static void gb_loopback_async_wait_all(struct gb_loopback *gb) 418 { 419 wait_event(gb->wq_completion, 420 !atomic_read(&gb->outstanding_operations)); 421 } 422 423 static void gb_loopback_async_operation_callback(struct gb_operation *operation) 424 { 425 struct gb_loopback_async_operation *op_async; 426 struct gb_loopback *gb; 427 ktime_t te; 428 int result; 429 430 te = ktime_get(); 431 result = gb_operation_result(operation); 432 op_async = gb_operation_get_data(operation); 433 gb = op_async->gb; 434 435 mutex_lock(&gb->mutex); 436 437 if (!result && op_async->completion) 438 result = op_async->completion(op_async); 439 440 if (!result) { 441 gb->elapsed_nsecs = gb_loopback_calc_latency(op_async->ts, te); 442 } else { 443 gb->error++; 444 if (result == -ETIMEDOUT) 445 gb->requests_timedout++; 446 } 447 448 gb->iteration_count++; 449 gb_loopback_calculate_stats(gb, result); 450 451 mutex_unlock(&gb->mutex); 452 453 dev_dbg(&gb->connection->bundle->dev, "complete operation %d\n", 454 operation->id); 455 456 /* Wake up waiters */ 457 atomic_dec(&op_async->gb->outstanding_operations); 458 wake_up(&gb->wq_completion); 459 460 /* Release resources */ 461 gb_operation_put(operation); 462 kfree(op_async); 463 } 464 465 static int gb_loopback_async_operation(struct gb_loopback *gb, int type, 466 void *request, int request_size, 467 int response_size, 468 void *completion) 469 { 470 struct gb_loopback_async_operation *op_async; 471 struct gb_operation *operation; 472 int ret; 473 474 op_async = kzalloc(sizeof(*op_async), GFP_KERNEL); 475 if (!op_async) 476 return -ENOMEM; 477 478 operation = gb_operation_create(gb->connection, type, request_size, 479 response_size, GFP_KERNEL); 480 if (!operation) { 481 kfree(op_async); 482 return -ENOMEM; 483 } 484 485 if (request_size) 486 memcpy(operation->request->payload, request, request_size); 487 488 gb_operation_set_data(operation, op_async); 489 490 op_async->gb = gb; 491 op_async->operation = operation; 492 op_async->completion = completion; 493 494 op_async->ts = ktime_get(); 495 496 atomic_inc(&gb->outstanding_operations); 497 ret = gb_operation_request_send(operation, 498 gb_loopback_async_operation_callback, 499 jiffies_to_msecs(gb->jiffy_timeout), 500 GFP_KERNEL); 501 if (ret) { 502 atomic_dec(&gb->outstanding_operations); 503 gb_operation_put(operation); 504 kfree(op_async); 505 } 506 return ret; 507 } 508 509 static int gb_loopback_sync_sink(struct gb_loopback *gb, u32 len) 510 { 511 struct gb_loopback_transfer_request *request; 512 int retval; 513 514 request = kmalloc(len + sizeof(*request), GFP_KERNEL); 515 if (!request) 516 return -ENOMEM; 517 518 request->len = cpu_to_le32(len); 519 retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_SINK, 520 request, len + sizeof(*request), 521 NULL, 0); 522 kfree(request); 523 return retval; 524 } 525 526 static int gb_loopback_sync_transfer(struct gb_loopback *gb, u32 len) 527 { 528 struct gb_loopback_transfer_request *request; 529 struct gb_loopback_transfer_response *response; 530 int retval; 531 532 gb->apbridge_latency_ts = 0; 533 gb->gbphy_latency_ts = 0; 534 535 request = kmalloc(len + sizeof(*request), GFP_KERNEL); 536 if (!request) 537 return -ENOMEM; 538 response = kmalloc(len + sizeof(*response), GFP_KERNEL); 539 if (!response) { 540 kfree(request); 541 return -ENOMEM; 542 } 543 544 memset(request->data, 0x5A, len); 545 546 request->len = cpu_to_le32(len); 547 retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_TRANSFER, 548 request, len + sizeof(*request), 549 response, len + sizeof(*response)); 550 if (retval) 551 goto gb_error; 552 553 if (memcmp(request->data, response->data, len)) { 554 dev_err(&gb->connection->bundle->dev, 555 "Loopback Data doesn't match\n"); 556 retval = -EREMOTEIO; 557 } 558 gb->apbridge_latency_ts = (u32)__le32_to_cpu(response->reserved0); 559 gb->gbphy_latency_ts = (u32)__le32_to_cpu(response->reserved1); 560 561 gb_error: 562 kfree(request); 563 kfree(response); 564 565 return retval; 566 } 567 568 static int gb_loopback_sync_ping(struct gb_loopback *gb) 569 { 570 return gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_PING, 571 NULL, 0, NULL, 0); 572 } 573 574 static int gb_loopback_async_sink(struct gb_loopback *gb, u32 len) 575 { 576 struct gb_loopback_transfer_request *request; 577 int retval; 578 579 request = kmalloc(len + sizeof(*request), GFP_KERNEL); 580 if (!request) 581 return -ENOMEM; 582 583 request->len = cpu_to_le32(len); 584 retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_SINK, 585 request, len + sizeof(*request), 586 0, NULL); 587 kfree(request); 588 return retval; 589 } 590 591 static int gb_loopback_async_transfer_complete( 592 struct gb_loopback_async_operation *op_async) 593 { 594 struct gb_loopback *gb; 595 struct gb_operation *operation; 596 struct gb_loopback_transfer_request *request; 597 struct gb_loopback_transfer_response *response; 598 size_t len; 599 int retval = 0; 600 601 gb = op_async->gb; 602 operation = op_async->operation; 603 request = operation->request->payload; 604 response = operation->response->payload; 605 len = le32_to_cpu(request->len); 606 607 if (memcmp(request->data, response->data, len)) { 608 dev_err(&gb->connection->bundle->dev, 609 "Loopback Data doesn't match operation id %d\n", 610 operation->id); 611 retval = -EREMOTEIO; 612 } else { 613 gb->apbridge_latency_ts = 614 (u32)__le32_to_cpu(response->reserved0); 615 gb->gbphy_latency_ts = 616 (u32)__le32_to_cpu(response->reserved1); 617 } 618 619 return retval; 620 } 621 622 static int gb_loopback_async_transfer(struct gb_loopback *gb, u32 len) 623 { 624 struct gb_loopback_transfer_request *request; 625 int retval, response_len; 626 627 request = kmalloc(len + sizeof(*request), GFP_KERNEL); 628 if (!request) 629 return -ENOMEM; 630 631 memset(request->data, 0x5A, len); 632 633 request->len = cpu_to_le32(len); 634 response_len = sizeof(struct gb_loopback_transfer_response); 635 retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_TRANSFER, 636 request, len + sizeof(*request), 637 len + response_len, 638 gb_loopback_async_transfer_complete); 639 if (retval) 640 goto gb_error; 641 642 gb_error: 643 kfree(request); 644 return retval; 645 } 646 647 static int gb_loopback_async_ping(struct gb_loopback *gb) 648 { 649 return gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_PING, 650 NULL, 0, 0, NULL); 651 } 652 653 static int gb_loopback_request_handler(struct gb_operation *operation) 654 { 655 struct gb_connection *connection = operation->connection; 656 struct gb_loopback_transfer_request *request; 657 struct gb_loopback_transfer_response *response; 658 struct device *dev = &connection->bundle->dev; 659 size_t len; 660 661 /* By convention, the AP initiates the version operation */ 662 switch (operation->type) { 663 case GB_LOOPBACK_TYPE_PING: 664 case GB_LOOPBACK_TYPE_SINK: 665 return 0; 666 case GB_LOOPBACK_TYPE_TRANSFER: 667 if (operation->request->payload_size < sizeof(*request)) { 668 dev_err(dev, "transfer request too small (%zu < %zu)\n", 669 operation->request->payload_size, 670 sizeof(*request)); 671 return -EINVAL; /* -EMSGSIZE */ 672 } 673 request = operation->request->payload; 674 len = le32_to_cpu(request->len); 675 if (len > gb_dev.size_max) { 676 dev_err(dev, "transfer request too large (%zu > %zu)\n", 677 len, gb_dev.size_max); 678 return -EINVAL; 679 } 680 681 if (!gb_operation_response_alloc(operation, 682 len + sizeof(*response), GFP_KERNEL)) { 683 dev_err(dev, "error allocating response\n"); 684 return -ENOMEM; 685 } 686 response = operation->response->payload; 687 response->len = cpu_to_le32(len); 688 if (len) 689 memcpy(response->data, request->data, len); 690 691 return 0; 692 default: 693 dev_err(dev, "unsupported request: %u\n", operation->type); 694 return -EINVAL; 695 } 696 } 697 698 static void gb_loopback_reset_stats(struct gb_loopback *gb) 699 { 700 struct gb_loopback_stats reset = { 701 .min = U32_MAX, 702 }; 703 704 /* Reset per-connection stats */ 705 memcpy(&gb->latency, &reset, 706 sizeof(struct gb_loopback_stats)); 707 memcpy(&gb->throughput, &reset, 708 sizeof(struct gb_loopback_stats)); 709 memcpy(&gb->requests_per_second, &reset, 710 sizeof(struct gb_loopback_stats)); 711 memcpy(&gb->apbridge_unipro_latency, &reset, 712 sizeof(struct gb_loopback_stats)); 713 memcpy(&gb->gbphy_firmware_latency, &reset, 714 sizeof(struct gb_loopback_stats)); 715 716 /* Should be initialized at least once per transaction set */ 717 gb->apbridge_latency_ts = 0; 718 gb->gbphy_latency_ts = 0; 719 gb->ts = ktime_set(0, 0); 720 } 721 722 static void gb_loopback_update_stats(struct gb_loopback_stats *stats, u32 val) 723 { 724 if (stats->min > val) 725 stats->min = val; 726 if (stats->max < val) 727 stats->max = val; 728 stats->sum += val; 729 stats->count++; 730 } 731 732 static void gb_loopback_update_stats_window(struct gb_loopback_stats *stats, 733 u64 val, u32 count) 734 { 735 stats->sum += val; 736 stats->count += count; 737 738 do_div(val, count); 739 if (stats->min > val) 740 stats->min = val; 741 if (stats->max < val) 742 stats->max = val; 743 } 744 745 static void gb_loopback_requests_update(struct gb_loopback *gb, u32 latency) 746 { 747 u64 req = gb->requests_completed * USEC_PER_SEC; 748 749 gb_loopback_update_stats_window(&gb->requests_per_second, req, latency); 750 } 751 752 static void gb_loopback_throughput_update(struct gb_loopback *gb, u32 latency) 753 { 754 u64 aggregate_size = sizeof(struct gb_operation_msg_hdr) * 2; 755 756 switch (gb->type) { 757 case GB_LOOPBACK_TYPE_PING: 758 break; 759 case GB_LOOPBACK_TYPE_SINK: 760 aggregate_size += sizeof(struct gb_loopback_transfer_request) + 761 gb->size; 762 break; 763 case GB_LOOPBACK_TYPE_TRANSFER: 764 aggregate_size += sizeof(struct gb_loopback_transfer_request) + 765 sizeof(struct gb_loopback_transfer_response) + 766 gb->size * 2; 767 break; 768 default: 769 return; 770 } 771 772 aggregate_size *= gb->requests_completed; 773 aggregate_size *= USEC_PER_SEC; 774 gb_loopback_update_stats_window(&gb->throughput, aggregate_size, 775 latency); 776 } 777 778 static void gb_loopback_calculate_latency_stats(struct gb_loopback *gb) 779 { 780 u32 lat; 781 782 /* Express latency in terms of microseconds */ 783 lat = gb_loopback_nsec_to_usec_latency(gb->elapsed_nsecs); 784 785 /* Log latency stastic */ 786 gb_loopback_update_stats(&gb->latency, lat); 787 788 /* Raw latency log on a per thread basis */ 789 kfifo_in(&gb->kfifo_lat, (unsigned char *)&lat, sizeof(lat)); 790 791 /* Log the firmware supplied latency values */ 792 gb_loopback_update_stats(&gb->apbridge_unipro_latency, 793 gb->apbridge_latency_ts); 794 gb_loopback_update_stats(&gb->gbphy_firmware_latency, 795 gb->gbphy_latency_ts); 796 } 797 798 static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error) 799 { 800 u64 nlat; 801 u32 lat; 802 ktime_t te; 803 804 if (!error) { 805 gb->requests_completed++; 806 gb_loopback_calculate_latency_stats(gb); 807 } 808 809 te = ktime_get(); 810 nlat = gb_loopback_calc_latency(gb->ts, te); 811 if (nlat >= NSEC_PER_SEC || gb->iteration_count == gb->iteration_max) { 812 lat = gb_loopback_nsec_to_usec_latency(nlat); 813 814 gb_loopback_throughput_update(gb, lat); 815 gb_loopback_requests_update(gb, lat); 816 817 if (gb->iteration_count != gb->iteration_max) { 818 gb->ts = te; 819 gb->requests_completed = 0; 820 } 821 } 822 } 823 824 static void gb_loopback_async_wait_to_send(struct gb_loopback *gb) 825 { 826 if (!(gb->async && gb->outstanding_operations_max)) 827 return; 828 wait_event_interruptible(gb->wq_completion, 829 (atomic_read(&gb->outstanding_operations) < 830 gb->outstanding_operations_max) || 831 kthread_should_stop()); 832 } 833 834 static int gb_loopback_fn(void *data) 835 { 836 int error = 0; 837 int us_wait = 0; 838 int type; 839 int ret; 840 u32 size; 841 842 struct gb_loopback *gb = data; 843 struct gb_bundle *bundle = gb->connection->bundle; 844 845 ret = gb_pm_runtime_get_sync(bundle); 846 if (ret) 847 return ret; 848 849 while (1) { 850 if (!gb->type) { 851 gb_pm_runtime_put_autosuspend(bundle); 852 wait_event_interruptible(gb->wq, gb->type || 853 kthread_should_stop()); 854 ret = gb_pm_runtime_get_sync(bundle); 855 if (ret) 856 return ret; 857 } 858 859 if (kthread_should_stop()) 860 break; 861 862 /* Limit the maximum number of in-flight async operations */ 863 gb_loopback_async_wait_to_send(gb); 864 if (kthread_should_stop()) 865 break; 866 867 mutex_lock(&gb->mutex); 868 869 /* Optionally terminate */ 870 if (gb->send_count == gb->iteration_max) { 871 mutex_unlock(&gb->mutex); 872 873 /* Wait for synchronous and asynchronus completion */ 874 gb_loopback_async_wait_all(gb); 875 876 /* Mark complete unless user-space has poked us */ 877 mutex_lock(&gb->mutex); 878 if (gb->iteration_count == gb->iteration_max) { 879 gb->type = 0; 880 gb->send_count = 0; 881 sysfs_notify(&gb->dev->kobj, NULL, 882 "iteration_count"); 883 dev_dbg(&bundle->dev, "load test complete\n"); 884 } else { 885 dev_dbg(&bundle->dev, 886 "continuing on with new test set\n"); 887 } 888 mutex_unlock(&gb->mutex); 889 continue; 890 } 891 size = gb->size; 892 us_wait = gb->us_wait; 893 type = gb->type; 894 if (ktime_to_ns(gb->ts) == 0) 895 gb->ts = ktime_get(); 896 897 /* Else operations to perform */ 898 if (gb->async) { 899 if (type == GB_LOOPBACK_TYPE_PING) 900 error = gb_loopback_async_ping(gb); 901 else if (type == GB_LOOPBACK_TYPE_TRANSFER) 902 error = gb_loopback_async_transfer(gb, size); 903 else if (type == GB_LOOPBACK_TYPE_SINK) 904 error = gb_loopback_async_sink(gb, size); 905 906 if (error) { 907 gb->error++; 908 gb->iteration_count++; 909 } 910 } else { 911 /* We are effectively single threaded here */ 912 if (type == GB_LOOPBACK_TYPE_PING) 913 error = gb_loopback_sync_ping(gb); 914 else if (type == GB_LOOPBACK_TYPE_TRANSFER) 915 error = gb_loopback_sync_transfer(gb, size); 916 else if (type == GB_LOOPBACK_TYPE_SINK) 917 error = gb_loopback_sync_sink(gb, size); 918 919 if (error) 920 gb->error++; 921 gb->iteration_count++; 922 gb_loopback_calculate_stats(gb, !!error); 923 } 924 gb->send_count++; 925 mutex_unlock(&gb->mutex); 926 927 if (us_wait) { 928 if (us_wait < 20000) 929 usleep_range(us_wait, us_wait + 100); 930 else 931 msleep(us_wait / 1000); 932 } 933 } 934 935 gb_pm_runtime_put_autosuspend(bundle); 936 937 return 0; 938 } 939 940 static int gb_loopback_dbgfs_latency_show_common(struct seq_file *s, 941 struct kfifo *kfifo, 942 struct mutex *mutex) 943 { 944 u32 latency; 945 int retval; 946 947 if (kfifo_len(kfifo) == 0) { 948 retval = -EAGAIN; 949 goto done; 950 } 951 952 mutex_lock(mutex); 953 retval = kfifo_out(kfifo, &latency, sizeof(latency)); 954 if (retval > 0) { 955 seq_printf(s, "%u", latency); 956 retval = 0; 957 } 958 mutex_unlock(mutex); 959 done: 960 return retval; 961 } 962 963 static int gb_loopback_dbgfs_latency_show(struct seq_file *s, void *unused) 964 { 965 struct gb_loopback *gb = s->private; 966 967 return gb_loopback_dbgfs_latency_show_common(s, &gb->kfifo_lat, 968 &gb->mutex); 969 } 970 DEFINE_SHOW_ATTRIBUTE(gb_loopback_dbgfs_latency); 971 972 #define DEBUGFS_NAMELEN 32 973 974 static int gb_loopback_probe(struct gb_bundle *bundle, 975 const struct greybus_bundle_id *id) 976 { 977 struct greybus_descriptor_cport *cport_desc; 978 struct gb_connection *connection; 979 struct gb_loopback *gb; 980 struct device *dev; 981 int retval; 982 char name[DEBUGFS_NAMELEN]; 983 unsigned long flags; 984 985 if (bundle->num_cports != 1) 986 return -ENODEV; 987 988 cport_desc = &bundle->cport_desc[0]; 989 if (cport_desc->protocol_id != GREYBUS_PROTOCOL_LOOPBACK) 990 return -ENODEV; 991 992 gb = kzalloc(sizeof(*gb), GFP_KERNEL); 993 if (!gb) 994 return -ENOMEM; 995 996 connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id), 997 gb_loopback_request_handler); 998 if (IS_ERR(connection)) { 999 retval = PTR_ERR(connection); 1000 goto out_kzalloc; 1001 } 1002 1003 gb->connection = connection; 1004 greybus_set_drvdata(bundle, gb); 1005 1006 init_waitqueue_head(&gb->wq); 1007 init_waitqueue_head(&gb->wq_completion); 1008 atomic_set(&gb->outstanding_operations, 0); 1009 gb_loopback_reset_stats(gb); 1010 1011 /* Reported values to user-space for min/max timeouts */ 1012 gb->timeout_min = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MIN); 1013 gb->timeout_max = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MAX); 1014 1015 if (!gb_dev.count) { 1016 /* Calculate maximum payload */ 1017 gb_dev.size_max = gb_operation_get_payload_size_max(connection); 1018 if (gb_dev.size_max <= 1019 sizeof(struct gb_loopback_transfer_request)) { 1020 retval = -EINVAL; 1021 goto out_connection_destroy; 1022 } 1023 gb_dev.size_max -= sizeof(struct gb_loopback_transfer_request); 1024 } 1025 1026 /* Create per-connection sysfs and debugfs data-points */ 1027 snprintf(name, sizeof(name), "raw_latency_%s", 1028 dev_name(&connection->bundle->dev)); 1029 gb->file = debugfs_create_file(name, S_IFREG | 0444, gb_dev.root, gb, 1030 &gb_loopback_dbgfs_latency_fops); 1031 1032 gb->id = ida_simple_get(&loopback_ida, 0, 0, GFP_KERNEL); 1033 if (gb->id < 0) { 1034 retval = gb->id; 1035 goto out_debugfs_remove; 1036 } 1037 1038 retval = gb_connection_enable(connection); 1039 if (retval) 1040 goto out_ida_remove; 1041 1042 dev = device_create_with_groups(&loopback_class, 1043 &connection->bundle->dev, 1044 MKDEV(0, 0), gb, loopback_groups, 1045 "gb_loopback%d", gb->id); 1046 if (IS_ERR(dev)) { 1047 retval = PTR_ERR(dev); 1048 goto out_connection_disable; 1049 } 1050 gb->dev = dev; 1051 1052 /* Allocate kfifo */ 1053 if (kfifo_alloc(&gb->kfifo_lat, kfifo_depth * sizeof(u32), 1054 GFP_KERNEL)) { 1055 retval = -ENOMEM; 1056 goto out_conn; 1057 } 1058 /* Fork worker thread */ 1059 mutex_init(&gb->mutex); 1060 gb->task = kthread_run(gb_loopback_fn, gb, "gb_loopback"); 1061 if (IS_ERR(gb->task)) { 1062 retval = PTR_ERR(gb->task); 1063 goto out_kfifo; 1064 } 1065 1066 spin_lock_irqsave(&gb_dev.lock, flags); 1067 gb_dev.count++; 1068 spin_unlock_irqrestore(&gb_dev.lock, flags); 1069 1070 gb_connection_latency_tag_enable(connection); 1071 1072 gb_pm_runtime_put_autosuspend(bundle); 1073 1074 return 0; 1075 1076 out_kfifo: 1077 kfifo_free(&gb->kfifo_lat); 1078 out_conn: 1079 device_unregister(dev); 1080 out_connection_disable: 1081 gb_connection_disable(connection); 1082 out_ida_remove: 1083 ida_simple_remove(&loopback_ida, gb->id); 1084 out_debugfs_remove: 1085 debugfs_remove(gb->file); 1086 out_connection_destroy: 1087 gb_connection_destroy(connection); 1088 out_kzalloc: 1089 kfree(gb); 1090 1091 return retval; 1092 } 1093 1094 static void gb_loopback_disconnect(struct gb_bundle *bundle) 1095 { 1096 struct gb_loopback *gb = greybus_get_drvdata(bundle); 1097 unsigned long flags; 1098 int ret; 1099 1100 ret = gb_pm_runtime_get_sync(bundle); 1101 if (ret) 1102 gb_pm_runtime_get_noresume(bundle); 1103 1104 gb_connection_disable(gb->connection); 1105 1106 if (!IS_ERR_OR_NULL(gb->task)) 1107 kthread_stop(gb->task); 1108 1109 kfifo_free(&gb->kfifo_lat); 1110 gb_connection_latency_tag_disable(gb->connection); 1111 debugfs_remove(gb->file); 1112 1113 /* 1114 * FIXME: gb_loopback_async_wait_all() is redundant now, as connection 1115 * is disabled at the beginning and so we can't have any more 1116 * incoming/outgoing requests. 1117 */ 1118 gb_loopback_async_wait_all(gb); 1119 1120 spin_lock_irqsave(&gb_dev.lock, flags); 1121 gb_dev.count--; 1122 spin_unlock_irqrestore(&gb_dev.lock, flags); 1123 1124 device_unregister(gb->dev); 1125 ida_simple_remove(&loopback_ida, gb->id); 1126 1127 gb_connection_destroy(gb->connection); 1128 kfree(gb); 1129 } 1130 1131 static const struct greybus_bundle_id gb_loopback_id_table[] = { 1132 { GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_LOOPBACK) }, 1133 { } 1134 }; 1135 MODULE_DEVICE_TABLE(greybus, gb_loopback_id_table); 1136 1137 static struct greybus_driver gb_loopback_driver = { 1138 .name = "loopback", 1139 .probe = gb_loopback_probe, 1140 .disconnect = gb_loopback_disconnect, 1141 .id_table = gb_loopback_id_table, 1142 }; 1143 1144 static int loopback_init(void) 1145 { 1146 int retval; 1147 1148 spin_lock_init(&gb_dev.lock); 1149 gb_dev.root = debugfs_create_dir("gb_loopback", NULL); 1150 1151 retval = class_register(&loopback_class); 1152 if (retval) 1153 goto err; 1154 1155 retval = greybus_register(&gb_loopback_driver); 1156 if (retval) 1157 goto err_unregister; 1158 1159 return 0; 1160 1161 err_unregister: 1162 class_unregister(&loopback_class); 1163 err: 1164 debugfs_remove_recursive(gb_dev.root); 1165 return retval; 1166 } 1167 module_init(loopback_init); 1168 1169 static void __exit loopback_exit(void) 1170 { 1171 debugfs_remove_recursive(gb_dev.root); 1172 greybus_deregister(&gb_loopback_driver); 1173 class_unregister(&loopback_class); 1174 ida_destroy(&loopback_ida); 1175 } 1176 module_exit(loopback_exit); 1177 1178 MODULE_LICENSE("GPL v2"); 1179