1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * ring buffer based function tracer 4 * 5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com> 6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 7 * 8 * Originally taken from the RT patch by: 9 * Arnaldo Carvalho de Melo <acme@redhat.com> 10 * 11 * Based on code from the latency_tracer, that is: 12 * Copyright (C) 2004-2006 Ingo Molnar 13 * Copyright (C) 2004 Nadia Yvette Chambers 14 */ 15 #include <linux/ring_buffer.h> 16 #include <generated/utsrelease.h> 17 #include <linux/stacktrace.h> 18 #include <linux/writeback.h> 19 #include <linux/kallsyms.h> 20 #include <linux/security.h> 21 #include <linux/seq_file.h> 22 #include <linux/irqflags.h> 23 #include <linux/debugfs.h> 24 #include <linux/tracefs.h> 25 #include <linux/pagemap.h> 26 #include <linux/hardirq.h> 27 #include <linux/linkage.h> 28 #include <linux/uaccess.h> 29 #include <linux/vmalloc.h> 30 #include <linux/ftrace.h> 31 #include <linux/module.h> 32 #include <linux/percpu.h> 33 #include <linux/splice.h> 34 #include <linux/kdebug.h> 35 #include <linux/string.h> 36 #include <linux/mount.h> 37 #include <linux/rwsem.h> 38 #include <linux/slab.h> 39 #include <linux/ctype.h> 40 #include <linux/init.h> 41 #include <linux/panic_notifier.h> 42 #include <linux/poll.h> 43 #include <linux/nmi.h> 44 #include <linux/fs.h> 45 #include <linux/trace.h> 46 #include <linux/sched/clock.h> 47 #include <linux/sched/rt.h> 48 #include <linux/fsnotify.h> 49 #include <linux/irq_work.h> 50 #include <linux/workqueue.h> 51 52 #include <asm/setup.h> /* COMMAND_LINE_SIZE */ 53 54 #include "trace.h" 55 #include "trace_output.h" 56 57 #ifdef CONFIG_FTRACE_STARTUP_TEST 58 /* 59 * We need to change this state when a selftest is running. 60 * A selftest will lurk into the ring-buffer to count the 61 * entries inserted during the selftest although some concurrent 62 * insertions into the ring-buffer such as trace_printk could occurred 63 * at the same time, giving false positive or negative results. 64 */ 65 static bool __read_mostly tracing_selftest_running; 66 67 /* 68 * If boot-time tracing including tracers/events via kernel cmdline 69 * is running, we do not want to run SELFTEST. 70 */ 71 bool __read_mostly tracing_selftest_disabled; 72 73 void __init disable_tracing_selftest(const char *reason) 74 { 75 if (!tracing_selftest_disabled) { 76 tracing_selftest_disabled = true; 77 pr_info("Ftrace startup test is disabled due to %s\n", reason); 78 } 79 } 80 #else 81 #define tracing_selftest_running 0 82 #define tracing_selftest_disabled 0 83 #endif 84 85 /* Pipe tracepoints to printk */ 86 static struct trace_iterator *tracepoint_print_iter; 87 int tracepoint_printk; 88 static bool tracepoint_printk_stop_on_boot __initdata; 89 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key); 90 91 /* For tracers that don't implement custom flags */ 92 static struct tracer_opt dummy_tracer_opt[] = { 93 { } 94 }; 95 96 static int 97 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 98 { 99 return 0; 100 } 101 102 /* 103 * To prevent the comm cache from being overwritten when no 104 * tracing is active, only save the comm when a trace event 105 * occurred. 106 */ 107 static DEFINE_PER_CPU(bool, trace_taskinfo_save); 108 109 /* 110 * Kill all tracing for good (never come back). 111 * It is initialized to 1 but will turn to zero if the initialization 112 * of the tracer is successful. But that is the only place that sets 113 * this back to zero. 114 */ 115 static int tracing_disabled = 1; 116 117 cpumask_var_t __read_mostly tracing_buffer_mask; 118 119 /* 120 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops 121 * 122 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops 123 * is set, then ftrace_dump is called. This will output the contents 124 * of the ftrace buffers to the console. This is very useful for 125 * capturing traces that lead to crashes and outputing it to a 126 * serial console. 127 * 128 * It is default off, but you can enable it with either specifying 129 * "ftrace_dump_on_oops" in the kernel command line, or setting 130 * /proc/sys/kernel/ftrace_dump_on_oops 131 * Set 1 if you want to dump buffers of all CPUs 132 * Set 2 if you want to dump the buffer of the CPU that triggered oops 133 */ 134 135 enum ftrace_dump_mode ftrace_dump_on_oops; 136 137 /* When set, tracing will stop when a WARN*() is hit */ 138 int __disable_trace_on_warning; 139 140 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 141 /* Map of enums to their values, for "eval_map" file */ 142 struct trace_eval_map_head { 143 struct module *mod; 144 unsigned long length; 145 }; 146 147 union trace_eval_map_item; 148 149 struct trace_eval_map_tail { 150 /* 151 * "end" is first and points to NULL as it must be different 152 * than "mod" or "eval_string" 153 */ 154 union trace_eval_map_item *next; 155 const char *end; /* points to NULL */ 156 }; 157 158 static DEFINE_MUTEX(trace_eval_mutex); 159 160 /* 161 * The trace_eval_maps are saved in an array with two extra elements, 162 * one at the beginning, and one at the end. The beginning item contains 163 * the count of the saved maps (head.length), and the module they 164 * belong to if not built in (head.mod). The ending item contains a 165 * pointer to the next array of saved eval_map items. 166 */ 167 union trace_eval_map_item { 168 struct trace_eval_map map; 169 struct trace_eval_map_head head; 170 struct trace_eval_map_tail tail; 171 }; 172 173 static union trace_eval_map_item *trace_eval_maps; 174 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */ 175 176 int tracing_set_tracer(struct trace_array *tr, const char *buf); 177 static void ftrace_trace_userstack(struct trace_array *tr, 178 struct trace_buffer *buffer, 179 unsigned int trace_ctx); 180 181 #define MAX_TRACER_SIZE 100 182 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; 183 static char *default_bootup_tracer; 184 185 static bool allocate_snapshot; 186 static bool snapshot_at_boot; 187 188 static char boot_instance_info[COMMAND_LINE_SIZE] __initdata; 189 static int boot_instance_index; 190 191 static char boot_snapshot_info[COMMAND_LINE_SIZE] __initdata; 192 static int boot_snapshot_index; 193 194 static int __init set_cmdline_ftrace(char *str) 195 { 196 strscpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); 197 default_bootup_tracer = bootup_tracer_buf; 198 /* We are using ftrace early, expand it */ 199 trace_set_ring_buffer_expanded(NULL); 200 return 1; 201 } 202 __setup("ftrace=", set_cmdline_ftrace); 203 204 static int __init set_ftrace_dump_on_oops(char *str) 205 { 206 if (*str++ != '=' || !*str || !strcmp("1", str)) { 207 ftrace_dump_on_oops = DUMP_ALL; 208 return 1; 209 } 210 211 if (!strcmp("orig_cpu", str) || !strcmp("2", str)) { 212 ftrace_dump_on_oops = DUMP_ORIG; 213 return 1; 214 } 215 216 return 0; 217 } 218 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); 219 220 static int __init stop_trace_on_warning(char *str) 221 { 222 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 223 __disable_trace_on_warning = 1; 224 return 1; 225 } 226 __setup("traceoff_on_warning", stop_trace_on_warning); 227 228 static int __init boot_alloc_snapshot(char *str) 229 { 230 char *slot = boot_snapshot_info + boot_snapshot_index; 231 int left = sizeof(boot_snapshot_info) - boot_snapshot_index; 232 int ret; 233 234 if (str[0] == '=') { 235 str++; 236 if (strlen(str) >= left) 237 return -1; 238 239 ret = snprintf(slot, left, "%s\t", str); 240 boot_snapshot_index += ret; 241 } else { 242 allocate_snapshot = true; 243 /* We also need the main ring buffer expanded */ 244 trace_set_ring_buffer_expanded(NULL); 245 } 246 return 1; 247 } 248 __setup("alloc_snapshot", boot_alloc_snapshot); 249 250 251 static int __init boot_snapshot(char *str) 252 { 253 snapshot_at_boot = true; 254 boot_alloc_snapshot(str); 255 return 1; 256 } 257 __setup("ftrace_boot_snapshot", boot_snapshot); 258 259 260 static int __init boot_instance(char *str) 261 { 262 char *slot = boot_instance_info + boot_instance_index; 263 int left = sizeof(boot_instance_info) - boot_instance_index; 264 int ret; 265 266 if (strlen(str) >= left) 267 return -1; 268 269 ret = snprintf(slot, left, "%s\t", str); 270 boot_instance_index += ret; 271 272 return 1; 273 } 274 __setup("trace_instance=", boot_instance); 275 276 277 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata; 278 279 static int __init set_trace_boot_options(char *str) 280 { 281 strscpy(trace_boot_options_buf, str, MAX_TRACER_SIZE); 282 return 1; 283 } 284 __setup("trace_options=", set_trace_boot_options); 285 286 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata; 287 static char *trace_boot_clock __initdata; 288 289 static int __init set_trace_boot_clock(char *str) 290 { 291 strscpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE); 292 trace_boot_clock = trace_boot_clock_buf; 293 return 1; 294 } 295 __setup("trace_clock=", set_trace_boot_clock); 296 297 static int __init set_tracepoint_printk(char *str) 298 { 299 /* Ignore the "tp_printk_stop_on_boot" param */ 300 if (*str == '_') 301 return 0; 302 303 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 304 tracepoint_printk = 1; 305 return 1; 306 } 307 __setup("tp_printk", set_tracepoint_printk); 308 309 static int __init set_tracepoint_printk_stop(char *str) 310 { 311 tracepoint_printk_stop_on_boot = true; 312 return 1; 313 } 314 __setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop); 315 316 unsigned long long ns2usecs(u64 nsec) 317 { 318 nsec += 500; 319 do_div(nsec, 1000); 320 return nsec; 321 } 322 323 static void 324 trace_process_export(struct trace_export *export, 325 struct ring_buffer_event *event, int flag) 326 { 327 struct trace_entry *entry; 328 unsigned int size = 0; 329 330 if (export->flags & flag) { 331 entry = ring_buffer_event_data(event); 332 size = ring_buffer_event_length(event); 333 export->write(export, entry, size); 334 } 335 } 336 337 static DEFINE_MUTEX(ftrace_export_lock); 338 339 static struct trace_export __rcu *ftrace_exports_list __read_mostly; 340 341 static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled); 342 static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled); 343 static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled); 344 345 static inline void ftrace_exports_enable(struct trace_export *export) 346 { 347 if (export->flags & TRACE_EXPORT_FUNCTION) 348 static_branch_inc(&trace_function_exports_enabled); 349 350 if (export->flags & TRACE_EXPORT_EVENT) 351 static_branch_inc(&trace_event_exports_enabled); 352 353 if (export->flags & TRACE_EXPORT_MARKER) 354 static_branch_inc(&trace_marker_exports_enabled); 355 } 356 357 static inline void ftrace_exports_disable(struct trace_export *export) 358 { 359 if (export->flags & TRACE_EXPORT_FUNCTION) 360 static_branch_dec(&trace_function_exports_enabled); 361 362 if (export->flags & TRACE_EXPORT_EVENT) 363 static_branch_dec(&trace_event_exports_enabled); 364 365 if (export->flags & TRACE_EXPORT_MARKER) 366 static_branch_dec(&trace_marker_exports_enabled); 367 } 368 369 static void ftrace_exports(struct ring_buffer_event *event, int flag) 370 { 371 struct trace_export *export; 372 373 preempt_disable_notrace(); 374 375 export = rcu_dereference_raw_check(ftrace_exports_list); 376 while (export) { 377 trace_process_export(export, event, flag); 378 export = rcu_dereference_raw_check(export->next); 379 } 380 381 preempt_enable_notrace(); 382 } 383 384 static inline void 385 add_trace_export(struct trace_export **list, struct trace_export *export) 386 { 387 rcu_assign_pointer(export->next, *list); 388 /* 389 * We are entering export into the list but another 390 * CPU might be walking that list. We need to make sure 391 * the export->next pointer is valid before another CPU sees 392 * the export pointer included into the list. 393 */ 394 rcu_assign_pointer(*list, export); 395 } 396 397 static inline int 398 rm_trace_export(struct trace_export **list, struct trace_export *export) 399 { 400 struct trace_export **p; 401 402 for (p = list; *p != NULL; p = &(*p)->next) 403 if (*p == export) 404 break; 405 406 if (*p != export) 407 return -1; 408 409 rcu_assign_pointer(*p, (*p)->next); 410 411 return 0; 412 } 413 414 static inline void 415 add_ftrace_export(struct trace_export **list, struct trace_export *export) 416 { 417 ftrace_exports_enable(export); 418 419 add_trace_export(list, export); 420 } 421 422 static inline int 423 rm_ftrace_export(struct trace_export **list, struct trace_export *export) 424 { 425 int ret; 426 427 ret = rm_trace_export(list, export); 428 ftrace_exports_disable(export); 429 430 return ret; 431 } 432 433 int register_ftrace_export(struct trace_export *export) 434 { 435 if (WARN_ON_ONCE(!export->write)) 436 return -1; 437 438 mutex_lock(&ftrace_export_lock); 439 440 add_ftrace_export(&ftrace_exports_list, export); 441 442 mutex_unlock(&ftrace_export_lock); 443 444 return 0; 445 } 446 EXPORT_SYMBOL_GPL(register_ftrace_export); 447 448 int unregister_ftrace_export(struct trace_export *export) 449 { 450 int ret; 451 452 mutex_lock(&ftrace_export_lock); 453 454 ret = rm_ftrace_export(&ftrace_exports_list, export); 455 456 mutex_unlock(&ftrace_export_lock); 457 458 return ret; 459 } 460 EXPORT_SYMBOL_GPL(unregister_ftrace_export); 461 462 /* trace_flags holds trace_options default values */ 463 #define TRACE_DEFAULT_FLAGS \ 464 (FUNCTION_DEFAULT_FLAGS | \ 465 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \ 466 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \ 467 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \ 468 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | \ 469 TRACE_ITER_HASH_PTR) 470 471 /* trace_options that are only supported by global_trace */ 472 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \ 473 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD) 474 475 /* trace_flags that are default zero for instances */ 476 #define ZEROED_TRACE_FLAGS \ 477 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK) 478 479 /* 480 * The global_trace is the descriptor that holds the top-level tracing 481 * buffers for the live tracing. 482 */ 483 static struct trace_array global_trace = { 484 .trace_flags = TRACE_DEFAULT_FLAGS, 485 }; 486 487 void trace_set_ring_buffer_expanded(struct trace_array *tr) 488 { 489 if (!tr) 490 tr = &global_trace; 491 tr->ring_buffer_expanded = true; 492 } 493 494 LIST_HEAD(ftrace_trace_arrays); 495 496 int trace_array_get(struct trace_array *this_tr) 497 { 498 struct trace_array *tr; 499 int ret = -ENODEV; 500 501 mutex_lock(&trace_types_lock); 502 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 503 if (tr == this_tr) { 504 tr->ref++; 505 ret = 0; 506 break; 507 } 508 } 509 mutex_unlock(&trace_types_lock); 510 511 return ret; 512 } 513 514 static void __trace_array_put(struct trace_array *this_tr) 515 { 516 WARN_ON(!this_tr->ref); 517 this_tr->ref--; 518 } 519 520 /** 521 * trace_array_put - Decrement the reference counter for this trace array. 522 * @this_tr : pointer to the trace array 523 * 524 * NOTE: Use this when we no longer need the trace array returned by 525 * trace_array_get_by_name(). This ensures the trace array can be later 526 * destroyed. 527 * 528 */ 529 void trace_array_put(struct trace_array *this_tr) 530 { 531 if (!this_tr) 532 return; 533 534 mutex_lock(&trace_types_lock); 535 __trace_array_put(this_tr); 536 mutex_unlock(&trace_types_lock); 537 } 538 EXPORT_SYMBOL_GPL(trace_array_put); 539 540 int tracing_check_open_get_tr(struct trace_array *tr) 541 { 542 int ret; 543 544 ret = security_locked_down(LOCKDOWN_TRACEFS); 545 if (ret) 546 return ret; 547 548 if (tracing_disabled) 549 return -ENODEV; 550 551 if (tr && trace_array_get(tr) < 0) 552 return -ENODEV; 553 554 return 0; 555 } 556 557 int call_filter_check_discard(struct trace_event_call *call, void *rec, 558 struct trace_buffer *buffer, 559 struct ring_buffer_event *event) 560 { 561 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) && 562 !filter_match_preds(call->filter, rec)) { 563 __trace_event_discard_commit(buffer, event); 564 return 1; 565 } 566 567 return 0; 568 } 569 570 /** 571 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list 572 * @filtered_pids: The list of pids to check 573 * @search_pid: The PID to find in @filtered_pids 574 * 575 * Returns true if @search_pid is found in @filtered_pids, and false otherwise. 576 */ 577 bool 578 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid) 579 { 580 return trace_pid_list_is_set(filtered_pids, search_pid); 581 } 582 583 /** 584 * trace_ignore_this_task - should a task be ignored for tracing 585 * @filtered_pids: The list of pids to check 586 * @filtered_no_pids: The list of pids not to be traced 587 * @task: The task that should be ignored if not filtered 588 * 589 * Checks if @task should be traced or not from @filtered_pids. 590 * Returns true if @task should *NOT* be traced. 591 * Returns false if @task should be traced. 592 */ 593 bool 594 trace_ignore_this_task(struct trace_pid_list *filtered_pids, 595 struct trace_pid_list *filtered_no_pids, 596 struct task_struct *task) 597 { 598 /* 599 * If filtered_no_pids is not empty, and the task's pid is listed 600 * in filtered_no_pids, then return true. 601 * Otherwise, if filtered_pids is empty, that means we can 602 * trace all tasks. If it has content, then only trace pids 603 * within filtered_pids. 604 */ 605 606 return (filtered_pids && 607 !trace_find_filtered_pid(filtered_pids, task->pid)) || 608 (filtered_no_pids && 609 trace_find_filtered_pid(filtered_no_pids, task->pid)); 610 } 611 612 /** 613 * trace_filter_add_remove_task - Add or remove a task from a pid_list 614 * @pid_list: The list to modify 615 * @self: The current task for fork or NULL for exit 616 * @task: The task to add or remove 617 * 618 * If adding a task, if @self is defined, the task is only added if @self 619 * is also included in @pid_list. This happens on fork and tasks should 620 * only be added when the parent is listed. If @self is NULL, then the 621 * @task pid will be removed from the list, which would happen on exit 622 * of a task. 623 */ 624 void trace_filter_add_remove_task(struct trace_pid_list *pid_list, 625 struct task_struct *self, 626 struct task_struct *task) 627 { 628 if (!pid_list) 629 return; 630 631 /* For forks, we only add if the forking task is listed */ 632 if (self) { 633 if (!trace_find_filtered_pid(pid_list, self->pid)) 634 return; 635 } 636 637 /* "self" is set for forks, and NULL for exits */ 638 if (self) 639 trace_pid_list_set(pid_list, task->pid); 640 else 641 trace_pid_list_clear(pid_list, task->pid); 642 } 643 644 /** 645 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list 646 * @pid_list: The pid list to show 647 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed) 648 * @pos: The position of the file 649 * 650 * This is used by the seq_file "next" operation to iterate the pids 651 * listed in a trace_pid_list structure. 652 * 653 * Returns the pid+1 as we want to display pid of zero, but NULL would 654 * stop the iteration. 655 */ 656 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos) 657 { 658 long pid = (unsigned long)v; 659 unsigned int next; 660 661 (*pos)++; 662 663 /* pid already is +1 of the actual previous bit */ 664 if (trace_pid_list_next(pid_list, pid, &next) < 0) 665 return NULL; 666 667 pid = next; 668 669 /* Return pid + 1 to allow zero to be represented */ 670 return (void *)(pid + 1); 671 } 672 673 /** 674 * trace_pid_start - Used for seq_file to start reading pid lists 675 * @pid_list: The pid list to show 676 * @pos: The position of the file 677 * 678 * This is used by seq_file "start" operation to start the iteration 679 * of listing pids. 680 * 681 * Returns the pid+1 as we want to display pid of zero, but NULL would 682 * stop the iteration. 683 */ 684 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos) 685 { 686 unsigned long pid; 687 unsigned int first; 688 loff_t l = 0; 689 690 if (trace_pid_list_first(pid_list, &first) < 0) 691 return NULL; 692 693 pid = first; 694 695 /* Return pid + 1 so that zero can be the exit value */ 696 for (pid++; pid && l < *pos; 697 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l)) 698 ; 699 return (void *)pid; 700 } 701 702 /** 703 * trace_pid_show - show the current pid in seq_file processing 704 * @m: The seq_file structure to write into 705 * @v: A void pointer of the pid (+1) value to display 706 * 707 * Can be directly used by seq_file operations to display the current 708 * pid value. 709 */ 710 int trace_pid_show(struct seq_file *m, void *v) 711 { 712 unsigned long pid = (unsigned long)v - 1; 713 714 seq_printf(m, "%lu\n", pid); 715 return 0; 716 } 717 718 /* 128 should be much more than enough */ 719 #define PID_BUF_SIZE 127 720 721 int trace_pid_write(struct trace_pid_list *filtered_pids, 722 struct trace_pid_list **new_pid_list, 723 const char __user *ubuf, size_t cnt) 724 { 725 struct trace_pid_list *pid_list; 726 struct trace_parser parser; 727 unsigned long val; 728 int nr_pids = 0; 729 ssize_t read = 0; 730 ssize_t ret; 731 loff_t pos; 732 pid_t pid; 733 734 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1)) 735 return -ENOMEM; 736 737 /* 738 * Always recreate a new array. The write is an all or nothing 739 * operation. Always create a new array when adding new pids by 740 * the user. If the operation fails, then the current list is 741 * not modified. 742 */ 743 pid_list = trace_pid_list_alloc(); 744 if (!pid_list) { 745 trace_parser_put(&parser); 746 return -ENOMEM; 747 } 748 749 if (filtered_pids) { 750 /* copy the current bits to the new max */ 751 ret = trace_pid_list_first(filtered_pids, &pid); 752 while (!ret) { 753 trace_pid_list_set(pid_list, pid); 754 ret = trace_pid_list_next(filtered_pids, pid + 1, &pid); 755 nr_pids++; 756 } 757 } 758 759 ret = 0; 760 while (cnt > 0) { 761 762 pos = 0; 763 764 ret = trace_get_user(&parser, ubuf, cnt, &pos); 765 if (ret < 0) 766 break; 767 768 read += ret; 769 ubuf += ret; 770 cnt -= ret; 771 772 if (!trace_parser_loaded(&parser)) 773 break; 774 775 ret = -EINVAL; 776 if (kstrtoul(parser.buffer, 0, &val)) 777 break; 778 779 pid = (pid_t)val; 780 781 if (trace_pid_list_set(pid_list, pid) < 0) { 782 ret = -1; 783 break; 784 } 785 nr_pids++; 786 787 trace_parser_clear(&parser); 788 ret = 0; 789 } 790 trace_parser_put(&parser); 791 792 if (ret < 0) { 793 trace_pid_list_free(pid_list); 794 return ret; 795 } 796 797 if (!nr_pids) { 798 /* Cleared the list of pids */ 799 trace_pid_list_free(pid_list); 800 pid_list = NULL; 801 } 802 803 *new_pid_list = pid_list; 804 805 return read; 806 } 807 808 static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu) 809 { 810 u64 ts; 811 812 /* Early boot up does not have a buffer yet */ 813 if (!buf->buffer) 814 return trace_clock_local(); 815 816 ts = ring_buffer_time_stamp(buf->buffer); 817 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts); 818 819 return ts; 820 } 821 822 u64 ftrace_now(int cpu) 823 { 824 return buffer_ftrace_now(&global_trace.array_buffer, cpu); 825 } 826 827 /** 828 * tracing_is_enabled - Show if global_trace has been enabled 829 * 830 * Shows if the global trace has been enabled or not. It uses the 831 * mirror flag "buffer_disabled" to be used in fast paths such as for 832 * the irqsoff tracer. But it may be inaccurate due to races. If you 833 * need to know the accurate state, use tracing_is_on() which is a little 834 * slower, but accurate. 835 */ 836 int tracing_is_enabled(void) 837 { 838 /* 839 * For quick access (irqsoff uses this in fast path), just 840 * return the mirror variable of the state of the ring buffer. 841 * It's a little racy, but we don't really care. 842 */ 843 smp_rmb(); 844 return !global_trace.buffer_disabled; 845 } 846 847 /* 848 * trace_buf_size is the size in bytes that is allocated 849 * for a buffer. Note, the number of bytes is always rounded 850 * to page size. 851 * 852 * This number is purposely set to a low number of 16384. 853 * If the dump on oops happens, it will be much appreciated 854 * to not have to wait for all that output. Anyway this can be 855 * boot time and run time configurable. 856 */ 857 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */ 858 859 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT; 860 861 /* trace_types holds a link list of available tracers. */ 862 static struct tracer *trace_types __read_mostly; 863 864 /* 865 * trace_types_lock is used to protect the trace_types list. 866 */ 867 DEFINE_MUTEX(trace_types_lock); 868 869 /* 870 * serialize the access of the ring buffer 871 * 872 * ring buffer serializes readers, but it is low level protection. 873 * The validity of the events (which returns by ring_buffer_peek() ..etc) 874 * are not protected by ring buffer. 875 * 876 * The content of events may become garbage if we allow other process consumes 877 * these events concurrently: 878 * A) the page of the consumed events may become a normal page 879 * (not reader page) in ring buffer, and this page will be rewritten 880 * by events producer. 881 * B) The page of the consumed events may become a page for splice_read, 882 * and this page will be returned to system. 883 * 884 * These primitives allow multi process access to different cpu ring buffer 885 * concurrently. 886 * 887 * These primitives don't distinguish read-only and read-consume access. 888 * Multi read-only access are also serialized. 889 */ 890 891 #ifdef CONFIG_SMP 892 static DECLARE_RWSEM(all_cpu_access_lock); 893 static DEFINE_PER_CPU(struct mutex, cpu_access_lock); 894 895 static inline void trace_access_lock(int cpu) 896 { 897 if (cpu == RING_BUFFER_ALL_CPUS) { 898 /* gain it for accessing the whole ring buffer. */ 899 down_write(&all_cpu_access_lock); 900 } else { 901 /* gain it for accessing a cpu ring buffer. */ 902 903 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */ 904 down_read(&all_cpu_access_lock); 905 906 /* Secondly block other access to this @cpu ring buffer. */ 907 mutex_lock(&per_cpu(cpu_access_lock, cpu)); 908 } 909 } 910 911 static inline void trace_access_unlock(int cpu) 912 { 913 if (cpu == RING_BUFFER_ALL_CPUS) { 914 up_write(&all_cpu_access_lock); 915 } else { 916 mutex_unlock(&per_cpu(cpu_access_lock, cpu)); 917 up_read(&all_cpu_access_lock); 918 } 919 } 920 921 static inline void trace_access_lock_init(void) 922 { 923 int cpu; 924 925 for_each_possible_cpu(cpu) 926 mutex_init(&per_cpu(cpu_access_lock, cpu)); 927 } 928 929 #else 930 931 static DEFINE_MUTEX(access_lock); 932 933 static inline void trace_access_lock(int cpu) 934 { 935 (void)cpu; 936 mutex_lock(&access_lock); 937 } 938 939 static inline void trace_access_unlock(int cpu) 940 { 941 (void)cpu; 942 mutex_unlock(&access_lock); 943 } 944 945 static inline void trace_access_lock_init(void) 946 { 947 } 948 949 #endif 950 951 #ifdef CONFIG_STACKTRACE 952 static void __ftrace_trace_stack(struct trace_buffer *buffer, 953 unsigned int trace_ctx, 954 int skip, struct pt_regs *regs); 955 static inline void ftrace_trace_stack(struct trace_array *tr, 956 struct trace_buffer *buffer, 957 unsigned int trace_ctx, 958 int skip, struct pt_regs *regs); 959 960 #else 961 static inline void __ftrace_trace_stack(struct trace_buffer *buffer, 962 unsigned int trace_ctx, 963 int skip, struct pt_regs *regs) 964 { 965 } 966 static inline void ftrace_trace_stack(struct trace_array *tr, 967 struct trace_buffer *buffer, 968 unsigned long trace_ctx, 969 int skip, struct pt_regs *regs) 970 { 971 } 972 973 #endif 974 975 static __always_inline void 976 trace_event_setup(struct ring_buffer_event *event, 977 int type, unsigned int trace_ctx) 978 { 979 struct trace_entry *ent = ring_buffer_event_data(event); 980 981 tracing_generic_entry_update(ent, type, trace_ctx); 982 } 983 984 static __always_inline struct ring_buffer_event * 985 __trace_buffer_lock_reserve(struct trace_buffer *buffer, 986 int type, 987 unsigned long len, 988 unsigned int trace_ctx) 989 { 990 struct ring_buffer_event *event; 991 992 event = ring_buffer_lock_reserve(buffer, len); 993 if (event != NULL) 994 trace_event_setup(event, type, trace_ctx); 995 996 return event; 997 } 998 999 void tracer_tracing_on(struct trace_array *tr) 1000 { 1001 if (tr->array_buffer.buffer) 1002 ring_buffer_record_on(tr->array_buffer.buffer); 1003 /* 1004 * This flag is looked at when buffers haven't been allocated 1005 * yet, or by some tracers (like irqsoff), that just want to 1006 * know if the ring buffer has been disabled, but it can handle 1007 * races of where it gets disabled but we still do a record. 1008 * As the check is in the fast path of the tracers, it is more 1009 * important to be fast than accurate. 1010 */ 1011 tr->buffer_disabled = 0; 1012 /* Make the flag seen by readers */ 1013 smp_wmb(); 1014 } 1015 1016 /** 1017 * tracing_on - enable tracing buffers 1018 * 1019 * This function enables tracing buffers that may have been 1020 * disabled with tracing_off. 1021 */ 1022 void tracing_on(void) 1023 { 1024 tracer_tracing_on(&global_trace); 1025 } 1026 EXPORT_SYMBOL_GPL(tracing_on); 1027 1028 1029 static __always_inline void 1030 __buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event) 1031 { 1032 __this_cpu_write(trace_taskinfo_save, true); 1033 1034 /* If this is the temp buffer, we need to commit fully */ 1035 if (this_cpu_read(trace_buffered_event) == event) { 1036 /* Length is in event->array[0] */ 1037 ring_buffer_write(buffer, event->array[0], &event->array[1]); 1038 /* Release the temp buffer */ 1039 this_cpu_dec(trace_buffered_event_cnt); 1040 /* ring_buffer_unlock_commit() enables preemption */ 1041 preempt_enable_notrace(); 1042 } else 1043 ring_buffer_unlock_commit(buffer); 1044 } 1045 1046 int __trace_array_puts(struct trace_array *tr, unsigned long ip, 1047 const char *str, int size) 1048 { 1049 struct ring_buffer_event *event; 1050 struct trace_buffer *buffer; 1051 struct print_entry *entry; 1052 unsigned int trace_ctx; 1053 int alloc; 1054 1055 if (!(tr->trace_flags & TRACE_ITER_PRINTK)) 1056 return 0; 1057 1058 if (unlikely(tracing_selftest_running && tr == &global_trace)) 1059 return 0; 1060 1061 if (unlikely(tracing_disabled)) 1062 return 0; 1063 1064 alloc = sizeof(*entry) + size + 2; /* possible \n added */ 1065 1066 trace_ctx = tracing_gen_ctx(); 1067 buffer = tr->array_buffer.buffer; 1068 ring_buffer_nest_start(buffer); 1069 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 1070 trace_ctx); 1071 if (!event) { 1072 size = 0; 1073 goto out; 1074 } 1075 1076 entry = ring_buffer_event_data(event); 1077 entry->ip = ip; 1078 1079 memcpy(&entry->buf, str, size); 1080 1081 /* Add a newline if necessary */ 1082 if (entry->buf[size - 1] != '\n') { 1083 entry->buf[size] = '\n'; 1084 entry->buf[size + 1] = '\0'; 1085 } else 1086 entry->buf[size] = '\0'; 1087 1088 __buffer_unlock_commit(buffer, event); 1089 ftrace_trace_stack(tr, buffer, trace_ctx, 4, NULL); 1090 out: 1091 ring_buffer_nest_end(buffer); 1092 return size; 1093 } 1094 EXPORT_SYMBOL_GPL(__trace_array_puts); 1095 1096 /** 1097 * __trace_puts - write a constant string into the trace buffer. 1098 * @ip: The address of the caller 1099 * @str: The constant string to write 1100 * @size: The size of the string. 1101 */ 1102 int __trace_puts(unsigned long ip, const char *str, int size) 1103 { 1104 return __trace_array_puts(&global_trace, ip, str, size); 1105 } 1106 EXPORT_SYMBOL_GPL(__trace_puts); 1107 1108 /** 1109 * __trace_bputs - write the pointer to a constant string into trace buffer 1110 * @ip: The address of the caller 1111 * @str: The constant string to write to the buffer to 1112 */ 1113 int __trace_bputs(unsigned long ip, const char *str) 1114 { 1115 struct ring_buffer_event *event; 1116 struct trace_buffer *buffer; 1117 struct bputs_entry *entry; 1118 unsigned int trace_ctx; 1119 int size = sizeof(struct bputs_entry); 1120 int ret = 0; 1121 1122 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 1123 return 0; 1124 1125 if (unlikely(tracing_selftest_running || tracing_disabled)) 1126 return 0; 1127 1128 trace_ctx = tracing_gen_ctx(); 1129 buffer = global_trace.array_buffer.buffer; 1130 1131 ring_buffer_nest_start(buffer); 1132 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size, 1133 trace_ctx); 1134 if (!event) 1135 goto out; 1136 1137 entry = ring_buffer_event_data(event); 1138 entry->ip = ip; 1139 entry->str = str; 1140 1141 __buffer_unlock_commit(buffer, event); 1142 ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL); 1143 1144 ret = 1; 1145 out: 1146 ring_buffer_nest_end(buffer); 1147 return ret; 1148 } 1149 EXPORT_SYMBOL_GPL(__trace_bputs); 1150 1151 #ifdef CONFIG_TRACER_SNAPSHOT 1152 static void tracing_snapshot_instance_cond(struct trace_array *tr, 1153 void *cond_data) 1154 { 1155 struct tracer *tracer = tr->current_trace; 1156 unsigned long flags; 1157 1158 if (in_nmi()) { 1159 trace_array_puts(tr, "*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n"); 1160 trace_array_puts(tr, "*** snapshot is being ignored ***\n"); 1161 return; 1162 } 1163 1164 if (!tr->allocated_snapshot) { 1165 trace_array_puts(tr, "*** SNAPSHOT NOT ALLOCATED ***\n"); 1166 trace_array_puts(tr, "*** stopping trace here! ***\n"); 1167 tracer_tracing_off(tr); 1168 return; 1169 } 1170 1171 /* Note, snapshot can not be used when the tracer uses it */ 1172 if (tracer->use_max_tr) { 1173 trace_array_puts(tr, "*** LATENCY TRACER ACTIVE ***\n"); 1174 trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n"); 1175 return; 1176 } 1177 1178 local_irq_save(flags); 1179 update_max_tr(tr, current, smp_processor_id(), cond_data); 1180 local_irq_restore(flags); 1181 } 1182 1183 void tracing_snapshot_instance(struct trace_array *tr) 1184 { 1185 tracing_snapshot_instance_cond(tr, NULL); 1186 } 1187 1188 /** 1189 * tracing_snapshot - take a snapshot of the current buffer. 1190 * 1191 * This causes a swap between the snapshot buffer and the current live 1192 * tracing buffer. You can use this to take snapshots of the live 1193 * trace when some condition is triggered, but continue to trace. 1194 * 1195 * Note, make sure to allocate the snapshot with either 1196 * a tracing_snapshot_alloc(), or by doing it manually 1197 * with: echo 1 > /sys/kernel/tracing/snapshot 1198 * 1199 * If the snapshot buffer is not allocated, it will stop tracing. 1200 * Basically making a permanent snapshot. 1201 */ 1202 void tracing_snapshot(void) 1203 { 1204 struct trace_array *tr = &global_trace; 1205 1206 tracing_snapshot_instance(tr); 1207 } 1208 EXPORT_SYMBOL_GPL(tracing_snapshot); 1209 1210 /** 1211 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer. 1212 * @tr: The tracing instance to snapshot 1213 * @cond_data: The data to be tested conditionally, and possibly saved 1214 * 1215 * This is the same as tracing_snapshot() except that the snapshot is 1216 * conditional - the snapshot will only happen if the 1217 * cond_snapshot.update() implementation receiving the cond_data 1218 * returns true, which means that the trace array's cond_snapshot 1219 * update() operation used the cond_data to determine whether the 1220 * snapshot should be taken, and if it was, presumably saved it along 1221 * with the snapshot. 1222 */ 1223 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) 1224 { 1225 tracing_snapshot_instance_cond(tr, cond_data); 1226 } 1227 EXPORT_SYMBOL_GPL(tracing_snapshot_cond); 1228 1229 /** 1230 * tracing_cond_snapshot_data - get the user data associated with a snapshot 1231 * @tr: The tracing instance 1232 * 1233 * When the user enables a conditional snapshot using 1234 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved 1235 * with the snapshot. This accessor is used to retrieve it. 1236 * 1237 * Should not be called from cond_snapshot.update(), since it takes 1238 * the tr->max_lock lock, which the code calling 1239 * cond_snapshot.update() has already done. 1240 * 1241 * Returns the cond_data associated with the trace array's snapshot. 1242 */ 1243 void *tracing_cond_snapshot_data(struct trace_array *tr) 1244 { 1245 void *cond_data = NULL; 1246 1247 local_irq_disable(); 1248 arch_spin_lock(&tr->max_lock); 1249 1250 if (tr->cond_snapshot) 1251 cond_data = tr->cond_snapshot->cond_data; 1252 1253 arch_spin_unlock(&tr->max_lock); 1254 local_irq_enable(); 1255 1256 return cond_data; 1257 } 1258 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data); 1259 1260 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf, 1261 struct array_buffer *size_buf, int cpu_id); 1262 static void set_buffer_entries(struct array_buffer *buf, unsigned long val); 1263 1264 int tracing_alloc_snapshot_instance(struct trace_array *tr) 1265 { 1266 int order; 1267 int ret; 1268 1269 if (!tr->allocated_snapshot) { 1270 1271 /* Make the snapshot buffer have the same order as main buffer */ 1272 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); 1273 ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order); 1274 if (ret < 0) 1275 return ret; 1276 1277 /* allocate spare buffer */ 1278 ret = resize_buffer_duplicate_size(&tr->max_buffer, 1279 &tr->array_buffer, RING_BUFFER_ALL_CPUS); 1280 if (ret < 0) 1281 return ret; 1282 1283 tr->allocated_snapshot = true; 1284 } 1285 1286 return 0; 1287 } 1288 1289 static void free_snapshot(struct trace_array *tr) 1290 { 1291 /* 1292 * We don't free the ring buffer. instead, resize it because 1293 * The max_tr ring buffer has some state (e.g. ring->clock) and 1294 * we want preserve it. 1295 */ 1296 ring_buffer_subbuf_order_set(tr->max_buffer.buffer, 0); 1297 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); 1298 set_buffer_entries(&tr->max_buffer, 1); 1299 tracing_reset_online_cpus(&tr->max_buffer); 1300 tr->allocated_snapshot = false; 1301 } 1302 1303 /** 1304 * tracing_alloc_snapshot - allocate snapshot buffer. 1305 * 1306 * This only allocates the snapshot buffer if it isn't already 1307 * allocated - it doesn't also take a snapshot. 1308 * 1309 * This is meant to be used in cases where the snapshot buffer needs 1310 * to be set up for events that can't sleep but need to be able to 1311 * trigger a snapshot. 1312 */ 1313 int tracing_alloc_snapshot(void) 1314 { 1315 struct trace_array *tr = &global_trace; 1316 int ret; 1317 1318 ret = tracing_alloc_snapshot_instance(tr); 1319 WARN_ON(ret < 0); 1320 1321 return ret; 1322 } 1323 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 1324 1325 /** 1326 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer. 1327 * 1328 * This is similar to tracing_snapshot(), but it will allocate the 1329 * snapshot buffer if it isn't already allocated. Use this only 1330 * where it is safe to sleep, as the allocation may sleep. 1331 * 1332 * This causes a swap between the snapshot buffer and the current live 1333 * tracing buffer. You can use this to take snapshots of the live 1334 * trace when some condition is triggered, but continue to trace. 1335 */ 1336 void tracing_snapshot_alloc(void) 1337 { 1338 int ret; 1339 1340 ret = tracing_alloc_snapshot(); 1341 if (ret < 0) 1342 return; 1343 1344 tracing_snapshot(); 1345 } 1346 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 1347 1348 /** 1349 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance 1350 * @tr: The tracing instance 1351 * @cond_data: User data to associate with the snapshot 1352 * @update: Implementation of the cond_snapshot update function 1353 * 1354 * Check whether the conditional snapshot for the given instance has 1355 * already been enabled, or if the current tracer is already using a 1356 * snapshot; if so, return -EBUSY, else create a cond_snapshot and 1357 * save the cond_data and update function inside. 1358 * 1359 * Returns 0 if successful, error otherwise. 1360 */ 1361 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, 1362 cond_update_fn_t update) 1363 { 1364 struct cond_snapshot *cond_snapshot; 1365 int ret = 0; 1366 1367 cond_snapshot = kzalloc(sizeof(*cond_snapshot), GFP_KERNEL); 1368 if (!cond_snapshot) 1369 return -ENOMEM; 1370 1371 cond_snapshot->cond_data = cond_data; 1372 cond_snapshot->update = update; 1373 1374 mutex_lock(&trace_types_lock); 1375 1376 ret = tracing_alloc_snapshot_instance(tr); 1377 if (ret) 1378 goto fail_unlock; 1379 1380 if (tr->current_trace->use_max_tr) { 1381 ret = -EBUSY; 1382 goto fail_unlock; 1383 } 1384 1385 /* 1386 * The cond_snapshot can only change to NULL without the 1387 * trace_types_lock. We don't care if we race with it going 1388 * to NULL, but we want to make sure that it's not set to 1389 * something other than NULL when we get here, which we can 1390 * do safely with only holding the trace_types_lock and not 1391 * having to take the max_lock. 1392 */ 1393 if (tr->cond_snapshot) { 1394 ret = -EBUSY; 1395 goto fail_unlock; 1396 } 1397 1398 local_irq_disable(); 1399 arch_spin_lock(&tr->max_lock); 1400 tr->cond_snapshot = cond_snapshot; 1401 arch_spin_unlock(&tr->max_lock); 1402 local_irq_enable(); 1403 1404 mutex_unlock(&trace_types_lock); 1405 1406 return ret; 1407 1408 fail_unlock: 1409 mutex_unlock(&trace_types_lock); 1410 kfree(cond_snapshot); 1411 return ret; 1412 } 1413 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable); 1414 1415 /** 1416 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance 1417 * @tr: The tracing instance 1418 * 1419 * Check whether the conditional snapshot for the given instance is 1420 * enabled; if so, free the cond_snapshot associated with it, 1421 * otherwise return -EINVAL. 1422 * 1423 * Returns 0 if successful, error otherwise. 1424 */ 1425 int tracing_snapshot_cond_disable(struct trace_array *tr) 1426 { 1427 int ret = 0; 1428 1429 local_irq_disable(); 1430 arch_spin_lock(&tr->max_lock); 1431 1432 if (!tr->cond_snapshot) 1433 ret = -EINVAL; 1434 else { 1435 kfree(tr->cond_snapshot); 1436 tr->cond_snapshot = NULL; 1437 } 1438 1439 arch_spin_unlock(&tr->max_lock); 1440 local_irq_enable(); 1441 1442 return ret; 1443 } 1444 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable); 1445 #else 1446 void tracing_snapshot(void) 1447 { 1448 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used"); 1449 } 1450 EXPORT_SYMBOL_GPL(tracing_snapshot); 1451 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) 1452 { 1453 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used"); 1454 } 1455 EXPORT_SYMBOL_GPL(tracing_snapshot_cond); 1456 int tracing_alloc_snapshot(void) 1457 { 1458 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used"); 1459 return -ENODEV; 1460 } 1461 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 1462 void tracing_snapshot_alloc(void) 1463 { 1464 /* Give warning */ 1465 tracing_snapshot(); 1466 } 1467 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 1468 void *tracing_cond_snapshot_data(struct trace_array *tr) 1469 { 1470 return NULL; 1471 } 1472 EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data); 1473 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update) 1474 { 1475 return -ENODEV; 1476 } 1477 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable); 1478 int tracing_snapshot_cond_disable(struct trace_array *tr) 1479 { 1480 return false; 1481 } 1482 EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable); 1483 #define free_snapshot(tr) do { } while (0) 1484 #endif /* CONFIG_TRACER_SNAPSHOT */ 1485 1486 void tracer_tracing_off(struct trace_array *tr) 1487 { 1488 if (tr->array_buffer.buffer) 1489 ring_buffer_record_off(tr->array_buffer.buffer); 1490 /* 1491 * This flag is looked at when buffers haven't been allocated 1492 * yet, or by some tracers (like irqsoff), that just want to 1493 * know if the ring buffer has been disabled, but it can handle 1494 * races of where it gets disabled but we still do a record. 1495 * As the check is in the fast path of the tracers, it is more 1496 * important to be fast than accurate. 1497 */ 1498 tr->buffer_disabled = 1; 1499 /* Make the flag seen by readers */ 1500 smp_wmb(); 1501 } 1502 1503 /** 1504 * tracing_off - turn off tracing buffers 1505 * 1506 * This function stops the tracing buffers from recording data. 1507 * It does not disable any overhead the tracers themselves may 1508 * be causing. This function simply causes all recording to 1509 * the ring buffers to fail. 1510 */ 1511 void tracing_off(void) 1512 { 1513 tracer_tracing_off(&global_trace); 1514 } 1515 EXPORT_SYMBOL_GPL(tracing_off); 1516 1517 void disable_trace_on_warning(void) 1518 { 1519 if (__disable_trace_on_warning) { 1520 trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_, 1521 "Disabling tracing due to warning\n"); 1522 tracing_off(); 1523 } 1524 } 1525 1526 /** 1527 * tracer_tracing_is_on - show real state of ring buffer enabled 1528 * @tr : the trace array to know if ring buffer is enabled 1529 * 1530 * Shows real state of the ring buffer if it is enabled or not. 1531 */ 1532 bool tracer_tracing_is_on(struct trace_array *tr) 1533 { 1534 if (tr->array_buffer.buffer) 1535 return ring_buffer_record_is_on(tr->array_buffer.buffer); 1536 return !tr->buffer_disabled; 1537 } 1538 1539 /** 1540 * tracing_is_on - show state of ring buffers enabled 1541 */ 1542 int tracing_is_on(void) 1543 { 1544 return tracer_tracing_is_on(&global_trace); 1545 } 1546 EXPORT_SYMBOL_GPL(tracing_is_on); 1547 1548 static int __init set_buf_size(char *str) 1549 { 1550 unsigned long buf_size; 1551 1552 if (!str) 1553 return 0; 1554 buf_size = memparse(str, &str); 1555 /* 1556 * nr_entries can not be zero and the startup 1557 * tests require some buffer space. Therefore 1558 * ensure we have at least 4096 bytes of buffer. 1559 */ 1560 trace_buf_size = max(4096UL, buf_size); 1561 return 1; 1562 } 1563 __setup("trace_buf_size=", set_buf_size); 1564 1565 static int __init set_tracing_thresh(char *str) 1566 { 1567 unsigned long threshold; 1568 int ret; 1569 1570 if (!str) 1571 return 0; 1572 ret = kstrtoul(str, 0, &threshold); 1573 if (ret < 0) 1574 return 0; 1575 tracing_thresh = threshold * 1000; 1576 return 1; 1577 } 1578 __setup("tracing_thresh=", set_tracing_thresh); 1579 1580 unsigned long nsecs_to_usecs(unsigned long nsecs) 1581 { 1582 return nsecs / 1000; 1583 } 1584 1585 /* 1586 * TRACE_FLAGS is defined as a tuple matching bit masks with strings. 1587 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that 1588 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list 1589 * of strings in the order that the evals (enum) were defined. 1590 */ 1591 #undef C 1592 #define C(a, b) b 1593 1594 /* These must match the bit positions in trace_iterator_flags */ 1595 static const char *trace_options[] = { 1596 TRACE_FLAGS 1597 NULL 1598 }; 1599 1600 static struct { 1601 u64 (*func)(void); 1602 const char *name; 1603 int in_ns; /* is this clock in nanoseconds? */ 1604 } trace_clocks[] = { 1605 { trace_clock_local, "local", 1 }, 1606 { trace_clock_global, "global", 1 }, 1607 { trace_clock_counter, "counter", 0 }, 1608 { trace_clock_jiffies, "uptime", 0 }, 1609 { trace_clock, "perf", 1 }, 1610 { ktime_get_mono_fast_ns, "mono", 1 }, 1611 { ktime_get_raw_fast_ns, "mono_raw", 1 }, 1612 { ktime_get_boot_fast_ns, "boot", 1 }, 1613 { ktime_get_tai_fast_ns, "tai", 1 }, 1614 ARCH_TRACE_CLOCKS 1615 }; 1616 1617 bool trace_clock_in_ns(struct trace_array *tr) 1618 { 1619 if (trace_clocks[tr->clock_id].in_ns) 1620 return true; 1621 1622 return false; 1623 } 1624 1625 /* 1626 * trace_parser_get_init - gets the buffer for trace parser 1627 */ 1628 int trace_parser_get_init(struct trace_parser *parser, int size) 1629 { 1630 memset(parser, 0, sizeof(*parser)); 1631 1632 parser->buffer = kmalloc(size, GFP_KERNEL); 1633 if (!parser->buffer) 1634 return 1; 1635 1636 parser->size = size; 1637 return 0; 1638 } 1639 1640 /* 1641 * trace_parser_put - frees the buffer for trace parser 1642 */ 1643 void trace_parser_put(struct trace_parser *parser) 1644 { 1645 kfree(parser->buffer); 1646 parser->buffer = NULL; 1647 } 1648 1649 /* 1650 * trace_get_user - reads the user input string separated by space 1651 * (matched by isspace(ch)) 1652 * 1653 * For each string found the 'struct trace_parser' is updated, 1654 * and the function returns. 1655 * 1656 * Returns number of bytes read. 1657 * 1658 * See kernel/trace/trace.h for 'struct trace_parser' details. 1659 */ 1660 int trace_get_user(struct trace_parser *parser, const char __user *ubuf, 1661 size_t cnt, loff_t *ppos) 1662 { 1663 char ch; 1664 size_t read = 0; 1665 ssize_t ret; 1666 1667 if (!*ppos) 1668 trace_parser_clear(parser); 1669 1670 ret = get_user(ch, ubuf++); 1671 if (ret) 1672 goto out; 1673 1674 read++; 1675 cnt--; 1676 1677 /* 1678 * The parser is not finished with the last write, 1679 * continue reading the user input without skipping spaces. 1680 */ 1681 if (!parser->cont) { 1682 /* skip white space */ 1683 while (cnt && isspace(ch)) { 1684 ret = get_user(ch, ubuf++); 1685 if (ret) 1686 goto out; 1687 read++; 1688 cnt--; 1689 } 1690 1691 parser->idx = 0; 1692 1693 /* only spaces were written */ 1694 if (isspace(ch) || !ch) { 1695 *ppos += read; 1696 ret = read; 1697 goto out; 1698 } 1699 } 1700 1701 /* read the non-space input */ 1702 while (cnt && !isspace(ch) && ch) { 1703 if (parser->idx < parser->size - 1) 1704 parser->buffer[parser->idx++] = ch; 1705 else { 1706 ret = -EINVAL; 1707 goto out; 1708 } 1709 ret = get_user(ch, ubuf++); 1710 if (ret) 1711 goto out; 1712 read++; 1713 cnt--; 1714 } 1715 1716 /* We either got finished input or we have to wait for another call. */ 1717 if (isspace(ch) || !ch) { 1718 parser->buffer[parser->idx] = 0; 1719 parser->cont = false; 1720 } else if (parser->idx < parser->size - 1) { 1721 parser->cont = true; 1722 parser->buffer[parser->idx++] = ch; 1723 /* Make sure the parsed string always terminates with '\0'. */ 1724 parser->buffer[parser->idx] = 0; 1725 } else { 1726 ret = -EINVAL; 1727 goto out; 1728 } 1729 1730 *ppos += read; 1731 ret = read; 1732 1733 out: 1734 return ret; 1735 } 1736 1737 /* TODO add a seq_buf_to_buffer() */ 1738 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) 1739 { 1740 int len; 1741 1742 if (trace_seq_used(s) <= s->readpos) 1743 return -EBUSY; 1744 1745 len = trace_seq_used(s) - s->readpos; 1746 if (cnt > len) 1747 cnt = len; 1748 memcpy(buf, s->buffer + s->readpos, cnt); 1749 1750 s->readpos += cnt; 1751 return cnt; 1752 } 1753 1754 unsigned long __read_mostly tracing_thresh; 1755 1756 #ifdef CONFIG_TRACER_MAX_TRACE 1757 static const struct file_operations tracing_max_lat_fops; 1758 1759 #ifdef LATENCY_FS_NOTIFY 1760 1761 static struct workqueue_struct *fsnotify_wq; 1762 1763 static void latency_fsnotify_workfn(struct work_struct *work) 1764 { 1765 struct trace_array *tr = container_of(work, struct trace_array, 1766 fsnotify_work); 1767 fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY); 1768 } 1769 1770 static void latency_fsnotify_workfn_irq(struct irq_work *iwork) 1771 { 1772 struct trace_array *tr = container_of(iwork, struct trace_array, 1773 fsnotify_irqwork); 1774 queue_work(fsnotify_wq, &tr->fsnotify_work); 1775 } 1776 1777 static void trace_create_maxlat_file(struct trace_array *tr, 1778 struct dentry *d_tracer) 1779 { 1780 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn); 1781 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq); 1782 tr->d_max_latency = trace_create_file("tracing_max_latency", 1783 TRACE_MODE_WRITE, 1784 d_tracer, tr, 1785 &tracing_max_lat_fops); 1786 } 1787 1788 __init static int latency_fsnotify_init(void) 1789 { 1790 fsnotify_wq = alloc_workqueue("tr_max_lat_wq", 1791 WQ_UNBOUND | WQ_HIGHPRI, 0); 1792 if (!fsnotify_wq) { 1793 pr_err("Unable to allocate tr_max_lat_wq\n"); 1794 return -ENOMEM; 1795 } 1796 return 0; 1797 } 1798 1799 late_initcall_sync(latency_fsnotify_init); 1800 1801 void latency_fsnotify(struct trace_array *tr) 1802 { 1803 if (!fsnotify_wq) 1804 return; 1805 /* 1806 * We cannot call queue_work(&tr->fsnotify_work) from here because it's 1807 * possible that we are called from __schedule() or do_idle(), which 1808 * could cause a deadlock. 1809 */ 1810 irq_work_queue(&tr->fsnotify_irqwork); 1811 } 1812 1813 #else /* !LATENCY_FS_NOTIFY */ 1814 1815 #define trace_create_maxlat_file(tr, d_tracer) \ 1816 trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, \ 1817 d_tracer, tr, &tracing_max_lat_fops) 1818 1819 #endif 1820 1821 /* 1822 * Copy the new maximum trace into the separate maximum-trace 1823 * structure. (this way the maximum trace is permanently saved, 1824 * for later retrieval via /sys/kernel/tracing/tracing_max_latency) 1825 */ 1826 static void 1827 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 1828 { 1829 struct array_buffer *trace_buf = &tr->array_buffer; 1830 struct array_buffer *max_buf = &tr->max_buffer; 1831 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu); 1832 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu); 1833 1834 max_buf->cpu = cpu; 1835 max_buf->time_start = data->preempt_timestamp; 1836 1837 max_data->saved_latency = tr->max_latency; 1838 max_data->critical_start = data->critical_start; 1839 max_data->critical_end = data->critical_end; 1840 1841 strncpy(max_data->comm, tsk->comm, TASK_COMM_LEN); 1842 max_data->pid = tsk->pid; 1843 /* 1844 * If tsk == current, then use current_uid(), as that does not use 1845 * RCU. The irq tracer can be called out of RCU scope. 1846 */ 1847 if (tsk == current) 1848 max_data->uid = current_uid(); 1849 else 1850 max_data->uid = task_uid(tsk); 1851 1852 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; 1853 max_data->policy = tsk->policy; 1854 max_data->rt_priority = tsk->rt_priority; 1855 1856 /* record this tasks comm */ 1857 tracing_record_cmdline(tsk); 1858 latency_fsnotify(tr); 1859 } 1860 1861 /** 1862 * update_max_tr - snapshot all trace buffers from global_trace to max_tr 1863 * @tr: tracer 1864 * @tsk: the task with the latency 1865 * @cpu: The cpu that initiated the trace. 1866 * @cond_data: User data associated with a conditional snapshot 1867 * 1868 * Flip the buffers between the @tr and the max_tr and record information 1869 * about which task was the cause of this latency. 1870 */ 1871 void 1872 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu, 1873 void *cond_data) 1874 { 1875 if (tr->stop_count) 1876 return; 1877 1878 WARN_ON_ONCE(!irqs_disabled()); 1879 1880 if (!tr->allocated_snapshot) { 1881 /* Only the nop tracer should hit this when disabling */ 1882 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1883 return; 1884 } 1885 1886 arch_spin_lock(&tr->max_lock); 1887 1888 /* Inherit the recordable setting from array_buffer */ 1889 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer)) 1890 ring_buffer_record_on(tr->max_buffer.buffer); 1891 else 1892 ring_buffer_record_off(tr->max_buffer.buffer); 1893 1894 #ifdef CONFIG_TRACER_SNAPSHOT 1895 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) { 1896 arch_spin_unlock(&tr->max_lock); 1897 return; 1898 } 1899 #endif 1900 swap(tr->array_buffer.buffer, tr->max_buffer.buffer); 1901 1902 __update_max_tr(tr, tsk, cpu); 1903 1904 arch_spin_unlock(&tr->max_lock); 1905 1906 /* Any waiters on the old snapshot buffer need to wake up */ 1907 ring_buffer_wake_waiters(tr->array_buffer.buffer, RING_BUFFER_ALL_CPUS); 1908 } 1909 1910 /** 1911 * update_max_tr_single - only copy one trace over, and reset the rest 1912 * @tr: tracer 1913 * @tsk: task with the latency 1914 * @cpu: the cpu of the buffer to copy. 1915 * 1916 * Flip the trace of a single CPU buffer between the @tr and the max_tr. 1917 */ 1918 void 1919 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) 1920 { 1921 int ret; 1922 1923 if (tr->stop_count) 1924 return; 1925 1926 WARN_ON_ONCE(!irqs_disabled()); 1927 if (!tr->allocated_snapshot) { 1928 /* Only the nop tracer should hit this when disabling */ 1929 WARN_ON_ONCE(tr->current_trace != &nop_trace); 1930 return; 1931 } 1932 1933 arch_spin_lock(&tr->max_lock); 1934 1935 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu); 1936 1937 if (ret == -EBUSY) { 1938 /* 1939 * We failed to swap the buffer due to a commit taking 1940 * place on this CPU. We fail to record, but we reset 1941 * the max trace buffer (no one writes directly to it) 1942 * and flag that it failed. 1943 * Another reason is resize is in progress. 1944 */ 1945 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_, 1946 "Failed to swap buffers due to commit or resize in progress\n"); 1947 } 1948 1949 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); 1950 1951 __update_max_tr(tr, tsk, cpu); 1952 arch_spin_unlock(&tr->max_lock); 1953 } 1954 1955 #endif /* CONFIG_TRACER_MAX_TRACE */ 1956 1957 static int wait_on_pipe(struct trace_iterator *iter, int full) 1958 { 1959 int ret; 1960 1961 /* Iterators are static, they should be filled or empty */ 1962 if (trace_buffer_iter(iter, iter->cpu_file)) 1963 return 0; 1964 1965 ret = ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, full); 1966 1967 #ifdef CONFIG_TRACER_MAX_TRACE 1968 /* 1969 * Make sure this is still the snapshot buffer, as if a snapshot were 1970 * to happen, this would now be the main buffer. 1971 */ 1972 if (iter->snapshot) 1973 iter->array_buffer = &iter->tr->max_buffer; 1974 #endif 1975 return ret; 1976 } 1977 1978 #ifdef CONFIG_FTRACE_STARTUP_TEST 1979 static bool selftests_can_run; 1980 1981 struct trace_selftests { 1982 struct list_head list; 1983 struct tracer *type; 1984 }; 1985 1986 static LIST_HEAD(postponed_selftests); 1987 1988 static int save_selftest(struct tracer *type) 1989 { 1990 struct trace_selftests *selftest; 1991 1992 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL); 1993 if (!selftest) 1994 return -ENOMEM; 1995 1996 selftest->type = type; 1997 list_add(&selftest->list, &postponed_selftests); 1998 return 0; 1999 } 2000 2001 static int run_tracer_selftest(struct tracer *type) 2002 { 2003 struct trace_array *tr = &global_trace; 2004 struct tracer *saved_tracer = tr->current_trace; 2005 int ret; 2006 2007 if (!type->selftest || tracing_selftest_disabled) 2008 return 0; 2009 2010 /* 2011 * If a tracer registers early in boot up (before scheduling is 2012 * initialized and such), then do not run its selftests yet. 2013 * Instead, run it a little later in the boot process. 2014 */ 2015 if (!selftests_can_run) 2016 return save_selftest(type); 2017 2018 if (!tracing_is_on()) { 2019 pr_warn("Selftest for tracer %s skipped due to tracing disabled\n", 2020 type->name); 2021 return 0; 2022 } 2023 2024 /* 2025 * Run a selftest on this tracer. 2026 * Here we reset the trace buffer, and set the current 2027 * tracer to be this tracer. The tracer can then run some 2028 * internal tracing to verify that everything is in order. 2029 * If we fail, we do not register this tracer. 2030 */ 2031 tracing_reset_online_cpus(&tr->array_buffer); 2032 2033 tr->current_trace = type; 2034 2035 #ifdef CONFIG_TRACER_MAX_TRACE 2036 if (type->use_max_tr) { 2037 /* If we expanded the buffers, make sure the max is expanded too */ 2038 if (tr->ring_buffer_expanded) 2039 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size, 2040 RING_BUFFER_ALL_CPUS); 2041 tr->allocated_snapshot = true; 2042 } 2043 #endif 2044 2045 /* the test is responsible for initializing and enabling */ 2046 pr_info("Testing tracer %s: ", type->name); 2047 ret = type->selftest(type, tr); 2048 /* the test is responsible for resetting too */ 2049 tr->current_trace = saved_tracer; 2050 if (ret) { 2051 printk(KERN_CONT "FAILED!\n"); 2052 /* Add the warning after printing 'FAILED' */ 2053 WARN_ON(1); 2054 return -1; 2055 } 2056 /* Only reset on passing, to avoid touching corrupted buffers */ 2057 tracing_reset_online_cpus(&tr->array_buffer); 2058 2059 #ifdef CONFIG_TRACER_MAX_TRACE 2060 if (type->use_max_tr) { 2061 tr->allocated_snapshot = false; 2062 2063 /* Shrink the max buffer again */ 2064 if (tr->ring_buffer_expanded) 2065 ring_buffer_resize(tr->max_buffer.buffer, 1, 2066 RING_BUFFER_ALL_CPUS); 2067 } 2068 #endif 2069 2070 printk(KERN_CONT "PASSED\n"); 2071 return 0; 2072 } 2073 2074 static int do_run_tracer_selftest(struct tracer *type) 2075 { 2076 int ret; 2077 2078 /* 2079 * Tests can take a long time, especially if they are run one after the 2080 * other, as does happen during bootup when all the tracers are 2081 * registered. This could cause the soft lockup watchdog to trigger. 2082 */ 2083 cond_resched(); 2084 2085 tracing_selftest_running = true; 2086 ret = run_tracer_selftest(type); 2087 tracing_selftest_running = false; 2088 2089 return ret; 2090 } 2091 2092 static __init int init_trace_selftests(void) 2093 { 2094 struct trace_selftests *p, *n; 2095 struct tracer *t, **last; 2096 int ret; 2097 2098 selftests_can_run = true; 2099 2100 mutex_lock(&trace_types_lock); 2101 2102 if (list_empty(&postponed_selftests)) 2103 goto out; 2104 2105 pr_info("Running postponed tracer tests:\n"); 2106 2107 tracing_selftest_running = true; 2108 list_for_each_entry_safe(p, n, &postponed_selftests, list) { 2109 /* This loop can take minutes when sanitizers are enabled, so 2110 * lets make sure we allow RCU processing. 2111 */ 2112 cond_resched(); 2113 ret = run_tracer_selftest(p->type); 2114 /* If the test fails, then warn and remove from available_tracers */ 2115 if (ret < 0) { 2116 WARN(1, "tracer: %s failed selftest, disabling\n", 2117 p->type->name); 2118 last = &trace_types; 2119 for (t = trace_types; t; t = t->next) { 2120 if (t == p->type) { 2121 *last = t->next; 2122 break; 2123 } 2124 last = &t->next; 2125 } 2126 } 2127 list_del(&p->list); 2128 kfree(p); 2129 } 2130 tracing_selftest_running = false; 2131 2132 out: 2133 mutex_unlock(&trace_types_lock); 2134 2135 return 0; 2136 } 2137 core_initcall(init_trace_selftests); 2138 #else 2139 static inline int run_tracer_selftest(struct tracer *type) 2140 { 2141 return 0; 2142 } 2143 static inline int do_run_tracer_selftest(struct tracer *type) 2144 { 2145 return 0; 2146 } 2147 #endif /* CONFIG_FTRACE_STARTUP_TEST */ 2148 2149 static void add_tracer_options(struct trace_array *tr, struct tracer *t); 2150 2151 static void __init apply_trace_boot_options(void); 2152 2153 /** 2154 * register_tracer - register a tracer with the ftrace system. 2155 * @type: the plugin for the tracer 2156 * 2157 * Register a new plugin tracer. 2158 */ 2159 int __init register_tracer(struct tracer *type) 2160 { 2161 struct tracer *t; 2162 int ret = 0; 2163 2164 if (!type->name) { 2165 pr_info("Tracer must have a name\n"); 2166 return -1; 2167 } 2168 2169 if (strlen(type->name) >= MAX_TRACER_SIZE) { 2170 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE); 2171 return -1; 2172 } 2173 2174 if (security_locked_down(LOCKDOWN_TRACEFS)) { 2175 pr_warn("Can not register tracer %s due to lockdown\n", 2176 type->name); 2177 return -EPERM; 2178 } 2179 2180 mutex_lock(&trace_types_lock); 2181 2182 for (t = trace_types; t; t = t->next) { 2183 if (strcmp(type->name, t->name) == 0) { 2184 /* already found */ 2185 pr_info("Tracer %s already registered\n", 2186 type->name); 2187 ret = -1; 2188 goto out; 2189 } 2190 } 2191 2192 if (!type->set_flag) 2193 type->set_flag = &dummy_set_flag; 2194 if (!type->flags) { 2195 /*allocate a dummy tracer_flags*/ 2196 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL); 2197 if (!type->flags) { 2198 ret = -ENOMEM; 2199 goto out; 2200 } 2201 type->flags->val = 0; 2202 type->flags->opts = dummy_tracer_opt; 2203 } else 2204 if (!type->flags->opts) 2205 type->flags->opts = dummy_tracer_opt; 2206 2207 /* store the tracer for __set_tracer_option */ 2208 type->flags->trace = type; 2209 2210 ret = do_run_tracer_selftest(type); 2211 if (ret < 0) 2212 goto out; 2213 2214 type->next = trace_types; 2215 trace_types = type; 2216 add_tracer_options(&global_trace, type); 2217 2218 out: 2219 mutex_unlock(&trace_types_lock); 2220 2221 if (ret || !default_bootup_tracer) 2222 goto out_unlock; 2223 2224 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) 2225 goto out_unlock; 2226 2227 printk(KERN_INFO "Starting tracer '%s'\n", type->name); 2228 /* Do we want this tracer to start on bootup? */ 2229 tracing_set_tracer(&global_trace, type->name); 2230 default_bootup_tracer = NULL; 2231 2232 apply_trace_boot_options(); 2233 2234 /* disable other selftests, since this will break it. */ 2235 disable_tracing_selftest("running a tracer"); 2236 2237 out_unlock: 2238 return ret; 2239 } 2240 2241 static void tracing_reset_cpu(struct array_buffer *buf, int cpu) 2242 { 2243 struct trace_buffer *buffer = buf->buffer; 2244 2245 if (!buffer) 2246 return; 2247 2248 ring_buffer_record_disable(buffer); 2249 2250 /* Make sure all commits have finished */ 2251 synchronize_rcu(); 2252 ring_buffer_reset_cpu(buffer, cpu); 2253 2254 ring_buffer_record_enable(buffer); 2255 } 2256 2257 void tracing_reset_online_cpus(struct array_buffer *buf) 2258 { 2259 struct trace_buffer *buffer = buf->buffer; 2260 2261 if (!buffer) 2262 return; 2263 2264 ring_buffer_record_disable(buffer); 2265 2266 /* Make sure all commits have finished */ 2267 synchronize_rcu(); 2268 2269 buf->time_start = buffer_ftrace_now(buf, buf->cpu); 2270 2271 ring_buffer_reset_online_cpus(buffer); 2272 2273 ring_buffer_record_enable(buffer); 2274 } 2275 2276 /* Must have trace_types_lock held */ 2277 void tracing_reset_all_online_cpus_unlocked(void) 2278 { 2279 struct trace_array *tr; 2280 2281 lockdep_assert_held(&trace_types_lock); 2282 2283 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 2284 if (!tr->clear_trace) 2285 continue; 2286 tr->clear_trace = false; 2287 tracing_reset_online_cpus(&tr->array_buffer); 2288 #ifdef CONFIG_TRACER_MAX_TRACE 2289 tracing_reset_online_cpus(&tr->max_buffer); 2290 #endif 2291 } 2292 } 2293 2294 void tracing_reset_all_online_cpus(void) 2295 { 2296 mutex_lock(&trace_types_lock); 2297 tracing_reset_all_online_cpus_unlocked(); 2298 mutex_unlock(&trace_types_lock); 2299 } 2300 2301 /* 2302 * The tgid_map array maps from pid to tgid; i.e. the value stored at index i 2303 * is the tgid last observed corresponding to pid=i. 2304 */ 2305 static int *tgid_map; 2306 2307 /* The maximum valid index into tgid_map. */ 2308 static size_t tgid_map_max; 2309 2310 #define SAVED_CMDLINES_DEFAULT 128 2311 #define NO_CMDLINE_MAP UINT_MAX 2312 /* 2313 * Preemption must be disabled before acquiring trace_cmdline_lock. 2314 * The various trace_arrays' max_lock must be acquired in a context 2315 * where interrupt is disabled. 2316 */ 2317 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; 2318 struct saved_cmdlines_buffer { 2319 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 2320 unsigned *map_cmdline_to_pid; 2321 unsigned cmdline_num; 2322 int cmdline_idx; 2323 char saved_cmdlines[]; 2324 }; 2325 static struct saved_cmdlines_buffer *savedcmd; 2326 2327 static inline char *get_saved_cmdlines(int idx) 2328 { 2329 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN]; 2330 } 2331 2332 static inline void set_cmdline(int idx, const char *cmdline) 2333 { 2334 strncpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN); 2335 } 2336 2337 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s) 2338 { 2339 int order = get_order(sizeof(*s) + s->cmdline_num * TASK_COMM_LEN); 2340 2341 kfree(s->map_cmdline_to_pid); 2342 free_pages((unsigned long)s, order); 2343 } 2344 2345 static struct saved_cmdlines_buffer *allocate_cmdlines_buffer(unsigned int val) 2346 { 2347 struct saved_cmdlines_buffer *s; 2348 struct page *page; 2349 int orig_size, size; 2350 int order; 2351 2352 /* Figure out how much is needed to hold the given number of cmdlines */ 2353 orig_size = sizeof(*s) + val * TASK_COMM_LEN; 2354 order = get_order(orig_size); 2355 size = 1 << (order + PAGE_SHIFT); 2356 page = alloc_pages(GFP_KERNEL, order); 2357 if (!page) 2358 return NULL; 2359 2360 s = page_address(page); 2361 memset(s, 0, sizeof(*s)); 2362 2363 /* Round up to actual allocation */ 2364 val = (size - sizeof(*s)) / TASK_COMM_LEN; 2365 s->cmdline_num = val; 2366 2367 s->map_cmdline_to_pid = kmalloc_array(val, 2368 sizeof(*s->map_cmdline_to_pid), 2369 GFP_KERNEL); 2370 if (!s->map_cmdline_to_pid) { 2371 free_saved_cmdlines_buffer(s); 2372 return NULL; 2373 } 2374 2375 s->cmdline_idx = 0; 2376 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP, 2377 sizeof(s->map_pid_to_cmdline)); 2378 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP, 2379 val * sizeof(*s->map_cmdline_to_pid)); 2380 2381 return s; 2382 } 2383 2384 static int trace_create_savedcmd(void) 2385 { 2386 savedcmd = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT); 2387 2388 return savedcmd ? 0 : -ENOMEM; 2389 } 2390 2391 int is_tracing_stopped(void) 2392 { 2393 return global_trace.stop_count; 2394 } 2395 2396 static void tracing_start_tr(struct trace_array *tr) 2397 { 2398 struct trace_buffer *buffer; 2399 unsigned long flags; 2400 2401 if (tracing_disabled) 2402 return; 2403 2404 raw_spin_lock_irqsave(&tr->start_lock, flags); 2405 if (--tr->stop_count) { 2406 if (WARN_ON_ONCE(tr->stop_count < 0)) { 2407 /* Someone screwed up their debugging */ 2408 tr->stop_count = 0; 2409 } 2410 goto out; 2411 } 2412 2413 /* Prevent the buffers from switching */ 2414 arch_spin_lock(&tr->max_lock); 2415 2416 buffer = tr->array_buffer.buffer; 2417 if (buffer) 2418 ring_buffer_record_enable(buffer); 2419 2420 #ifdef CONFIG_TRACER_MAX_TRACE 2421 buffer = tr->max_buffer.buffer; 2422 if (buffer) 2423 ring_buffer_record_enable(buffer); 2424 #endif 2425 2426 arch_spin_unlock(&tr->max_lock); 2427 2428 out: 2429 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 2430 } 2431 2432 /** 2433 * tracing_start - quick start of the tracer 2434 * 2435 * If tracing is enabled but was stopped by tracing_stop, 2436 * this will start the tracer back up. 2437 */ 2438 void tracing_start(void) 2439 2440 { 2441 return tracing_start_tr(&global_trace); 2442 } 2443 2444 static void tracing_stop_tr(struct trace_array *tr) 2445 { 2446 struct trace_buffer *buffer; 2447 unsigned long flags; 2448 2449 raw_spin_lock_irqsave(&tr->start_lock, flags); 2450 if (tr->stop_count++) 2451 goto out; 2452 2453 /* Prevent the buffers from switching */ 2454 arch_spin_lock(&tr->max_lock); 2455 2456 buffer = tr->array_buffer.buffer; 2457 if (buffer) 2458 ring_buffer_record_disable(buffer); 2459 2460 #ifdef CONFIG_TRACER_MAX_TRACE 2461 buffer = tr->max_buffer.buffer; 2462 if (buffer) 2463 ring_buffer_record_disable(buffer); 2464 #endif 2465 2466 arch_spin_unlock(&tr->max_lock); 2467 2468 out: 2469 raw_spin_unlock_irqrestore(&tr->start_lock, flags); 2470 } 2471 2472 /** 2473 * tracing_stop - quick stop of the tracer 2474 * 2475 * Light weight way to stop tracing. Use in conjunction with 2476 * tracing_start. 2477 */ 2478 void tracing_stop(void) 2479 { 2480 return tracing_stop_tr(&global_trace); 2481 } 2482 2483 static int trace_save_cmdline(struct task_struct *tsk) 2484 { 2485 unsigned tpid, idx; 2486 2487 /* treat recording of idle task as a success */ 2488 if (!tsk->pid) 2489 return 1; 2490 2491 tpid = tsk->pid & (PID_MAX_DEFAULT - 1); 2492 2493 /* 2494 * It's not the end of the world if we don't get 2495 * the lock, but we also don't want to spin 2496 * nor do we want to disable interrupts, 2497 * so if we miss here, then better luck next time. 2498 * 2499 * This is called within the scheduler and wake up, so interrupts 2500 * had better been disabled and run queue lock been held. 2501 */ 2502 lockdep_assert_preemption_disabled(); 2503 if (!arch_spin_trylock(&trace_cmdline_lock)) 2504 return 0; 2505 2506 idx = savedcmd->map_pid_to_cmdline[tpid]; 2507 if (idx == NO_CMDLINE_MAP) { 2508 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num; 2509 2510 savedcmd->map_pid_to_cmdline[tpid] = idx; 2511 savedcmd->cmdline_idx = idx; 2512 } 2513 2514 savedcmd->map_cmdline_to_pid[idx] = tsk->pid; 2515 set_cmdline(idx, tsk->comm); 2516 2517 arch_spin_unlock(&trace_cmdline_lock); 2518 2519 return 1; 2520 } 2521 2522 static void __trace_find_cmdline(int pid, char comm[]) 2523 { 2524 unsigned map; 2525 int tpid; 2526 2527 if (!pid) { 2528 strcpy(comm, "<idle>"); 2529 return; 2530 } 2531 2532 if (WARN_ON_ONCE(pid < 0)) { 2533 strcpy(comm, "<XXX>"); 2534 return; 2535 } 2536 2537 tpid = pid & (PID_MAX_DEFAULT - 1); 2538 map = savedcmd->map_pid_to_cmdline[tpid]; 2539 if (map != NO_CMDLINE_MAP) { 2540 tpid = savedcmd->map_cmdline_to_pid[map]; 2541 if (tpid == pid) { 2542 strscpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN); 2543 return; 2544 } 2545 } 2546 strcpy(comm, "<...>"); 2547 } 2548 2549 void trace_find_cmdline(int pid, char comm[]) 2550 { 2551 preempt_disable(); 2552 arch_spin_lock(&trace_cmdline_lock); 2553 2554 __trace_find_cmdline(pid, comm); 2555 2556 arch_spin_unlock(&trace_cmdline_lock); 2557 preempt_enable(); 2558 } 2559 2560 static int *trace_find_tgid_ptr(int pid) 2561 { 2562 /* 2563 * Pairs with the smp_store_release in set_tracer_flag() to ensure that 2564 * if we observe a non-NULL tgid_map then we also observe the correct 2565 * tgid_map_max. 2566 */ 2567 int *map = smp_load_acquire(&tgid_map); 2568 2569 if (unlikely(!map || pid > tgid_map_max)) 2570 return NULL; 2571 2572 return &map[pid]; 2573 } 2574 2575 int trace_find_tgid(int pid) 2576 { 2577 int *ptr = trace_find_tgid_ptr(pid); 2578 2579 return ptr ? *ptr : 0; 2580 } 2581 2582 static int trace_save_tgid(struct task_struct *tsk) 2583 { 2584 int *ptr; 2585 2586 /* treat recording of idle task as a success */ 2587 if (!tsk->pid) 2588 return 1; 2589 2590 ptr = trace_find_tgid_ptr(tsk->pid); 2591 if (!ptr) 2592 return 0; 2593 2594 *ptr = tsk->tgid; 2595 return 1; 2596 } 2597 2598 static bool tracing_record_taskinfo_skip(int flags) 2599 { 2600 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID)))) 2601 return true; 2602 if (!__this_cpu_read(trace_taskinfo_save)) 2603 return true; 2604 return false; 2605 } 2606 2607 /** 2608 * tracing_record_taskinfo - record the task info of a task 2609 * 2610 * @task: task to record 2611 * @flags: TRACE_RECORD_CMDLINE for recording comm 2612 * TRACE_RECORD_TGID for recording tgid 2613 */ 2614 void tracing_record_taskinfo(struct task_struct *task, int flags) 2615 { 2616 bool done; 2617 2618 if (tracing_record_taskinfo_skip(flags)) 2619 return; 2620 2621 /* 2622 * Record as much task information as possible. If some fail, continue 2623 * to try to record the others. 2624 */ 2625 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task); 2626 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task); 2627 2628 /* If recording any information failed, retry again soon. */ 2629 if (!done) 2630 return; 2631 2632 __this_cpu_write(trace_taskinfo_save, false); 2633 } 2634 2635 /** 2636 * tracing_record_taskinfo_sched_switch - record task info for sched_switch 2637 * 2638 * @prev: previous task during sched_switch 2639 * @next: next task during sched_switch 2640 * @flags: TRACE_RECORD_CMDLINE for recording comm 2641 * TRACE_RECORD_TGID for recording tgid 2642 */ 2643 void tracing_record_taskinfo_sched_switch(struct task_struct *prev, 2644 struct task_struct *next, int flags) 2645 { 2646 bool done; 2647 2648 if (tracing_record_taskinfo_skip(flags)) 2649 return; 2650 2651 /* 2652 * Record as much task information as possible. If some fail, continue 2653 * to try to record the others. 2654 */ 2655 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev); 2656 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next); 2657 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev); 2658 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next); 2659 2660 /* If recording any information failed, retry again soon. */ 2661 if (!done) 2662 return; 2663 2664 __this_cpu_write(trace_taskinfo_save, false); 2665 } 2666 2667 /* Helpers to record a specific task information */ 2668 void tracing_record_cmdline(struct task_struct *task) 2669 { 2670 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE); 2671 } 2672 2673 void tracing_record_tgid(struct task_struct *task) 2674 { 2675 tracing_record_taskinfo(task, TRACE_RECORD_TGID); 2676 } 2677 2678 /* 2679 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq 2680 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function 2681 * simplifies those functions and keeps them in sync. 2682 */ 2683 enum print_line_t trace_handle_return(struct trace_seq *s) 2684 { 2685 return trace_seq_has_overflowed(s) ? 2686 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED; 2687 } 2688 EXPORT_SYMBOL_GPL(trace_handle_return); 2689 2690 static unsigned short migration_disable_value(void) 2691 { 2692 #if defined(CONFIG_SMP) 2693 return current->migration_disabled; 2694 #else 2695 return 0; 2696 #endif 2697 } 2698 2699 unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status) 2700 { 2701 unsigned int trace_flags = irqs_status; 2702 unsigned int pc; 2703 2704 pc = preempt_count(); 2705 2706 if (pc & NMI_MASK) 2707 trace_flags |= TRACE_FLAG_NMI; 2708 if (pc & HARDIRQ_MASK) 2709 trace_flags |= TRACE_FLAG_HARDIRQ; 2710 if (in_serving_softirq()) 2711 trace_flags |= TRACE_FLAG_SOFTIRQ; 2712 if (softirq_count() >> (SOFTIRQ_SHIFT + 1)) 2713 trace_flags |= TRACE_FLAG_BH_OFF; 2714 2715 if (tif_need_resched()) 2716 trace_flags |= TRACE_FLAG_NEED_RESCHED; 2717 if (test_preempt_need_resched()) 2718 trace_flags |= TRACE_FLAG_PREEMPT_RESCHED; 2719 return (trace_flags << 16) | (min_t(unsigned int, pc & 0xff, 0xf)) | 2720 (min_t(unsigned int, migration_disable_value(), 0xf)) << 4; 2721 } 2722 2723 struct ring_buffer_event * 2724 trace_buffer_lock_reserve(struct trace_buffer *buffer, 2725 int type, 2726 unsigned long len, 2727 unsigned int trace_ctx) 2728 { 2729 return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx); 2730 } 2731 2732 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event); 2733 DEFINE_PER_CPU(int, trace_buffered_event_cnt); 2734 static int trace_buffered_event_ref; 2735 2736 /** 2737 * trace_buffered_event_enable - enable buffering events 2738 * 2739 * When events are being filtered, it is quicker to use a temporary 2740 * buffer to write the event data into if there's a likely chance 2741 * that it will not be committed. The discard of the ring buffer 2742 * is not as fast as committing, and is much slower than copying 2743 * a commit. 2744 * 2745 * When an event is to be filtered, allocate per cpu buffers to 2746 * write the event data into, and if the event is filtered and discarded 2747 * it is simply dropped, otherwise, the entire data is to be committed 2748 * in one shot. 2749 */ 2750 void trace_buffered_event_enable(void) 2751 { 2752 struct ring_buffer_event *event; 2753 struct page *page; 2754 int cpu; 2755 2756 WARN_ON_ONCE(!mutex_is_locked(&event_mutex)); 2757 2758 if (trace_buffered_event_ref++) 2759 return; 2760 2761 for_each_tracing_cpu(cpu) { 2762 page = alloc_pages_node(cpu_to_node(cpu), 2763 GFP_KERNEL | __GFP_NORETRY, 0); 2764 /* This is just an optimization and can handle failures */ 2765 if (!page) { 2766 pr_err("Failed to allocate event buffer\n"); 2767 break; 2768 } 2769 2770 event = page_address(page); 2771 memset(event, 0, sizeof(*event)); 2772 2773 per_cpu(trace_buffered_event, cpu) = event; 2774 2775 preempt_disable(); 2776 if (cpu == smp_processor_id() && 2777 __this_cpu_read(trace_buffered_event) != 2778 per_cpu(trace_buffered_event, cpu)) 2779 WARN_ON_ONCE(1); 2780 preempt_enable(); 2781 } 2782 } 2783 2784 static void enable_trace_buffered_event(void *data) 2785 { 2786 /* Probably not needed, but do it anyway */ 2787 smp_rmb(); 2788 this_cpu_dec(trace_buffered_event_cnt); 2789 } 2790 2791 static void disable_trace_buffered_event(void *data) 2792 { 2793 this_cpu_inc(trace_buffered_event_cnt); 2794 } 2795 2796 /** 2797 * trace_buffered_event_disable - disable buffering events 2798 * 2799 * When a filter is removed, it is faster to not use the buffered 2800 * events, and to commit directly into the ring buffer. Free up 2801 * the temp buffers when there are no more users. This requires 2802 * special synchronization with current events. 2803 */ 2804 void trace_buffered_event_disable(void) 2805 { 2806 int cpu; 2807 2808 WARN_ON_ONCE(!mutex_is_locked(&event_mutex)); 2809 2810 if (WARN_ON_ONCE(!trace_buffered_event_ref)) 2811 return; 2812 2813 if (--trace_buffered_event_ref) 2814 return; 2815 2816 /* For each CPU, set the buffer as used. */ 2817 on_each_cpu_mask(tracing_buffer_mask, disable_trace_buffered_event, 2818 NULL, true); 2819 2820 /* Wait for all current users to finish */ 2821 synchronize_rcu(); 2822 2823 for_each_tracing_cpu(cpu) { 2824 free_page((unsigned long)per_cpu(trace_buffered_event, cpu)); 2825 per_cpu(trace_buffered_event, cpu) = NULL; 2826 } 2827 2828 /* 2829 * Wait for all CPUs that potentially started checking if they can use 2830 * their event buffer only after the previous synchronize_rcu() call and 2831 * they still read a valid pointer from trace_buffered_event. It must be 2832 * ensured they don't see cleared trace_buffered_event_cnt else they 2833 * could wrongly decide to use the pointed-to buffer which is now freed. 2834 */ 2835 synchronize_rcu(); 2836 2837 /* For each CPU, relinquish the buffer */ 2838 on_each_cpu_mask(tracing_buffer_mask, enable_trace_buffered_event, NULL, 2839 true); 2840 } 2841 2842 static struct trace_buffer *temp_buffer; 2843 2844 struct ring_buffer_event * 2845 trace_event_buffer_lock_reserve(struct trace_buffer **current_rb, 2846 struct trace_event_file *trace_file, 2847 int type, unsigned long len, 2848 unsigned int trace_ctx) 2849 { 2850 struct ring_buffer_event *entry; 2851 struct trace_array *tr = trace_file->tr; 2852 int val; 2853 2854 *current_rb = tr->array_buffer.buffer; 2855 2856 if (!tr->no_filter_buffering_ref && 2857 (trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED))) { 2858 preempt_disable_notrace(); 2859 /* 2860 * Filtering is on, so try to use the per cpu buffer first. 2861 * This buffer will simulate a ring_buffer_event, 2862 * where the type_len is zero and the array[0] will 2863 * hold the full length. 2864 * (see include/linux/ring-buffer.h for details on 2865 * how the ring_buffer_event is structured). 2866 * 2867 * Using a temp buffer during filtering and copying it 2868 * on a matched filter is quicker than writing directly 2869 * into the ring buffer and then discarding it when 2870 * it doesn't match. That is because the discard 2871 * requires several atomic operations to get right. 2872 * Copying on match and doing nothing on a failed match 2873 * is still quicker than no copy on match, but having 2874 * to discard out of the ring buffer on a failed match. 2875 */ 2876 if ((entry = __this_cpu_read(trace_buffered_event))) { 2877 int max_len = PAGE_SIZE - struct_size(entry, array, 1); 2878 2879 val = this_cpu_inc_return(trace_buffered_event_cnt); 2880 2881 /* 2882 * Preemption is disabled, but interrupts and NMIs 2883 * can still come in now. If that happens after 2884 * the above increment, then it will have to go 2885 * back to the old method of allocating the event 2886 * on the ring buffer, and if the filter fails, it 2887 * will have to call ring_buffer_discard_commit() 2888 * to remove it. 2889 * 2890 * Need to also check the unlikely case that the 2891 * length is bigger than the temp buffer size. 2892 * If that happens, then the reserve is pretty much 2893 * guaranteed to fail, as the ring buffer currently 2894 * only allows events less than a page. But that may 2895 * change in the future, so let the ring buffer reserve 2896 * handle the failure in that case. 2897 */ 2898 if (val == 1 && likely(len <= max_len)) { 2899 trace_event_setup(entry, type, trace_ctx); 2900 entry->array[0] = len; 2901 /* Return with preemption disabled */ 2902 return entry; 2903 } 2904 this_cpu_dec(trace_buffered_event_cnt); 2905 } 2906 /* __trace_buffer_lock_reserve() disables preemption */ 2907 preempt_enable_notrace(); 2908 } 2909 2910 entry = __trace_buffer_lock_reserve(*current_rb, type, len, 2911 trace_ctx); 2912 /* 2913 * If tracing is off, but we have triggers enabled 2914 * we still need to look at the event data. Use the temp_buffer 2915 * to store the trace event for the trigger to use. It's recursive 2916 * safe and will not be recorded anywhere. 2917 */ 2918 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) { 2919 *current_rb = temp_buffer; 2920 entry = __trace_buffer_lock_reserve(*current_rb, type, len, 2921 trace_ctx); 2922 } 2923 return entry; 2924 } 2925 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve); 2926 2927 static DEFINE_RAW_SPINLOCK(tracepoint_iter_lock); 2928 static DEFINE_MUTEX(tracepoint_printk_mutex); 2929 2930 static void output_printk(struct trace_event_buffer *fbuffer) 2931 { 2932 struct trace_event_call *event_call; 2933 struct trace_event_file *file; 2934 struct trace_event *event; 2935 unsigned long flags; 2936 struct trace_iterator *iter = tracepoint_print_iter; 2937 2938 /* We should never get here if iter is NULL */ 2939 if (WARN_ON_ONCE(!iter)) 2940 return; 2941 2942 event_call = fbuffer->trace_file->event_call; 2943 if (!event_call || !event_call->event.funcs || 2944 !event_call->event.funcs->trace) 2945 return; 2946 2947 file = fbuffer->trace_file; 2948 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) || 2949 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) && 2950 !filter_match_preds(file->filter, fbuffer->entry))) 2951 return; 2952 2953 event = &fbuffer->trace_file->event_call->event; 2954 2955 raw_spin_lock_irqsave(&tracepoint_iter_lock, flags); 2956 trace_seq_init(&iter->seq); 2957 iter->ent = fbuffer->entry; 2958 event_call->event.funcs->trace(iter, 0, event); 2959 trace_seq_putc(&iter->seq, 0); 2960 printk("%s", iter->seq.buffer); 2961 2962 raw_spin_unlock_irqrestore(&tracepoint_iter_lock, flags); 2963 } 2964 2965 int tracepoint_printk_sysctl(struct ctl_table *table, int write, 2966 void *buffer, size_t *lenp, 2967 loff_t *ppos) 2968 { 2969 int save_tracepoint_printk; 2970 int ret; 2971 2972 mutex_lock(&tracepoint_printk_mutex); 2973 save_tracepoint_printk = tracepoint_printk; 2974 2975 ret = proc_dointvec(table, write, buffer, lenp, ppos); 2976 2977 /* 2978 * This will force exiting early, as tracepoint_printk 2979 * is always zero when tracepoint_printk_iter is not allocated 2980 */ 2981 if (!tracepoint_print_iter) 2982 tracepoint_printk = 0; 2983 2984 if (save_tracepoint_printk == tracepoint_printk) 2985 goto out; 2986 2987 if (tracepoint_printk) 2988 static_key_enable(&tracepoint_printk_key.key); 2989 else 2990 static_key_disable(&tracepoint_printk_key.key); 2991 2992 out: 2993 mutex_unlock(&tracepoint_printk_mutex); 2994 2995 return ret; 2996 } 2997 2998 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer) 2999 { 3000 enum event_trigger_type tt = ETT_NONE; 3001 struct trace_event_file *file = fbuffer->trace_file; 3002 3003 if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event, 3004 fbuffer->entry, &tt)) 3005 goto discard; 3006 3007 if (static_key_false(&tracepoint_printk_key.key)) 3008 output_printk(fbuffer); 3009 3010 if (static_branch_unlikely(&trace_event_exports_enabled)) 3011 ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT); 3012 3013 trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer, 3014 fbuffer->event, fbuffer->trace_ctx, fbuffer->regs); 3015 3016 discard: 3017 if (tt) 3018 event_triggers_post_call(file, tt); 3019 3020 } 3021 EXPORT_SYMBOL_GPL(trace_event_buffer_commit); 3022 3023 /* 3024 * Skip 3: 3025 * 3026 * trace_buffer_unlock_commit_regs() 3027 * trace_event_buffer_commit() 3028 * trace_event_raw_event_xxx() 3029 */ 3030 # define STACK_SKIP 3 3031 3032 void trace_buffer_unlock_commit_regs(struct trace_array *tr, 3033 struct trace_buffer *buffer, 3034 struct ring_buffer_event *event, 3035 unsigned int trace_ctx, 3036 struct pt_regs *regs) 3037 { 3038 __buffer_unlock_commit(buffer, event); 3039 3040 /* 3041 * If regs is not set, then skip the necessary functions. 3042 * Note, we can still get here via blktrace, wakeup tracer 3043 * and mmiotrace, but that's ok if they lose a function or 3044 * two. They are not that meaningful. 3045 */ 3046 ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs); 3047 ftrace_trace_userstack(tr, buffer, trace_ctx); 3048 } 3049 3050 /* 3051 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack. 3052 */ 3053 void 3054 trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer, 3055 struct ring_buffer_event *event) 3056 { 3057 __buffer_unlock_commit(buffer, event); 3058 } 3059 3060 void 3061 trace_function(struct trace_array *tr, unsigned long ip, unsigned long 3062 parent_ip, unsigned int trace_ctx) 3063 { 3064 struct trace_event_call *call = &event_function; 3065 struct trace_buffer *buffer = tr->array_buffer.buffer; 3066 struct ring_buffer_event *event; 3067 struct ftrace_entry *entry; 3068 3069 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), 3070 trace_ctx); 3071 if (!event) 3072 return; 3073 entry = ring_buffer_event_data(event); 3074 entry->ip = ip; 3075 entry->parent_ip = parent_ip; 3076 3077 if (!call_filter_check_discard(call, entry, buffer, event)) { 3078 if (static_branch_unlikely(&trace_function_exports_enabled)) 3079 ftrace_exports(event, TRACE_EXPORT_FUNCTION); 3080 __buffer_unlock_commit(buffer, event); 3081 } 3082 } 3083 3084 #ifdef CONFIG_STACKTRACE 3085 3086 /* Allow 4 levels of nesting: normal, softirq, irq, NMI */ 3087 #define FTRACE_KSTACK_NESTING 4 3088 3089 #define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING) 3090 3091 struct ftrace_stack { 3092 unsigned long calls[FTRACE_KSTACK_ENTRIES]; 3093 }; 3094 3095 3096 struct ftrace_stacks { 3097 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING]; 3098 }; 3099 3100 static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks); 3101 static DEFINE_PER_CPU(int, ftrace_stack_reserve); 3102 3103 static void __ftrace_trace_stack(struct trace_buffer *buffer, 3104 unsigned int trace_ctx, 3105 int skip, struct pt_regs *regs) 3106 { 3107 struct trace_event_call *call = &event_kernel_stack; 3108 struct ring_buffer_event *event; 3109 unsigned int size, nr_entries; 3110 struct ftrace_stack *fstack; 3111 struct stack_entry *entry; 3112 int stackidx; 3113 3114 /* 3115 * Add one, for this function and the call to save_stack_trace() 3116 * If regs is set, then these functions will not be in the way. 3117 */ 3118 #ifndef CONFIG_UNWINDER_ORC 3119 if (!regs) 3120 skip++; 3121 #endif 3122 3123 preempt_disable_notrace(); 3124 3125 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1; 3126 3127 /* This should never happen. If it does, yell once and skip */ 3128 if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING)) 3129 goto out; 3130 3131 /* 3132 * The above __this_cpu_inc_return() is 'atomic' cpu local. An 3133 * interrupt will either see the value pre increment or post 3134 * increment. If the interrupt happens pre increment it will have 3135 * restored the counter when it returns. We just need a barrier to 3136 * keep gcc from moving things around. 3137 */ 3138 barrier(); 3139 3140 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx; 3141 size = ARRAY_SIZE(fstack->calls); 3142 3143 if (regs) { 3144 nr_entries = stack_trace_save_regs(regs, fstack->calls, 3145 size, skip); 3146 } else { 3147 nr_entries = stack_trace_save(fstack->calls, size, skip); 3148 } 3149 3150 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK, 3151 struct_size(entry, caller, nr_entries), 3152 trace_ctx); 3153 if (!event) 3154 goto out; 3155 entry = ring_buffer_event_data(event); 3156 3157 entry->size = nr_entries; 3158 memcpy(&entry->caller, fstack->calls, 3159 flex_array_size(entry, caller, nr_entries)); 3160 3161 if (!call_filter_check_discard(call, entry, buffer, event)) 3162 __buffer_unlock_commit(buffer, event); 3163 3164 out: 3165 /* Again, don't let gcc optimize things here */ 3166 barrier(); 3167 __this_cpu_dec(ftrace_stack_reserve); 3168 preempt_enable_notrace(); 3169 3170 } 3171 3172 static inline void ftrace_trace_stack(struct trace_array *tr, 3173 struct trace_buffer *buffer, 3174 unsigned int trace_ctx, 3175 int skip, struct pt_regs *regs) 3176 { 3177 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE)) 3178 return; 3179 3180 __ftrace_trace_stack(buffer, trace_ctx, skip, regs); 3181 } 3182 3183 void __trace_stack(struct trace_array *tr, unsigned int trace_ctx, 3184 int skip) 3185 { 3186 struct trace_buffer *buffer = tr->array_buffer.buffer; 3187 3188 if (rcu_is_watching()) { 3189 __ftrace_trace_stack(buffer, trace_ctx, skip, NULL); 3190 return; 3191 } 3192 3193 if (WARN_ON_ONCE(IS_ENABLED(CONFIG_GENERIC_ENTRY))) 3194 return; 3195 3196 /* 3197 * When an NMI triggers, RCU is enabled via ct_nmi_enter(), 3198 * but if the above rcu_is_watching() failed, then the NMI 3199 * triggered someplace critical, and ct_irq_enter() should 3200 * not be called from NMI. 3201 */ 3202 if (unlikely(in_nmi())) 3203 return; 3204 3205 ct_irq_enter_irqson(); 3206 __ftrace_trace_stack(buffer, trace_ctx, skip, NULL); 3207 ct_irq_exit_irqson(); 3208 } 3209 3210 /** 3211 * trace_dump_stack - record a stack back trace in the trace buffer 3212 * @skip: Number of functions to skip (helper handlers) 3213 */ 3214 void trace_dump_stack(int skip) 3215 { 3216 if (tracing_disabled || tracing_selftest_running) 3217 return; 3218 3219 #ifndef CONFIG_UNWINDER_ORC 3220 /* Skip 1 to skip this function. */ 3221 skip++; 3222 #endif 3223 __ftrace_trace_stack(global_trace.array_buffer.buffer, 3224 tracing_gen_ctx(), skip, NULL); 3225 } 3226 EXPORT_SYMBOL_GPL(trace_dump_stack); 3227 3228 #ifdef CONFIG_USER_STACKTRACE_SUPPORT 3229 static DEFINE_PER_CPU(int, user_stack_count); 3230 3231 static void 3232 ftrace_trace_userstack(struct trace_array *tr, 3233 struct trace_buffer *buffer, unsigned int trace_ctx) 3234 { 3235 struct trace_event_call *call = &event_user_stack; 3236 struct ring_buffer_event *event; 3237 struct userstack_entry *entry; 3238 3239 if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE)) 3240 return; 3241 3242 /* 3243 * NMIs can not handle page faults, even with fix ups. 3244 * The save user stack can (and often does) fault. 3245 */ 3246 if (unlikely(in_nmi())) 3247 return; 3248 3249 /* 3250 * prevent recursion, since the user stack tracing may 3251 * trigger other kernel events. 3252 */ 3253 preempt_disable(); 3254 if (__this_cpu_read(user_stack_count)) 3255 goto out; 3256 3257 __this_cpu_inc(user_stack_count); 3258 3259 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, 3260 sizeof(*entry), trace_ctx); 3261 if (!event) 3262 goto out_drop_count; 3263 entry = ring_buffer_event_data(event); 3264 3265 entry->tgid = current->tgid; 3266 memset(&entry->caller, 0, sizeof(entry->caller)); 3267 3268 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES); 3269 if (!call_filter_check_discard(call, entry, buffer, event)) 3270 __buffer_unlock_commit(buffer, event); 3271 3272 out_drop_count: 3273 __this_cpu_dec(user_stack_count); 3274 out: 3275 preempt_enable(); 3276 } 3277 #else /* CONFIG_USER_STACKTRACE_SUPPORT */ 3278 static void ftrace_trace_userstack(struct trace_array *tr, 3279 struct trace_buffer *buffer, 3280 unsigned int trace_ctx) 3281 { 3282 } 3283 #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */ 3284 3285 #endif /* CONFIG_STACKTRACE */ 3286 3287 static inline void 3288 func_repeats_set_delta_ts(struct func_repeats_entry *entry, 3289 unsigned long long delta) 3290 { 3291 entry->bottom_delta_ts = delta & U32_MAX; 3292 entry->top_delta_ts = (delta >> 32); 3293 } 3294 3295 void trace_last_func_repeats(struct trace_array *tr, 3296 struct trace_func_repeats *last_info, 3297 unsigned int trace_ctx) 3298 { 3299 struct trace_buffer *buffer = tr->array_buffer.buffer; 3300 struct func_repeats_entry *entry; 3301 struct ring_buffer_event *event; 3302 u64 delta; 3303 3304 event = __trace_buffer_lock_reserve(buffer, TRACE_FUNC_REPEATS, 3305 sizeof(*entry), trace_ctx); 3306 if (!event) 3307 return; 3308 3309 delta = ring_buffer_event_time_stamp(buffer, event) - 3310 last_info->ts_last_call; 3311 3312 entry = ring_buffer_event_data(event); 3313 entry->ip = last_info->ip; 3314 entry->parent_ip = last_info->parent_ip; 3315 entry->count = last_info->count; 3316 func_repeats_set_delta_ts(entry, delta); 3317 3318 __buffer_unlock_commit(buffer, event); 3319 } 3320 3321 /* created for use with alloc_percpu */ 3322 struct trace_buffer_struct { 3323 int nesting; 3324 char buffer[4][TRACE_BUF_SIZE]; 3325 }; 3326 3327 static struct trace_buffer_struct __percpu *trace_percpu_buffer; 3328 3329 /* 3330 * This allows for lockless recording. If we're nested too deeply, then 3331 * this returns NULL. 3332 */ 3333 static char *get_trace_buf(void) 3334 { 3335 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer); 3336 3337 if (!trace_percpu_buffer || buffer->nesting >= 4) 3338 return NULL; 3339 3340 buffer->nesting++; 3341 3342 /* Interrupts must see nesting incremented before we use the buffer */ 3343 barrier(); 3344 return &buffer->buffer[buffer->nesting - 1][0]; 3345 } 3346 3347 static void put_trace_buf(void) 3348 { 3349 /* Don't let the decrement of nesting leak before this */ 3350 barrier(); 3351 this_cpu_dec(trace_percpu_buffer->nesting); 3352 } 3353 3354 static int alloc_percpu_trace_buffer(void) 3355 { 3356 struct trace_buffer_struct __percpu *buffers; 3357 3358 if (trace_percpu_buffer) 3359 return 0; 3360 3361 buffers = alloc_percpu(struct trace_buffer_struct); 3362 if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer")) 3363 return -ENOMEM; 3364 3365 trace_percpu_buffer = buffers; 3366 return 0; 3367 } 3368 3369 static int buffers_allocated; 3370 3371 void trace_printk_init_buffers(void) 3372 { 3373 if (buffers_allocated) 3374 return; 3375 3376 if (alloc_percpu_trace_buffer()) 3377 return; 3378 3379 /* trace_printk() is for debug use only. Don't use it in production. */ 3380 3381 pr_warn("\n"); 3382 pr_warn("**********************************************************\n"); 3383 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 3384 pr_warn("** **\n"); 3385 pr_warn("** trace_printk() being used. Allocating extra memory. **\n"); 3386 pr_warn("** **\n"); 3387 pr_warn("** This means that this is a DEBUG kernel and it is **\n"); 3388 pr_warn("** unsafe for production use. **\n"); 3389 pr_warn("** **\n"); 3390 pr_warn("** If you see this message and you are not debugging **\n"); 3391 pr_warn("** the kernel, report this immediately to your vendor! **\n"); 3392 pr_warn("** **\n"); 3393 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 3394 pr_warn("**********************************************************\n"); 3395 3396 /* Expand the buffers to set size */ 3397 tracing_update_buffers(&global_trace); 3398 3399 buffers_allocated = 1; 3400 3401 /* 3402 * trace_printk_init_buffers() can be called by modules. 3403 * If that happens, then we need to start cmdline recording 3404 * directly here. If the global_trace.buffer is already 3405 * allocated here, then this was called by module code. 3406 */ 3407 if (global_trace.array_buffer.buffer) 3408 tracing_start_cmdline_record(); 3409 } 3410 EXPORT_SYMBOL_GPL(trace_printk_init_buffers); 3411 3412 void trace_printk_start_comm(void) 3413 { 3414 /* Start tracing comms if trace printk is set */ 3415 if (!buffers_allocated) 3416 return; 3417 tracing_start_cmdline_record(); 3418 } 3419 3420 static void trace_printk_start_stop_comm(int enabled) 3421 { 3422 if (!buffers_allocated) 3423 return; 3424 3425 if (enabled) 3426 tracing_start_cmdline_record(); 3427 else 3428 tracing_stop_cmdline_record(); 3429 } 3430 3431 /** 3432 * trace_vbprintk - write binary msg to tracing buffer 3433 * @ip: The address of the caller 3434 * @fmt: The string format to write to the buffer 3435 * @args: Arguments for @fmt 3436 */ 3437 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) 3438 { 3439 struct trace_event_call *call = &event_bprint; 3440 struct ring_buffer_event *event; 3441 struct trace_buffer *buffer; 3442 struct trace_array *tr = &global_trace; 3443 struct bprint_entry *entry; 3444 unsigned int trace_ctx; 3445 char *tbuffer; 3446 int len = 0, size; 3447 3448 if (unlikely(tracing_selftest_running || tracing_disabled)) 3449 return 0; 3450 3451 /* Don't pollute graph traces with trace_vprintk internals */ 3452 pause_graph_tracing(); 3453 3454 trace_ctx = tracing_gen_ctx(); 3455 preempt_disable_notrace(); 3456 3457 tbuffer = get_trace_buf(); 3458 if (!tbuffer) { 3459 len = 0; 3460 goto out_nobuffer; 3461 } 3462 3463 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args); 3464 3465 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) 3466 goto out_put; 3467 3468 size = sizeof(*entry) + sizeof(u32) * len; 3469 buffer = tr->array_buffer.buffer; 3470 ring_buffer_nest_start(buffer); 3471 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, 3472 trace_ctx); 3473 if (!event) 3474 goto out; 3475 entry = ring_buffer_event_data(event); 3476 entry->ip = ip; 3477 entry->fmt = fmt; 3478 3479 memcpy(entry->buf, tbuffer, sizeof(u32) * len); 3480 if (!call_filter_check_discard(call, entry, buffer, event)) { 3481 __buffer_unlock_commit(buffer, event); 3482 ftrace_trace_stack(tr, buffer, trace_ctx, 6, NULL); 3483 } 3484 3485 out: 3486 ring_buffer_nest_end(buffer); 3487 out_put: 3488 put_trace_buf(); 3489 3490 out_nobuffer: 3491 preempt_enable_notrace(); 3492 unpause_graph_tracing(); 3493 3494 return len; 3495 } 3496 EXPORT_SYMBOL_GPL(trace_vbprintk); 3497 3498 __printf(3, 0) 3499 static int 3500 __trace_array_vprintk(struct trace_buffer *buffer, 3501 unsigned long ip, const char *fmt, va_list args) 3502 { 3503 struct trace_event_call *call = &event_print; 3504 struct ring_buffer_event *event; 3505 int len = 0, size; 3506 struct print_entry *entry; 3507 unsigned int trace_ctx; 3508 char *tbuffer; 3509 3510 if (tracing_disabled) 3511 return 0; 3512 3513 /* Don't pollute graph traces with trace_vprintk internals */ 3514 pause_graph_tracing(); 3515 3516 trace_ctx = tracing_gen_ctx(); 3517 preempt_disable_notrace(); 3518 3519 3520 tbuffer = get_trace_buf(); 3521 if (!tbuffer) { 3522 len = 0; 3523 goto out_nobuffer; 3524 } 3525 3526 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args); 3527 3528 size = sizeof(*entry) + len + 1; 3529 ring_buffer_nest_start(buffer); 3530 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 3531 trace_ctx); 3532 if (!event) 3533 goto out; 3534 entry = ring_buffer_event_data(event); 3535 entry->ip = ip; 3536 3537 memcpy(&entry->buf, tbuffer, len + 1); 3538 if (!call_filter_check_discard(call, entry, buffer, event)) { 3539 __buffer_unlock_commit(buffer, event); 3540 ftrace_trace_stack(&global_trace, buffer, trace_ctx, 6, NULL); 3541 } 3542 3543 out: 3544 ring_buffer_nest_end(buffer); 3545 put_trace_buf(); 3546 3547 out_nobuffer: 3548 preempt_enable_notrace(); 3549 unpause_graph_tracing(); 3550 3551 return len; 3552 } 3553 3554 __printf(3, 0) 3555 int trace_array_vprintk(struct trace_array *tr, 3556 unsigned long ip, const char *fmt, va_list args) 3557 { 3558 if (tracing_selftest_running && tr == &global_trace) 3559 return 0; 3560 3561 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args); 3562 } 3563 3564 /** 3565 * trace_array_printk - Print a message to a specific instance 3566 * @tr: The instance trace_array descriptor 3567 * @ip: The instruction pointer that this is called from. 3568 * @fmt: The format to print (printf format) 3569 * 3570 * If a subsystem sets up its own instance, they have the right to 3571 * printk strings into their tracing instance buffer using this 3572 * function. Note, this function will not write into the top level 3573 * buffer (use trace_printk() for that), as writing into the top level 3574 * buffer should only have events that can be individually disabled. 3575 * trace_printk() is only used for debugging a kernel, and should not 3576 * be ever incorporated in normal use. 3577 * 3578 * trace_array_printk() can be used, as it will not add noise to the 3579 * top level tracing buffer. 3580 * 3581 * Note, trace_array_init_printk() must be called on @tr before this 3582 * can be used. 3583 */ 3584 __printf(3, 0) 3585 int trace_array_printk(struct trace_array *tr, 3586 unsigned long ip, const char *fmt, ...) 3587 { 3588 int ret; 3589 va_list ap; 3590 3591 if (!tr) 3592 return -ENOENT; 3593 3594 /* This is only allowed for created instances */ 3595 if (tr == &global_trace) 3596 return 0; 3597 3598 if (!(tr->trace_flags & TRACE_ITER_PRINTK)) 3599 return 0; 3600 3601 va_start(ap, fmt); 3602 ret = trace_array_vprintk(tr, ip, fmt, ap); 3603 va_end(ap); 3604 return ret; 3605 } 3606 EXPORT_SYMBOL_GPL(trace_array_printk); 3607 3608 /** 3609 * trace_array_init_printk - Initialize buffers for trace_array_printk() 3610 * @tr: The trace array to initialize the buffers for 3611 * 3612 * As trace_array_printk() only writes into instances, they are OK to 3613 * have in the kernel (unlike trace_printk()). This needs to be called 3614 * before trace_array_printk() can be used on a trace_array. 3615 */ 3616 int trace_array_init_printk(struct trace_array *tr) 3617 { 3618 if (!tr) 3619 return -ENOENT; 3620 3621 /* This is only allowed for created instances */ 3622 if (tr == &global_trace) 3623 return -EINVAL; 3624 3625 return alloc_percpu_trace_buffer(); 3626 } 3627 EXPORT_SYMBOL_GPL(trace_array_init_printk); 3628 3629 __printf(3, 4) 3630 int trace_array_printk_buf(struct trace_buffer *buffer, 3631 unsigned long ip, const char *fmt, ...) 3632 { 3633 int ret; 3634 va_list ap; 3635 3636 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) 3637 return 0; 3638 3639 va_start(ap, fmt); 3640 ret = __trace_array_vprintk(buffer, ip, fmt, ap); 3641 va_end(ap); 3642 return ret; 3643 } 3644 3645 __printf(2, 0) 3646 int trace_vprintk(unsigned long ip, const char *fmt, va_list args) 3647 { 3648 return trace_array_vprintk(&global_trace, ip, fmt, args); 3649 } 3650 EXPORT_SYMBOL_GPL(trace_vprintk); 3651 3652 static void trace_iterator_increment(struct trace_iterator *iter) 3653 { 3654 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); 3655 3656 iter->idx++; 3657 if (buf_iter) 3658 ring_buffer_iter_advance(buf_iter); 3659 } 3660 3661 static struct trace_entry * 3662 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, 3663 unsigned long *lost_events) 3664 { 3665 struct ring_buffer_event *event; 3666 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu); 3667 3668 if (buf_iter) { 3669 event = ring_buffer_iter_peek(buf_iter, ts); 3670 if (lost_events) 3671 *lost_events = ring_buffer_iter_dropped(buf_iter) ? 3672 (unsigned long)-1 : 0; 3673 } else { 3674 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts, 3675 lost_events); 3676 } 3677 3678 if (event) { 3679 iter->ent_size = ring_buffer_event_length(event); 3680 return ring_buffer_event_data(event); 3681 } 3682 iter->ent_size = 0; 3683 return NULL; 3684 } 3685 3686 static struct trace_entry * 3687 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, 3688 unsigned long *missing_events, u64 *ent_ts) 3689 { 3690 struct trace_buffer *buffer = iter->array_buffer->buffer; 3691 struct trace_entry *ent, *next = NULL; 3692 unsigned long lost_events = 0, next_lost = 0; 3693 int cpu_file = iter->cpu_file; 3694 u64 next_ts = 0, ts; 3695 int next_cpu = -1; 3696 int next_size = 0; 3697 int cpu; 3698 3699 /* 3700 * If we are in a per_cpu trace file, don't bother by iterating over 3701 * all cpu and peek directly. 3702 */ 3703 if (cpu_file > RING_BUFFER_ALL_CPUS) { 3704 if (ring_buffer_empty_cpu(buffer, cpu_file)) 3705 return NULL; 3706 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events); 3707 if (ent_cpu) 3708 *ent_cpu = cpu_file; 3709 3710 return ent; 3711 } 3712 3713 for_each_tracing_cpu(cpu) { 3714 3715 if (ring_buffer_empty_cpu(buffer, cpu)) 3716 continue; 3717 3718 ent = peek_next_entry(iter, cpu, &ts, &lost_events); 3719 3720 /* 3721 * Pick the entry with the smallest timestamp: 3722 */ 3723 if (ent && (!next || ts < next_ts)) { 3724 next = ent; 3725 next_cpu = cpu; 3726 next_ts = ts; 3727 next_lost = lost_events; 3728 next_size = iter->ent_size; 3729 } 3730 } 3731 3732 iter->ent_size = next_size; 3733 3734 if (ent_cpu) 3735 *ent_cpu = next_cpu; 3736 3737 if (ent_ts) 3738 *ent_ts = next_ts; 3739 3740 if (missing_events) 3741 *missing_events = next_lost; 3742 3743 return next; 3744 } 3745 3746 #define STATIC_FMT_BUF_SIZE 128 3747 static char static_fmt_buf[STATIC_FMT_BUF_SIZE]; 3748 3749 char *trace_iter_expand_format(struct trace_iterator *iter) 3750 { 3751 char *tmp; 3752 3753 /* 3754 * iter->tr is NULL when used with tp_printk, which makes 3755 * this get called where it is not safe to call krealloc(). 3756 */ 3757 if (!iter->tr || iter->fmt == static_fmt_buf) 3758 return NULL; 3759 3760 tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE, 3761 GFP_KERNEL); 3762 if (tmp) { 3763 iter->fmt_size += STATIC_FMT_BUF_SIZE; 3764 iter->fmt = tmp; 3765 } 3766 3767 return tmp; 3768 } 3769 3770 /* Returns true if the string is safe to dereference from an event */ 3771 static bool trace_safe_str(struct trace_iterator *iter, const char *str, 3772 bool star, int len) 3773 { 3774 unsigned long addr = (unsigned long)str; 3775 struct trace_event *trace_event; 3776 struct trace_event_call *event; 3777 3778 /* Ignore strings with no length */ 3779 if (star && !len) 3780 return true; 3781 3782 /* OK if part of the event data */ 3783 if ((addr >= (unsigned long)iter->ent) && 3784 (addr < (unsigned long)iter->ent + iter->ent_size)) 3785 return true; 3786 3787 /* OK if part of the temp seq buffer */ 3788 if ((addr >= (unsigned long)iter->tmp_seq.buffer) && 3789 (addr < (unsigned long)iter->tmp_seq.buffer + TRACE_SEQ_BUFFER_SIZE)) 3790 return true; 3791 3792 /* Core rodata can not be freed */ 3793 if (is_kernel_rodata(addr)) 3794 return true; 3795 3796 if (trace_is_tracepoint_string(str)) 3797 return true; 3798 3799 /* 3800 * Now this could be a module event, referencing core module 3801 * data, which is OK. 3802 */ 3803 if (!iter->ent) 3804 return false; 3805 3806 trace_event = ftrace_find_event(iter->ent->type); 3807 if (!trace_event) 3808 return false; 3809 3810 event = container_of(trace_event, struct trace_event_call, event); 3811 if ((event->flags & TRACE_EVENT_FL_DYNAMIC) || !event->module) 3812 return false; 3813 3814 /* Would rather have rodata, but this will suffice */ 3815 if (within_module_core(addr, event->module)) 3816 return true; 3817 3818 return false; 3819 } 3820 3821 static DEFINE_STATIC_KEY_FALSE(trace_no_verify); 3822 3823 static int test_can_verify_check(const char *fmt, ...) 3824 { 3825 char buf[16]; 3826 va_list ap; 3827 int ret; 3828 3829 /* 3830 * The verifier is dependent on vsnprintf() modifies the va_list 3831 * passed to it, where it is sent as a reference. Some architectures 3832 * (like x86_32) passes it by value, which means that vsnprintf() 3833 * does not modify the va_list passed to it, and the verifier 3834 * would then need to be able to understand all the values that 3835 * vsnprintf can use. If it is passed by value, then the verifier 3836 * is disabled. 3837 */ 3838 va_start(ap, fmt); 3839 vsnprintf(buf, 16, "%d", ap); 3840 ret = va_arg(ap, int); 3841 va_end(ap); 3842 3843 return ret; 3844 } 3845 3846 static void test_can_verify(void) 3847 { 3848 if (!test_can_verify_check("%d %d", 0, 1)) { 3849 pr_info("trace event string verifier disabled\n"); 3850 static_branch_inc(&trace_no_verify); 3851 } 3852 } 3853 3854 /** 3855 * trace_check_vprintf - Check dereferenced strings while writing to the seq buffer 3856 * @iter: The iterator that holds the seq buffer and the event being printed 3857 * @fmt: The format used to print the event 3858 * @ap: The va_list holding the data to print from @fmt. 3859 * 3860 * This writes the data into the @iter->seq buffer using the data from 3861 * @fmt and @ap. If the format has a %s, then the source of the string 3862 * is examined to make sure it is safe to print, otherwise it will 3863 * warn and print "[UNSAFE MEMORY]" in place of the dereferenced string 3864 * pointer. 3865 */ 3866 void trace_check_vprintf(struct trace_iterator *iter, const char *fmt, 3867 va_list ap) 3868 { 3869 const char *p = fmt; 3870 const char *str; 3871 int i, j; 3872 3873 if (WARN_ON_ONCE(!fmt)) 3874 return; 3875 3876 if (static_branch_unlikely(&trace_no_verify)) 3877 goto print; 3878 3879 /* Don't bother checking when doing a ftrace_dump() */ 3880 if (iter->fmt == static_fmt_buf) 3881 goto print; 3882 3883 while (*p) { 3884 bool star = false; 3885 int len = 0; 3886 3887 j = 0; 3888 3889 /* We only care about %s and variants */ 3890 for (i = 0; p[i]; i++) { 3891 if (i + 1 >= iter->fmt_size) { 3892 /* 3893 * If we can't expand the copy buffer, 3894 * just print it. 3895 */ 3896 if (!trace_iter_expand_format(iter)) 3897 goto print; 3898 } 3899 3900 if (p[i] == '\\' && p[i+1]) { 3901 i++; 3902 continue; 3903 } 3904 if (p[i] == '%') { 3905 /* Need to test cases like %08.*s */ 3906 for (j = 1; p[i+j]; j++) { 3907 if (isdigit(p[i+j]) || 3908 p[i+j] == '.') 3909 continue; 3910 if (p[i+j] == '*') { 3911 star = true; 3912 continue; 3913 } 3914 break; 3915 } 3916 if (p[i+j] == 's') 3917 break; 3918 star = false; 3919 } 3920 j = 0; 3921 } 3922 /* If no %s found then just print normally */ 3923 if (!p[i]) 3924 break; 3925 3926 /* Copy up to the %s, and print that */ 3927 strncpy(iter->fmt, p, i); 3928 iter->fmt[i] = '\0'; 3929 trace_seq_vprintf(&iter->seq, iter->fmt, ap); 3930 3931 /* 3932 * If iter->seq is full, the above call no longer guarantees 3933 * that ap is in sync with fmt processing, and further calls 3934 * to va_arg() can return wrong positional arguments. 3935 * 3936 * Ensure that ap is no longer used in this case. 3937 */ 3938 if (iter->seq.full) { 3939 p = ""; 3940 break; 3941 } 3942 3943 if (star) 3944 len = va_arg(ap, int); 3945 3946 /* The ap now points to the string data of the %s */ 3947 str = va_arg(ap, const char *); 3948 3949 /* 3950 * If you hit this warning, it is likely that the 3951 * trace event in question used %s on a string that 3952 * was saved at the time of the event, but may not be 3953 * around when the trace is read. Use __string(), 3954 * __assign_str() and __get_str() helpers in the TRACE_EVENT() 3955 * instead. See samples/trace_events/trace-events-sample.h 3956 * for reference. 3957 */ 3958 if (WARN_ONCE(!trace_safe_str(iter, str, star, len), 3959 "fmt: '%s' current_buffer: '%s'", 3960 fmt, seq_buf_str(&iter->seq.seq))) { 3961 int ret; 3962 3963 /* Try to safely read the string */ 3964 if (star) { 3965 if (len + 1 > iter->fmt_size) 3966 len = iter->fmt_size - 1; 3967 if (len < 0) 3968 len = 0; 3969 ret = copy_from_kernel_nofault(iter->fmt, str, len); 3970 iter->fmt[len] = 0; 3971 star = false; 3972 } else { 3973 ret = strncpy_from_kernel_nofault(iter->fmt, str, 3974 iter->fmt_size); 3975 } 3976 if (ret < 0) 3977 trace_seq_printf(&iter->seq, "(0x%px)", str); 3978 else 3979 trace_seq_printf(&iter->seq, "(0x%px:%s)", 3980 str, iter->fmt); 3981 str = "[UNSAFE-MEMORY]"; 3982 strcpy(iter->fmt, "%s"); 3983 } else { 3984 strncpy(iter->fmt, p + i, j + 1); 3985 iter->fmt[j+1] = '\0'; 3986 } 3987 if (star) 3988 trace_seq_printf(&iter->seq, iter->fmt, len, str); 3989 else 3990 trace_seq_printf(&iter->seq, iter->fmt, str); 3991 3992 p += i + j + 1; 3993 } 3994 print: 3995 if (*p) 3996 trace_seq_vprintf(&iter->seq, p, ap); 3997 } 3998 3999 const char *trace_event_format(struct trace_iterator *iter, const char *fmt) 4000 { 4001 const char *p, *new_fmt; 4002 char *q; 4003 4004 if (WARN_ON_ONCE(!fmt)) 4005 return fmt; 4006 4007 if (!iter->tr || iter->tr->trace_flags & TRACE_ITER_HASH_PTR) 4008 return fmt; 4009 4010 p = fmt; 4011 new_fmt = q = iter->fmt; 4012 while (*p) { 4013 if (unlikely(q - new_fmt + 3 > iter->fmt_size)) { 4014 if (!trace_iter_expand_format(iter)) 4015 return fmt; 4016 4017 q += iter->fmt - new_fmt; 4018 new_fmt = iter->fmt; 4019 } 4020 4021 *q++ = *p++; 4022 4023 /* Replace %p with %px */ 4024 if (p[-1] == '%') { 4025 if (p[0] == '%') { 4026 *q++ = *p++; 4027 } else if (p[0] == 'p' && !isalnum(p[1])) { 4028 *q++ = *p++; 4029 *q++ = 'x'; 4030 } 4031 } 4032 } 4033 *q = '\0'; 4034 4035 return new_fmt; 4036 } 4037 4038 #define STATIC_TEMP_BUF_SIZE 128 4039 static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4); 4040 4041 /* Find the next real entry, without updating the iterator itself */ 4042 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, 4043 int *ent_cpu, u64 *ent_ts) 4044 { 4045 /* __find_next_entry will reset ent_size */ 4046 int ent_size = iter->ent_size; 4047 struct trace_entry *entry; 4048 4049 /* 4050 * If called from ftrace_dump(), then the iter->temp buffer 4051 * will be the static_temp_buf and not created from kmalloc. 4052 * If the entry size is greater than the buffer, we can 4053 * not save it. Just return NULL in that case. This is only 4054 * used to add markers when two consecutive events' time 4055 * stamps have a large delta. See trace_print_lat_context() 4056 */ 4057 if (iter->temp == static_temp_buf && 4058 STATIC_TEMP_BUF_SIZE < ent_size) 4059 return NULL; 4060 4061 /* 4062 * The __find_next_entry() may call peek_next_entry(), which may 4063 * call ring_buffer_peek() that may make the contents of iter->ent 4064 * undefined. Need to copy iter->ent now. 4065 */ 4066 if (iter->ent && iter->ent != iter->temp) { 4067 if ((!iter->temp || iter->temp_size < iter->ent_size) && 4068 !WARN_ON_ONCE(iter->temp == static_temp_buf)) { 4069 void *temp; 4070 temp = kmalloc(iter->ent_size, GFP_KERNEL); 4071 if (!temp) 4072 return NULL; 4073 kfree(iter->temp); 4074 iter->temp = temp; 4075 iter->temp_size = iter->ent_size; 4076 } 4077 memcpy(iter->temp, iter->ent, iter->ent_size); 4078 iter->ent = iter->temp; 4079 } 4080 entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts); 4081 /* Put back the original ent_size */ 4082 iter->ent_size = ent_size; 4083 4084 return entry; 4085 } 4086 4087 /* Find the next real entry, and increment the iterator to the next entry */ 4088 void *trace_find_next_entry_inc(struct trace_iterator *iter) 4089 { 4090 iter->ent = __find_next_entry(iter, &iter->cpu, 4091 &iter->lost_events, &iter->ts); 4092 4093 if (iter->ent) 4094 trace_iterator_increment(iter); 4095 4096 return iter->ent ? iter : NULL; 4097 } 4098 4099 static void trace_consume(struct trace_iterator *iter) 4100 { 4101 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts, 4102 &iter->lost_events); 4103 } 4104 4105 static void *s_next(struct seq_file *m, void *v, loff_t *pos) 4106 { 4107 struct trace_iterator *iter = m->private; 4108 int i = (int)*pos; 4109 void *ent; 4110 4111 WARN_ON_ONCE(iter->leftover); 4112 4113 (*pos)++; 4114 4115 /* can't go backwards */ 4116 if (iter->idx > i) 4117 return NULL; 4118 4119 if (iter->idx < 0) 4120 ent = trace_find_next_entry_inc(iter); 4121 else 4122 ent = iter; 4123 4124 while (ent && iter->idx < i) 4125 ent = trace_find_next_entry_inc(iter); 4126 4127 iter->pos = *pos; 4128 4129 return ent; 4130 } 4131 4132 void tracing_iter_reset(struct trace_iterator *iter, int cpu) 4133 { 4134 struct ring_buffer_iter *buf_iter; 4135 unsigned long entries = 0; 4136 u64 ts; 4137 4138 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0; 4139 4140 buf_iter = trace_buffer_iter(iter, cpu); 4141 if (!buf_iter) 4142 return; 4143 4144 ring_buffer_iter_reset(buf_iter); 4145 4146 /* 4147 * We could have the case with the max latency tracers 4148 * that a reset never took place on a cpu. This is evident 4149 * by the timestamp being before the start of the buffer. 4150 */ 4151 while (ring_buffer_iter_peek(buf_iter, &ts)) { 4152 if (ts >= iter->array_buffer->time_start) 4153 break; 4154 entries++; 4155 ring_buffer_iter_advance(buf_iter); 4156 } 4157 4158 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries; 4159 } 4160 4161 /* 4162 * The current tracer is copied to avoid a global locking 4163 * all around. 4164 */ 4165 static void *s_start(struct seq_file *m, loff_t *pos) 4166 { 4167 struct trace_iterator *iter = m->private; 4168 struct trace_array *tr = iter->tr; 4169 int cpu_file = iter->cpu_file; 4170 void *p = NULL; 4171 loff_t l = 0; 4172 int cpu; 4173 4174 mutex_lock(&trace_types_lock); 4175 if (unlikely(tr->current_trace != iter->trace)) { 4176 /* Close iter->trace before switching to the new current tracer */ 4177 if (iter->trace->close) 4178 iter->trace->close(iter); 4179 iter->trace = tr->current_trace; 4180 /* Reopen the new current tracer */ 4181 if (iter->trace->open) 4182 iter->trace->open(iter); 4183 } 4184 mutex_unlock(&trace_types_lock); 4185 4186 #ifdef CONFIG_TRACER_MAX_TRACE 4187 if (iter->snapshot && iter->trace->use_max_tr) 4188 return ERR_PTR(-EBUSY); 4189 #endif 4190 4191 if (*pos != iter->pos) { 4192 iter->ent = NULL; 4193 iter->cpu = 0; 4194 iter->idx = -1; 4195 4196 if (cpu_file == RING_BUFFER_ALL_CPUS) { 4197 for_each_tracing_cpu(cpu) 4198 tracing_iter_reset(iter, cpu); 4199 } else 4200 tracing_iter_reset(iter, cpu_file); 4201 4202 iter->leftover = 0; 4203 for (p = iter; p && l < *pos; p = s_next(m, p, &l)) 4204 ; 4205 4206 } else { 4207 /* 4208 * If we overflowed the seq_file before, then we want 4209 * to just reuse the trace_seq buffer again. 4210 */ 4211 if (iter->leftover) 4212 p = iter; 4213 else { 4214 l = *pos - 1; 4215 p = s_next(m, p, &l); 4216 } 4217 } 4218 4219 trace_event_read_lock(); 4220 trace_access_lock(cpu_file); 4221 return p; 4222 } 4223 4224 static void s_stop(struct seq_file *m, void *p) 4225 { 4226 struct trace_iterator *iter = m->private; 4227 4228 #ifdef CONFIG_TRACER_MAX_TRACE 4229 if (iter->snapshot && iter->trace->use_max_tr) 4230 return; 4231 #endif 4232 4233 trace_access_unlock(iter->cpu_file); 4234 trace_event_read_unlock(); 4235 } 4236 4237 static void 4238 get_total_entries_cpu(struct array_buffer *buf, unsigned long *total, 4239 unsigned long *entries, int cpu) 4240 { 4241 unsigned long count; 4242 4243 count = ring_buffer_entries_cpu(buf->buffer, cpu); 4244 /* 4245 * If this buffer has skipped entries, then we hold all 4246 * entries for the trace and we need to ignore the 4247 * ones before the time stamp. 4248 */ 4249 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) { 4250 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries; 4251 /* total is the same as the entries */ 4252 *total = count; 4253 } else 4254 *total = count + 4255 ring_buffer_overrun_cpu(buf->buffer, cpu); 4256 *entries = count; 4257 } 4258 4259 static void 4260 get_total_entries(struct array_buffer *buf, 4261 unsigned long *total, unsigned long *entries) 4262 { 4263 unsigned long t, e; 4264 int cpu; 4265 4266 *total = 0; 4267 *entries = 0; 4268 4269 for_each_tracing_cpu(cpu) { 4270 get_total_entries_cpu(buf, &t, &e, cpu); 4271 *total += t; 4272 *entries += e; 4273 } 4274 } 4275 4276 unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu) 4277 { 4278 unsigned long total, entries; 4279 4280 if (!tr) 4281 tr = &global_trace; 4282 4283 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu); 4284 4285 return entries; 4286 } 4287 4288 unsigned long trace_total_entries(struct trace_array *tr) 4289 { 4290 unsigned long total, entries; 4291 4292 if (!tr) 4293 tr = &global_trace; 4294 4295 get_total_entries(&tr->array_buffer, &total, &entries); 4296 4297 return entries; 4298 } 4299 4300 static void print_lat_help_header(struct seq_file *m) 4301 { 4302 seq_puts(m, "# _------=> CPU# \n" 4303 "# / _-----=> irqs-off/BH-disabled\n" 4304 "# | / _----=> need-resched \n" 4305 "# || / _---=> hardirq/softirq \n" 4306 "# ||| / _--=> preempt-depth \n" 4307 "# |||| / _-=> migrate-disable \n" 4308 "# ||||| / delay \n" 4309 "# cmd pid |||||| time | caller \n" 4310 "# \\ / |||||| \\ | / \n"); 4311 } 4312 4313 static void print_event_info(struct array_buffer *buf, struct seq_file *m) 4314 { 4315 unsigned long total; 4316 unsigned long entries; 4317 4318 get_total_entries(buf, &total, &entries); 4319 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n", 4320 entries, total, num_online_cpus()); 4321 seq_puts(m, "#\n"); 4322 } 4323 4324 static void print_func_help_header(struct array_buffer *buf, struct seq_file *m, 4325 unsigned int flags) 4326 { 4327 bool tgid = flags & TRACE_ITER_RECORD_TGID; 4328 4329 print_event_info(buf, m); 4330 4331 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : ""); 4332 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : ""); 4333 } 4334 4335 static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m, 4336 unsigned int flags) 4337 { 4338 bool tgid = flags & TRACE_ITER_RECORD_TGID; 4339 static const char space[] = " "; 4340 int prec = tgid ? 12 : 2; 4341 4342 print_event_info(buf, m); 4343 4344 seq_printf(m, "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space); 4345 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space); 4346 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space); 4347 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space); 4348 seq_printf(m, "# %.*s||| / _-=> migrate-disable\n", prec, space); 4349 seq_printf(m, "# %.*s|||| / delay\n", prec, space); 4350 seq_printf(m, "# TASK-PID %.*s CPU# ||||| TIMESTAMP FUNCTION\n", prec, " TGID "); 4351 seq_printf(m, "# | | %.*s | ||||| | |\n", prec, " | "); 4352 } 4353 4354 void 4355 print_trace_header(struct seq_file *m, struct trace_iterator *iter) 4356 { 4357 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK); 4358 struct array_buffer *buf = iter->array_buffer; 4359 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu); 4360 struct tracer *type = iter->trace; 4361 unsigned long entries; 4362 unsigned long total; 4363 const char *name = type->name; 4364 4365 get_total_entries(buf, &total, &entries); 4366 4367 seq_printf(m, "# %s latency trace v1.1.5 on %s\n", 4368 name, UTS_RELEASE); 4369 seq_puts(m, "# -----------------------------------" 4370 "---------------------------------\n"); 4371 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |" 4372 " (M:%s VP:%d, KP:%d, SP:%d HP:%d", 4373 nsecs_to_usecs(data->saved_latency), 4374 entries, 4375 total, 4376 buf->cpu, 4377 preempt_model_none() ? "server" : 4378 preempt_model_voluntary() ? "desktop" : 4379 preempt_model_full() ? "preempt" : 4380 preempt_model_rt() ? "preempt_rt" : 4381 "unknown", 4382 /* These are reserved for later use */ 4383 0, 0, 0, 0); 4384 #ifdef CONFIG_SMP 4385 seq_printf(m, " #P:%d)\n", num_online_cpus()); 4386 #else 4387 seq_puts(m, ")\n"); 4388 #endif 4389 seq_puts(m, "# -----------------\n"); 4390 seq_printf(m, "# | task: %.16s-%d " 4391 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n", 4392 data->comm, data->pid, 4393 from_kuid_munged(seq_user_ns(m), data->uid), data->nice, 4394 data->policy, data->rt_priority); 4395 seq_puts(m, "# -----------------\n"); 4396 4397 if (data->critical_start) { 4398 seq_puts(m, "# => started at: "); 4399 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); 4400 trace_print_seq(m, &iter->seq); 4401 seq_puts(m, "\n# => ended at: "); 4402 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 4403 trace_print_seq(m, &iter->seq); 4404 seq_puts(m, "\n#\n"); 4405 } 4406 4407 seq_puts(m, "#\n"); 4408 } 4409 4410 static void test_cpu_buff_start(struct trace_iterator *iter) 4411 { 4412 struct trace_seq *s = &iter->seq; 4413 struct trace_array *tr = iter->tr; 4414 4415 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE)) 4416 return; 4417 4418 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) 4419 return; 4420 4421 if (cpumask_available(iter->started) && 4422 cpumask_test_cpu(iter->cpu, iter->started)) 4423 return; 4424 4425 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries) 4426 return; 4427 4428 if (cpumask_available(iter->started)) 4429 cpumask_set_cpu(iter->cpu, iter->started); 4430 4431 /* Don't print started cpu buffer for the first entry of the trace */ 4432 if (iter->idx > 1) 4433 trace_seq_printf(s, "##### CPU %u buffer started ####\n", 4434 iter->cpu); 4435 } 4436 4437 static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 4438 { 4439 struct trace_array *tr = iter->tr; 4440 struct trace_seq *s = &iter->seq; 4441 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); 4442 struct trace_entry *entry; 4443 struct trace_event *event; 4444 4445 entry = iter->ent; 4446 4447 test_cpu_buff_start(iter); 4448 4449 event = ftrace_find_event(entry->type); 4450 4451 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 4452 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 4453 trace_print_lat_context(iter); 4454 else 4455 trace_print_context(iter); 4456 } 4457 4458 if (trace_seq_has_overflowed(s)) 4459 return TRACE_TYPE_PARTIAL_LINE; 4460 4461 if (event) { 4462 if (tr->trace_flags & TRACE_ITER_FIELDS) 4463 return print_event_fields(iter, event); 4464 return event->funcs->trace(iter, sym_flags, event); 4465 } 4466 4467 trace_seq_printf(s, "Unknown type %d\n", entry->type); 4468 4469 return trace_handle_return(s); 4470 } 4471 4472 static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 4473 { 4474 struct trace_array *tr = iter->tr; 4475 struct trace_seq *s = &iter->seq; 4476 struct trace_entry *entry; 4477 struct trace_event *event; 4478 4479 entry = iter->ent; 4480 4481 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) 4482 trace_seq_printf(s, "%d %d %llu ", 4483 entry->pid, iter->cpu, iter->ts); 4484 4485 if (trace_seq_has_overflowed(s)) 4486 return TRACE_TYPE_PARTIAL_LINE; 4487 4488 event = ftrace_find_event(entry->type); 4489 if (event) 4490 return event->funcs->raw(iter, 0, event); 4491 4492 trace_seq_printf(s, "%d ?\n", entry->type); 4493 4494 return trace_handle_return(s); 4495 } 4496 4497 static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 4498 { 4499 struct trace_array *tr = iter->tr; 4500 struct trace_seq *s = &iter->seq; 4501 unsigned char newline = '\n'; 4502 struct trace_entry *entry; 4503 struct trace_event *event; 4504 4505 entry = iter->ent; 4506 4507 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 4508 SEQ_PUT_HEX_FIELD(s, entry->pid); 4509 SEQ_PUT_HEX_FIELD(s, iter->cpu); 4510 SEQ_PUT_HEX_FIELD(s, iter->ts); 4511 if (trace_seq_has_overflowed(s)) 4512 return TRACE_TYPE_PARTIAL_LINE; 4513 } 4514 4515 event = ftrace_find_event(entry->type); 4516 if (event) { 4517 enum print_line_t ret = event->funcs->hex(iter, 0, event); 4518 if (ret != TRACE_TYPE_HANDLED) 4519 return ret; 4520 } 4521 4522 SEQ_PUT_FIELD(s, newline); 4523 4524 return trace_handle_return(s); 4525 } 4526 4527 static enum print_line_t print_bin_fmt(struct trace_iterator *iter) 4528 { 4529 struct trace_array *tr = iter->tr; 4530 struct trace_seq *s = &iter->seq; 4531 struct trace_entry *entry; 4532 struct trace_event *event; 4533 4534 entry = iter->ent; 4535 4536 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { 4537 SEQ_PUT_FIELD(s, entry->pid); 4538 SEQ_PUT_FIELD(s, iter->cpu); 4539 SEQ_PUT_FIELD(s, iter->ts); 4540 if (trace_seq_has_overflowed(s)) 4541 return TRACE_TYPE_PARTIAL_LINE; 4542 } 4543 4544 event = ftrace_find_event(entry->type); 4545 return event ? event->funcs->binary(iter, 0, event) : 4546 TRACE_TYPE_HANDLED; 4547 } 4548 4549 int trace_empty(struct trace_iterator *iter) 4550 { 4551 struct ring_buffer_iter *buf_iter; 4552 int cpu; 4553 4554 /* If we are looking at one CPU buffer, only check that one */ 4555 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 4556 cpu = iter->cpu_file; 4557 buf_iter = trace_buffer_iter(iter, cpu); 4558 if (buf_iter) { 4559 if (!ring_buffer_iter_empty(buf_iter)) 4560 return 0; 4561 } else { 4562 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) 4563 return 0; 4564 } 4565 return 1; 4566 } 4567 4568 for_each_tracing_cpu(cpu) { 4569 buf_iter = trace_buffer_iter(iter, cpu); 4570 if (buf_iter) { 4571 if (!ring_buffer_iter_empty(buf_iter)) 4572 return 0; 4573 } else { 4574 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) 4575 return 0; 4576 } 4577 } 4578 4579 return 1; 4580 } 4581 4582 /* Called with trace_event_read_lock() held. */ 4583 enum print_line_t print_trace_line(struct trace_iterator *iter) 4584 { 4585 struct trace_array *tr = iter->tr; 4586 unsigned long trace_flags = tr->trace_flags; 4587 enum print_line_t ret; 4588 4589 if (iter->lost_events) { 4590 if (iter->lost_events == (unsigned long)-1) 4591 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n", 4592 iter->cpu); 4593 else 4594 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", 4595 iter->cpu, iter->lost_events); 4596 if (trace_seq_has_overflowed(&iter->seq)) 4597 return TRACE_TYPE_PARTIAL_LINE; 4598 } 4599 4600 if (iter->trace && iter->trace->print_line) { 4601 ret = iter->trace->print_line(iter); 4602 if (ret != TRACE_TYPE_UNHANDLED) 4603 return ret; 4604 } 4605 4606 if (iter->ent->type == TRACE_BPUTS && 4607 trace_flags & TRACE_ITER_PRINTK && 4608 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 4609 return trace_print_bputs_msg_only(iter); 4610 4611 if (iter->ent->type == TRACE_BPRINT && 4612 trace_flags & TRACE_ITER_PRINTK && 4613 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 4614 return trace_print_bprintk_msg_only(iter); 4615 4616 if (iter->ent->type == TRACE_PRINT && 4617 trace_flags & TRACE_ITER_PRINTK && 4618 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 4619 return trace_print_printk_msg_only(iter); 4620 4621 if (trace_flags & TRACE_ITER_BIN) 4622 return print_bin_fmt(iter); 4623 4624 if (trace_flags & TRACE_ITER_HEX) 4625 return print_hex_fmt(iter); 4626 4627 if (trace_flags & TRACE_ITER_RAW) 4628 return print_raw_fmt(iter); 4629 4630 return print_trace_fmt(iter); 4631 } 4632 4633 void trace_latency_header(struct seq_file *m) 4634 { 4635 struct trace_iterator *iter = m->private; 4636 struct trace_array *tr = iter->tr; 4637 4638 /* print nothing if the buffers are empty */ 4639 if (trace_empty(iter)) 4640 return; 4641 4642 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 4643 print_trace_header(m, iter); 4644 4645 if (!(tr->trace_flags & TRACE_ITER_VERBOSE)) 4646 print_lat_help_header(m); 4647 } 4648 4649 void trace_default_header(struct seq_file *m) 4650 { 4651 struct trace_iterator *iter = m->private; 4652 struct trace_array *tr = iter->tr; 4653 unsigned long trace_flags = tr->trace_flags; 4654 4655 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO)) 4656 return; 4657 4658 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 4659 /* print nothing if the buffers are empty */ 4660 if (trace_empty(iter)) 4661 return; 4662 print_trace_header(m, iter); 4663 if (!(trace_flags & TRACE_ITER_VERBOSE)) 4664 print_lat_help_header(m); 4665 } else { 4666 if (!(trace_flags & TRACE_ITER_VERBOSE)) { 4667 if (trace_flags & TRACE_ITER_IRQ_INFO) 4668 print_func_help_header_irq(iter->array_buffer, 4669 m, trace_flags); 4670 else 4671 print_func_help_header(iter->array_buffer, m, 4672 trace_flags); 4673 } 4674 } 4675 } 4676 4677 static void test_ftrace_alive(struct seq_file *m) 4678 { 4679 if (!ftrace_is_dead()) 4680 return; 4681 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n" 4682 "# MAY BE MISSING FUNCTION EVENTS\n"); 4683 } 4684 4685 #ifdef CONFIG_TRACER_MAX_TRACE 4686 static void show_snapshot_main_help(struct seq_file *m) 4687 { 4688 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n" 4689 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 4690 "# Takes a snapshot of the main buffer.\n" 4691 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n" 4692 "# (Doesn't have to be '2' works with any number that\n" 4693 "# is not a '0' or '1')\n"); 4694 } 4695 4696 static void show_snapshot_percpu_help(struct seq_file *m) 4697 { 4698 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n"); 4699 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP 4700 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 4701 "# Takes a snapshot of the main buffer for this cpu.\n"); 4702 #else 4703 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n" 4704 "# Must use main snapshot file to allocate.\n"); 4705 #endif 4706 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n" 4707 "# (Doesn't have to be '2' works with any number that\n" 4708 "# is not a '0' or '1')\n"); 4709 } 4710 4711 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) 4712 { 4713 if (iter->tr->allocated_snapshot) 4714 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n"); 4715 else 4716 seq_puts(m, "#\n# * Snapshot is freed *\n#\n"); 4717 4718 seq_puts(m, "# Snapshot commands:\n"); 4719 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 4720 show_snapshot_main_help(m); 4721 else 4722 show_snapshot_percpu_help(m); 4723 } 4724 #else 4725 /* Should never be called */ 4726 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { } 4727 #endif 4728 4729 static int s_show(struct seq_file *m, void *v) 4730 { 4731 struct trace_iterator *iter = v; 4732 int ret; 4733 4734 if (iter->ent == NULL) { 4735 if (iter->tr) { 4736 seq_printf(m, "# tracer: %s\n", iter->trace->name); 4737 seq_puts(m, "#\n"); 4738 test_ftrace_alive(m); 4739 } 4740 if (iter->snapshot && trace_empty(iter)) 4741 print_snapshot_help(m, iter); 4742 else if (iter->trace && iter->trace->print_header) 4743 iter->trace->print_header(m); 4744 else 4745 trace_default_header(m); 4746 4747 } else if (iter->leftover) { 4748 /* 4749 * If we filled the seq_file buffer earlier, we 4750 * want to just show it now. 4751 */ 4752 ret = trace_print_seq(m, &iter->seq); 4753 4754 /* ret should this time be zero, but you never know */ 4755 iter->leftover = ret; 4756 4757 } else { 4758 ret = print_trace_line(iter); 4759 if (ret == TRACE_TYPE_PARTIAL_LINE) { 4760 iter->seq.full = 0; 4761 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n"); 4762 } 4763 ret = trace_print_seq(m, &iter->seq); 4764 /* 4765 * If we overflow the seq_file buffer, then it will 4766 * ask us for this data again at start up. 4767 * Use that instead. 4768 * ret is 0 if seq_file write succeeded. 4769 * -1 otherwise. 4770 */ 4771 iter->leftover = ret; 4772 } 4773 4774 return 0; 4775 } 4776 4777 /* 4778 * Should be used after trace_array_get(), trace_types_lock 4779 * ensures that i_cdev was already initialized. 4780 */ 4781 static inline int tracing_get_cpu(struct inode *inode) 4782 { 4783 if (inode->i_cdev) /* See trace_create_cpu_file() */ 4784 return (long)inode->i_cdev - 1; 4785 return RING_BUFFER_ALL_CPUS; 4786 } 4787 4788 static const struct seq_operations tracer_seq_ops = { 4789 .start = s_start, 4790 .next = s_next, 4791 .stop = s_stop, 4792 .show = s_show, 4793 }; 4794 4795 /* 4796 * Note, as iter itself can be allocated and freed in different 4797 * ways, this function is only used to free its content, and not 4798 * the iterator itself. The only requirement to all the allocations 4799 * is that it must zero all fields (kzalloc), as freeing works with 4800 * ethier allocated content or NULL. 4801 */ 4802 static void free_trace_iter_content(struct trace_iterator *iter) 4803 { 4804 /* The fmt is either NULL, allocated or points to static_fmt_buf */ 4805 if (iter->fmt != static_fmt_buf) 4806 kfree(iter->fmt); 4807 4808 kfree(iter->temp); 4809 kfree(iter->buffer_iter); 4810 mutex_destroy(&iter->mutex); 4811 free_cpumask_var(iter->started); 4812 } 4813 4814 static struct trace_iterator * 4815 __tracing_open(struct inode *inode, struct file *file, bool snapshot) 4816 { 4817 struct trace_array *tr = inode->i_private; 4818 struct trace_iterator *iter; 4819 int cpu; 4820 4821 if (tracing_disabled) 4822 return ERR_PTR(-ENODEV); 4823 4824 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter)); 4825 if (!iter) 4826 return ERR_PTR(-ENOMEM); 4827 4828 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter), 4829 GFP_KERNEL); 4830 if (!iter->buffer_iter) 4831 goto release; 4832 4833 /* 4834 * trace_find_next_entry() may need to save off iter->ent. 4835 * It will place it into the iter->temp buffer. As most 4836 * events are less than 128, allocate a buffer of that size. 4837 * If one is greater, then trace_find_next_entry() will 4838 * allocate a new buffer to adjust for the bigger iter->ent. 4839 * It's not critical if it fails to get allocated here. 4840 */ 4841 iter->temp = kmalloc(128, GFP_KERNEL); 4842 if (iter->temp) 4843 iter->temp_size = 128; 4844 4845 /* 4846 * trace_event_printf() may need to modify given format 4847 * string to replace %p with %px so that it shows real address 4848 * instead of hash value. However, that is only for the event 4849 * tracing, other tracer may not need. Defer the allocation 4850 * until it is needed. 4851 */ 4852 iter->fmt = NULL; 4853 iter->fmt_size = 0; 4854 4855 mutex_lock(&trace_types_lock); 4856 iter->trace = tr->current_trace; 4857 4858 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) 4859 goto fail; 4860 4861 iter->tr = tr; 4862 4863 #ifdef CONFIG_TRACER_MAX_TRACE 4864 /* Currently only the top directory has a snapshot */ 4865 if (tr->current_trace->print_max || snapshot) 4866 iter->array_buffer = &tr->max_buffer; 4867 else 4868 #endif 4869 iter->array_buffer = &tr->array_buffer; 4870 iter->snapshot = snapshot; 4871 iter->pos = -1; 4872 iter->cpu_file = tracing_get_cpu(inode); 4873 mutex_init(&iter->mutex); 4874 4875 /* Notify the tracer early; before we stop tracing. */ 4876 if (iter->trace->open) 4877 iter->trace->open(iter); 4878 4879 /* Annotate start of buffers if we had overruns */ 4880 if (ring_buffer_overruns(iter->array_buffer->buffer)) 4881 iter->iter_flags |= TRACE_FILE_ANNOTATE; 4882 4883 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 4884 if (trace_clocks[tr->clock_id].in_ns) 4885 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 4886 4887 /* 4888 * If pause-on-trace is enabled, then stop the trace while 4889 * dumping, unless this is the "snapshot" file 4890 */ 4891 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE)) 4892 tracing_stop_tr(tr); 4893 4894 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { 4895 for_each_tracing_cpu(cpu) { 4896 iter->buffer_iter[cpu] = 4897 ring_buffer_read_prepare(iter->array_buffer->buffer, 4898 cpu, GFP_KERNEL); 4899 } 4900 ring_buffer_read_prepare_sync(); 4901 for_each_tracing_cpu(cpu) { 4902 ring_buffer_read_start(iter->buffer_iter[cpu]); 4903 tracing_iter_reset(iter, cpu); 4904 } 4905 } else { 4906 cpu = iter->cpu_file; 4907 iter->buffer_iter[cpu] = 4908 ring_buffer_read_prepare(iter->array_buffer->buffer, 4909 cpu, GFP_KERNEL); 4910 ring_buffer_read_prepare_sync(); 4911 ring_buffer_read_start(iter->buffer_iter[cpu]); 4912 tracing_iter_reset(iter, cpu); 4913 } 4914 4915 mutex_unlock(&trace_types_lock); 4916 4917 return iter; 4918 4919 fail: 4920 mutex_unlock(&trace_types_lock); 4921 free_trace_iter_content(iter); 4922 release: 4923 seq_release_private(inode, file); 4924 return ERR_PTR(-ENOMEM); 4925 } 4926 4927 int tracing_open_generic(struct inode *inode, struct file *filp) 4928 { 4929 int ret; 4930 4931 ret = tracing_check_open_get_tr(NULL); 4932 if (ret) 4933 return ret; 4934 4935 filp->private_data = inode->i_private; 4936 return 0; 4937 } 4938 4939 bool tracing_is_disabled(void) 4940 { 4941 return (tracing_disabled) ? true: false; 4942 } 4943 4944 /* 4945 * Open and update trace_array ref count. 4946 * Must have the current trace_array passed to it. 4947 */ 4948 int tracing_open_generic_tr(struct inode *inode, struct file *filp) 4949 { 4950 struct trace_array *tr = inode->i_private; 4951 int ret; 4952 4953 ret = tracing_check_open_get_tr(tr); 4954 if (ret) 4955 return ret; 4956 4957 filp->private_data = inode->i_private; 4958 4959 return 0; 4960 } 4961 4962 /* 4963 * The private pointer of the inode is the trace_event_file. 4964 * Update the tr ref count associated to it. 4965 */ 4966 int tracing_open_file_tr(struct inode *inode, struct file *filp) 4967 { 4968 struct trace_event_file *file = inode->i_private; 4969 int ret; 4970 4971 ret = tracing_check_open_get_tr(file->tr); 4972 if (ret) 4973 return ret; 4974 4975 mutex_lock(&event_mutex); 4976 4977 /* Fail if the file is marked for removal */ 4978 if (file->flags & EVENT_FILE_FL_FREED) { 4979 trace_array_put(file->tr); 4980 ret = -ENODEV; 4981 } else { 4982 event_file_get(file); 4983 } 4984 4985 mutex_unlock(&event_mutex); 4986 if (ret) 4987 return ret; 4988 4989 filp->private_data = inode->i_private; 4990 4991 return 0; 4992 } 4993 4994 int tracing_release_file_tr(struct inode *inode, struct file *filp) 4995 { 4996 struct trace_event_file *file = inode->i_private; 4997 4998 trace_array_put(file->tr); 4999 event_file_put(file); 5000 5001 return 0; 5002 } 5003 5004 int tracing_single_release_file_tr(struct inode *inode, struct file *filp) 5005 { 5006 tracing_release_file_tr(inode, filp); 5007 return single_release(inode, filp); 5008 } 5009 5010 static int tracing_mark_open(struct inode *inode, struct file *filp) 5011 { 5012 stream_open(inode, filp); 5013 return tracing_open_generic_tr(inode, filp); 5014 } 5015 5016 static int tracing_release(struct inode *inode, struct file *file) 5017 { 5018 struct trace_array *tr = inode->i_private; 5019 struct seq_file *m = file->private_data; 5020 struct trace_iterator *iter; 5021 int cpu; 5022 5023 if (!(file->f_mode & FMODE_READ)) { 5024 trace_array_put(tr); 5025 return 0; 5026 } 5027 5028 /* Writes do not use seq_file */ 5029 iter = m->private; 5030 mutex_lock(&trace_types_lock); 5031 5032 for_each_tracing_cpu(cpu) { 5033 if (iter->buffer_iter[cpu]) 5034 ring_buffer_read_finish(iter->buffer_iter[cpu]); 5035 } 5036 5037 if (iter->trace && iter->trace->close) 5038 iter->trace->close(iter); 5039 5040 if (!iter->snapshot && tr->stop_count) 5041 /* reenable tracing if it was previously enabled */ 5042 tracing_start_tr(tr); 5043 5044 __trace_array_put(tr); 5045 5046 mutex_unlock(&trace_types_lock); 5047 5048 free_trace_iter_content(iter); 5049 seq_release_private(inode, file); 5050 5051 return 0; 5052 } 5053 5054 int tracing_release_generic_tr(struct inode *inode, struct file *file) 5055 { 5056 struct trace_array *tr = inode->i_private; 5057 5058 trace_array_put(tr); 5059 return 0; 5060 } 5061 5062 static int tracing_single_release_tr(struct inode *inode, struct file *file) 5063 { 5064 struct trace_array *tr = inode->i_private; 5065 5066 trace_array_put(tr); 5067 5068 return single_release(inode, file); 5069 } 5070 5071 static int tracing_open(struct inode *inode, struct file *file) 5072 { 5073 struct trace_array *tr = inode->i_private; 5074 struct trace_iterator *iter; 5075 int ret; 5076 5077 ret = tracing_check_open_get_tr(tr); 5078 if (ret) 5079 return ret; 5080 5081 /* If this file was open for write, then erase contents */ 5082 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 5083 int cpu = tracing_get_cpu(inode); 5084 struct array_buffer *trace_buf = &tr->array_buffer; 5085 5086 #ifdef CONFIG_TRACER_MAX_TRACE 5087 if (tr->current_trace->print_max) 5088 trace_buf = &tr->max_buffer; 5089 #endif 5090 5091 if (cpu == RING_BUFFER_ALL_CPUS) 5092 tracing_reset_online_cpus(trace_buf); 5093 else 5094 tracing_reset_cpu(trace_buf, cpu); 5095 } 5096 5097 if (file->f_mode & FMODE_READ) { 5098 iter = __tracing_open(inode, file, false); 5099 if (IS_ERR(iter)) 5100 ret = PTR_ERR(iter); 5101 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 5102 iter->iter_flags |= TRACE_FILE_LAT_FMT; 5103 } 5104 5105 if (ret < 0) 5106 trace_array_put(tr); 5107 5108 return ret; 5109 } 5110 5111 /* 5112 * Some tracers are not suitable for instance buffers. 5113 * A tracer is always available for the global array (toplevel) 5114 * or if it explicitly states that it is. 5115 */ 5116 static bool 5117 trace_ok_for_array(struct tracer *t, struct trace_array *tr) 5118 { 5119 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances; 5120 } 5121 5122 /* Find the next tracer that this trace array may use */ 5123 static struct tracer * 5124 get_tracer_for_array(struct trace_array *tr, struct tracer *t) 5125 { 5126 while (t && !trace_ok_for_array(t, tr)) 5127 t = t->next; 5128 5129 return t; 5130 } 5131 5132 static void * 5133 t_next(struct seq_file *m, void *v, loff_t *pos) 5134 { 5135 struct trace_array *tr = m->private; 5136 struct tracer *t = v; 5137 5138 (*pos)++; 5139 5140 if (t) 5141 t = get_tracer_for_array(tr, t->next); 5142 5143 return t; 5144 } 5145 5146 static void *t_start(struct seq_file *m, loff_t *pos) 5147 { 5148 struct trace_array *tr = m->private; 5149 struct tracer *t; 5150 loff_t l = 0; 5151 5152 mutex_lock(&trace_types_lock); 5153 5154 t = get_tracer_for_array(tr, trace_types); 5155 for (; t && l < *pos; t = t_next(m, t, &l)) 5156 ; 5157 5158 return t; 5159 } 5160 5161 static void t_stop(struct seq_file *m, void *p) 5162 { 5163 mutex_unlock(&trace_types_lock); 5164 } 5165 5166 static int t_show(struct seq_file *m, void *v) 5167 { 5168 struct tracer *t = v; 5169 5170 if (!t) 5171 return 0; 5172 5173 seq_puts(m, t->name); 5174 if (t->next) 5175 seq_putc(m, ' '); 5176 else 5177 seq_putc(m, '\n'); 5178 5179 return 0; 5180 } 5181 5182 static const struct seq_operations show_traces_seq_ops = { 5183 .start = t_start, 5184 .next = t_next, 5185 .stop = t_stop, 5186 .show = t_show, 5187 }; 5188 5189 static int show_traces_open(struct inode *inode, struct file *file) 5190 { 5191 struct trace_array *tr = inode->i_private; 5192 struct seq_file *m; 5193 int ret; 5194 5195 ret = tracing_check_open_get_tr(tr); 5196 if (ret) 5197 return ret; 5198 5199 ret = seq_open(file, &show_traces_seq_ops); 5200 if (ret) { 5201 trace_array_put(tr); 5202 return ret; 5203 } 5204 5205 m = file->private_data; 5206 m->private = tr; 5207 5208 return 0; 5209 } 5210 5211 static int show_traces_release(struct inode *inode, struct file *file) 5212 { 5213 struct trace_array *tr = inode->i_private; 5214 5215 trace_array_put(tr); 5216 return seq_release(inode, file); 5217 } 5218 5219 static ssize_t 5220 tracing_write_stub(struct file *filp, const char __user *ubuf, 5221 size_t count, loff_t *ppos) 5222 { 5223 return count; 5224 } 5225 5226 loff_t tracing_lseek(struct file *file, loff_t offset, int whence) 5227 { 5228 int ret; 5229 5230 if (file->f_mode & FMODE_READ) 5231 ret = seq_lseek(file, offset, whence); 5232 else 5233 file->f_pos = ret = 0; 5234 5235 return ret; 5236 } 5237 5238 static const struct file_operations tracing_fops = { 5239 .open = tracing_open, 5240 .read = seq_read, 5241 .read_iter = seq_read_iter, 5242 .splice_read = copy_splice_read, 5243 .write = tracing_write_stub, 5244 .llseek = tracing_lseek, 5245 .release = tracing_release, 5246 }; 5247 5248 static const struct file_operations show_traces_fops = { 5249 .open = show_traces_open, 5250 .read = seq_read, 5251 .llseek = seq_lseek, 5252 .release = show_traces_release, 5253 }; 5254 5255 static ssize_t 5256 tracing_cpumask_read(struct file *filp, char __user *ubuf, 5257 size_t count, loff_t *ppos) 5258 { 5259 struct trace_array *tr = file_inode(filp)->i_private; 5260 char *mask_str; 5261 int len; 5262 5263 len = snprintf(NULL, 0, "%*pb\n", 5264 cpumask_pr_args(tr->tracing_cpumask)) + 1; 5265 mask_str = kmalloc(len, GFP_KERNEL); 5266 if (!mask_str) 5267 return -ENOMEM; 5268 5269 len = snprintf(mask_str, len, "%*pb\n", 5270 cpumask_pr_args(tr->tracing_cpumask)); 5271 if (len >= count) { 5272 count = -EINVAL; 5273 goto out_err; 5274 } 5275 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len); 5276 5277 out_err: 5278 kfree(mask_str); 5279 5280 return count; 5281 } 5282 5283 int tracing_set_cpumask(struct trace_array *tr, 5284 cpumask_var_t tracing_cpumask_new) 5285 { 5286 int cpu; 5287 5288 if (!tr) 5289 return -EINVAL; 5290 5291 local_irq_disable(); 5292 arch_spin_lock(&tr->max_lock); 5293 for_each_tracing_cpu(cpu) { 5294 /* 5295 * Increase/decrease the disabled counter if we are 5296 * about to flip a bit in the cpumask: 5297 */ 5298 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) && 5299 !cpumask_test_cpu(cpu, tracing_cpumask_new)) { 5300 atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled); 5301 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu); 5302 #ifdef CONFIG_TRACER_MAX_TRACE 5303 ring_buffer_record_disable_cpu(tr->max_buffer.buffer, cpu); 5304 #endif 5305 } 5306 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) && 5307 cpumask_test_cpu(cpu, tracing_cpumask_new)) { 5308 atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled); 5309 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu); 5310 #ifdef CONFIG_TRACER_MAX_TRACE 5311 ring_buffer_record_enable_cpu(tr->max_buffer.buffer, cpu); 5312 #endif 5313 } 5314 } 5315 arch_spin_unlock(&tr->max_lock); 5316 local_irq_enable(); 5317 5318 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new); 5319 5320 return 0; 5321 } 5322 5323 static ssize_t 5324 tracing_cpumask_write(struct file *filp, const char __user *ubuf, 5325 size_t count, loff_t *ppos) 5326 { 5327 struct trace_array *tr = file_inode(filp)->i_private; 5328 cpumask_var_t tracing_cpumask_new; 5329 int err; 5330 5331 if (!zalloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL)) 5332 return -ENOMEM; 5333 5334 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); 5335 if (err) 5336 goto err_free; 5337 5338 err = tracing_set_cpumask(tr, tracing_cpumask_new); 5339 if (err) 5340 goto err_free; 5341 5342 free_cpumask_var(tracing_cpumask_new); 5343 5344 return count; 5345 5346 err_free: 5347 free_cpumask_var(tracing_cpumask_new); 5348 5349 return err; 5350 } 5351 5352 static const struct file_operations tracing_cpumask_fops = { 5353 .open = tracing_open_generic_tr, 5354 .read = tracing_cpumask_read, 5355 .write = tracing_cpumask_write, 5356 .release = tracing_release_generic_tr, 5357 .llseek = generic_file_llseek, 5358 }; 5359 5360 static int tracing_trace_options_show(struct seq_file *m, void *v) 5361 { 5362 struct tracer_opt *trace_opts; 5363 struct trace_array *tr = m->private; 5364 u32 tracer_flags; 5365 int i; 5366 5367 mutex_lock(&trace_types_lock); 5368 tracer_flags = tr->current_trace->flags->val; 5369 trace_opts = tr->current_trace->flags->opts; 5370 5371 for (i = 0; trace_options[i]; i++) { 5372 if (tr->trace_flags & (1 << i)) 5373 seq_printf(m, "%s\n", trace_options[i]); 5374 else 5375 seq_printf(m, "no%s\n", trace_options[i]); 5376 } 5377 5378 for (i = 0; trace_opts[i].name; i++) { 5379 if (tracer_flags & trace_opts[i].bit) 5380 seq_printf(m, "%s\n", trace_opts[i].name); 5381 else 5382 seq_printf(m, "no%s\n", trace_opts[i].name); 5383 } 5384 mutex_unlock(&trace_types_lock); 5385 5386 return 0; 5387 } 5388 5389 static int __set_tracer_option(struct trace_array *tr, 5390 struct tracer_flags *tracer_flags, 5391 struct tracer_opt *opts, int neg) 5392 { 5393 struct tracer *trace = tracer_flags->trace; 5394 int ret; 5395 5396 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg); 5397 if (ret) 5398 return ret; 5399 5400 if (neg) 5401 tracer_flags->val &= ~opts->bit; 5402 else 5403 tracer_flags->val |= opts->bit; 5404 return 0; 5405 } 5406 5407 /* Try to assign a tracer specific option */ 5408 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg) 5409 { 5410 struct tracer *trace = tr->current_trace; 5411 struct tracer_flags *tracer_flags = trace->flags; 5412 struct tracer_opt *opts = NULL; 5413 int i; 5414 5415 for (i = 0; tracer_flags->opts[i].name; i++) { 5416 opts = &tracer_flags->opts[i]; 5417 5418 if (strcmp(cmp, opts->name) == 0) 5419 return __set_tracer_option(tr, trace->flags, opts, neg); 5420 } 5421 5422 return -EINVAL; 5423 } 5424 5425 /* Some tracers require overwrite to stay enabled */ 5426 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set) 5427 { 5428 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set) 5429 return -1; 5430 5431 return 0; 5432 } 5433 5434 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled) 5435 { 5436 int *map; 5437 5438 if ((mask == TRACE_ITER_RECORD_TGID) || 5439 (mask == TRACE_ITER_RECORD_CMD)) 5440 lockdep_assert_held(&event_mutex); 5441 5442 /* do nothing if flag is already set */ 5443 if (!!(tr->trace_flags & mask) == !!enabled) 5444 return 0; 5445 5446 /* Give the tracer a chance to approve the change */ 5447 if (tr->current_trace->flag_changed) 5448 if (tr->current_trace->flag_changed(tr, mask, !!enabled)) 5449 return -EINVAL; 5450 5451 if (enabled) 5452 tr->trace_flags |= mask; 5453 else 5454 tr->trace_flags &= ~mask; 5455 5456 if (mask == TRACE_ITER_RECORD_CMD) 5457 trace_event_enable_cmd_record(enabled); 5458 5459 if (mask == TRACE_ITER_RECORD_TGID) { 5460 if (!tgid_map) { 5461 tgid_map_max = pid_max; 5462 map = kvcalloc(tgid_map_max + 1, sizeof(*tgid_map), 5463 GFP_KERNEL); 5464 5465 /* 5466 * Pairs with smp_load_acquire() in 5467 * trace_find_tgid_ptr() to ensure that if it observes 5468 * the tgid_map we just allocated then it also observes 5469 * the corresponding tgid_map_max value. 5470 */ 5471 smp_store_release(&tgid_map, map); 5472 } 5473 if (!tgid_map) { 5474 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID; 5475 return -ENOMEM; 5476 } 5477 5478 trace_event_enable_tgid_record(enabled); 5479 } 5480 5481 if (mask == TRACE_ITER_EVENT_FORK) 5482 trace_event_follow_fork(tr, enabled); 5483 5484 if (mask == TRACE_ITER_FUNC_FORK) 5485 ftrace_pid_follow_fork(tr, enabled); 5486 5487 if (mask == TRACE_ITER_OVERWRITE) { 5488 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled); 5489 #ifdef CONFIG_TRACER_MAX_TRACE 5490 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled); 5491 #endif 5492 } 5493 5494 if (mask == TRACE_ITER_PRINTK) { 5495 trace_printk_start_stop_comm(enabled); 5496 trace_printk_control(enabled); 5497 } 5498 5499 return 0; 5500 } 5501 5502 int trace_set_options(struct trace_array *tr, char *option) 5503 { 5504 char *cmp; 5505 int neg = 0; 5506 int ret; 5507 size_t orig_len = strlen(option); 5508 int len; 5509 5510 cmp = strstrip(option); 5511 5512 len = str_has_prefix(cmp, "no"); 5513 if (len) 5514 neg = 1; 5515 5516 cmp += len; 5517 5518 mutex_lock(&event_mutex); 5519 mutex_lock(&trace_types_lock); 5520 5521 ret = match_string(trace_options, -1, cmp); 5522 /* If no option could be set, test the specific tracer options */ 5523 if (ret < 0) 5524 ret = set_tracer_option(tr, cmp, neg); 5525 else 5526 ret = set_tracer_flag(tr, 1 << ret, !neg); 5527 5528 mutex_unlock(&trace_types_lock); 5529 mutex_unlock(&event_mutex); 5530 5531 /* 5532 * If the first trailing whitespace is replaced with '\0' by strstrip, 5533 * turn it back into a space. 5534 */ 5535 if (orig_len > strlen(option)) 5536 option[strlen(option)] = ' '; 5537 5538 return ret; 5539 } 5540 5541 static void __init apply_trace_boot_options(void) 5542 { 5543 char *buf = trace_boot_options_buf; 5544 char *option; 5545 5546 while (true) { 5547 option = strsep(&buf, ","); 5548 5549 if (!option) 5550 break; 5551 5552 if (*option) 5553 trace_set_options(&global_trace, option); 5554 5555 /* Put back the comma to allow this to be called again */ 5556 if (buf) 5557 *(buf - 1) = ','; 5558 } 5559 } 5560 5561 static ssize_t 5562 tracing_trace_options_write(struct file *filp, const char __user *ubuf, 5563 size_t cnt, loff_t *ppos) 5564 { 5565 struct seq_file *m = filp->private_data; 5566 struct trace_array *tr = m->private; 5567 char buf[64]; 5568 int ret; 5569 5570 if (cnt >= sizeof(buf)) 5571 return -EINVAL; 5572 5573 if (copy_from_user(buf, ubuf, cnt)) 5574 return -EFAULT; 5575 5576 buf[cnt] = 0; 5577 5578 ret = trace_set_options(tr, buf); 5579 if (ret < 0) 5580 return ret; 5581 5582 *ppos += cnt; 5583 5584 return cnt; 5585 } 5586 5587 static int tracing_trace_options_open(struct inode *inode, struct file *file) 5588 { 5589 struct trace_array *tr = inode->i_private; 5590 int ret; 5591 5592 ret = tracing_check_open_get_tr(tr); 5593 if (ret) 5594 return ret; 5595 5596 ret = single_open(file, tracing_trace_options_show, inode->i_private); 5597 if (ret < 0) 5598 trace_array_put(tr); 5599 5600 return ret; 5601 } 5602 5603 static const struct file_operations tracing_iter_fops = { 5604 .open = tracing_trace_options_open, 5605 .read = seq_read, 5606 .llseek = seq_lseek, 5607 .release = tracing_single_release_tr, 5608 .write = tracing_trace_options_write, 5609 }; 5610 5611 static const char readme_msg[] = 5612 "tracing mini-HOWTO:\n\n" 5613 "# echo 0 > tracing_on : quick way to disable tracing\n" 5614 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n" 5615 " Important files:\n" 5616 " trace\t\t\t- The static contents of the buffer\n" 5617 "\t\t\t To clear the buffer write into this file: echo > trace\n" 5618 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n" 5619 " current_tracer\t- function and latency tracers\n" 5620 " available_tracers\t- list of configured tracers for current_tracer\n" 5621 " error_log\t- error log for failed commands (that support it)\n" 5622 " buffer_size_kb\t- view and modify size of per cpu buffer\n" 5623 " buffer_total_size_kb - view total size of all cpu buffers\n\n" 5624 " trace_clock\t\t- change the clock used to order events\n" 5625 " local: Per cpu clock but may not be synced across CPUs\n" 5626 " global: Synced across CPUs but slows tracing down.\n" 5627 " counter: Not a clock, but just an increment\n" 5628 " uptime: Jiffy counter from time of boot\n" 5629 " perf: Same clock that perf events use\n" 5630 #ifdef CONFIG_X86_64 5631 " x86-tsc: TSC cycle counter\n" 5632 #endif 5633 "\n timestamp_mode\t- view the mode used to timestamp events\n" 5634 " delta: Delta difference against a buffer-wide timestamp\n" 5635 " absolute: Absolute (standalone) timestamp\n" 5636 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n" 5637 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n" 5638 " tracing_cpumask\t- Limit which CPUs to trace\n" 5639 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n" 5640 "\t\t\t Remove sub-buffer with rmdir\n" 5641 " trace_options\t\t- Set format or modify how tracing happens\n" 5642 "\t\t\t Disable an option by prefixing 'no' to the\n" 5643 "\t\t\t option name\n" 5644 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n" 5645 #ifdef CONFIG_DYNAMIC_FTRACE 5646 "\n available_filter_functions - list of functions that can be filtered on\n" 5647 " set_ftrace_filter\t- echo function name in here to only trace these\n" 5648 "\t\t\t functions\n" 5649 "\t accepts: func_full_name or glob-matching-pattern\n" 5650 "\t modules: Can select a group via module\n" 5651 "\t Format: :mod:<module-name>\n" 5652 "\t example: echo :mod:ext3 > set_ftrace_filter\n" 5653 "\t triggers: a command to perform when function is hit\n" 5654 "\t Format: <function>:<trigger>[:count]\n" 5655 "\t trigger: traceon, traceoff\n" 5656 "\t\t enable_event:<system>:<event>\n" 5657 "\t\t disable_event:<system>:<event>\n" 5658 #ifdef CONFIG_STACKTRACE 5659 "\t\t stacktrace\n" 5660 #endif 5661 #ifdef CONFIG_TRACER_SNAPSHOT 5662 "\t\t snapshot\n" 5663 #endif 5664 "\t\t dump\n" 5665 "\t\t cpudump\n" 5666 "\t example: echo do_fault:traceoff > set_ftrace_filter\n" 5667 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n" 5668 "\t The first one will disable tracing every time do_fault is hit\n" 5669 "\t The second will disable tracing at most 3 times when do_trap is hit\n" 5670 "\t The first time do trap is hit and it disables tracing, the\n" 5671 "\t counter will decrement to 2. If tracing is already disabled,\n" 5672 "\t the counter will not decrement. It only decrements when the\n" 5673 "\t trigger did work\n" 5674 "\t To remove trigger without count:\n" 5675 "\t echo '!<function>:<trigger> > set_ftrace_filter\n" 5676 "\t To remove trigger with a count:\n" 5677 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n" 5678 " set_ftrace_notrace\t- echo function name in here to never trace.\n" 5679 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n" 5680 "\t modules: Can select a group via module command :mod:\n" 5681 "\t Does not accept triggers\n" 5682 #endif /* CONFIG_DYNAMIC_FTRACE */ 5683 #ifdef CONFIG_FUNCTION_TRACER 5684 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n" 5685 "\t\t (function)\n" 5686 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n" 5687 "\t\t (function)\n" 5688 #endif 5689 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 5690 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n" 5691 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n" 5692 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n" 5693 #endif 5694 #ifdef CONFIG_TRACER_SNAPSHOT 5695 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n" 5696 "\t\t\t snapshot buffer. Read the contents for more\n" 5697 "\t\t\t information\n" 5698 #endif 5699 #ifdef CONFIG_STACK_TRACER 5700 " stack_trace\t\t- Shows the max stack trace when active\n" 5701 " stack_max_size\t- Shows current max stack size that was traced\n" 5702 "\t\t\t Write into this file to reset the max size (trigger a\n" 5703 "\t\t\t new trace)\n" 5704 #ifdef CONFIG_DYNAMIC_FTRACE 5705 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n" 5706 "\t\t\t traces\n" 5707 #endif 5708 #endif /* CONFIG_STACK_TRACER */ 5709 #ifdef CONFIG_DYNAMIC_EVENTS 5710 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n" 5711 "\t\t\t Write into this file to define/undefine new trace events.\n" 5712 #endif 5713 #ifdef CONFIG_KPROBE_EVENTS 5714 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n" 5715 "\t\t\t Write into this file to define/undefine new trace events.\n" 5716 #endif 5717 #ifdef CONFIG_UPROBE_EVENTS 5718 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n" 5719 "\t\t\t Write into this file to define/undefine new trace events.\n" 5720 #endif 5721 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) || \ 5722 defined(CONFIG_FPROBE_EVENTS) 5723 "\t accepts: event-definitions (one definition per line)\n" 5724 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) 5725 "\t Format: p[:[<group>/][<event>]] <place> [<args>]\n" 5726 "\t r[maxactive][:[<group>/][<event>]] <place> [<args>]\n" 5727 #endif 5728 #ifdef CONFIG_FPROBE_EVENTS 5729 "\t f[:[<group>/][<event>]] <func-name>[%return] [<args>]\n" 5730 "\t t[:[<group>/][<event>]] <tracepoint> [<args>]\n" 5731 #endif 5732 #ifdef CONFIG_HIST_TRIGGERS 5733 "\t s:[synthetic/]<event> <field> [<field>]\n" 5734 #endif 5735 "\t e[:[<group>/][<event>]] <attached-group>.<attached-event> [<args>] [if <filter>]\n" 5736 "\t -:[<group>/][<event>]\n" 5737 #ifdef CONFIG_KPROBE_EVENTS 5738 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n" 5739 "place (kretprobe): [<module>:]<symbol>[+<offset>]%return|<memaddr>\n" 5740 #endif 5741 #ifdef CONFIG_UPROBE_EVENTS 5742 " place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n" 5743 #endif 5744 "\t args: <name>=fetcharg[:type]\n" 5745 "\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n" 5746 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 5747 #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS 5748 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n" 5749 "\t <argname>[->field[->field|.field...]],\n" 5750 #else 5751 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n" 5752 #endif 5753 #else 5754 "\t $stack<index>, $stack, $retval, $comm,\n" 5755 #endif 5756 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n" 5757 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n" 5758 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n" 5759 "\t symstr, <type>\\[<array-size>\\]\n" 5760 #ifdef CONFIG_HIST_TRIGGERS 5761 "\t field: <stype> <name>;\n" 5762 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n" 5763 "\t [unsigned] char/int/long\n" 5764 #endif 5765 "\t efield: For event probes ('e' types), the field is on of the fields\n" 5766 "\t of the <attached-group>/<attached-event>.\n" 5767 #endif 5768 " events/\t\t- Directory containing all trace event subsystems:\n" 5769 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" 5770 " events/<system>/\t- Directory containing all trace events for <system>:\n" 5771 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n" 5772 "\t\t\t events\n" 5773 " filter\t\t- If set, only events passing filter are traced\n" 5774 " events/<system>/<event>/\t- Directory containing control files for\n" 5775 "\t\t\t <event>:\n" 5776 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n" 5777 " filter\t\t- If set, only events passing filter are traced\n" 5778 " trigger\t\t- If set, a command to perform when event is hit\n" 5779 "\t Format: <trigger>[:count][if <filter>]\n" 5780 "\t trigger: traceon, traceoff\n" 5781 "\t enable_event:<system>:<event>\n" 5782 "\t disable_event:<system>:<event>\n" 5783 #ifdef CONFIG_HIST_TRIGGERS 5784 "\t enable_hist:<system>:<event>\n" 5785 "\t disable_hist:<system>:<event>\n" 5786 #endif 5787 #ifdef CONFIG_STACKTRACE 5788 "\t\t stacktrace\n" 5789 #endif 5790 #ifdef CONFIG_TRACER_SNAPSHOT 5791 "\t\t snapshot\n" 5792 #endif 5793 #ifdef CONFIG_HIST_TRIGGERS 5794 "\t\t hist (see below)\n" 5795 #endif 5796 "\t example: echo traceoff > events/block/block_unplug/trigger\n" 5797 "\t echo traceoff:3 > events/block/block_unplug/trigger\n" 5798 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n" 5799 "\t events/block/block_unplug/trigger\n" 5800 "\t The first disables tracing every time block_unplug is hit.\n" 5801 "\t The second disables tracing the first 3 times block_unplug is hit.\n" 5802 "\t The third enables the kmalloc event the first 3 times block_unplug\n" 5803 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n" 5804 "\t Like function triggers, the counter is only decremented if it\n" 5805 "\t enabled or disabled tracing.\n" 5806 "\t To remove a trigger without a count:\n" 5807 "\t echo '!<trigger> > <system>/<event>/trigger\n" 5808 "\t To remove a trigger with a count:\n" 5809 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n" 5810 "\t Filters can be ignored when removing a trigger.\n" 5811 #ifdef CONFIG_HIST_TRIGGERS 5812 " hist trigger\t- If set, event hits are aggregated into a hash table\n" 5813 "\t Format: hist:keys=<field1[,field2,...]>\n" 5814 "\t [:<var1>=<field|var_ref|numeric_literal>[,<var2>=...]]\n" 5815 "\t [:values=<field1[,field2,...]>]\n" 5816 "\t [:sort=<field1[,field2,...]>]\n" 5817 "\t [:size=#entries]\n" 5818 "\t [:pause][:continue][:clear]\n" 5819 "\t [:name=histname1]\n" 5820 "\t [:nohitcount]\n" 5821 "\t [:<handler>.<action>]\n" 5822 "\t [if <filter>]\n\n" 5823 "\t Note, special fields can be used as well:\n" 5824 "\t common_timestamp - to record current timestamp\n" 5825 "\t common_cpu - to record the CPU the event happened on\n" 5826 "\n" 5827 "\t A hist trigger variable can be:\n" 5828 "\t - a reference to a field e.g. x=current_timestamp,\n" 5829 "\t - a reference to another variable e.g. y=$x,\n" 5830 "\t - a numeric literal: e.g. ms_per_sec=1000,\n" 5831 "\t - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n" 5832 "\n" 5833 "\t hist trigger arithmetic expressions support addition(+), subtraction(-),\n" 5834 "\t multiplication(*) and division(/) operators. An operand can be either a\n" 5835 "\t variable reference, field or numeric literal.\n" 5836 "\n" 5837 "\t When a matching event is hit, an entry is added to a hash\n" 5838 "\t table using the key(s) and value(s) named, and the value of a\n" 5839 "\t sum called 'hitcount' is incremented. Keys and values\n" 5840 "\t correspond to fields in the event's format description. Keys\n" 5841 "\t can be any field, or the special string 'common_stacktrace'.\n" 5842 "\t Compound keys consisting of up to two fields can be specified\n" 5843 "\t by the 'keys' keyword. Values must correspond to numeric\n" 5844 "\t fields. Sort keys consisting of up to two fields can be\n" 5845 "\t specified using the 'sort' keyword. The sort direction can\n" 5846 "\t be modified by appending '.descending' or '.ascending' to a\n" 5847 "\t sort field. The 'size' parameter can be used to specify more\n" 5848 "\t or fewer than the default 2048 entries for the hashtable size.\n" 5849 "\t If a hist trigger is given a name using the 'name' parameter,\n" 5850 "\t its histogram data will be shared with other triggers of the\n" 5851 "\t same name, and trigger hits will update this common data.\n\n" 5852 "\t Reading the 'hist' file for the event will dump the hash\n" 5853 "\t table in its entirety to stdout. If there are multiple hist\n" 5854 "\t triggers attached to an event, there will be a table for each\n" 5855 "\t trigger in the output. The table displayed for a named\n" 5856 "\t trigger will be the same as any other instance having the\n" 5857 "\t same name. The default format used to display a given field\n" 5858 "\t can be modified by appending any of the following modifiers\n" 5859 "\t to the field name, as applicable:\n\n" 5860 "\t .hex display a number as a hex value\n" 5861 "\t .sym display an address as a symbol\n" 5862 "\t .sym-offset display an address as a symbol and offset\n" 5863 "\t .execname display a common_pid as a program name\n" 5864 "\t .syscall display a syscall id as a syscall name\n" 5865 "\t .log2 display log2 value rather than raw number\n" 5866 "\t .buckets=size display values in groups of size rather than raw number\n" 5867 "\t .usecs display a common_timestamp in microseconds\n" 5868 "\t .percent display a number of percentage value\n" 5869 "\t .graph display a bar-graph of a value\n\n" 5870 "\t The 'pause' parameter can be used to pause an existing hist\n" 5871 "\t trigger or to start a hist trigger but not log any events\n" 5872 "\t until told to do so. 'continue' can be used to start or\n" 5873 "\t restart a paused hist trigger.\n\n" 5874 "\t The 'clear' parameter will clear the contents of a running\n" 5875 "\t hist trigger and leave its current paused/active state\n" 5876 "\t unchanged.\n\n" 5877 "\t The 'nohitcount' (or NOHC) parameter will suppress display of\n" 5878 "\t raw hitcount in the histogram.\n\n" 5879 "\t The enable_hist and disable_hist triggers can be used to\n" 5880 "\t have one event conditionally start and stop another event's\n" 5881 "\t already-attached hist trigger. The syntax is analogous to\n" 5882 "\t the enable_event and disable_event triggers.\n\n" 5883 "\t Hist trigger handlers and actions are executed whenever a\n" 5884 "\t a histogram entry is added or updated. They take the form:\n\n" 5885 "\t <handler>.<action>\n\n" 5886 "\t The available handlers are:\n\n" 5887 "\t onmatch(matching.event) - invoke on addition or update\n" 5888 "\t onmax(var) - invoke if var exceeds current max\n" 5889 "\t onchange(var) - invoke action if var changes\n\n" 5890 "\t The available actions are:\n\n" 5891 "\t trace(<synthetic_event>,param list) - generate synthetic event\n" 5892 "\t save(field,...) - save current event fields\n" 5893 #ifdef CONFIG_TRACER_SNAPSHOT 5894 "\t snapshot() - snapshot the trace buffer\n\n" 5895 #endif 5896 #ifdef CONFIG_SYNTH_EVENTS 5897 " events/synthetic_events\t- Create/append/remove/show synthetic events\n" 5898 "\t Write into this file to define/undefine new synthetic events.\n" 5899 "\t example: echo 'myevent u64 lat; char name[]; long[] stack' >> synthetic_events\n" 5900 #endif 5901 #endif 5902 ; 5903 5904 static ssize_t 5905 tracing_readme_read(struct file *filp, char __user *ubuf, 5906 size_t cnt, loff_t *ppos) 5907 { 5908 return simple_read_from_buffer(ubuf, cnt, ppos, 5909 readme_msg, strlen(readme_msg)); 5910 } 5911 5912 static const struct file_operations tracing_readme_fops = { 5913 .open = tracing_open_generic, 5914 .read = tracing_readme_read, 5915 .llseek = generic_file_llseek, 5916 }; 5917 5918 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos) 5919 { 5920 int pid = ++(*pos); 5921 5922 return trace_find_tgid_ptr(pid); 5923 } 5924 5925 static void *saved_tgids_start(struct seq_file *m, loff_t *pos) 5926 { 5927 int pid = *pos; 5928 5929 return trace_find_tgid_ptr(pid); 5930 } 5931 5932 static void saved_tgids_stop(struct seq_file *m, void *v) 5933 { 5934 } 5935 5936 static int saved_tgids_show(struct seq_file *m, void *v) 5937 { 5938 int *entry = (int *)v; 5939 int pid = entry - tgid_map; 5940 int tgid = *entry; 5941 5942 if (tgid == 0) 5943 return SEQ_SKIP; 5944 5945 seq_printf(m, "%d %d\n", pid, tgid); 5946 return 0; 5947 } 5948 5949 static const struct seq_operations tracing_saved_tgids_seq_ops = { 5950 .start = saved_tgids_start, 5951 .stop = saved_tgids_stop, 5952 .next = saved_tgids_next, 5953 .show = saved_tgids_show, 5954 }; 5955 5956 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp) 5957 { 5958 int ret; 5959 5960 ret = tracing_check_open_get_tr(NULL); 5961 if (ret) 5962 return ret; 5963 5964 return seq_open(filp, &tracing_saved_tgids_seq_ops); 5965 } 5966 5967 5968 static const struct file_operations tracing_saved_tgids_fops = { 5969 .open = tracing_saved_tgids_open, 5970 .read = seq_read, 5971 .llseek = seq_lseek, 5972 .release = seq_release, 5973 }; 5974 5975 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos) 5976 { 5977 unsigned int *ptr = v; 5978 5979 if (*pos || m->count) 5980 ptr++; 5981 5982 (*pos)++; 5983 5984 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num]; 5985 ptr++) { 5986 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP) 5987 continue; 5988 5989 return ptr; 5990 } 5991 5992 return NULL; 5993 } 5994 5995 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos) 5996 { 5997 void *v; 5998 loff_t l = 0; 5999 6000 preempt_disable(); 6001 arch_spin_lock(&trace_cmdline_lock); 6002 6003 v = &savedcmd->map_cmdline_to_pid[0]; 6004 while (l <= *pos) { 6005 v = saved_cmdlines_next(m, v, &l); 6006 if (!v) 6007 return NULL; 6008 } 6009 6010 return v; 6011 } 6012 6013 static void saved_cmdlines_stop(struct seq_file *m, void *v) 6014 { 6015 arch_spin_unlock(&trace_cmdline_lock); 6016 preempt_enable(); 6017 } 6018 6019 static int saved_cmdlines_show(struct seq_file *m, void *v) 6020 { 6021 char buf[TASK_COMM_LEN]; 6022 unsigned int *pid = v; 6023 6024 __trace_find_cmdline(*pid, buf); 6025 seq_printf(m, "%d %s\n", *pid, buf); 6026 return 0; 6027 } 6028 6029 static const struct seq_operations tracing_saved_cmdlines_seq_ops = { 6030 .start = saved_cmdlines_start, 6031 .next = saved_cmdlines_next, 6032 .stop = saved_cmdlines_stop, 6033 .show = saved_cmdlines_show, 6034 }; 6035 6036 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp) 6037 { 6038 int ret; 6039 6040 ret = tracing_check_open_get_tr(NULL); 6041 if (ret) 6042 return ret; 6043 6044 return seq_open(filp, &tracing_saved_cmdlines_seq_ops); 6045 } 6046 6047 static const struct file_operations tracing_saved_cmdlines_fops = { 6048 .open = tracing_saved_cmdlines_open, 6049 .read = seq_read, 6050 .llseek = seq_lseek, 6051 .release = seq_release, 6052 }; 6053 6054 static ssize_t 6055 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf, 6056 size_t cnt, loff_t *ppos) 6057 { 6058 char buf[64]; 6059 int r; 6060 6061 preempt_disable(); 6062 arch_spin_lock(&trace_cmdline_lock); 6063 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num); 6064 arch_spin_unlock(&trace_cmdline_lock); 6065 preempt_enable(); 6066 6067 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6068 } 6069 6070 static int tracing_resize_saved_cmdlines(unsigned int val) 6071 { 6072 struct saved_cmdlines_buffer *s, *savedcmd_temp; 6073 6074 s = allocate_cmdlines_buffer(val); 6075 if (!s) 6076 return -ENOMEM; 6077 6078 preempt_disable(); 6079 arch_spin_lock(&trace_cmdline_lock); 6080 savedcmd_temp = savedcmd; 6081 savedcmd = s; 6082 arch_spin_unlock(&trace_cmdline_lock); 6083 preempt_enable(); 6084 free_saved_cmdlines_buffer(savedcmd_temp); 6085 6086 return 0; 6087 } 6088 6089 static ssize_t 6090 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf, 6091 size_t cnt, loff_t *ppos) 6092 { 6093 unsigned long val; 6094 int ret; 6095 6096 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6097 if (ret) 6098 return ret; 6099 6100 /* must have at least 1 entry or less than PID_MAX_DEFAULT */ 6101 if (!val || val > PID_MAX_DEFAULT) 6102 return -EINVAL; 6103 6104 ret = tracing_resize_saved_cmdlines((unsigned int)val); 6105 if (ret < 0) 6106 return ret; 6107 6108 *ppos += cnt; 6109 6110 return cnt; 6111 } 6112 6113 static const struct file_operations tracing_saved_cmdlines_size_fops = { 6114 .open = tracing_open_generic, 6115 .read = tracing_saved_cmdlines_size_read, 6116 .write = tracing_saved_cmdlines_size_write, 6117 }; 6118 6119 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 6120 static union trace_eval_map_item * 6121 update_eval_map(union trace_eval_map_item *ptr) 6122 { 6123 if (!ptr->map.eval_string) { 6124 if (ptr->tail.next) { 6125 ptr = ptr->tail.next; 6126 /* Set ptr to the next real item (skip head) */ 6127 ptr++; 6128 } else 6129 return NULL; 6130 } 6131 return ptr; 6132 } 6133 6134 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos) 6135 { 6136 union trace_eval_map_item *ptr = v; 6137 6138 /* 6139 * Paranoid! If ptr points to end, we don't want to increment past it. 6140 * This really should never happen. 6141 */ 6142 (*pos)++; 6143 ptr = update_eval_map(ptr); 6144 if (WARN_ON_ONCE(!ptr)) 6145 return NULL; 6146 6147 ptr++; 6148 ptr = update_eval_map(ptr); 6149 6150 return ptr; 6151 } 6152 6153 static void *eval_map_start(struct seq_file *m, loff_t *pos) 6154 { 6155 union trace_eval_map_item *v; 6156 loff_t l = 0; 6157 6158 mutex_lock(&trace_eval_mutex); 6159 6160 v = trace_eval_maps; 6161 if (v) 6162 v++; 6163 6164 while (v && l < *pos) { 6165 v = eval_map_next(m, v, &l); 6166 } 6167 6168 return v; 6169 } 6170 6171 static void eval_map_stop(struct seq_file *m, void *v) 6172 { 6173 mutex_unlock(&trace_eval_mutex); 6174 } 6175 6176 static int eval_map_show(struct seq_file *m, void *v) 6177 { 6178 union trace_eval_map_item *ptr = v; 6179 6180 seq_printf(m, "%s %ld (%s)\n", 6181 ptr->map.eval_string, ptr->map.eval_value, 6182 ptr->map.system); 6183 6184 return 0; 6185 } 6186 6187 static const struct seq_operations tracing_eval_map_seq_ops = { 6188 .start = eval_map_start, 6189 .next = eval_map_next, 6190 .stop = eval_map_stop, 6191 .show = eval_map_show, 6192 }; 6193 6194 static int tracing_eval_map_open(struct inode *inode, struct file *filp) 6195 { 6196 int ret; 6197 6198 ret = tracing_check_open_get_tr(NULL); 6199 if (ret) 6200 return ret; 6201 6202 return seq_open(filp, &tracing_eval_map_seq_ops); 6203 } 6204 6205 static const struct file_operations tracing_eval_map_fops = { 6206 .open = tracing_eval_map_open, 6207 .read = seq_read, 6208 .llseek = seq_lseek, 6209 .release = seq_release, 6210 }; 6211 6212 static inline union trace_eval_map_item * 6213 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr) 6214 { 6215 /* Return tail of array given the head */ 6216 return ptr + ptr->head.length + 1; 6217 } 6218 6219 static void 6220 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start, 6221 int len) 6222 { 6223 struct trace_eval_map **stop; 6224 struct trace_eval_map **map; 6225 union trace_eval_map_item *map_array; 6226 union trace_eval_map_item *ptr; 6227 6228 stop = start + len; 6229 6230 /* 6231 * The trace_eval_maps contains the map plus a head and tail item, 6232 * where the head holds the module and length of array, and the 6233 * tail holds a pointer to the next list. 6234 */ 6235 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL); 6236 if (!map_array) { 6237 pr_warn("Unable to allocate trace eval mapping\n"); 6238 return; 6239 } 6240 6241 mutex_lock(&trace_eval_mutex); 6242 6243 if (!trace_eval_maps) 6244 trace_eval_maps = map_array; 6245 else { 6246 ptr = trace_eval_maps; 6247 for (;;) { 6248 ptr = trace_eval_jmp_to_tail(ptr); 6249 if (!ptr->tail.next) 6250 break; 6251 ptr = ptr->tail.next; 6252 6253 } 6254 ptr->tail.next = map_array; 6255 } 6256 map_array->head.mod = mod; 6257 map_array->head.length = len; 6258 map_array++; 6259 6260 for (map = start; (unsigned long)map < (unsigned long)stop; map++) { 6261 map_array->map = **map; 6262 map_array++; 6263 } 6264 memset(map_array, 0, sizeof(*map_array)); 6265 6266 mutex_unlock(&trace_eval_mutex); 6267 } 6268 6269 static void trace_create_eval_file(struct dentry *d_tracer) 6270 { 6271 trace_create_file("eval_map", TRACE_MODE_READ, d_tracer, 6272 NULL, &tracing_eval_map_fops); 6273 } 6274 6275 #else /* CONFIG_TRACE_EVAL_MAP_FILE */ 6276 static inline void trace_create_eval_file(struct dentry *d_tracer) { } 6277 static inline void trace_insert_eval_map_file(struct module *mod, 6278 struct trace_eval_map **start, int len) { } 6279 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */ 6280 6281 static void trace_insert_eval_map(struct module *mod, 6282 struct trace_eval_map **start, int len) 6283 { 6284 struct trace_eval_map **map; 6285 6286 if (len <= 0) 6287 return; 6288 6289 map = start; 6290 6291 trace_event_eval_update(map, len); 6292 6293 trace_insert_eval_map_file(mod, start, len); 6294 } 6295 6296 static ssize_t 6297 tracing_set_trace_read(struct file *filp, char __user *ubuf, 6298 size_t cnt, loff_t *ppos) 6299 { 6300 struct trace_array *tr = filp->private_data; 6301 char buf[MAX_TRACER_SIZE+2]; 6302 int r; 6303 6304 mutex_lock(&trace_types_lock); 6305 r = sprintf(buf, "%s\n", tr->current_trace->name); 6306 mutex_unlock(&trace_types_lock); 6307 6308 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6309 } 6310 6311 int tracer_init(struct tracer *t, struct trace_array *tr) 6312 { 6313 tracing_reset_online_cpus(&tr->array_buffer); 6314 return t->init(tr); 6315 } 6316 6317 static void set_buffer_entries(struct array_buffer *buf, unsigned long val) 6318 { 6319 int cpu; 6320 6321 for_each_tracing_cpu(cpu) 6322 per_cpu_ptr(buf->data, cpu)->entries = val; 6323 } 6324 6325 static void update_buffer_entries(struct array_buffer *buf, int cpu) 6326 { 6327 if (cpu == RING_BUFFER_ALL_CPUS) { 6328 set_buffer_entries(buf, ring_buffer_size(buf->buffer, 0)); 6329 } else { 6330 per_cpu_ptr(buf->data, cpu)->entries = ring_buffer_size(buf->buffer, cpu); 6331 } 6332 } 6333 6334 #ifdef CONFIG_TRACER_MAX_TRACE 6335 /* resize @tr's buffer to the size of @size_tr's entries */ 6336 static int resize_buffer_duplicate_size(struct array_buffer *trace_buf, 6337 struct array_buffer *size_buf, int cpu_id) 6338 { 6339 int cpu, ret = 0; 6340 6341 if (cpu_id == RING_BUFFER_ALL_CPUS) { 6342 for_each_tracing_cpu(cpu) { 6343 ret = ring_buffer_resize(trace_buf->buffer, 6344 per_cpu_ptr(size_buf->data, cpu)->entries, cpu); 6345 if (ret < 0) 6346 break; 6347 per_cpu_ptr(trace_buf->data, cpu)->entries = 6348 per_cpu_ptr(size_buf->data, cpu)->entries; 6349 } 6350 } else { 6351 ret = ring_buffer_resize(trace_buf->buffer, 6352 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id); 6353 if (ret == 0) 6354 per_cpu_ptr(trace_buf->data, cpu_id)->entries = 6355 per_cpu_ptr(size_buf->data, cpu_id)->entries; 6356 } 6357 6358 return ret; 6359 } 6360 #endif /* CONFIG_TRACER_MAX_TRACE */ 6361 6362 static int __tracing_resize_ring_buffer(struct trace_array *tr, 6363 unsigned long size, int cpu) 6364 { 6365 int ret; 6366 6367 /* 6368 * If kernel or user changes the size of the ring buffer 6369 * we use the size that was given, and we can forget about 6370 * expanding it later. 6371 */ 6372 trace_set_ring_buffer_expanded(tr); 6373 6374 /* May be called before buffers are initialized */ 6375 if (!tr->array_buffer.buffer) 6376 return 0; 6377 6378 /* Do not allow tracing while resizing ring buffer */ 6379 tracing_stop_tr(tr); 6380 6381 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu); 6382 if (ret < 0) 6383 goto out_start; 6384 6385 #ifdef CONFIG_TRACER_MAX_TRACE 6386 if (!tr->allocated_snapshot) 6387 goto out; 6388 6389 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu); 6390 if (ret < 0) { 6391 int r = resize_buffer_duplicate_size(&tr->array_buffer, 6392 &tr->array_buffer, cpu); 6393 if (r < 0) { 6394 /* 6395 * AARGH! We are left with different 6396 * size max buffer!!!! 6397 * The max buffer is our "snapshot" buffer. 6398 * When a tracer needs a snapshot (one of the 6399 * latency tracers), it swaps the max buffer 6400 * with the saved snap shot. We succeeded to 6401 * update the size of the main buffer, but failed to 6402 * update the size of the max buffer. But when we tried 6403 * to reset the main buffer to the original size, we 6404 * failed there too. This is very unlikely to 6405 * happen, but if it does, warn and kill all 6406 * tracing. 6407 */ 6408 WARN_ON(1); 6409 tracing_disabled = 1; 6410 } 6411 goto out_start; 6412 } 6413 6414 update_buffer_entries(&tr->max_buffer, cpu); 6415 6416 out: 6417 #endif /* CONFIG_TRACER_MAX_TRACE */ 6418 6419 update_buffer_entries(&tr->array_buffer, cpu); 6420 out_start: 6421 tracing_start_tr(tr); 6422 return ret; 6423 } 6424 6425 ssize_t tracing_resize_ring_buffer(struct trace_array *tr, 6426 unsigned long size, int cpu_id) 6427 { 6428 int ret; 6429 6430 mutex_lock(&trace_types_lock); 6431 6432 if (cpu_id != RING_BUFFER_ALL_CPUS) { 6433 /* make sure, this cpu is enabled in the mask */ 6434 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) { 6435 ret = -EINVAL; 6436 goto out; 6437 } 6438 } 6439 6440 ret = __tracing_resize_ring_buffer(tr, size, cpu_id); 6441 if (ret < 0) 6442 ret = -ENOMEM; 6443 6444 out: 6445 mutex_unlock(&trace_types_lock); 6446 6447 return ret; 6448 } 6449 6450 6451 /** 6452 * tracing_update_buffers - used by tracing facility to expand ring buffers 6453 * @tr: The tracing instance 6454 * 6455 * To save on memory when the tracing is never used on a system with it 6456 * configured in. The ring buffers are set to a minimum size. But once 6457 * a user starts to use the tracing facility, then they need to grow 6458 * to their default size. 6459 * 6460 * This function is to be called when a tracer is about to be used. 6461 */ 6462 int tracing_update_buffers(struct trace_array *tr) 6463 { 6464 int ret = 0; 6465 6466 mutex_lock(&trace_types_lock); 6467 if (!tr->ring_buffer_expanded) 6468 ret = __tracing_resize_ring_buffer(tr, trace_buf_size, 6469 RING_BUFFER_ALL_CPUS); 6470 mutex_unlock(&trace_types_lock); 6471 6472 return ret; 6473 } 6474 6475 struct trace_option_dentry; 6476 6477 static void 6478 create_trace_option_files(struct trace_array *tr, struct tracer *tracer); 6479 6480 /* 6481 * Used to clear out the tracer before deletion of an instance. 6482 * Must have trace_types_lock held. 6483 */ 6484 static void tracing_set_nop(struct trace_array *tr) 6485 { 6486 if (tr->current_trace == &nop_trace) 6487 return; 6488 6489 tr->current_trace->enabled--; 6490 6491 if (tr->current_trace->reset) 6492 tr->current_trace->reset(tr); 6493 6494 tr->current_trace = &nop_trace; 6495 } 6496 6497 static bool tracer_options_updated; 6498 6499 static void add_tracer_options(struct trace_array *tr, struct tracer *t) 6500 { 6501 /* Only enable if the directory has been created already. */ 6502 if (!tr->dir) 6503 return; 6504 6505 /* Only create trace option files after update_tracer_options finish */ 6506 if (!tracer_options_updated) 6507 return; 6508 6509 create_trace_option_files(tr, t); 6510 } 6511 6512 int tracing_set_tracer(struct trace_array *tr, const char *buf) 6513 { 6514 struct tracer *t; 6515 #ifdef CONFIG_TRACER_MAX_TRACE 6516 bool had_max_tr; 6517 #endif 6518 int ret = 0; 6519 6520 mutex_lock(&trace_types_lock); 6521 6522 if (!tr->ring_buffer_expanded) { 6523 ret = __tracing_resize_ring_buffer(tr, trace_buf_size, 6524 RING_BUFFER_ALL_CPUS); 6525 if (ret < 0) 6526 goto out; 6527 ret = 0; 6528 } 6529 6530 for (t = trace_types; t; t = t->next) { 6531 if (strcmp(t->name, buf) == 0) 6532 break; 6533 } 6534 if (!t) { 6535 ret = -EINVAL; 6536 goto out; 6537 } 6538 if (t == tr->current_trace) 6539 goto out; 6540 6541 #ifdef CONFIG_TRACER_SNAPSHOT 6542 if (t->use_max_tr) { 6543 local_irq_disable(); 6544 arch_spin_lock(&tr->max_lock); 6545 if (tr->cond_snapshot) 6546 ret = -EBUSY; 6547 arch_spin_unlock(&tr->max_lock); 6548 local_irq_enable(); 6549 if (ret) 6550 goto out; 6551 } 6552 #endif 6553 /* Some tracers won't work on kernel command line */ 6554 if (system_state < SYSTEM_RUNNING && t->noboot) { 6555 pr_warn("Tracer '%s' is not allowed on command line, ignored\n", 6556 t->name); 6557 goto out; 6558 } 6559 6560 /* Some tracers are only allowed for the top level buffer */ 6561 if (!trace_ok_for_array(t, tr)) { 6562 ret = -EINVAL; 6563 goto out; 6564 } 6565 6566 /* If trace pipe files are being read, we can't change the tracer */ 6567 if (tr->trace_ref) { 6568 ret = -EBUSY; 6569 goto out; 6570 } 6571 6572 trace_branch_disable(); 6573 6574 tr->current_trace->enabled--; 6575 6576 if (tr->current_trace->reset) 6577 tr->current_trace->reset(tr); 6578 6579 #ifdef CONFIG_TRACER_MAX_TRACE 6580 had_max_tr = tr->current_trace->use_max_tr; 6581 6582 /* Current trace needs to be nop_trace before synchronize_rcu */ 6583 tr->current_trace = &nop_trace; 6584 6585 if (had_max_tr && !t->use_max_tr) { 6586 /* 6587 * We need to make sure that the update_max_tr sees that 6588 * current_trace changed to nop_trace to keep it from 6589 * swapping the buffers after we resize it. 6590 * The update_max_tr is called from interrupts disabled 6591 * so a synchronized_sched() is sufficient. 6592 */ 6593 synchronize_rcu(); 6594 free_snapshot(tr); 6595 } 6596 6597 if (t->use_max_tr && !tr->allocated_snapshot) { 6598 ret = tracing_alloc_snapshot_instance(tr); 6599 if (ret < 0) 6600 goto out; 6601 } 6602 #else 6603 tr->current_trace = &nop_trace; 6604 #endif 6605 6606 if (t->init) { 6607 ret = tracer_init(t, tr); 6608 if (ret) 6609 goto out; 6610 } 6611 6612 tr->current_trace = t; 6613 tr->current_trace->enabled++; 6614 trace_branch_enable(tr); 6615 out: 6616 mutex_unlock(&trace_types_lock); 6617 6618 return ret; 6619 } 6620 6621 static ssize_t 6622 tracing_set_trace_write(struct file *filp, const char __user *ubuf, 6623 size_t cnt, loff_t *ppos) 6624 { 6625 struct trace_array *tr = filp->private_data; 6626 char buf[MAX_TRACER_SIZE+1]; 6627 char *name; 6628 size_t ret; 6629 int err; 6630 6631 ret = cnt; 6632 6633 if (cnt > MAX_TRACER_SIZE) 6634 cnt = MAX_TRACER_SIZE; 6635 6636 if (copy_from_user(buf, ubuf, cnt)) 6637 return -EFAULT; 6638 6639 buf[cnt] = 0; 6640 6641 name = strim(buf); 6642 6643 err = tracing_set_tracer(tr, name); 6644 if (err) 6645 return err; 6646 6647 *ppos += ret; 6648 6649 return ret; 6650 } 6651 6652 static ssize_t 6653 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf, 6654 size_t cnt, loff_t *ppos) 6655 { 6656 char buf[64]; 6657 int r; 6658 6659 r = snprintf(buf, sizeof(buf), "%ld\n", 6660 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr)); 6661 if (r > sizeof(buf)) 6662 r = sizeof(buf); 6663 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6664 } 6665 6666 static ssize_t 6667 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf, 6668 size_t cnt, loff_t *ppos) 6669 { 6670 unsigned long val; 6671 int ret; 6672 6673 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6674 if (ret) 6675 return ret; 6676 6677 *ptr = val * 1000; 6678 6679 return cnt; 6680 } 6681 6682 static ssize_t 6683 tracing_thresh_read(struct file *filp, char __user *ubuf, 6684 size_t cnt, loff_t *ppos) 6685 { 6686 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos); 6687 } 6688 6689 static ssize_t 6690 tracing_thresh_write(struct file *filp, const char __user *ubuf, 6691 size_t cnt, loff_t *ppos) 6692 { 6693 struct trace_array *tr = filp->private_data; 6694 int ret; 6695 6696 mutex_lock(&trace_types_lock); 6697 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos); 6698 if (ret < 0) 6699 goto out; 6700 6701 if (tr->current_trace->update_thresh) { 6702 ret = tr->current_trace->update_thresh(tr); 6703 if (ret < 0) 6704 goto out; 6705 } 6706 6707 ret = cnt; 6708 out: 6709 mutex_unlock(&trace_types_lock); 6710 6711 return ret; 6712 } 6713 6714 #ifdef CONFIG_TRACER_MAX_TRACE 6715 6716 static ssize_t 6717 tracing_max_lat_read(struct file *filp, char __user *ubuf, 6718 size_t cnt, loff_t *ppos) 6719 { 6720 struct trace_array *tr = filp->private_data; 6721 6722 return tracing_nsecs_read(&tr->max_latency, ubuf, cnt, ppos); 6723 } 6724 6725 static ssize_t 6726 tracing_max_lat_write(struct file *filp, const char __user *ubuf, 6727 size_t cnt, loff_t *ppos) 6728 { 6729 struct trace_array *tr = filp->private_data; 6730 6731 return tracing_nsecs_write(&tr->max_latency, ubuf, cnt, ppos); 6732 } 6733 6734 #endif 6735 6736 static int open_pipe_on_cpu(struct trace_array *tr, int cpu) 6737 { 6738 if (cpu == RING_BUFFER_ALL_CPUS) { 6739 if (cpumask_empty(tr->pipe_cpumask)) { 6740 cpumask_setall(tr->pipe_cpumask); 6741 return 0; 6742 } 6743 } else if (!cpumask_test_cpu(cpu, tr->pipe_cpumask)) { 6744 cpumask_set_cpu(cpu, tr->pipe_cpumask); 6745 return 0; 6746 } 6747 return -EBUSY; 6748 } 6749 6750 static void close_pipe_on_cpu(struct trace_array *tr, int cpu) 6751 { 6752 if (cpu == RING_BUFFER_ALL_CPUS) { 6753 WARN_ON(!cpumask_full(tr->pipe_cpumask)); 6754 cpumask_clear(tr->pipe_cpumask); 6755 } else { 6756 WARN_ON(!cpumask_test_cpu(cpu, tr->pipe_cpumask)); 6757 cpumask_clear_cpu(cpu, tr->pipe_cpumask); 6758 } 6759 } 6760 6761 static int tracing_open_pipe(struct inode *inode, struct file *filp) 6762 { 6763 struct trace_array *tr = inode->i_private; 6764 struct trace_iterator *iter; 6765 int cpu; 6766 int ret; 6767 6768 ret = tracing_check_open_get_tr(tr); 6769 if (ret) 6770 return ret; 6771 6772 mutex_lock(&trace_types_lock); 6773 cpu = tracing_get_cpu(inode); 6774 ret = open_pipe_on_cpu(tr, cpu); 6775 if (ret) 6776 goto fail_pipe_on_cpu; 6777 6778 /* create a buffer to store the information to pass to userspace */ 6779 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 6780 if (!iter) { 6781 ret = -ENOMEM; 6782 goto fail_alloc_iter; 6783 } 6784 6785 trace_seq_init(&iter->seq); 6786 iter->trace = tr->current_trace; 6787 6788 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { 6789 ret = -ENOMEM; 6790 goto fail; 6791 } 6792 6793 /* trace pipe does not show start of buffer */ 6794 cpumask_setall(iter->started); 6795 6796 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) 6797 iter->iter_flags |= TRACE_FILE_LAT_FMT; 6798 6799 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 6800 if (trace_clocks[tr->clock_id].in_ns) 6801 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 6802 6803 iter->tr = tr; 6804 iter->array_buffer = &tr->array_buffer; 6805 iter->cpu_file = cpu; 6806 mutex_init(&iter->mutex); 6807 filp->private_data = iter; 6808 6809 if (iter->trace->pipe_open) 6810 iter->trace->pipe_open(iter); 6811 6812 nonseekable_open(inode, filp); 6813 6814 tr->trace_ref++; 6815 6816 mutex_unlock(&trace_types_lock); 6817 return ret; 6818 6819 fail: 6820 kfree(iter); 6821 fail_alloc_iter: 6822 close_pipe_on_cpu(tr, cpu); 6823 fail_pipe_on_cpu: 6824 __trace_array_put(tr); 6825 mutex_unlock(&trace_types_lock); 6826 return ret; 6827 } 6828 6829 static int tracing_release_pipe(struct inode *inode, struct file *file) 6830 { 6831 struct trace_iterator *iter = file->private_data; 6832 struct trace_array *tr = inode->i_private; 6833 6834 mutex_lock(&trace_types_lock); 6835 6836 tr->trace_ref--; 6837 6838 if (iter->trace->pipe_close) 6839 iter->trace->pipe_close(iter); 6840 close_pipe_on_cpu(tr, iter->cpu_file); 6841 mutex_unlock(&trace_types_lock); 6842 6843 free_trace_iter_content(iter); 6844 kfree(iter); 6845 6846 trace_array_put(tr); 6847 6848 return 0; 6849 } 6850 6851 static __poll_t 6852 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table) 6853 { 6854 struct trace_array *tr = iter->tr; 6855 6856 /* Iterators are static, they should be filled or empty */ 6857 if (trace_buffer_iter(iter, iter->cpu_file)) 6858 return EPOLLIN | EPOLLRDNORM; 6859 6860 if (tr->trace_flags & TRACE_ITER_BLOCK) 6861 /* 6862 * Always select as readable when in blocking mode 6863 */ 6864 return EPOLLIN | EPOLLRDNORM; 6865 else 6866 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file, 6867 filp, poll_table, iter->tr->buffer_percent); 6868 } 6869 6870 static __poll_t 6871 tracing_poll_pipe(struct file *filp, poll_table *poll_table) 6872 { 6873 struct trace_iterator *iter = filp->private_data; 6874 6875 return trace_poll(iter, filp, poll_table); 6876 } 6877 6878 /* Must be called with iter->mutex held. */ 6879 static int tracing_wait_pipe(struct file *filp) 6880 { 6881 struct trace_iterator *iter = filp->private_data; 6882 int ret; 6883 6884 while (trace_empty(iter)) { 6885 6886 if ((filp->f_flags & O_NONBLOCK)) { 6887 return -EAGAIN; 6888 } 6889 6890 /* 6891 * We block until we read something and tracing is disabled. 6892 * We still block if tracing is disabled, but we have never 6893 * read anything. This allows a user to cat this file, and 6894 * then enable tracing. But after we have read something, 6895 * we give an EOF when tracing is again disabled. 6896 * 6897 * iter->pos will be 0 if we haven't read anything. 6898 */ 6899 if (!tracer_tracing_is_on(iter->tr) && iter->pos) 6900 break; 6901 6902 mutex_unlock(&iter->mutex); 6903 6904 ret = wait_on_pipe(iter, 0); 6905 6906 mutex_lock(&iter->mutex); 6907 6908 if (ret) 6909 return ret; 6910 } 6911 6912 return 1; 6913 } 6914 6915 /* 6916 * Consumer reader. 6917 */ 6918 static ssize_t 6919 tracing_read_pipe(struct file *filp, char __user *ubuf, 6920 size_t cnt, loff_t *ppos) 6921 { 6922 struct trace_iterator *iter = filp->private_data; 6923 ssize_t sret; 6924 6925 /* 6926 * Avoid more than one consumer on a single file descriptor 6927 * This is just a matter of traces coherency, the ring buffer itself 6928 * is protected. 6929 */ 6930 mutex_lock(&iter->mutex); 6931 6932 /* return any leftover data */ 6933 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 6934 if (sret != -EBUSY) 6935 goto out; 6936 6937 trace_seq_init(&iter->seq); 6938 6939 if (iter->trace->read) { 6940 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 6941 if (sret) 6942 goto out; 6943 } 6944 6945 waitagain: 6946 sret = tracing_wait_pipe(filp); 6947 if (sret <= 0) 6948 goto out; 6949 6950 /* stop when tracing is finished */ 6951 if (trace_empty(iter)) { 6952 sret = 0; 6953 goto out; 6954 } 6955 6956 if (cnt >= TRACE_SEQ_BUFFER_SIZE) 6957 cnt = TRACE_SEQ_BUFFER_SIZE - 1; 6958 6959 /* reset all but tr, trace, and overruns */ 6960 trace_iterator_reset(iter); 6961 cpumask_clear(iter->started); 6962 trace_seq_init(&iter->seq); 6963 6964 trace_event_read_lock(); 6965 trace_access_lock(iter->cpu_file); 6966 while (trace_find_next_entry_inc(iter) != NULL) { 6967 enum print_line_t ret; 6968 int save_len = iter->seq.seq.len; 6969 6970 ret = print_trace_line(iter); 6971 if (ret == TRACE_TYPE_PARTIAL_LINE) { 6972 /* 6973 * If one print_trace_line() fills entire trace_seq in one shot, 6974 * trace_seq_to_user() will returns -EBUSY because save_len == 0, 6975 * In this case, we need to consume it, otherwise, loop will peek 6976 * this event next time, resulting in an infinite loop. 6977 */ 6978 if (save_len == 0) { 6979 iter->seq.full = 0; 6980 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n"); 6981 trace_consume(iter); 6982 break; 6983 } 6984 6985 /* In other cases, don't print partial lines */ 6986 iter->seq.seq.len = save_len; 6987 break; 6988 } 6989 if (ret != TRACE_TYPE_NO_CONSUME) 6990 trace_consume(iter); 6991 6992 if (trace_seq_used(&iter->seq) >= cnt) 6993 break; 6994 6995 /* 6996 * Setting the full flag means we reached the trace_seq buffer 6997 * size and we should leave by partial output condition above. 6998 * One of the trace_seq_* functions is not used properly. 6999 */ 7000 WARN_ONCE(iter->seq.full, "full flag set for trace type %d", 7001 iter->ent->type); 7002 } 7003 trace_access_unlock(iter->cpu_file); 7004 trace_event_read_unlock(); 7005 7006 /* Now copy what we have to the user */ 7007 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 7008 if (iter->seq.readpos >= trace_seq_used(&iter->seq)) 7009 trace_seq_init(&iter->seq); 7010 7011 /* 7012 * If there was nothing to send to user, in spite of consuming trace 7013 * entries, go back to wait for more entries. 7014 */ 7015 if (sret == -EBUSY) 7016 goto waitagain; 7017 7018 out: 7019 mutex_unlock(&iter->mutex); 7020 7021 return sret; 7022 } 7023 7024 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd, 7025 unsigned int idx) 7026 { 7027 __free_page(spd->pages[idx]); 7028 } 7029 7030 static size_t 7031 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter) 7032 { 7033 size_t count; 7034 int save_len; 7035 int ret; 7036 7037 /* Seq buffer is page-sized, exactly what we need. */ 7038 for (;;) { 7039 save_len = iter->seq.seq.len; 7040 ret = print_trace_line(iter); 7041 7042 if (trace_seq_has_overflowed(&iter->seq)) { 7043 iter->seq.seq.len = save_len; 7044 break; 7045 } 7046 7047 /* 7048 * This should not be hit, because it should only 7049 * be set if the iter->seq overflowed. But check it 7050 * anyway to be safe. 7051 */ 7052 if (ret == TRACE_TYPE_PARTIAL_LINE) { 7053 iter->seq.seq.len = save_len; 7054 break; 7055 } 7056 7057 count = trace_seq_used(&iter->seq) - save_len; 7058 if (rem < count) { 7059 rem = 0; 7060 iter->seq.seq.len = save_len; 7061 break; 7062 } 7063 7064 if (ret != TRACE_TYPE_NO_CONSUME) 7065 trace_consume(iter); 7066 rem -= count; 7067 if (!trace_find_next_entry_inc(iter)) { 7068 rem = 0; 7069 iter->ent = NULL; 7070 break; 7071 } 7072 } 7073 7074 return rem; 7075 } 7076 7077 static ssize_t tracing_splice_read_pipe(struct file *filp, 7078 loff_t *ppos, 7079 struct pipe_inode_info *pipe, 7080 size_t len, 7081 unsigned int flags) 7082 { 7083 struct page *pages_def[PIPE_DEF_BUFFERS]; 7084 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 7085 struct trace_iterator *iter = filp->private_data; 7086 struct splice_pipe_desc spd = { 7087 .pages = pages_def, 7088 .partial = partial_def, 7089 .nr_pages = 0, /* This gets updated below. */ 7090 .nr_pages_max = PIPE_DEF_BUFFERS, 7091 .ops = &default_pipe_buf_ops, 7092 .spd_release = tracing_spd_release_pipe, 7093 }; 7094 ssize_t ret; 7095 size_t rem; 7096 unsigned int i; 7097 7098 if (splice_grow_spd(pipe, &spd)) 7099 return -ENOMEM; 7100 7101 mutex_lock(&iter->mutex); 7102 7103 if (iter->trace->splice_read) { 7104 ret = iter->trace->splice_read(iter, filp, 7105 ppos, pipe, len, flags); 7106 if (ret) 7107 goto out_err; 7108 } 7109 7110 ret = tracing_wait_pipe(filp); 7111 if (ret <= 0) 7112 goto out_err; 7113 7114 if (!iter->ent && !trace_find_next_entry_inc(iter)) { 7115 ret = -EFAULT; 7116 goto out_err; 7117 } 7118 7119 trace_event_read_lock(); 7120 trace_access_lock(iter->cpu_file); 7121 7122 /* Fill as many pages as possible. */ 7123 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) { 7124 spd.pages[i] = alloc_page(GFP_KERNEL); 7125 if (!spd.pages[i]) 7126 break; 7127 7128 rem = tracing_fill_pipe_page(rem, iter); 7129 7130 /* Copy the data into the page, so we can start over. */ 7131 ret = trace_seq_to_buffer(&iter->seq, 7132 page_address(spd.pages[i]), 7133 trace_seq_used(&iter->seq)); 7134 if (ret < 0) { 7135 __free_page(spd.pages[i]); 7136 break; 7137 } 7138 spd.partial[i].offset = 0; 7139 spd.partial[i].len = trace_seq_used(&iter->seq); 7140 7141 trace_seq_init(&iter->seq); 7142 } 7143 7144 trace_access_unlock(iter->cpu_file); 7145 trace_event_read_unlock(); 7146 mutex_unlock(&iter->mutex); 7147 7148 spd.nr_pages = i; 7149 7150 if (i) 7151 ret = splice_to_pipe(pipe, &spd); 7152 else 7153 ret = 0; 7154 out: 7155 splice_shrink_spd(&spd); 7156 return ret; 7157 7158 out_err: 7159 mutex_unlock(&iter->mutex); 7160 goto out; 7161 } 7162 7163 static ssize_t 7164 tracing_entries_read(struct file *filp, char __user *ubuf, 7165 size_t cnt, loff_t *ppos) 7166 { 7167 struct inode *inode = file_inode(filp); 7168 struct trace_array *tr = inode->i_private; 7169 int cpu = tracing_get_cpu(inode); 7170 char buf[64]; 7171 int r = 0; 7172 ssize_t ret; 7173 7174 mutex_lock(&trace_types_lock); 7175 7176 if (cpu == RING_BUFFER_ALL_CPUS) { 7177 int cpu, buf_size_same; 7178 unsigned long size; 7179 7180 size = 0; 7181 buf_size_same = 1; 7182 /* check if all cpu sizes are same */ 7183 for_each_tracing_cpu(cpu) { 7184 /* fill in the size from first enabled cpu */ 7185 if (size == 0) 7186 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries; 7187 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) { 7188 buf_size_same = 0; 7189 break; 7190 } 7191 } 7192 7193 if (buf_size_same) { 7194 if (!tr->ring_buffer_expanded) 7195 r = sprintf(buf, "%lu (expanded: %lu)\n", 7196 size >> 10, 7197 trace_buf_size >> 10); 7198 else 7199 r = sprintf(buf, "%lu\n", size >> 10); 7200 } else 7201 r = sprintf(buf, "X\n"); 7202 } else 7203 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10); 7204 7205 mutex_unlock(&trace_types_lock); 7206 7207 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 7208 return ret; 7209 } 7210 7211 static ssize_t 7212 tracing_entries_write(struct file *filp, const char __user *ubuf, 7213 size_t cnt, loff_t *ppos) 7214 { 7215 struct inode *inode = file_inode(filp); 7216 struct trace_array *tr = inode->i_private; 7217 unsigned long val; 7218 int ret; 7219 7220 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 7221 if (ret) 7222 return ret; 7223 7224 /* must have at least 1 entry */ 7225 if (!val) 7226 return -EINVAL; 7227 7228 /* value is in KB */ 7229 val <<= 10; 7230 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode)); 7231 if (ret < 0) 7232 return ret; 7233 7234 *ppos += cnt; 7235 7236 return cnt; 7237 } 7238 7239 static ssize_t 7240 tracing_total_entries_read(struct file *filp, char __user *ubuf, 7241 size_t cnt, loff_t *ppos) 7242 { 7243 struct trace_array *tr = filp->private_data; 7244 char buf[64]; 7245 int r, cpu; 7246 unsigned long size = 0, expanded_size = 0; 7247 7248 mutex_lock(&trace_types_lock); 7249 for_each_tracing_cpu(cpu) { 7250 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10; 7251 if (!tr->ring_buffer_expanded) 7252 expanded_size += trace_buf_size >> 10; 7253 } 7254 if (tr->ring_buffer_expanded) 7255 r = sprintf(buf, "%lu\n", size); 7256 else 7257 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size); 7258 mutex_unlock(&trace_types_lock); 7259 7260 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 7261 } 7262 7263 static ssize_t 7264 tracing_free_buffer_write(struct file *filp, const char __user *ubuf, 7265 size_t cnt, loff_t *ppos) 7266 { 7267 /* 7268 * There is no need to read what the user has written, this function 7269 * is just to make sure that there is no error when "echo" is used 7270 */ 7271 7272 *ppos += cnt; 7273 7274 return cnt; 7275 } 7276 7277 static int 7278 tracing_free_buffer_release(struct inode *inode, struct file *filp) 7279 { 7280 struct trace_array *tr = inode->i_private; 7281 7282 /* disable tracing ? */ 7283 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE) 7284 tracer_tracing_off(tr); 7285 /* resize the ring buffer to 0 */ 7286 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS); 7287 7288 trace_array_put(tr); 7289 7290 return 0; 7291 } 7292 7293 static ssize_t 7294 tracing_mark_write(struct file *filp, const char __user *ubuf, 7295 size_t cnt, loff_t *fpos) 7296 { 7297 struct trace_array *tr = filp->private_data; 7298 struct ring_buffer_event *event; 7299 enum event_trigger_type tt = ETT_NONE; 7300 struct trace_buffer *buffer; 7301 struct print_entry *entry; 7302 int meta_size; 7303 ssize_t written; 7304 size_t size; 7305 int len; 7306 7307 /* Used in tracing_mark_raw_write() as well */ 7308 #define FAULTED_STR "<faulted>" 7309 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */ 7310 7311 if (tracing_disabled) 7312 return -EINVAL; 7313 7314 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) 7315 return -EINVAL; 7316 7317 if ((ssize_t)cnt < 0) 7318 return -EINVAL; 7319 7320 meta_size = sizeof(*entry) + 2; /* add '\0' and possible '\n' */ 7321 again: 7322 size = cnt + meta_size; 7323 7324 /* If less than "<faulted>", then make sure we can still add that */ 7325 if (cnt < FAULTED_SIZE) 7326 size += FAULTED_SIZE - cnt; 7327 7328 if (size > TRACE_SEQ_BUFFER_SIZE) { 7329 cnt -= size - TRACE_SEQ_BUFFER_SIZE; 7330 goto again; 7331 } 7332 7333 buffer = tr->array_buffer.buffer; 7334 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 7335 tracing_gen_ctx()); 7336 if (unlikely(!event)) { 7337 /* 7338 * If the size was greater than what was allowed, then 7339 * make it smaller and try again. 7340 */ 7341 if (size > ring_buffer_max_event_size(buffer)) { 7342 /* cnt < FAULTED size should never be bigger than max */ 7343 if (WARN_ON_ONCE(cnt < FAULTED_SIZE)) 7344 return -EBADF; 7345 cnt = ring_buffer_max_event_size(buffer) - meta_size; 7346 /* The above should only happen once */ 7347 if (WARN_ON_ONCE(cnt + meta_size == size)) 7348 return -EBADF; 7349 goto again; 7350 } 7351 7352 /* Ring buffer disabled, return as if not open for write */ 7353 return -EBADF; 7354 } 7355 7356 entry = ring_buffer_event_data(event); 7357 entry->ip = _THIS_IP_; 7358 7359 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt); 7360 if (len) { 7361 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE); 7362 cnt = FAULTED_SIZE; 7363 written = -EFAULT; 7364 } else 7365 written = cnt; 7366 7367 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) { 7368 /* do not add \n before testing triggers, but add \0 */ 7369 entry->buf[cnt] = '\0'; 7370 tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event); 7371 } 7372 7373 if (entry->buf[cnt - 1] != '\n') { 7374 entry->buf[cnt] = '\n'; 7375 entry->buf[cnt + 1] = '\0'; 7376 } else 7377 entry->buf[cnt] = '\0'; 7378 7379 if (static_branch_unlikely(&trace_marker_exports_enabled)) 7380 ftrace_exports(event, TRACE_EXPORT_MARKER); 7381 __buffer_unlock_commit(buffer, event); 7382 7383 if (tt) 7384 event_triggers_post_call(tr->trace_marker_file, tt); 7385 7386 return written; 7387 } 7388 7389 static ssize_t 7390 tracing_mark_raw_write(struct file *filp, const char __user *ubuf, 7391 size_t cnt, loff_t *fpos) 7392 { 7393 struct trace_array *tr = filp->private_data; 7394 struct ring_buffer_event *event; 7395 struct trace_buffer *buffer; 7396 struct raw_data_entry *entry; 7397 ssize_t written; 7398 int size; 7399 int len; 7400 7401 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int)) 7402 7403 if (tracing_disabled) 7404 return -EINVAL; 7405 7406 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) 7407 return -EINVAL; 7408 7409 /* The marker must at least have a tag id */ 7410 if (cnt < sizeof(unsigned int)) 7411 return -EINVAL; 7412 7413 size = sizeof(*entry) + cnt; 7414 if (cnt < FAULT_SIZE_ID) 7415 size += FAULT_SIZE_ID - cnt; 7416 7417 buffer = tr->array_buffer.buffer; 7418 7419 if (size > ring_buffer_max_event_size(buffer)) 7420 return -EINVAL; 7421 7422 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size, 7423 tracing_gen_ctx()); 7424 if (!event) 7425 /* Ring buffer disabled, return as if not open for write */ 7426 return -EBADF; 7427 7428 entry = ring_buffer_event_data(event); 7429 7430 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt); 7431 if (len) { 7432 entry->id = -1; 7433 memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE); 7434 written = -EFAULT; 7435 } else 7436 written = cnt; 7437 7438 __buffer_unlock_commit(buffer, event); 7439 7440 return written; 7441 } 7442 7443 static int tracing_clock_show(struct seq_file *m, void *v) 7444 { 7445 struct trace_array *tr = m->private; 7446 int i; 7447 7448 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) 7449 seq_printf(m, 7450 "%s%s%s%s", i ? " " : "", 7451 i == tr->clock_id ? "[" : "", trace_clocks[i].name, 7452 i == tr->clock_id ? "]" : ""); 7453 seq_putc(m, '\n'); 7454 7455 return 0; 7456 } 7457 7458 int tracing_set_clock(struct trace_array *tr, const char *clockstr) 7459 { 7460 int i; 7461 7462 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) { 7463 if (strcmp(trace_clocks[i].name, clockstr) == 0) 7464 break; 7465 } 7466 if (i == ARRAY_SIZE(trace_clocks)) 7467 return -EINVAL; 7468 7469 mutex_lock(&trace_types_lock); 7470 7471 tr->clock_id = i; 7472 7473 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func); 7474 7475 /* 7476 * New clock may not be consistent with the previous clock. 7477 * Reset the buffer so that it doesn't have incomparable timestamps. 7478 */ 7479 tracing_reset_online_cpus(&tr->array_buffer); 7480 7481 #ifdef CONFIG_TRACER_MAX_TRACE 7482 if (tr->max_buffer.buffer) 7483 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); 7484 tracing_reset_online_cpus(&tr->max_buffer); 7485 #endif 7486 7487 mutex_unlock(&trace_types_lock); 7488 7489 return 0; 7490 } 7491 7492 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf, 7493 size_t cnt, loff_t *fpos) 7494 { 7495 struct seq_file *m = filp->private_data; 7496 struct trace_array *tr = m->private; 7497 char buf[64]; 7498 const char *clockstr; 7499 int ret; 7500 7501 if (cnt >= sizeof(buf)) 7502 return -EINVAL; 7503 7504 if (copy_from_user(buf, ubuf, cnt)) 7505 return -EFAULT; 7506 7507 buf[cnt] = 0; 7508 7509 clockstr = strstrip(buf); 7510 7511 ret = tracing_set_clock(tr, clockstr); 7512 if (ret) 7513 return ret; 7514 7515 *fpos += cnt; 7516 7517 return cnt; 7518 } 7519 7520 static int tracing_clock_open(struct inode *inode, struct file *file) 7521 { 7522 struct trace_array *tr = inode->i_private; 7523 int ret; 7524 7525 ret = tracing_check_open_get_tr(tr); 7526 if (ret) 7527 return ret; 7528 7529 ret = single_open(file, tracing_clock_show, inode->i_private); 7530 if (ret < 0) 7531 trace_array_put(tr); 7532 7533 return ret; 7534 } 7535 7536 static int tracing_time_stamp_mode_show(struct seq_file *m, void *v) 7537 { 7538 struct trace_array *tr = m->private; 7539 7540 mutex_lock(&trace_types_lock); 7541 7542 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer)) 7543 seq_puts(m, "delta [absolute]\n"); 7544 else 7545 seq_puts(m, "[delta] absolute\n"); 7546 7547 mutex_unlock(&trace_types_lock); 7548 7549 return 0; 7550 } 7551 7552 static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file) 7553 { 7554 struct trace_array *tr = inode->i_private; 7555 int ret; 7556 7557 ret = tracing_check_open_get_tr(tr); 7558 if (ret) 7559 return ret; 7560 7561 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private); 7562 if (ret < 0) 7563 trace_array_put(tr); 7564 7565 return ret; 7566 } 7567 7568 u64 tracing_event_time_stamp(struct trace_buffer *buffer, struct ring_buffer_event *rbe) 7569 { 7570 if (rbe == this_cpu_read(trace_buffered_event)) 7571 return ring_buffer_time_stamp(buffer); 7572 7573 return ring_buffer_event_time_stamp(buffer, rbe); 7574 } 7575 7576 /* 7577 * Set or disable using the per CPU trace_buffer_event when possible. 7578 */ 7579 int tracing_set_filter_buffering(struct trace_array *tr, bool set) 7580 { 7581 int ret = 0; 7582 7583 mutex_lock(&trace_types_lock); 7584 7585 if (set && tr->no_filter_buffering_ref++) 7586 goto out; 7587 7588 if (!set) { 7589 if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) { 7590 ret = -EINVAL; 7591 goto out; 7592 } 7593 7594 --tr->no_filter_buffering_ref; 7595 } 7596 out: 7597 mutex_unlock(&trace_types_lock); 7598 7599 return ret; 7600 } 7601 7602 struct ftrace_buffer_info { 7603 struct trace_iterator iter; 7604 void *spare; 7605 unsigned int spare_cpu; 7606 unsigned int spare_size; 7607 unsigned int read; 7608 }; 7609 7610 #ifdef CONFIG_TRACER_SNAPSHOT 7611 static int tracing_snapshot_open(struct inode *inode, struct file *file) 7612 { 7613 struct trace_array *tr = inode->i_private; 7614 struct trace_iterator *iter; 7615 struct seq_file *m; 7616 int ret; 7617 7618 ret = tracing_check_open_get_tr(tr); 7619 if (ret) 7620 return ret; 7621 7622 if (file->f_mode & FMODE_READ) { 7623 iter = __tracing_open(inode, file, true); 7624 if (IS_ERR(iter)) 7625 ret = PTR_ERR(iter); 7626 } else { 7627 /* Writes still need the seq_file to hold the private data */ 7628 ret = -ENOMEM; 7629 m = kzalloc(sizeof(*m), GFP_KERNEL); 7630 if (!m) 7631 goto out; 7632 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 7633 if (!iter) { 7634 kfree(m); 7635 goto out; 7636 } 7637 ret = 0; 7638 7639 iter->tr = tr; 7640 iter->array_buffer = &tr->max_buffer; 7641 iter->cpu_file = tracing_get_cpu(inode); 7642 m->private = iter; 7643 file->private_data = m; 7644 } 7645 out: 7646 if (ret < 0) 7647 trace_array_put(tr); 7648 7649 return ret; 7650 } 7651 7652 static void tracing_swap_cpu_buffer(void *tr) 7653 { 7654 update_max_tr_single((struct trace_array *)tr, current, smp_processor_id()); 7655 } 7656 7657 static ssize_t 7658 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, 7659 loff_t *ppos) 7660 { 7661 struct seq_file *m = filp->private_data; 7662 struct trace_iterator *iter = m->private; 7663 struct trace_array *tr = iter->tr; 7664 unsigned long val; 7665 int ret; 7666 7667 ret = tracing_update_buffers(tr); 7668 if (ret < 0) 7669 return ret; 7670 7671 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 7672 if (ret) 7673 return ret; 7674 7675 mutex_lock(&trace_types_lock); 7676 7677 if (tr->current_trace->use_max_tr) { 7678 ret = -EBUSY; 7679 goto out; 7680 } 7681 7682 local_irq_disable(); 7683 arch_spin_lock(&tr->max_lock); 7684 if (tr->cond_snapshot) 7685 ret = -EBUSY; 7686 arch_spin_unlock(&tr->max_lock); 7687 local_irq_enable(); 7688 if (ret) 7689 goto out; 7690 7691 switch (val) { 7692 case 0: 7693 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 7694 ret = -EINVAL; 7695 break; 7696 } 7697 if (tr->allocated_snapshot) 7698 free_snapshot(tr); 7699 break; 7700 case 1: 7701 /* Only allow per-cpu swap if the ring buffer supports it */ 7702 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP 7703 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 7704 ret = -EINVAL; 7705 break; 7706 } 7707 #endif 7708 if (tr->allocated_snapshot) 7709 ret = resize_buffer_duplicate_size(&tr->max_buffer, 7710 &tr->array_buffer, iter->cpu_file); 7711 else 7712 ret = tracing_alloc_snapshot_instance(tr); 7713 if (ret < 0) 7714 break; 7715 /* Now, we're going to swap */ 7716 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { 7717 local_irq_disable(); 7718 update_max_tr(tr, current, smp_processor_id(), NULL); 7719 local_irq_enable(); 7720 } else { 7721 smp_call_function_single(iter->cpu_file, tracing_swap_cpu_buffer, 7722 (void *)tr, 1); 7723 } 7724 break; 7725 default: 7726 if (tr->allocated_snapshot) { 7727 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 7728 tracing_reset_online_cpus(&tr->max_buffer); 7729 else 7730 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file); 7731 } 7732 break; 7733 } 7734 7735 if (ret >= 0) { 7736 *ppos += cnt; 7737 ret = cnt; 7738 } 7739 out: 7740 mutex_unlock(&trace_types_lock); 7741 return ret; 7742 } 7743 7744 static int tracing_snapshot_release(struct inode *inode, struct file *file) 7745 { 7746 struct seq_file *m = file->private_data; 7747 int ret; 7748 7749 ret = tracing_release(inode, file); 7750 7751 if (file->f_mode & FMODE_READ) 7752 return ret; 7753 7754 /* If write only, the seq_file is just a stub */ 7755 if (m) 7756 kfree(m->private); 7757 kfree(m); 7758 7759 return 0; 7760 } 7761 7762 static int tracing_buffers_open(struct inode *inode, struct file *filp); 7763 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, 7764 size_t count, loff_t *ppos); 7765 static int tracing_buffers_release(struct inode *inode, struct file *file); 7766 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, 7767 struct pipe_inode_info *pipe, size_t len, unsigned int flags); 7768 7769 static int snapshot_raw_open(struct inode *inode, struct file *filp) 7770 { 7771 struct ftrace_buffer_info *info; 7772 int ret; 7773 7774 /* The following checks for tracefs lockdown */ 7775 ret = tracing_buffers_open(inode, filp); 7776 if (ret < 0) 7777 return ret; 7778 7779 info = filp->private_data; 7780 7781 if (info->iter.trace->use_max_tr) { 7782 tracing_buffers_release(inode, filp); 7783 return -EBUSY; 7784 } 7785 7786 info->iter.snapshot = true; 7787 info->iter.array_buffer = &info->iter.tr->max_buffer; 7788 7789 return ret; 7790 } 7791 7792 #endif /* CONFIG_TRACER_SNAPSHOT */ 7793 7794 7795 static const struct file_operations tracing_thresh_fops = { 7796 .open = tracing_open_generic, 7797 .read = tracing_thresh_read, 7798 .write = tracing_thresh_write, 7799 .llseek = generic_file_llseek, 7800 }; 7801 7802 #ifdef CONFIG_TRACER_MAX_TRACE 7803 static const struct file_operations tracing_max_lat_fops = { 7804 .open = tracing_open_generic_tr, 7805 .read = tracing_max_lat_read, 7806 .write = tracing_max_lat_write, 7807 .llseek = generic_file_llseek, 7808 .release = tracing_release_generic_tr, 7809 }; 7810 #endif 7811 7812 static const struct file_operations set_tracer_fops = { 7813 .open = tracing_open_generic_tr, 7814 .read = tracing_set_trace_read, 7815 .write = tracing_set_trace_write, 7816 .llseek = generic_file_llseek, 7817 .release = tracing_release_generic_tr, 7818 }; 7819 7820 static const struct file_operations tracing_pipe_fops = { 7821 .open = tracing_open_pipe, 7822 .poll = tracing_poll_pipe, 7823 .read = tracing_read_pipe, 7824 .splice_read = tracing_splice_read_pipe, 7825 .release = tracing_release_pipe, 7826 .llseek = no_llseek, 7827 }; 7828 7829 static const struct file_operations tracing_entries_fops = { 7830 .open = tracing_open_generic_tr, 7831 .read = tracing_entries_read, 7832 .write = tracing_entries_write, 7833 .llseek = generic_file_llseek, 7834 .release = tracing_release_generic_tr, 7835 }; 7836 7837 static const struct file_operations tracing_total_entries_fops = { 7838 .open = tracing_open_generic_tr, 7839 .read = tracing_total_entries_read, 7840 .llseek = generic_file_llseek, 7841 .release = tracing_release_generic_tr, 7842 }; 7843 7844 static const struct file_operations tracing_free_buffer_fops = { 7845 .open = tracing_open_generic_tr, 7846 .write = tracing_free_buffer_write, 7847 .release = tracing_free_buffer_release, 7848 }; 7849 7850 static const struct file_operations tracing_mark_fops = { 7851 .open = tracing_mark_open, 7852 .write = tracing_mark_write, 7853 .release = tracing_release_generic_tr, 7854 }; 7855 7856 static const struct file_operations tracing_mark_raw_fops = { 7857 .open = tracing_mark_open, 7858 .write = tracing_mark_raw_write, 7859 .release = tracing_release_generic_tr, 7860 }; 7861 7862 static const struct file_operations trace_clock_fops = { 7863 .open = tracing_clock_open, 7864 .read = seq_read, 7865 .llseek = seq_lseek, 7866 .release = tracing_single_release_tr, 7867 .write = tracing_clock_write, 7868 }; 7869 7870 static const struct file_operations trace_time_stamp_mode_fops = { 7871 .open = tracing_time_stamp_mode_open, 7872 .read = seq_read, 7873 .llseek = seq_lseek, 7874 .release = tracing_single_release_tr, 7875 }; 7876 7877 #ifdef CONFIG_TRACER_SNAPSHOT 7878 static const struct file_operations snapshot_fops = { 7879 .open = tracing_snapshot_open, 7880 .read = seq_read, 7881 .write = tracing_snapshot_write, 7882 .llseek = tracing_lseek, 7883 .release = tracing_snapshot_release, 7884 }; 7885 7886 static const struct file_operations snapshot_raw_fops = { 7887 .open = snapshot_raw_open, 7888 .read = tracing_buffers_read, 7889 .release = tracing_buffers_release, 7890 .splice_read = tracing_buffers_splice_read, 7891 .llseek = no_llseek, 7892 }; 7893 7894 #endif /* CONFIG_TRACER_SNAPSHOT */ 7895 7896 /* 7897 * trace_min_max_write - Write a u64 value to a trace_min_max_param struct 7898 * @filp: The active open file structure 7899 * @ubuf: The userspace provided buffer to read value into 7900 * @cnt: The maximum number of bytes to read 7901 * @ppos: The current "file" position 7902 * 7903 * This function implements the write interface for a struct trace_min_max_param. 7904 * The filp->private_data must point to a trace_min_max_param structure that 7905 * defines where to write the value, the min and the max acceptable values, 7906 * and a lock to protect the write. 7907 */ 7908 static ssize_t 7909 trace_min_max_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) 7910 { 7911 struct trace_min_max_param *param = filp->private_data; 7912 u64 val; 7913 int err; 7914 7915 if (!param) 7916 return -EFAULT; 7917 7918 err = kstrtoull_from_user(ubuf, cnt, 10, &val); 7919 if (err) 7920 return err; 7921 7922 if (param->lock) 7923 mutex_lock(param->lock); 7924 7925 if (param->min && val < *param->min) 7926 err = -EINVAL; 7927 7928 if (param->max && val > *param->max) 7929 err = -EINVAL; 7930 7931 if (!err) 7932 *param->val = val; 7933 7934 if (param->lock) 7935 mutex_unlock(param->lock); 7936 7937 if (err) 7938 return err; 7939 7940 return cnt; 7941 } 7942 7943 /* 7944 * trace_min_max_read - Read a u64 value from a trace_min_max_param struct 7945 * @filp: The active open file structure 7946 * @ubuf: The userspace provided buffer to read value into 7947 * @cnt: The maximum number of bytes to read 7948 * @ppos: The current "file" position 7949 * 7950 * This function implements the read interface for a struct trace_min_max_param. 7951 * The filp->private_data must point to a trace_min_max_param struct with valid 7952 * data. 7953 */ 7954 static ssize_t 7955 trace_min_max_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) 7956 { 7957 struct trace_min_max_param *param = filp->private_data; 7958 char buf[U64_STR_SIZE]; 7959 int len; 7960 u64 val; 7961 7962 if (!param) 7963 return -EFAULT; 7964 7965 val = *param->val; 7966 7967 if (cnt > sizeof(buf)) 7968 cnt = sizeof(buf); 7969 7970 len = snprintf(buf, sizeof(buf), "%llu\n", val); 7971 7972 return simple_read_from_buffer(ubuf, cnt, ppos, buf, len); 7973 } 7974 7975 const struct file_operations trace_min_max_fops = { 7976 .open = tracing_open_generic, 7977 .read = trace_min_max_read, 7978 .write = trace_min_max_write, 7979 }; 7980 7981 #define TRACING_LOG_ERRS_MAX 8 7982 #define TRACING_LOG_LOC_MAX 128 7983 7984 #define CMD_PREFIX " Command: " 7985 7986 struct err_info { 7987 const char **errs; /* ptr to loc-specific array of err strings */ 7988 u8 type; /* index into errs -> specific err string */ 7989 u16 pos; /* caret position */ 7990 u64 ts; 7991 }; 7992 7993 struct tracing_log_err { 7994 struct list_head list; 7995 struct err_info info; 7996 char loc[TRACING_LOG_LOC_MAX]; /* err location */ 7997 char *cmd; /* what caused err */ 7998 }; 7999 8000 static DEFINE_MUTEX(tracing_err_log_lock); 8001 8002 static struct tracing_log_err *alloc_tracing_log_err(int len) 8003 { 8004 struct tracing_log_err *err; 8005 8006 err = kzalloc(sizeof(*err), GFP_KERNEL); 8007 if (!err) 8008 return ERR_PTR(-ENOMEM); 8009 8010 err->cmd = kzalloc(len, GFP_KERNEL); 8011 if (!err->cmd) { 8012 kfree(err); 8013 return ERR_PTR(-ENOMEM); 8014 } 8015 8016 return err; 8017 } 8018 8019 static void free_tracing_log_err(struct tracing_log_err *err) 8020 { 8021 kfree(err->cmd); 8022 kfree(err); 8023 } 8024 8025 static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr, 8026 int len) 8027 { 8028 struct tracing_log_err *err; 8029 char *cmd; 8030 8031 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) { 8032 err = alloc_tracing_log_err(len); 8033 if (PTR_ERR(err) != -ENOMEM) 8034 tr->n_err_log_entries++; 8035 8036 return err; 8037 } 8038 cmd = kzalloc(len, GFP_KERNEL); 8039 if (!cmd) 8040 return ERR_PTR(-ENOMEM); 8041 err = list_first_entry(&tr->err_log, struct tracing_log_err, list); 8042 kfree(err->cmd); 8043 err->cmd = cmd; 8044 list_del(&err->list); 8045 8046 return err; 8047 } 8048 8049 /** 8050 * err_pos - find the position of a string within a command for error careting 8051 * @cmd: The tracing command that caused the error 8052 * @str: The string to position the caret at within @cmd 8053 * 8054 * Finds the position of the first occurrence of @str within @cmd. The 8055 * return value can be passed to tracing_log_err() for caret placement 8056 * within @cmd. 8057 * 8058 * Returns the index within @cmd of the first occurrence of @str or 0 8059 * if @str was not found. 8060 */ 8061 unsigned int err_pos(char *cmd, const char *str) 8062 { 8063 char *found; 8064 8065 if (WARN_ON(!strlen(cmd))) 8066 return 0; 8067 8068 found = strstr(cmd, str); 8069 if (found) 8070 return found - cmd; 8071 8072 return 0; 8073 } 8074 8075 /** 8076 * tracing_log_err - write an error to the tracing error log 8077 * @tr: The associated trace array for the error (NULL for top level array) 8078 * @loc: A string describing where the error occurred 8079 * @cmd: The tracing command that caused the error 8080 * @errs: The array of loc-specific static error strings 8081 * @type: The index into errs[], which produces the specific static err string 8082 * @pos: The position the caret should be placed in the cmd 8083 * 8084 * Writes an error into tracing/error_log of the form: 8085 * 8086 * <loc>: error: <text> 8087 * Command: <cmd> 8088 * ^ 8089 * 8090 * tracing/error_log is a small log file containing the last 8091 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated 8092 * unless there has been a tracing error, and the error log can be 8093 * cleared and have its memory freed by writing the empty string in 8094 * truncation mode to it i.e. echo > tracing/error_log. 8095 * 8096 * NOTE: the @errs array along with the @type param are used to 8097 * produce a static error string - this string is not copied and saved 8098 * when the error is logged - only a pointer to it is saved. See 8099 * existing callers for examples of how static strings are typically 8100 * defined for use with tracing_log_err(). 8101 */ 8102 void tracing_log_err(struct trace_array *tr, 8103 const char *loc, const char *cmd, 8104 const char **errs, u8 type, u16 pos) 8105 { 8106 struct tracing_log_err *err; 8107 int len = 0; 8108 8109 if (!tr) 8110 tr = &global_trace; 8111 8112 len += sizeof(CMD_PREFIX) + 2 * sizeof("\n") + strlen(cmd) + 1; 8113 8114 mutex_lock(&tracing_err_log_lock); 8115 err = get_tracing_log_err(tr, len); 8116 if (PTR_ERR(err) == -ENOMEM) { 8117 mutex_unlock(&tracing_err_log_lock); 8118 return; 8119 } 8120 8121 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc); 8122 snprintf(err->cmd, len, "\n" CMD_PREFIX "%s\n", cmd); 8123 8124 err->info.errs = errs; 8125 err->info.type = type; 8126 err->info.pos = pos; 8127 err->info.ts = local_clock(); 8128 8129 list_add_tail(&err->list, &tr->err_log); 8130 mutex_unlock(&tracing_err_log_lock); 8131 } 8132 8133 static void clear_tracing_err_log(struct trace_array *tr) 8134 { 8135 struct tracing_log_err *err, *next; 8136 8137 mutex_lock(&tracing_err_log_lock); 8138 list_for_each_entry_safe(err, next, &tr->err_log, list) { 8139 list_del(&err->list); 8140 free_tracing_log_err(err); 8141 } 8142 8143 tr->n_err_log_entries = 0; 8144 mutex_unlock(&tracing_err_log_lock); 8145 } 8146 8147 static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos) 8148 { 8149 struct trace_array *tr = m->private; 8150 8151 mutex_lock(&tracing_err_log_lock); 8152 8153 return seq_list_start(&tr->err_log, *pos); 8154 } 8155 8156 static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos) 8157 { 8158 struct trace_array *tr = m->private; 8159 8160 return seq_list_next(v, &tr->err_log, pos); 8161 } 8162 8163 static void tracing_err_log_seq_stop(struct seq_file *m, void *v) 8164 { 8165 mutex_unlock(&tracing_err_log_lock); 8166 } 8167 8168 static void tracing_err_log_show_pos(struct seq_file *m, u16 pos) 8169 { 8170 u16 i; 8171 8172 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++) 8173 seq_putc(m, ' '); 8174 for (i = 0; i < pos; i++) 8175 seq_putc(m, ' '); 8176 seq_puts(m, "^\n"); 8177 } 8178 8179 static int tracing_err_log_seq_show(struct seq_file *m, void *v) 8180 { 8181 struct tracing_log_err *err = v; 8182 8183 if (err) { 8184 const char *err_text = err->info.errs[err->info.type]; 8185 u64 sec = err->info.ts; 8186 u32 nsec; 8187 8188 nsec = do_div(sec, NSEC_PER_SEC); 8189 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000, 8190 err->loc, err_text); 8191 seq_printf(m, "%s", err->cmd); 8192 tracing_err_log_show_pos(m, err->info.pos); 8193 } 8194 8195 return 0; 8196 } 8197 8198 static const struct seq_operations tracing_err_log_seq_ops = { 8199 .start = tracing_err_log_seq_start, 8200 .next = tracing_err_log_seq_next, 8201 .stop = tracing_err_log_seq_stop, 8202 .show = tracing_err_log_seq_show 8203 }; 8204 8205 static int tracing_err_log_open(struct inode *inode, struct file *file) 8206 { 8207 struct trace_array *tr = inode->i_private; 8208 int ret = 0; 8209 8210 ret = tracing_check_open_get_tr(tr); 8211 if (ret) 8212 return ret; 8213 8214 /* If this file was opened for write, then erase contents */ 8215 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) 8216 clear_tracing_err_log(tr); 8217 8218 if (file->f_mode & FMODE_READ) { 8219 ret = seq_open(file, &tracing_err_log_seq_ops); 8220 if (!ret) { 8221 struct seq_file *m = file->private_data; 8222 m->private = tr; 8223 } else { 8224 trace_array_put(tr); 8225 } 8226 } 8227 return ret; 8228 } 8229 8230 static ssize_t tracing_err_log_write(struct file *file, 8231 const char __user *buffer, 8232 size_t count, loff_t *ppos) 8233 { 8234 return count; 8235 } 8236 8237 static int tracing_err_log_release(struct inode *inode, struct file *file) 8238 { 8239 struct trace_array *tr = inode->i_private; 8240 8241 trace_array_put(tr); 8242 8243 if (file->f_mode & FMODE_READ) 8244 seq_release(inode, file); 8245 8246 return 0; 8247 } 8248 8249 static const struct file_operations tracing_err_log_fops = { 8250 .open = tracing_err_log_open, 8251 .write = tracing_err_log_write, 8252 .read = seq_read, 8253 .llseek = tracing_lseek, 8254 .release = tracing_err_log_release, 8255 }; 8256 8257 static int tracing_buffers_open(struct inode *inode, struct file *filp) 8258 { 8259 struct trace_array *tr = inode->i_private; 8260 struct ftrace_buffer_info *info; 8261 int ret; 8262 8263 ret = tracing_check_open_get_tr(tr); 8264 if (ret) 8265 return ret; 8266 8267 info = kvzalloc(sizeof(*info), GFP_KERNEL); 8268 if (!info) { 8269 trace_array_put(tr); 8270 return -ENOMEM; 8271 } 8272 8273 mutex_lock(&trace_types_lock); 8274 8275 info->iter.tr = tr; 8276 info->iter.cpu_file = tracing_get_cpu(inode); 8277 info->iter.trace = tr->current_trace; 8278 info->iter.array_buffer = &tr->array_buffer; 8279 info->spare = NULL; 8280 /* Force reading ring buffer for first read */ 8281 info->read = (unsigned int)-1; 8282 8283 filp->private_data = info; 8284 8285 tr->trace_ref++; 8286 8287 mutex_unlock(&trace_types_lock); 8288 8289 ret = nonseekable_open(inode, filp); 8290 if (ret < 0) 8291 trace_array_put(tr); 8292 8293 return ret; 8294 } 8295 8296 static __poll_t 8297 tracing_buffers_poll(struct file *filp, poll_table *poll_table) 8298 { 8299 struct ftrace_buffer_info *info = filp->private_data; 8300 struct trace_iterator *iter = &info->iter; 8301 8302 return trace_poll(iter, filp, poll_table); 8303 } 8304 8305 static ssize_t 8306 tracing_buffers_read(struct file *filp, char __user *ubuf, 8307 size_t count, loff_t *ppos) 8308 { 8309 struct ftrace_buffer_info *info = filp->private_data; 8310 struct trace_iterator *iter = &info->iter; 8311 void *trace_data; 8312 int page_size; 8313 ssize_t ret = 0; 8314 ssize_t size; 8315 8316 if (!count) 8317 return 0; 8318 8319 #ifdef CONFIG_TRACER_MAX_TRACE 8320 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 8321 return -EBUSY; 8322 #endif 8323 8324 page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); 8325 8326 /* Make sure the spare matches the current sub buffer size */ 8327 if (info->spare) { 8328 if (page_size != info->spare_size) { 8329 ring_buffer_free_read_page(iter->array_buffer->buffer, 8330 info->spare_cpu, info->spare); 8331 info->spare = NULL; 8332 } 8333 } 8334 8335 if (!info->spare) { 8336 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer, 8337 iter->cpu_file); 8338 if (IS_ERR(info->spare)) { 8339 ret = PTR_ERR(info->spare); 8340 info->spare = NULL; 8341 } else { 8342 info->spare_cpu = iter->cpu_file; 8343 info->spare_size = page_size; 8344 } 8345 } 8346 if (!info->spare) 8347 return ret; 8348 8349 /* Do we have previous read data to read? */ 8350 if (info->read < page_size) 8351 goto read; 8352 8353 again: 8354 trace_access_lock(iter->cpu_file); 8355 ret = ring_buffer_read_page(iter->array_buffer->buffer, 8356 info->spare, 8357 count, 8358 iter->cpu_file, 0); 8359 trace_access_unlock(iter->cpu_file); 8360 8361 if (ret < 0) { 8362 if (trace_empty(iter)) { 8363 if ((filp->f_flags & O_NONBLOCK)) 8364 return -EAGAIN; 8365 8366 ret = wait_on_pipe(iter, 0); 8367 if (ret) 8368 return ret; 8369 8370 goto again; 8371 } 8372 return 0; 8373 } 8374 8375 info->read = 0; 8376 read: 8377 size = page_size - info->read; 8378 if (size > count) 8379 size = count; 8380 trace_data = ring_buffer_read_page_data(info->spare); 8381 ret = copy_to_user(ubuf, trace_data + info->read, size); 8382 if (ret == size) 8383 return -EFAULT; 8384 8385 size -= ret; 8386 8387 *ppos += size; 8388 info->read += size; 8389 8390 return size; 8391 } 8392 8393 static int tracing_buffers_release(struct inode *inode, struct file *file) 8394 { 8395 struct ftrace_buffer_info *info = file->private_data; 8396 struct trace_iterator *iter = &info->iter; 8397 8398 mutex_lock(&trace_types_lock); 8399 8400 iter->tr->trace_ref--; 8401 8402 __trace_array_put(iter->tr); 8403 8404 iter->wait_index++; 8405 /* Make sure the waiters see the new wait_index */ 8406 smp_wmb(); 8407 8408 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); 8409 8410 if (info->spare) 8411 ring_buffer_free_read_page(iter->array_buffer->buffer, 8412 info->spare_cpu, info->spare); 8413 kvfree(info); 8414 8415 mutex_unlock(&trace_types_lock); 8416 8417 return 0; 8418 } 8419 8420 struct buffer_ref { 8421 struct trace_buffer *buffer; 8422 void *page; 8423 int cpu; 8424 refcount_t refcount; 8425 }; 8426 8427 static void buffer_ref_release(struct buffer_ref *ref) 8428 { 8429 if (!refcount_dec_and_test(&ref->refcount)) 8430 return; 8431 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); 8432 kfree(ref); 8433 } 8434 8435 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe, 8436 struct pipe_buffer *buf) 8437 { 8438 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 8439 8440 buffer_ref_release(ref); 8441 buf->private = 0; 8442 } 8443 8444 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe, 8445 struct pipe_buffer *buf) 8446 { 8447 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 8448 8449 if (refcount_read(&ref->refcount) > INT_MAX/2) 8450 return false; 8451 8452 refcount_inc(&ref->refcount); 8453 return true; 8454 } 8455 8456 /* Pipe buffer operations for a buffer. */ 8457 static const struct pipe_buf_operations buffer_pipe_buf_ops = { 8458 .release = buffer_pipe_buf_release, 8459 .get = buffer_pipe_buf_get, 8460 }; 8461 8462 /* 8463 * Callback from splice_to_pipe(), if we need to release some pages 8464 * at the end of the spd in case we error'ed out in filling the pipe. 8465 */ 8466 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i) 8467 { 8468 struct buffer_ref *ref = 8469 (struct buffer_ref *)spd->partial[i].private; 8470 8471 buffer_ref_release(ref); 8472 spd->partial[i].private = 0; 8473 } 8474 8475 static ssize_t 8476 tracing_buffers_splice_read(struct file *file, loff_t *ppos, 8477 struct pipe_inode_info *pipe, size_t len, 8478 unsigned int flags) 8479 { 8480 struct ftrace_buffer_info *info = file->private_data; 8481 struct trace_iterator *iter = &info->iter; 8482 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 8483 struct page *pages_def[PIPE_DEF_BUFFERS]; 8484 struct splice_pipe_desc spd = { 8485 .pages = pages_def, 8486 .partial = partial_def, 8487 .nr_pages_max = PIPE_DEF_BUFFERS, 8488 .ops = &buffer_pipe_buf_ops, 8489 .spd_release = buffer_spd_release, 8490 }; 8491 struct buffer_ref *ref; 8492 int page_size; 8493 int entries, i; 8494 ssize_t ret = 0; 8495 8496 #ifdef CONFIG_TRACER_MAX_TRACE 8497 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 8498 return -EBUSY; 8499 #endif 8500 8501 page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); 8502 if (*ppos & (page_size - 1)) 8503 return -EINVAL; 8504 8505 if (len & (page_size - 1)) { 8506 if (len < page_size) 8507 return -EINVAL; 8508 len &= (~(page_size - 1)); 8509 } 8510 8511 if (splice_grow_spd(pipe, &spd)) 8512 return -ENOMEM; 8513 8514 again: 8515 trace_access_lock(iter->cpu_file); 8516 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); 8517 8518 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= page_size) { 8519 struct page *page; 8520 int r; 8521 8522 ref = kzalloc(sizeof(*ref), GFP_KERNEL); 8523 if (!ref) { 8524 ret = -ENOMEM; 8525 break; 8526 } 8527 8528 refcount_set(&ref->refcount, 1); 8529 ref->buffer = iter->array_buffer->buffer; 8530 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); 8531 if (IS_ERR(ref->page)) { 8532 ret = PTR_ERR(ref->page); 8533 ref->page = NULL; 8534 kfree(ref); 8535 break; 8536 } 8537 ref->cpu = iter->cpu_file; 8538 8539 r = ring_buffer_read_page(ref->buffer, ref->page, 8540 len, iter->cpu_file, 1); 8541 if (r < 0) { 8542 ring_buffer_free_read_page(ref->buffer, ref->cpu, 8543 ref->page); 8544 kfree(ref); 8545 break; 8546 } 8547 8548 page = virt_to_page(ring_buffer_read_page_data(ref->page)); 8549 8550 spd.pages[i] = page; 8551 spd.partial[i].len = page_size; 8552 spd.partial[i].offset = 0; 8553 spd.partial[i].private = (unsigned long)ref; 8554 spd.nr_pages++; 8555 *ppos += page_size; 8556 8557 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); 8558 } 8559 8560 trace_access_unlock(iter->cpu_file); 8561 spd.nr_pages = i; 8562 8563 /* did we read anything? */ 8564 if (!spd.nr_pages) { 8565 long wait_index; 8566 8567 if (ret) 8568 goto out; 8569 8570 ret = -EAGAIN; 8571 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) 8572 goto out; 8573 8574 wait_index = READ_ONCE(iter->wait_index); 8575 8576 ret = wait_on_pipe(iter, iter->snapshot ? 0 : iter->tr->buffer_percent); 8577 if (ret) 8578 goto out; 8579 8580 /* No need to wait after waking up when tracing is off */ 8581 if (!tracer_tracing_is_on(iter->tr)) 8582 goto out; 8583 8584 /* Make sure we see the new wait_index */ 8585 smp_rmb(); 8586 if (wait_index != iter->wait_index) 8587 goto out; 8588 8589 goto again; 8590 } 8591 8592 ret = splice_to_pipe(pipe, &spd); 8593 out: 8594 splice_shrink_spd(&spd); 8595 8596 return ret; 8597 } 8598 8599 /* An ioctl call with cmd 0 to the ring buffer file will wake up all waiters */ 8600 static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 8601 { 8602 struct ftrace_buffer_info *info = file->private_data; 8603 struct trace_iterator *iter = &info->iter; 8604 8605 if (cmd) 8606 return -ENOIOCTLCMD; 8607 8608 mutex_lock(&trace_types_lock); 8609 8610 iter->wait_index++; 8611 /* Make sure the waiters see the new wait_index */ 8612 smp_wmb(); 8613 8614 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); 8615 8616 mutex_unlock(&trace_types_lock); 8617 return 0; 8618 } 8619 8620 static const struct file_operations tracing_buffers_fops = { 8621 .open = tracing_buffers_open, 8622 .read = tracing_buffers_read, 8623 .poll = tracing_buffers_poll, 8624 .release = tracing_buffers_release, 8625 .splice_read = tracing_buffers_splice_read, 8626 .unlocked_ioctl = tracing_buffers_ioctl, 8627 .llseek = no_llseek, 8628 }; 8629 8630 static ssize_t 8631 tracing_stats_read(struct file *filp, char __user *ubuf, 8632 size_t count, loff_t *ppos) 8633 { 8634 struct inode *inode = file_inode(filp); 8635 struct trace_array *tr = inode->i_private; 8636 struct array_buffer *trace_buf = &tr->array_buffer; 8637 int cpu = tracing_get_cpu(inode); 8638 struct trace_seq *s; 8639 unsigned long cnt; 8640 unsigned long long t; 8641 unsigned long usec_rem; 8642 8643 s = kmalloc(sizeof(*s), GFP_KERNEL); 8644 if (!s) 8645 return -ENOMEM; 8646 8647 trace_seq_init(s); 8648 8649 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu); 8650 trace_seq_printf(s, "entries: %ld\n", cnt); 8651 8652 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu); 8653 trace_seq_printf(s, "overrun: %ld\n", cnt); 8654 8655 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu); 8656 trace_seq_printf(s, "commit overrun: %ld\n", cnt); 8657 8658 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu); 8659 trace_seq_printf(s, "bytes: %ld\n", cnt); 8660 8661 if (trace_clocks[tr->clock_id].in_ns) { 8662 /* local or global for trace_clock */ 8663 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 8664 usec_rem = do_div(t, USEC_PER_SEC); 8665 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", 8666 t, usec_rem); 8667 8668 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer)); 8669 usec_rem = do_div(t, USEC_PER_SEC); 8670 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem); 8671 } else { 8672 /* counter or tsc mode for trace_clock */ 8673 trace_seq_printf(s, "oldest event ts: %llu\n", 8674 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 8675 8676 trace_seq_printf(s, "now ts: %llu\n", 8677 ring_buffer_time_stamp(trace_buf->buffer)); 8678 } 8679 8680 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu); 8681 trace_seq_printf(s, "dropped events: %ld\n", cnt); 8682 8683 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu); 8684 trace_seq_printf(s, "read events: %ld\n", cnt); 8685 8686 count = simple_read_from_buffer(ubuf, count, ppos, 8687 s->buffer, trace_seq_used(s)); 8688 8689 kfree(s); 8690 8691 return count; 8692 } 8693 8694 static const struct file_operations tracing_stats_fops = { 8695 .open = tracing_open_generic_tr, 8696 .read = tracing_stats_read, 8697 .llseek = generic_file_llseek, 8698 .release = tracing_release_generic_tr, 8699 }; 8700 8701 #ifdef CONFIG_DYNAMIC_FTRACE 8702 8703 static ssize_t 8704 tracing_read_dyn_info(struct file *filp, char __user *ubuf, 8705 size_t cnt, loff_t *ppos) 8706 { 8707 ssize_t ret; 8708 char *buf; 8709 int r; 8710 8711 /* 256 should be plenty to hold the amount needed */ 8712 buf = kmalloc(256, GFP_KERNEL); 8713 if (!buf) 8714 return -ENOMEM; 8715 8716 r = scnprintf(buf, 256, "%ld pages:%ld groups: %ld\n", 8717 ftrace_update_tot_cnt, 8718 ftrace_number_of_pages, 8719 ftrace_number_of_groups); 8720 8721 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 8722 kfree(buf); 8723 return ret; 8724 } 8725 8726 static const struct file_operations tracing_dyn_info_fops = { 8727 .open = tracing_open_generic, 8728 .read = tracing_read_dyn_info, 8729 .llseek = generic_file_llseek, 8730 }; 8731 #endif /* CONFIG_DYNAMIC_FTRACE */ 8732 8733 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) 8734 static void 8735 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, 8736 struct trace_array *tr, struct ftrace_probe_ops *ops, 8737 void *data) 8738 { 8739 tracing_snapshot_instance(tr); 8740 } 8741 8742 static void 8743 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, 8744 struct trace_array *tr, struct ftrace_probe_ops *ops, 8745 void *data) 8746 { 8747 struct ftrace_func_mapper *mapper = data; 8748 long *count = NULL; 8749 8750 if (mapper) 8751 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 8752 8753 if (count) { 8754 8755 if (*count <= 0) 8756 return; 8757 8758 (*count)--; 8759 } 8760 8761 tracing_snapshot_instance(tr); 8762 } 8763 8764 static int 8765 ftrace_snapshot_print(struct seq_file *m, unsigned long ip, 8766 struct ftrace_probe_ops *ops, void *data) 8767 { 8768 struct ftrace_func_mapper *mapper = data; 8769 long *count = NULL; 8770 8771 seq_printf(m, "%ps:", (void *)ip); 8772 8773 seq_puts(m, "snapshot"); 8774 8775 if (mapper) 8776 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 8777 8778 if (count) 8779 seq_printf(m, ":count=%ld\n", *count); 8780 else 8781 seq_puts(m, ":unlimited\n"); 8782 8783 return 0; 8784 } 8785 8786 static int 8787 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr, 8788 unsigned long ip, void *init_data, void **data) 8789 { 8790 struct ftrace_func_mapper *mapper = *data; 8791 8792 if (!mapper) { 8793 mapper = allocate_ftrace_func_mapper(); 8794 if (!mapper) 8795 return -ENOMEM; 8796 *data = mapper; 8797 } 8798 8799 return ftrace_func_mapper_add_ip(mapper, ip, init_data); 8800 } 8801 8802 static void 8803 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr, 8804 unsigned long ip, void *data) 8805 { 8806 struct ftrace_func_mapper *mapper = data; 8807 8808 if (!ip) { 8809 if (!mapper) 8810 return; 8811 free_ftrace_func_mapper(mapper, NULL); 8812 return; 8813 } 8814 8815 ftrace_func_mapper_remove_ip(mapper, ip); 8816 } 8817 8818 static struct ftrace_probe_ops snapshot_probe_ops = { 8819 .func = ftrace_snapshot, 8820 .print = ftrace_snapshot_print, 8821 }; 8822 8823 static struct ftrace_probe_ops snapshot_count_probe_ops = { 8824 .func = ftrace_count_snapshot, 8825 .print = ftrace_snapshot_print, 8826 .init = ftrace_snapshot_init, 8827 .free = ftrace_snapshot_free, 8828 }; 8829 8830 static int 8831 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, 8832 char *glob, char *cmd, char *param, int enable) 8833 { 8834 struct ftrace_probe_ops *ops; 8835 void *count = (void *)-1; 8836 char *number; 8837 int ret; 8838 8839 if (!tr) 8840 return -ENODEV; 8841 8842 /* hash funcs only work with set_ftrace_filter */ 8843 if (!enable) 8844 return -EINVAL; 8845 8846 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops; 8847 8848 if (glob[0] == '!') 8849 return unregister_ftrace_function_probe_func(glob+1, tr, ops); 8850 8851 if (!param) 8852 goto out_reg; 8853 8854 number = strsep(¶m, ":"); 8855 8856 if (!strlen(number)) 8857 goto out_reg; 8858 8859 /* 8860 * We use the callback data field (which is a pointer) 8861 * as our counter. 8862 */ 8863 ret = kstrtoul(number, 0, (unsigned long *)&count); 8864 if (ret) 8865 return ret; 8866 8867 out_reg: 8868 ret = tracing_alloc_snapshot_instance(tr); 8869 if (ret < 0) 8870 goto out; 8871 8872 ret = register_ftrace_function_probe(glob, tr, ops, count); 8873 8874 out: 8875 return ret < 0 ? ret : 0; 8876 } 8877 8878 static struct ftrace_func_command ftrace_snapshot_cmd = { 8879 .name = "snapshot", 8880 .func = ftrace_trace_snapshot_callback, 8881 }; 8882 8883 static __init int register_snapshot_cmd(void) 8884 { 8885 return register_ftrace_command(&ftrace_snapshot_cmd); 8886 } 8887 #else 8888 static inline __init int register_snapshot_cmd(void) { return 0; } 8889 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */ 8890 8891 static struct dentry *tracing_get_dentry(struct trace_array *tr) 8892 { 8893 if (WARN_ON(!tr->dir)) 8894 return ERR_PTR(-ENODEV); 8895 8896 /* Top directory uses NULL as the parent */ 8897 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 8898 return NULL; 8899 8900 /* All sub buffers have a descriptor */ 8901 return tr->dir; 8902 } 8903 8904 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu) 8905 { 8906 struct dentry *d_tracer; 8907 8908 if (tr->percpu_dir) 8909 return tr->percpu_dir; 8910 8911 d_tracer = tracing_get_dentry(tr); 8912 if (IS_ERR(d_tracer)) 8913 return NULL; 8914 8915 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer); 8916 8917 MEM_FAIL(!tr->percpu_dir, 8918 "Could not create tracefs directory 'per_cpu/%d'\n", cpu); 8919 8920 return tr->percpu_dir; 8921 } 8922 8923 static struct dentry * 8924 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent, 8925 void *data, long cpu, const struct file_operations *fops) 8926 { 8927 struct dentry *ret = trace_create_file(name, mode, parent, data, fops); 8928 8929 if (ret) /* See tracing_get_cpu() */ 8930 d_inode(ret)->i_cdev = (void *)(cpu + 1); 8931 return ret; 8932 } 8933 8934 static void 8935 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu) 8936 { 8937 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu); 8938 struct dentry *d_cpu; 8939 char cpu_dir[30]; /* 30 characters should be more than enough */ 8940 8941 if (!d_percpu) 8942 return; 8943 8944 snprintf(cpu_dir, 30, "cpu%ld", cpu); 8945 d_cpu = tracefs_create_dir(cpu_dir, d_percpu); 8946 if (!d_cpu) { 8947 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir); 8948 return; 8949 } 8950 8951 /* per cpu trace_pipe */ 8952 trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu, 8953 tr, cpu, &tracing_pipe_fops); 8954 8955 /* per cpu trace */ 8956 trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu, 8957 tr, cpu, &tracing_fops); 8958 8959 trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu, 8960 tr, cpu, &tracing_buffers_fops); 8961 8962 trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu, 8963 tr, cpu, &tracing_stats_fops); 8964 8965 trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu, 8966 tr, cpu, &tracing_entries_fops); 8967 8968 #ifdef CONFIG_TRACER_SNAPSHOT 8969 trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu, 8970 tr, cpu, &snapshot_fops); 8971 8972 trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu, 8973 tr, cpu, &snapshot_raw_fops); 8974 #endif 8975 } 8976 8977 #ifdef CONFIG_FTRACE_SELFTEST 8978 /* Let selftest have access to static functions in this file */ 8979 #include "trace_selftest.c" 8980 #endif 8981 8982 static ssize_t 8983 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt, 8984 loff_t *ppos) 8985 { 8986 struct trace_option_dentry *topt = filp->private_data; 8987 char *buf; 8988 8989 if (topt->flags->val & topt->opt->bit) 8990 buf = "1\n"; 8991 else 8992 buf = "0\n"; 8993 8994 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 8995 } 8996 8997 static ssize_t 8998 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt, 8999 loff_t *ppos) 9000 { 9001 struct trace_option_dentry *topt = filp->private_data; 9002 unsigned long val; 9003 int ret; 9004 9005 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 9006 if (ret) 9007 return ret; 9008 9009 if (val != 0 && val != 1) 9010 return -EINVAL; 9011 9012 if (!!(topt->flags->val & topt->opt->bit) != val) { 9013 mutex_lock(&trace_types_lock); 9014 ret = __set_tracer_option(topt->tr, topt->flags, 9015 topt->opt, !val); 9016 mutex_unlock(&trace_types_lock); 9017 if (ret) 9018 return ret; 9019 } 9020 9021 *ppos += cnt; 9022 9023 return cnt; 9024 } 9025 9026 static int tracing_open_options(struct inode *inode, struct file *filp) 9027 { 9028 struct trace_option_dentry *topt = inode->i_private; 9029 int ret; 9030 9031 ret = tracing_check_open_get_tr(topt->tr); 9032 if (ret) 9033 return ret; 9034 9035 filp->private_data = inode->i_private; 9036 return 0; 9037 } 9038 9039 static int tracing_release_options(struct inode *inode, struct file *file) 9040 { 9041 struct trace_option_dentry *topt = file->private_data; 9042 9043 trace_array_put(topt->tr); 9044 return 0; 9045 } 9046 9047 static const struct file_operations trace_options_fops = { 9048 .open = tracing_open_options, 9049 .read = trace_options_read, 9050 .write = trace_options_write, 9051 .llseek = generic_file_llseek, 9052 .release = tracing_release_options, 9053 }; 9054 9055 /* 9056 * In order to pass in both the trace_array descriptor as well as the index 9057 * to the flag that the trace option file represents, the trace_array 9058 * has a character array of trace_flags_index[], which holds the index 9059 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc. 9060 * The address of this character array is passed to the flag option file 9061 * read/write callbacks. 9062 * 9063 * In order to extract both the index and the trace_array descriptor, 9064 * get_tr_index() uses the following algorithm. 9065 * 9066 * idx = *ptr; 9067 * 9068 * As the pointer itself contains the address of the index (remember 9069 * index[1] == 1). 9070 * 9071 * Then to get the trace_array descriptor, by subtracting that index 9072 * from the ptr, we get to the start of the index itself. 9073 * 9074 * ptr - idx == &index[0] 9075 * 9076 * Then a simple container_of() from that pointer gets us to the 9077 * trace_array descriptor. 9078 */ 9079 static void get_tr_index(void *data, struct trace_array **ptr, 9080 unsigned int *pindex) 9081 { 9082 *pindex = *(unsigned char *)data; 9083 9084 *ptr = container_of(data - *pindex, struct trace_array, 9085 trace_flags_index); 9086 } 9087 9088 static ssize_t 9089 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt, 9090 loff_t *ppos) 9091 { 9092 void *tr_index = filp->private_data; 9093 struct trace_array *tr; 9094 unsigned int index; 9095 char *buf; 9096 9097 get_tr_index(tr_index, &tr, &index); 9098 9099 if (tr->trace_flags & (1 << index)) 9100 buf = "1\n"; 9101 else 9102 buf = "0\n"; 9103 9104 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 9105 } 9106 9107 static ssize_t 9108 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, 9109 loff_t *ppos) 9110 { 9111 void *tr_index = filp->private_data; 9112 struct trace_array *tr; 9113 unsigned int index; 9114 unsigned long val; 9115 int ret; 9116 9117 get_tr_index(tr_index, &tr, &index); 9118 9119 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 9120 if (ret) 9121 return ret; 9122 9123 if (val != 0 && val != 1) 9124 return -EINVAL; 9125 9126 mutex_lock(&event_mutex); 9127 mutex_lock(&trace_types_lock); 9128 ret = set_tracer_flag(tr, 1 << index, val); 9129 mutex_unlock(&trace_types_lock); 9130 mutex_unlock(&event_mutex); 9131 9132 if (ret < 0) 9133 return ret; 9134 9135 *ppos += cnt; 9136 9137 return cnt; 9138 } 9139 9140 static const struct file_operations trace_options_core_fops = { 9141 .open = tracing_open_generic, 9142 .read = trace_options_core_read, 9143 .write = trace_options_core_write, 9144 .llseek = generic_file_llseek, 9145 }; 9146 9147 struct dentry *trace_create_file(const char *name, 9148 umode_t mode, 9149 struct dentry *parent, 9150 void *data, 9151 const struct file_operations *fops) 9152 { 9153 struct dentry *ret; 9154 9155 ret = tracefs_create_file(name, mode, parent, data, fops); 9156 if (!ret) 9157 pr_warn("Could not create tracefs '%s' entry\n", name); 9158 9159 return ret; 9160 } 9161 9162 9163 static struct dentry *trace_options_init_dentry(struct trace_array *tr) 9164 { 9165 struct dentry *d_tracer; 9166 9167 if (tr->options) 9168 return tr->options; 9169 9170 d_tracer = tracing_get_dentry(tr); 9171 if (IS_ERR(d_tracer)) 9172 return NULL; 9173 9174 tr->options = tracefs_create_dir("options", d_tracer); 9175 if (!tr->options) { 9176 pr_warn("Could not create tracefs directory 'options'\n"); 9177 return NULL; 9178 } 9179 9180 return tr->options; 9181 } 9182 9183 static void 9184 create_trace_option_file(struct trace_array *tr, 9185 struct trace_option_dentry *topt, 9186 struct tracer_flags *flags, 9187 struct tracer_opt *opt) 9188 { 9189 struct dentry *t_options; 9190 9191 t_options = trace_options_init_dentry(tr); 9192 if (!t_options) 9193 return; 9194 9195 topt->flags = flags; 9196 topt->opt = opt; 9197 topt->tr = tr; 9198 9199 topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE, 9200 t_options, topt, &trace_options_fops); 9201 9202 } 9203 9204 static void 9205 create_trace_option_files(struct trace_array *tr, struct tracer *tracer) 9206 { 9207 struct trace_option_dentry *topts; 9208 struct trace_options *tr_topts; 9209 struct tracer_flags *flags; 9210 struct tracer_opt *opts; 9211 int cnt; 9212 int i; 9213 9214 if (!tracer) 9215 return; 9216 9217 flags = tracer->flags; 9218 9219 if (!flags || !flags->opts) 9220 return; 9221 9222 /* 9223 * If this is an instance, only create flags for tracers 9224 * the instance may have. 9225 */ 9226 if (!trace_ok_for_array(tracer, tr)) 9227 return; 9228 9229 for (i = 0; i < tr->nr_topts; i++) { 9230 /* Make sure there's no duplicate flags. */ 9231 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags)) 9232 return; 9233 } 9234 9235 opts = flags->opts; 9236 9237 for (cnt = 0; opts[cnt].name; cnt++) 9238 ; 9239 9240 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL); 9241 if (!topts) 9242 return; 9243 9244 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1), 9245 GFP_KERNEL); 9246 if (!tr_topts) { 9247 kfree(topts); 9248 return; 9249 } 9250 9251 tr->topts = tr_topts; 9252 tr->topts[tr->nr_topts].tracer = tracer; 9253 tr->topts[tr->nr_topts].topts = topts; 9254 tr->nr_topts++; 9255 9256 for (cnt = 0; opts[cnt].name; cnt++) { 9257 create_trace_option_file(tr, &topts[cnt], flags, 9258 &opts[cnt]); 9259 MEM_FAIL(topts[cnt].entry == NULL, 9260 "Failed to create trace option: %s", 9261 opts[cnt].name); 9262 } 9263 } 9264 9265 static struct dentry * 9266 create_trace_option_core_file(struct trace_array *tr, 9267 const char *option, long index) 9268 { 9269 struct dentry *t_options; 9270 9271 t_options = trace_options_init_dentry(tr); 9272 if (!t_options) 9273 return NULL; 9274 9275 return trace_create_file(option, TRACE_MODE_WRITE, t_options, 9276 (void *)&tr->trace_flags_index[index], 9277 &trace_options_core_fops); 9278 } 9279 9280 static void create_trace_options_dir(struct trace_array *tr) 9281 { 9282 struct dentry *t_options; 9283 bool top_level = tr == &global_trace; 9284 int i; 9285 9286 t_options = trace_options_init_dentry(tr); 9287 if (!t_options) 9288 return; 9289 9290 for (i = 0; trace_options[i]; i++) { 9291 if (top_level || 9292 !((1 << i) & TOP_LEVEL_TRACE_FLAGS)) 9293 create_trace_option_core_file(tr, trace_options[i], i); 9294 } 9295 } 9296 9297 static ssize_t 9298 rb_simple_read(struct file *filp, char __user *ubuf, 9299 size_t cnt, loff_t *ppos) 9300 { 9301 struct trace_array *tr = filp->private_data; 9302 char buf[64]; 9303 int r; 9304 9305 r = tracer_tracing_is_on(tr); 9306 r = sprintf(buf, "%d\n", r); 9307 9308 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 9309 } 9310 9311 static ssize_t 9312 rb_simple_write(struct file *filp, const char __user *ubuf, 9313 size_t cnt, loff_t *ppos) 9314 { 9315 struct trace_array *tr = filp->private_data; 9316 struct trace_buffer *buffer = tr->array_buffer.buffer; 9317 unsigned long val; 9318 int ret; 9319 9320 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 9321 if (ret) 9322 return ret; 9323 9324 if (buffer) { 9325 mutex_lock(&trace_types_lock); 9326 if (!!val == tracer_tracing_is_on(tr)) { 9327 val = 0; /* do nothing */ 9328 } else if (val) { 9329 tracer_tracing_on(tr); 9330 if (tr->current_trace->start) 9331 tr->current_trace->start(tr); 9332 } else { 9333 tracer_tracing_off(tr); 9334 if (tr->current_trace->stop) 9335 tr->current_trace->stop(tr); 9336 /* Wake up any waiters */ 9337 ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS); 9338 } 9339 mutex_unlock(&trace_types_lock); 9340 } 9341 9342 (*ppos)++; 9343 9344 return cnt; 9345 } 9346 9347 static const struct file_operations rb_simple_fops = { 9348 .open = tracing_open_generic_tr, 9349 .read = rb_simple_read, 9350 .write = rb_simple_write, 9351 .release = tracing_release_generic_tr, 9352 .llseek = default_llseek, 9353 }; 9354 9355 static ssize_t 9356 buffer_percent_read(struct file *filp, char __user *ubuf, 9357 size_t cnt, loff_t *ppos) 9358 { 9359 struct trace_array *tr = filp->private_data; 9360 char buf[64]; 9361 int r; 9362 9363 r = tr->buffer_percent; 9364 r = sprintf(buf, "%d\n", r); 9365 9366 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 9367 } 9368 9369 static ssize_t 9370 buffer_percent_write(struct file *filp, const char __user *ubuf, 9371 size_t cnt, loff_t *ppos) 9372 { 9373 struct trace_array *tr = filp->private_data; 9374 unsigned long val; 9375 int ret; 9376 9377 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 9378 if (ret) 9379 return ret; 9380 9381 if (val > 100) 9382 return -EINVAL; 9383 9384 tr->buffer_percent = val; 9385 9386 (*ppos)++; 9387 9388 return cnt; 9389 } 9390 9391 static const struct file_operations buffer_percent_fops = { 9392 .open = tracing_open_generic_tr, 9393 .read = buffer_percent_read, 9394 .write = buffer_percent_write, 9395 .release = tracing_release_generic_tr, 9396 .llseek = default_llseek, 9397 }; 9398 9399 static ssize_t 9400 buffer_subbuf_size_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) 9401 { 9402 struct trace_array *tr = filp->private_data; 9403 size_t size; 9404 char buf[64]; 9405 int order; 9406 int r; 9407 9408 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); 9409 size = (PAGE_SIZE << order) / 1024; 9410 9411 r = sprintf(buf, "%zd\n", size); 9412 9413 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 9414 } 9415 9416 static ssize_t 9417 buffer_subbuf_size_write(struct file *filp, const char __user *ubuf, 9418 size_t cnt, loff_t *ppos) 9419 { 9420 struct trace_array *tr = filp->private_data; 9421 unsigned long val; 9422 int old_order; 9423 int order; 9424 int pages; 9425 int ret; 9426 9427 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 9428 if (ret) 9429 return ret; 9430 9431 val *= 1024; /* value passed in is in KB */ 9432 9433 pages = DIV_ROUND_UP(val, PAGE_SIZE); 9434 order = fls(pages - 1); 9435 9436 /* limit between 1 and 128 system pages */ 9437 if (order < 0 || order > 7) 9438 return -EINVAL; 9439 9440 /* Do not allow tracing while changing the order of the ring buffer */ 9441 tracing_stop_tr(tr); 9442 9443 old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); 9444 if (old_order == order) 9445 goto out; 9446 9447 ret = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, order); 9448 if (ret) 9449 goto out; 9450 9451 #ifdef CONFIG_TRACER_MAX_TRACE 9452 9453 if (!tr->allocated_snapshot) 9454 goto out_max; 9455 9456 ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order); 9457 if (ret) { 9458 /* Put back the old order */ 9459 cnt = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, old_order); 9460 if (WARN_ON_ONCE(cnt)) { 9461 /* 9462 * AARGH! We are left with different orders! 9463 * The max buffer is our "snapshot" buffer. 9464 * When a tracer needs a snapshot (one of the 9465 * latency tracers), it swaps the max buffer 9466 * with the saved snap shot. We succeeded to 9467 * update the order of the main buffer, but failed to 9468 * update the order of the max buffer. But when we tried 9469 * to reset the main buffer to the original size, we 9470 * failed there too. This is very unlikely to 9471 * happen, but if it does, warn and kill all 9472 * tracing. 9473 */ 9474 tracing_disabled = 1; 9475 } 9476 goto out; 9477 } 9478 out_max: 9479 #endif 9480 (*ppos)++; 9481 out: 9482 if (ret) 9483 cnt = ret; 9484 tracing_start_tr(tr); 9485 return cnt; 9486 } 9487 9488 static const struct file_operations buffer_subbuf_size_fops = { 9489 .open = tracing_open_generic_tr, 9490 .read = buffer_subbuf_size_read, 9491 .write = buffer_subbuf_size_write, 9492 .release = tracing_release_generic_tr, 9493 .llseek = default_llseek, 9494 }; 9495 9496 static struct dentry *trace_instance_dir; 9497 9498 static void 9499 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer); 9500 9501 static int 9502 allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size) 9503 { 9504 enum ring_buffer_flags rb_flags; 9505 9506 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0; 9507 9508 buf->tr = tr; 9509 9510 buf->buffer = ring_buffer_alloc(size, rb_flags); 9511 if (!buf->buffer) 9512 return -ENOMEM; 9513 9514 buf->data = alloc_percpu(struct trace_array_cpu); 9515 if (!buf->data) { 9516 ring_buffer_free(buf->buffer); 9517 buf->buffer = NULL; 9518 return -ENOMEM; 9519 } 9520 9521 /* Allocate the first page for all buffers */ 9522 set_buffer_entries(&tr->array_buffer, 9523 ring_buffer_size(tr->array_buffer.buffer, 0)); 9524 9525 return 0; 9526 } 9527 9528 static void free_trace_buffer(struct array_buffer *buf) 9529 { 9530 if (buf->buffer) { 9531 ring_buffer_free(buf->buffer); 9532 buf->buffer = NULL; 9533 free_percpu(buf->data); 9534 buf->data = NULL; 9535 } 9536 } 9537 9538 static int allocate_trace_buffers(struct trace_array *tr, int size) 9539 { 9540 int ret; 9541 9542 ret = allocate_trace_buffer(tr, &tr->array_buffer, size); 9543 if (ret) 9544 return ret; 9545 9546 #ifdef CONFIG_TRACER_MAX_TRACE 9547 ret = allocate_trace_buffer(tr, &tr->max_buffer, 9548 allocate_snapshot ? size : 1); 9549 if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) { 9550 free_trace_buffer(&tr->array_buffer); 9551 return -ENOMEM; 9552 } 9553 tr->allocated_snapshot = allocate_snapshot; 9554 9555 allocate_snapshot = false; 9556 #endif 9557 9558 return 0; 9559 } 9560 9561 static void free_trace_buffers(struct trace_array *tr) 9562 { 9563 if (!tr) 9564 return; 9565 9566 free_trace_buffer(&tr->array_buffer); 9567 9568 #ifdef CONFIG_TRACER_MAX_TRACE 9569 free_trace_buffer(&tr->max_buffer); 9570 #endif 9571 } 9572 9573 static void init_trace_flags_index(struct trace_array *tr) 9574 { 9575 int i; 9576 9577 /* Used by the trace options files */ 9578 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) 9579 tr->trace_flags_index[i] = i; 9580 } 9581 9582 static void __update_tracer_options(struct trace_array *tr) 9583 { 9584 struct tracer *t; 9585 9586 for (t = trace_types; t; t = t->next) 9587 add_tracer_options(tr, t); 9588 } 9589 9590 static void update_tracer_options(struct trace_array *tr) 9591 { 9592 mutex_lock(&trace_types_lock); 9593 tracer_options_updated = true; 9594 __update_tracer_options(tr); 9595 mutex_unlock(&trace_types_lock); 9596 } 9597 9598 /* Must have trace_types_lock held */ 9599 struct trace_array *trace_array_find(const char *instance) 9600 { 9601 struct trace_array *tr, *found = NULL; 9602 9603 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 9604 if (tr->name && strcmp(tr->name, instance) == 0) { 9605 found = tr; 9606 break; 9607 } 9608 } 9609 9610 return found; 9611 } 9612 9613 struct trace_array *trace_array_find_get(const char *instance) 9614 { 9615 struct trace_array *tr; 9616 9617 mutex_lock(&trace_types_lock); 9618 tr = trace_array_find(instance); 9619 if (tr) 9620 tr->ref++; 9621 mutex_unlock(&trace_types_lock); 9622 9623 return tr; 9624 } 9625 9626 static int trace_array_create_dir(struct trace_array *tr) 9627 { 9628 int ret; 9629 9630 tr->dir = tracefs_create_dir(tr->name, trace_instance_dir); 9631 if (!tr->dir) 9632 return -EINVAL; 9633 9634 ret = event_trace_add_tracer(tr->dir, tr); 9635 if (ret) { 9636 tracefs_remove(tr->dir); 9637 return ret; 9638 } 9639 9640 init_tracer_tracefs(tr, tr->dir); 9641 __update_tracer_options(tr); 9642 9643 return ret; 9644 } 9645 9646 static struct trace_array * 9647 trace_array_create_systems(const char *name, const char *systems) 9648 { 9649 struct trace_array *tr; 9650 int ret; 9651 9652 ret = -ENOMEM; 9653 tr = kzalloc(sizeof(*tr), GFP_KERNEL); 9654 if (!tr) 9655 return ERR_PTR(ret); 9656 9657 tr->name = kstrdup(name, GFP_KERNEL); 9658 if (!tr->name) 9659 goto out_free_tr; 9660 9661 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL)) 9662 goto out_free_tr; 9663 9664 if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL)) 9665 goto out_free_tr; 9666 9667 if (systems) { 9668 tr->system_names = kstrdup_const(systems, GFP_KERNEL); 9669 if (!tr->system_names) 9670 goto out_free_tr; 9671 } 9672 9673 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS; 9674 9675 cpumask_copy(tr->tracing_cpumask, cpu_all_mask); 9676 9677 raw_spin_lock_init(&tr->start_lock); 9678 9679 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 9680 9681 tr->current_trace = &nop_trace; 9682 9683 INIT_LIST_HEAD(&tr->systems); 9684 INIT_LIST_HEAD(&tr->events); 9685 INIT_LIST_HEAD(&tr->hist_vars); 9686 INIT_LIST_HEAD(&tr->err_log); 9687 9688 if (allocate_trace_buffers(tr, trace_buf_size) < 0) 9689 goto out_free_tr; 9690 9691 /* The ring buffer is defaultly expanded */ 9692 trace_set_ring_buffer_expanded(tr); 9693 9694 if (ftrace_allocate_ftrace_ops(tr) < 0) 9695 goto out_free_tr; 9696 9697 ftrace_init_trace_array(tr); 9698 9699 init_trace_flags_index(tr); 9700 9701 if (trace_instance_dir) { 9702 ret = trace_array_create_dir(tr); 9703 if (ret) 9704 goto out_free_tr; 9705 } else 9706 __trace_early_add_events(tr); 9707 9708 list_add(&tr->list, &ftrace_trace_arrays); 9709 9710 tr->ref++; 9711 9712 return tr; 9713 9714 out_free_tr: 9715 ftrace_free_ftrace_ops(tr); 9716 free_trace_buffers(tr); 9717 free_cpumask_var(tr->pipe_cpumask); 9718 free_cpumask_var(tr->tracing_cpumask); 9719 kfree_const(tr->system_names); 9720 kfree(tr->name); 9721 kfree(tr); 9722 9723 return ERR_PTR(ret); 9724 } 9725 9726 static struct trace_array *trace_array_create(const char *name) 9727 { 9728 return trace_array_create_systems(name, NULL); 9729 } 9730 9731 static int instance_mkdir(const char *name) 9732 { 9733 struct trace_array *tr; 9734 int ret; 9735 9736 mutex_lock(&event_mutex); 9737 mutex_lock(&trace_types_lock); 9738 9739 ret = -EEXIST; 9740 if (trace_array_find(name)) 9741 goto out_unlock; 9742 9743 tr = trace_array_create(name); 9744 9745 ret = PTR_ERR_OR_ZERO(tr); 9746 9747 out_unlock: 9748 mutex_unlock(&trace_types_lock); 9749 mutex_unlock(&event_mutex); 9750 return ret; 9751 } 9752 9753 /** 9754 * trace_array_get_by_name - Create/Lookup a trace array, given its name. 9755 * @name: The name of the trace array to be looked up/created. 9756 * @systems: A list of systems to create event directories for (NULL for all) 9757 * 9758 * Returns pointer to trace array with given name. 9759 * NULL, if it cannot be created. 9760 * 9761 * NOTE: This function increments the reference counter associated with the 9762 * trace array returned. This makes sure it cannot be freed while in use. 9763 * Use trace_array_put() once the trace array is no longer needed. 9764 * If the trace_array is to be freed, trace_array_destroy() needs to 9765 * be called after the trace_array_put(), or simply let user space delete 9766 * it from the tracefs instances directory. But until the 9767 * trace_array_put() is called, user space can not delete it. 9768 * 9769 */ 9770 struct trace_array *trace_array_get_by_name(const char *name, const char *systems) 9771 { 9772 struct trace_array *tr; 9773 9774 mutex_lock(&event_mutex); 9775 mutex_lock(&trace_types_lock); 9776 9777 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 9778 if (tr->name && strcmp(tr->name, name) == 0) 9779 goto out_unlock; 9780 } 9781 9782 tr = trace_array_create_systems(name, systems); 9783 9784 if (IS_ERR(tr)) 9785 tr = NULL; 9786 out_unlock: 9787 if (tr) 9788 tr->ref++; 9789 9790 mutex_unlock(&trace_types_lock); 9791 mutex_unlock(&event_mutex); 9792 return tr; 9793 } 9794 EXPORT_SYMBOL_GPL(trace_array_get_by_name); 9795 9796 static int __remove_instance(struct trace_array *tr) 9797 { 9798 int i; 9799 9800 /* Reference counter for a newly created trace array = 1. */ 9801 if (tr->ref > 1 || (tr->current_trace && tr->trace_ref)) 9802 return -EBUSY; 9803 9804 list_del(&tr->list); 9805 9806 /* Disable all the flags that were enabled coming in */ 9807 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) { 9808 if ((1 << i) & ZEROED_TRACE_FLAGS) 9809 set_tracer_flag(tr, 1 << i, 0); 9810 } 9811 9812 tracing_set_nop(tr); 9813 clear_ftrace_function_probes(tr); 9814 event_trace_del_tracer(tr); 9815 ftrace_clear_pids(tr); 9816 ftrace_destroy_function_files(tr); 9817 tracefs_remove(tr->dir); 9818 free_percpu(tr->last_func_repeats); 9819 free_trace_buffers(tr); 9820 clear_tracing_err_log(tr); 9821 9822 for (i = 0; i < tr->nr_topts; i++) { 9823 kfree(tr->topts[i].topts); 9824 } 9825 kfree(tr->topts); 9826 9827 free_cpumask_var(tr->pipe_cpumask); 9828 free_cpumask_var(tr->tracing_cpumask); 9829 kfree_const(tr->system_names); 9830 kfree(tr->name); 9831 kfree(tr); 9832 9833 return 0; 9834 } 9835 9836 int trace_array_destroy(struct trace_array *this_tr) 9837 { 9838 struct trace_array *tr; 9839 int ret; 9840 9841 if (!this_tr) 9842 return -EINVAL; 9843 9844 mutex_lock(&event_mutex); 9845 mutex_lock(&trace_types_lock); 9846 9847 ret = -ENODEV; 9848 9849 /* Making sure trace array exists before destroying it. */ 9850 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 9851 if (tr == this_tr) { 9852 ret = __remove_instance(tr); 9853 break; 9854 } 9855 } 9856 9857 mutex_unlock(&trace_types_lock); 9858 mutex_unlock(&event_mutex); 9859 9860 return ret; 9861 } 9862 EXPORT_SYMBOL_GPL(trace_array_destroy); 9863 9864 static int instance_rmdir(const char *name) 9865 { 9866 struct trace_array *tr; 9867 int ret; 9868 9869 mutex_lock(&event_mutex); 9870 mutex_lock(&trace_types_lock); 9871 9872 ret = -ENODEV; 9873 tr = trace_array_find(name); 9874 if (tr) 9875 ret = __remove_instance(tr); 9876 9877 mutex_unlock(&trace_types_lock); 9878 mutex_unlock(&event_mutex); 9879 9880 return ret; 9881 } 9882 9883 static __init void create_trace_instances(struct dentry *d_tracer) 9884 { 9885 struct trace_array *tr; 9886 9887 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer, 9888 instance_mkdir, 9889 instance_rmdir); 9890 if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n")) 9891 return; 9892 9893 mutex_lock(&event_mutex); 9894 mutex_lock(&trace_types_lock); 9895 9896 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 9897 if (!tr->name) 9898 continue; 9899 if (MEM_FAIL(trace_array_create_dir(tr) < 0, 9900 "Failed to create instance directory\n")) 9901 break; 9902 } 9903 9904 mutex_unlock(&trace_types_lock); 9905 mutex_unlock(&event_mutex); 9906 } 9907 9908 static void 9909 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) 9910 { 9911 int cpu; 9912 9913 trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer, 9914 tr, &show_traces_fops); 9915 9916 trace_create_file("current_tracer", TRACE_MODE_WRITE, d_tracer, 9917 tr, &set_tracer_fops); 9918 9919 trace_create_file("tracing_cpumask", TRACE_MODE_WRITE, d_tracer, 9920 tr, &tracing_cpumask_fops); 9921 9922 trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer, 9923 tr, &tracing_iter_fops); 9924 9925 trace_create_file("trace", TRACE_MODE_WRITE, d_tracer, 9926 tr, &tracing_fops); 9927 9928 trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer, 9929 tr, &tracing_pipe_fops); 9930 9931 trace_create_file("buffer_size_kb", TRACE_MODE_WRITE, d_tracer, 9932 tr, &tracing_entries_fops); 9933 9934 trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer, 9935 tr, &tracing_total_entries_fops); 9936 9937 trace_create_file("free_buffer", 0200, d_tracer, 9938 tr, &tracing_free_buffer_fops); 9939 9940 trace_create_file("trace_marker", 0220, d_tracer, 9941 tr, &tracing_mark_fops); 9942 9943 tr->trace_marker_file = __find_event_file(tr, "ftrace", "print"); 9944 9945 trace_create_file("trace_marker_raw", 0220, d_tracer, 9946 tr, &tracing_mark_raw_fops); 9947 9948 trace_create_file("trace_clock", TRACE_MODE_WRITE, d_tracer, tr, 9949 &trace_clock_fops); 9950 9951 trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer, 9952 tr, &rb_simple_fops); 9953 9954 trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr, 9955 &trace_time_stamp_mode_fops); 9956 9957 tr->buffer_percent = 50; 9958 9959 trace_create_file("buffer_percent", TRACE_MODE_WRITE, d_tracer, 9960 tr, &buffer_percent_fops); 9961 9962 trace_create_file("buffer_subbuf_size_kb", TRACE_MODE_WRITE, d_tracer, 9963 tr, &buffer_subbuf_size_fops); 9964 9965 create_trace_options_dir(tr); 9966 9967 #ifdef CONFIG_TRACER_MAX_TRACE 9968 trace_create_maxlat_file(tr, d_tracer); 9969 #endif 9970 9971 if (ftrace_create_function_files(tr, d_tracer)) 9972 MEM_FAIL(1, "Could not allocate function filter files"); 9973 9974 #ifdef CONFIG_TRACER_SNAPSHOT 9975 trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer, 9976 tr, &snapshot_fops); 9977 #endif 9978 9979 trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer, 9980 tr, &tracing_err_log_fops); 9981 9982 for_each_tracing_cpu(cpu) 9983 tracing_init_tracefs_percpu(tr, cpu); 9984 9985 ftrace_init_tracefs(tr, d_tracer); 9986 } 9987 9988 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore) 9989 { 9990 struct vfsmount *mnt; 9991 struct file_system_type *type; 9992 9993 /* 9994 * To maintain backward compatibility for tools that mount 9995 * debugfs to get to the tracing facility, tracefs is automatically 9996 * mounted to the debugfs/tracing directory. 9997 */ 9998 type = get_fs_type("tracefs"); 9999 if (!type) 10000 return NULL; 10001 mnt = vfs_submount(mntpt, type, "tracefs", NULL); 10002 put_filesystem(type); 10003 if (IS_ERR(mnt)) 10004 return NULL; 10005 mntget(mnt); 10006 10007 return mnt; 10008 } 10009 10010 /** 10011 * tracing_init_dentry - initialize top level trace array 10012 * 10013 * This is called when creating files or directories in the tracing 10014 * directory. It is called via fs_initcall() by any of the boot up code 10015 * and expects to return the dentry of the top level tracing directory. 10016 */ 10017 int tracing_init_dentry(void) 10018 { 10019 struct trace_array *tr = &global_trace; 10020 10021 if (security_locked_down(LOCKDOWN_TRACEFS)) { 10022 pr_warn("Tracing disabled due to lockdown\n"); 10023 return -EPERM; 10024 } 10025 10026 /* The top level trace array uses NULL as parent */ 10027 if (tr->dir) 10028 return 0; 10029 10030 if (WARN_ON(!tracefs_initialized())) 10031 return -ENODEV; 10032 10033 /* 10034 * As there may still be users that expect the tracing 10035 * files to exist in debugfs/tracing, we must automount 10036 * the tracefs file system there, so older tools still 10037 * work with the newer kernel. 10038 */ 10039 tr->dir = debugfs_create_automount("tracing", NULL, 10040 trace_automount, NULL); 10041 10042 return 0; 10043 } 10044 10045 extern struct trace_eval_map *__start_ftrace_eval_maps[]; 10046 extern struct trace_eval_map *__stop_ftrace_eval_maps[]; 10047 10048 static struct workqueue_struct *eval_map_wq __initdata; 10049 static struct work_struct eval_map_work __initdata; 10050 static struct work_struct tracerfs_init_work __initdata; 10051 10052 static void __init eval_map_work_func(struct work_struct *work) 10053 { 10054 int len; 10055 10056 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps; 10057 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len); 10058 } 10059 10060 static int __init trace_eval_init(void) 10061 { 10062 INIT_WORK(&eval_map_work, eval_map_work_func); 10063 10064 eval_map_wq = alloc_workqueue("eval_map_wq", WQ_UNBOUND, 0); 10065 if (!eval_map_wq) { 10066 pr_err("Unable to allocate eval_map_wq\n"); 10067 /* Do work here */ 10068 eval_map_work_func(&eval_map_work); 10069 return -ENOMEM; 10070 } 10071 10072 queue_work(eval_map_wq, &eval_map_work); 10073 return 0; 10074 } 10075 10076 subsys_initcall(trace_eval_init); 10077 10078 static int __init trace_eval_sync(void) 10079 { 10080 /* Make sure the eval map updates are finished */ 10081 if (eval_map_wq) 10082 destroy_workqueue(eval_map_wq); 10083 return 0; 10084 } 10085 10086 late_initcall_sync(trace_eval_sync); 10087 10088 10089 #ifdef CONFIG_MODULES 10090 static void trace_module_add_evals(struct module *mod) 10091 { 10092 if (!mod->num_trace_evals) 10093 return; 10094 10095 /* 10096 * Modules with bad taint do not have events created, do 10097 * not bother with enums either. 10098 */ 10099 if (trace_module_has_bad_taint(mod)) 10100 return; 10101 10102 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals); 10103 } 10104 10105 #ifdef CONFIG_TRACE_EVAL_MAP_FILE 10106 static void trace_module_remove_evals(struct module *mod) 10107 { 10108 union trace_eval_map_item *map; 10109 union trace_eval_map_item **last = &trace_eval_maps; 10110 10111 if (!mod->num_trace_evals) 10112 return; 10113 10114 mutex_lock(&trace_eval_mutex); 10115 10116 map = trace_eval_maps; 10117 10118 while (map) { 10119 if (map->head.mod == mod) 10120 break; 10121 map = trace_eval_jmp_to_tail(map); 10122 last = &map->tail.next; 10123 map = map->tail.next; 10124 } 10125 if (!map) 10126 goto out; 10127 10128 *last = trace_eval_jmp_to_tail(map)->tail.next; 10129 kfree(map); 10130 out: 10131 mutex_unlock(&trace_eval_mutex); 10132 } 10133 #else 10134 static inline void trace_module_remove_evals(struct module *mod) { } 10135 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */ 10136 10137 static int trace_module_notify(struct notifier_block *self, 10138 unsigned long val, void *data) 10139 { 10140 struct module *mod = data; 10141 10142 switch (val) { 10143 case MODULE_STATE_COMING: 10144 trace_module_add_evals(mod); 10145 break; 10146 case MODULE_STATE_GOING: 10147 trace_module_remove_evals(mod); 10148 break; 10149 } 10150 10151 return NOTIFY_OK; 10152 } 10153 10154 static struct notifier_block trace_module_nb = { 10155 .notifier_call = trace_module_notify, 10156 .priority = 0, 10157 }; 10158 #endif /* CONFIG_MODULES */ 10159 10160 static __init void tracer_init_tracefs_work_func(struct work_struct *work) 10161 { 10162 10163 event_trace_init(); 10164 10165 init_tracer_tracefs(&global_trace, NULL); 10166 ftrace_init_tracefs_toplevel(&global_trace, NULL); 10167 10168 trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL, 10169 &global_trace, &tracing_thresh_fops); 10170 10171 trace_create_file("README", TRACE_MODE_READ, NULL, 10172 NULL, &tracing_readme_fops); 10173 10174 trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL, 10175 NULL, &tracing_saved_cmdlines_fops); 10176 10177 trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL, 10178 NULL, &tracing_saved_cmdlines_size_fops); 10179 10180 trace_create_file("saved_tgids", TRACE_MODE_READ, NULL, 10181 NULL, &tracing_saved_tgids_fops); 10182 10183 trace_create_eval_file(NULL); 10184 10185 #ifdef CONFIG_MODULES 10186 register_module_notifier(&trace_module_nb); 10187 #endif 10188 10189 #ifdef CONFIG_DYNAMIC_FTRACE 10190 trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL, 10191 NULL, &tracing_dyn_info_fops); 10192 #endif 10193 10194 create_trace_instances(NULL); 10195 10196 update_tracer_options(&global_trace); 10197 } 10198 10199 static __init int tracer_init_tracefs(void) 10200 { 10201 int ret; 10202 10203 trace_access_lock_init(); 10204 10205 ret = tracing_init_dentry(); 10206 if (ret) 10207 return 0; 10208 10209 if (eval_map_wq) { 10210 INIT_WORK(&tracerfs_init_work, tracer_init_tracefs_work_func); 10211 queue_work(eval_map_wq, &tracerfs_init_work); 10212 } else { 10213 tracer_init_tracefs_work_func(NULL); 10214 } 10215 10216 rv_init_interface(); 10217 10218 return 0; 10219 } 10220 10221 fs_initcall(tracer_init_tracefs); 10222 10223 static int trace_die_panic_handler(struct notifier_block *self, 10224 unsigned long ev, void *unused); 10225 10226 static struct notifier_block trace_panic_notifier = { 10227 .notifier_call = trace_die_panic_handler, 10228 .priority = INT_MAX - 1, 10229 }; 10230 10231 static struct notifier_block trace_die_notifier = { 10232 .notifier_call = trace_die_panic_handler, 10233 .priority = INT_MAX - 1, 10234 }; 10235 10236 /* 10237 * The idea is to execute the following die/panic callback early, in order 10238 * to avoid showing irrelevant information in the trace (like other panic 10239 * notifier functions); we are the 2nd to run, after hung_task/rcu_stall 10240 * warnings get disabled (to prevent potential log flooding). 10241 */ 10242 static int trace_die_panic_handler(struct notifier_block *self, 10243 unsigned long ev, void *unused) 10244 { 10245 if (!ftrace_dump_on_oops) 10246 return NOTIFY_DONE; 10247 10248 /* The die notifier requires DIE_OOPS to trigger */ 10249 if (self == &trace_die_notifier && ev != DIE_OOPS) 10250 return NOTIFY_DONE; 10251 10252 ftrace_dump(ftrace_dump_on_oops); 10253 10254 return NOTIFY_DONE; 10255 } 10256 10257 /* 10258 * printk is set to max of 1024, we really don't need it that big. 10259 * Nothing should be printing 1000 characters anyway. 10260 */ 10261 #define TRACE_MAX_PRINT 1000 10262 10263 /* 10264 * Define here KERN_TRACE so that we have one place to modify 10265 * it if we decide to change what log level the ftrace dump 10266 * should be at. 10267 */ 10268 #define KERN_TRACE KERN_EMERG 10269 10270 void 10271 trace_printk_seq(struct trace_seq *s) 10272 { 10273 /* Probably should print a warning here. */ 10274 if (s->seq.len >= TRACE_MAX_PRINT) 10275 s->seq.len = TRACE_MAX_PRINT; 10276 10277 /* 10278 * More paranoid code. Although the buffer size is set to 10279 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just 10280 * an extra layer of protection. 10281 */ 10282 if (WARN_ON_ONCE(s->seq.len >= s->seq.size)) 10283 s->seq.len = s->seq.size - 1; 10284 10285 /* should be zero ended, but we are paranoid. */ 10286 s->buffer[s->seq.len] = 0; 10287 10288 printk(KERN_TRACE "%s", s->buffer); 10289 10290 trace_seq_init(s); 10291 } 10292 10293 void trace_init_global_iter(struct trace_iterator *iter) 10294 { 10295 iter->tr = &global_trace; 10296 iter->trace = iter->tr->current_trace; 10297 iter->cpu_file = RING_BUFFER_ALL_CPUS; 10298 iter->array_buffer = &global_trace.array_buffer; 10299 10300 if (iter->trace && iter->trace->open) 10301 iter->trace->open(iter); 10302 10303 /* Annotate start of buffers if we had overruns */ 10304 if (ring_buffer_overruns(iter->array_buffer->buffer)) 10305 iter->iter_flags |= TRACE_FILE_ANNOTATE; 10306 10307 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 10308 if (trace_clocks[iter->tr->clock_id].in_ns) 10309 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 10310 10311 /* Can not use kmalloc for iter.temp and iter.fmt */ 10312 iter->temp = static_temp_buf; 10313 iter->temp_size = STATIC_TEMP_BUF_SIZE; 10314 iter->fmt = static_fmt_buf; 10315 iter->fmt_size = STATIC_FMT_BUF_SIZE; 10316 } 10317 10318 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) 10319 { 10320 /* use static because iter can be a bit big for the stack */ 10321 static struct trace_iterator iter; 10322 static atomic_t dump_running; 10323 struct trace_array *tr = &global_trace; 10324 unsigned int old_userobj; 10325 unsigned long flags; 10326 int cnt = 0, cpu; 10327 10328 /* Only allow one dump user at a time. */ 10329 if (atomic_inc_return(&dump_running) != 1) { 10330 atomic_dec(&dump_running); 10331 return; 10332 } 10333 10334 /* 10335 * Always turn off tracing when we dump. 10336 * We don't need to show trace output of what happens 10337 * between multiple crashes. 10338 * 10339 * If the user does a sysrq-z, then they can re-enable 10340 * tracing with echo 1 > tracing_on. 10341 */ 10342 tracing_off(); 10343 10344 local_irq_save(flags); 10345 10346 /* Simulate the iterator */ 10347 trace_init_global_iter(&iter); 10348 10349 for_each_tracing_cpu(cpu) { 10350 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); 10351 } 10352 10353 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ; 10354 10355 /* don't look at user memory in panic mode */ 10356 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ; 10357 10358 switch (oops_dump_mode) { 10359 case DUMP_ALL: 10360 iter.cpu_file = RING_BUFFER_ALL_CPUS; 10361 break; 10362 case DUMP_ORIG: 10363 iter.cpu_file = raw_smp_processor_id(); 10364 break; 10365 case DUMP_NONE: 10366 goto out_enable; 10367 default: 10368 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); 10369 iter.cpu_file = RING_BUFFER_ALL_CPUS; 10370 } 10371 10372 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 10373 10374 /* Did function tracer already get disabled? */ 10375 if (ftrace_is_dead()) { 10376 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n"); 10377 printk("# MAY BE MISSING FUNCTION EVENTS\n"); 10378 } 10379 10380 /* 10381 * We need to stop all tracing on all CPUS to read 10382 * the next buffer. This is a bit expensive, but is 10383 * not done often. We fill all what we can read, 10384 * and then release the locks again. 10385 */ 10386 10387 while (!trace_empty(&iter)) { 10388 10389 if (!cnt) 10390 printk(KERN_TRACE "---------------------------------\n"); 10391 10392 cnt++; 10393 10394 trace_iterator_reset(&iter); 10395 iter.iter_flags |= TRACE_FILE_LAT_FMT; 10396 10397 if (trace_find_next_entry_inc(&iter) != NULL) { 10398 int ret; 10399 10400 ret = print_trace_line(&iter); 10401 if (ret != TRACE_TYPE_NO_CONSUME) 10402 trace_consume(&iter); 10403 } 10404 touch_nmi_watchdog(); 10405 10406 trace_printk_seq(&iter.seq); 10407 } 10408 10409 if (!cnt) 10410 printk(KERN_TRACE " (ftrace buffer empty)\n"); 10411 else 10412 printk(KERN_TRACE "---------------------------------\n"); 10413 10414 out_enable: 10415 tr->trace_flags |= old_userobj; 10416 10417 for_each_tracing_cpu(cpu) { 10418 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled); 10419 } 10420 atomic_dec(&dump_running); 10421 local_irq_restore(flags); 10422 } 10423 EXPORT_SYMBOL_GPL(ftrace_dump); 10424 10425 #define WRITE_BUFSIZE 4096 10426 10427 ssize_t trace_parse_run_command(struct file *file, const char __user *buffer, 10428 size_t count, loff_t *ppos, 10429 int (*createfn)(const char *)) 10430 { 10431 char *kbuf, *buf, *tmp; 10432 int ret = 0; 10433 size_t done = 0; 10434 size_t size; 10435 10436 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL); 10437 if (!kbuf) 10438 return -ENOMEM; 10439 10440 while (done < count) { 10441 size = count - done; 10442 10443 if (size >= WRITE_BUFSIZE) 10444 size = WRITE_BUFSIZE - 1; 10445 10446 if (copy_from_user(kbuf, buffer + done, size)) { 10447 ret = -EFAULT; 10448 goto out; 10449 } 10450 kbuf[size] = '\0'; 10451 buf = kbuf; 10452 do { 10453 tmp = strchr(buf, '\n'); 10454 if (tmp) { 10455 *tmp = '\0'; 10456 size = tmp - buf + 1; 10457 } else { 10458 size = strlen(buf); 10459 if (done + size < count) { 10460 if (buf != kbuf) 10461 break; 10462 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */ 10463 pr_warn("Line length is too long: Should be less than %d\n", 10464 WRITE_BUFSIZE - 2); 10465 ret = -EINVAL; 10466 goto out; 10467 } 10468 } 10469 done += size; 10470 10471 /* Remove comments */ 10472 tmp = strchr(buf, '#'); 10473 10474 if (tmp) 10475 *tmp = '\0'; 10476 10477 ret = createfn(buf); 10478 if (ret) 10479 goto out; 10480 buf += size; 10481 10482 } while (done < count); 10483 } 10484 ret = done; 10485 10486 out: 10487 kfree(kbuf); 10488 10489 return ret; 10490 } 10491 10492 #ifdef CONFIG_TRACER_MAX_TRACE 10493 __init static bool tr_needs_alloc_snapshot(const char *name) 10494 { 10495 char *test; 10496 int len = strlen(name); 10497 bool ret; 10498 10499 if (!boot_snapshot_index) 10500 return false; 10501 10502 if (strncmp(name, boot_snapshot_info, len) == 0 && 10503 boot_snapshot_info[len] == '\t') 10504 return true; 10505 10506 test = kmalloc(strlen(name) + 3, GFP_KERNEL); 10507 if (!test) 10508 return false; 10509 10510 sprintf(test, "\t%s\t", name); 10511 ret = strstr(boot_snapshot_info, test) == NULL; 10512 kfree(test); 10513 return ret; 10514 } 10515 10516 __init static void do_allocate_snapshot(const char *name) 10517 { 10518 if (!tr_needs_alloc_snapshot(name)) 10519 return; 10520 10521 /* 10522 * When allocate_snapshot is set, the next call to 10523 * allocate_trace_buffers() (called by trace_array_get_by_name()) 10524 * will allocate the snapshot buffer. That will alse clear 10525 * this flag. 10526 */ 10527 allocate_snapshot = true; 10528 } 10529 #else 10530 static inline void do_allocate_snapshot(const char *name) { } 10531 #endif 10532 10533 __init static void enable_instances(void) 10534 { 10535 struct trace_array *tr; 10536 char *curr_str; 10537 char *str; 10538 char *tok; 10539 10540 /* A tab is always appended */ 10541 boot_instance_info[boot_instance_index - 1] = '\0'; 10542 str = boot_instance_info; 10543 10544 while ((curr_str = strsep(&str, "\t"))) { 10545 10546 tok = strsep(&curr_str, ","); 10547 10548 if (IS_ENABLED(CONFIG_TRACER_MAX_TRACE)) 10549 do_allocate_snapshot(tok); 10550 10551 tr = trace_array_get_by_name(tok, NULL); 10552 if (!tr) { 10553 pr_warn("Failed to create instance buffer %s\n", curr_str); 10554 continue; 10555 } 10556 /* Allow user space to delete it */ 10557 trace_array_put(tr); 10558 10559 while ((tok = strsep(&curr_str, ","))) { 10560 early_enable_events(tr, tok, true); 10561 } 10562 } 10563 } 10564 10565 __init static int tracer_alloc_buffers(void) 10566 { 10567 int ring_buf_size; 10568 int ret = -ENOMEM; 10569 10570 10571 if (security_locked_down(LOCKDOWN_TRACEFS)) { 10572 pr_warn("Tracing disabled due to lockdown\n"); 10573 return -EPERM; 10574 } 10575 10576 /* 10577 * Make sure we don't accidentally add more trace options 10578 * than we have bits for. 10579 */ 10580 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE); 10581 10582 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL)) 10583 goto out; 10584 10585 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL)) 10586 goto out_free_buffer_mask; 10587 10588 /* Only allocate trace_printk buffers if a trace_printk exists */ 10589 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt) 10590 /* Must be called before global_trace.buffer is allocated */ 10591 trace_printk_init_buffers(); 10592 10593 /* To save memory, keep the ring buffer size to its minimum */ 10594 if (global_trace.ring_buffer_expanded) 10595 ring_buf_size = trace_buf_size; 10596 else 10597 ring_buf_size = 1; 10598 10599 cpumask_copy(tracing_buffer_mask, cpu_possible_mask); 10600 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask); 10601 10602 raw_spin_lock_init(&global_trace.start_lock); 10603 10604 /* 10605 * The prepare callbacks allocates some memory for the ring buffer. We 10606 * don't free the buffer if the CPU goes down. If we were to free 10607 * the buffer, then the user would lose any trace that was in the 10608 * buffer. The memory will be removed once the "instance" is removed. 10609 */ 10610 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE, 10611 "trace/RB:prepare", trace_rb_cpu_prepare, 10612 NULL); 10613 if (ret < 0) 10614 goto out_free_cpumask; 10615 /* Used for event triggers */ 10616 ret = -ENOMEM; 10617 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE); 10618 if (!temp_buffer) 10619 goto out_rm_hp_state; 10620 10621 if (trace_create_savedcmd() < 0) 10622 goto out_free_temp_buffer; 10623 10624 if (!zalloc_cpumask_var(&global_trace.pipe_cpumask, GFP_KERNEL)) 10625 goto out_free_savedcmd; 10626 10627 /* TODO: make the number of buffers hot pluggable with CPUS */ 10628 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) { 10629 MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n"); 10630 goto out_free_pipe_cpumask; 10631 } 10632 if (global_trace.buffer_disabled) 10633 tracing_off(); 10634 10635 if (trace_boot_clock) { 10636 ret = tracing_set_clock(&global_trace, trace_boot_clock); 10637 if (ret < 0) 10638 pr_warn("Trace clock %s not defined, going back to default\n", 10639 trace_boot_clock); 10640 } 10641 10642 /* 10643 * register_tracer() might reference current_trace, so it 10644 * needs to be set before we register anything. This is 10645 * just a bootstrap of current_trace anyway. 10646 */ 10647 global_trace.current_trace = &nop_trace; 10648 10649 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 10650 10651 ftrace_init_global_array_ops(&global_trace); 10652 10653 init_trace_flags_index(&global_trace); 10654 10655 register_tracer(&nop_trace); 10656 10657 /* Function tracing may start here (via kernel command line) */ 10658 init_function_trace(); 10659 10660 /* All seems OK, enable tracing */ 10661 tracing_disabled = 0; 10662 10663 atomic_notifier_chain_register(&panic_notifier_list, 10664 &trace_panic_notifier); 10665 10666 register_die_notifier(&trace_die_notifier); 10667 10668 global_trace.flags = TRACE_ARRAY_FL_GLOBAL; 10669 10670 INIT_LIST_HEAD(&global_trace.systems); 10671 INIT_LIST_HEAD(&global_trace.events); 10672 INIT_LIST_HEAD(&global_trace.hist_vars); 10673 INIT_LIST_HEAD(&global_trace.err_log); 10674 list_add(&global_trace.list, &ftrace_trace_arrays); 10675 10676 apply_trace_boot_options(); 10677 10678 register_snapshot_cmd(); 10679 10680 test_can_verify(); 10681 10682 return 0; 10683 10684 out_free_pipe_cpumask: 10685 free_cpumask_var(global_trace.pipe_cpumask); 10686 out_free_savedcmd: 10687 free_saved_cmdlines_buffer(savedcmd); 10688 out_free_temp_buffer: 10689 ring_buffer_free(temp_buffer); 10690 out_rm_hp_state: 10691 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE); 10692 out_free_cpumask: 10693 free_cpumask_var(global_trace.tracing_cpumask); 10694 out_free_buffer_mask: 10695 free_cpumask_var(tracing_buffer_mask); 10696 out: 10697 return ret; 10698 } 10699 10700 void __init ftrace_boot_snapshot(void) 10701 { 10702 #ifdef CONFIG_TRACER_MAX_TRACE 10703 struct trace_array *tr; 10704 10705 if (!snapshot_at_boot) 10706 return; 10707 10708 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 10709 if (!tr->allocated_snapshot) 10710 continue; 10711 10712 tracing_snapshot_instance(tr); 10713 trace_array_puts(tr, "** Boot snapshot taken **\n"); 10714 } 10715 #endif 10716 } 10717 10718 void __init early_trace_init(void) 10719 { 10720 if (tracepoint_printk) { 10721 tracepoint_print_iter = 10722 kzalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL); 10723 if (MEM_FAIL(!tracepoint_print_iter, 10724 "Failed to allocate trace iterator\n")) 10725 tracepoint_printk = 0; 10726 else 10727 static_key_enable(&tracepoint_printk_key.key); 10728 } 10729 tracer_alloc_buffers(); 10730 10731 init_events(); 10732 } 10733 10734 void __init trace_init(void) 10735 { 10736 trace_event_init(); 10737 10738 if (boot_instance_index) 10739 enable_instances(); 10740 } 10741 10742 __init static void clear_boot_tracer(void) 10743 { 10744 /* 10745 * The default tracer at boot buffer is an init section. 10746 * This function is called in lateinit. If we did not 10747 * find the boot tracer, then clear it out, to prevent 10748 * later registration from accessing the buffer that is 10749 * about to be freed. 10750 */ 10751 if (!default_bootup_tracer) 10752 return; 10753 10754 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n", 10755 default_bootup_tracer); 10756 default_bootup_tracer = NULL; 10757 } 10758 10759 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 10760 __init static void tracing_set_default_clock(void) 10761 { 10762 /* sched_clock_stable() is determined in late_initcall */ 10763 if (!trace_boot_clock && !sched_clock_stable()) { 10764 if (security_locked_down(LOCKDOWN_TRACEFS)) { 10765 pr_warn("Can not set tracing clock due to lockdown\n"); 10766 return; 10767 } 10768 10769 printk(KERN_WARNING 10770 "Unstable clock detected, switching default tracing clock to \"global\"\n" 10771 "If you want to keep using the local clock, then add:\n" 10772 " \"trace_clock=local\"\n" 10773 "on the kernel command line\n"); 10774 tracing_set_clock(&global_trace, "global"); 10775 } 10776 } 10777 #else 10778 static inline void tracing_set_default_clock(void) { } 10779 #endif 10780 10781 __init static int late_trace_init(void) 10782 { 10783 if (tracepoint_printk && tracepoint_printk_stop_on_boot) { 10784 static_key_disable(&tracepoint_printk_key.key); 10785 tracepoint_printk = 0; 10786 } 10787 10788 tracing_set_default_clock(); 10789 clear_boot_tracer(); 10790 return 0; 10791 } 10792 10793 late_initcall_sync(late_trace_init); 10794