1""" 2yappi.py - Yet Another Python Profiler 3""" 4import os 5import sys 6import _yappi 7import pickle 8import threading 9import warnings 10import types 11import inspect 12import itertools 13try: 14 from thread import get_ident # Python 2 15except ImportError: 16 from threading import get_ident # Python 3 17 18from contextlib import contextmanager 19 20 21class YappiError(Exception): 22 pass 23 24 25__all__ = [ 26 'start', 'stop', 'get_func_stats', 'get_thread_stats', 'clear_stats', 27 'is_running', 'get_clock_time', 'get_clock_type', 'set_clock_type', 28 'get_clock_info', 'get_mem_usage', 'set_context_backend' 29] 30 31LINESEP = os.linesep 32COLUMN_GAP = 2 33YPICKLE_PROTOCOL = 2 34 35# this dict holds {full_name: code object or PyCfunctionobject}. We did not hold 36# this in YStat because it makes it unpickable. I played with some code to make it 37# unpickable by NULLifying the fn_descriptor attrib. but there were lots of happening 38# and some multithread tests were failing, I switched back to a simpler design: 39# do not hold fn_descriptor inside YStats. This is also better design since YFuncStats 40# will have this value only optionally because of unpickling problems of CodeObjects. 41_fn_descriptor_dict = {} 42 43COLUMNS_FUNCSTATS = ["name", "ncall", "ttot", "tsub", "tavg"] 44SORT_TYPES_FUNCSTATS = { 45 "name": 0, 46 "callcount": 3, 47 "totaltime": 6, 48 "subtime": 7, 49 "avgtime": 14, 50 "ncall": 3, 51 "ttot": 6, 52 "tsub": 7, 53 "tavg": 14 54} 55SORT_TYPES_CHILDFUNCSTATS = { 56 "name": 10, 57 "callcount": 1, 58 "totaltime": 3, 59 "subtime": 4, 60 "avgtime": 5, 61 "ncall": 1, 62 "ttot": 3, 63 "tsub": 4, 64 "tavg": 5 65} 66 67SORT_ORDERS = {"ascending": 0, "asc": 0, "descending": 1, "desc": 1} 68DEFAULT_SORT_TYPE = "totaltime" 69DEFAULT_SORT_ORDER = "desc" 70 71CLOCK_TYPES = {"WALL": 0, "CPU": 1} 72NATIVE_THREAD = "NATIVE_THREAD" 73GREENLET = "GREENLET" 74BACKEND_TYPES = {NATIVE_THREAD: 0, GREENLET: 1} 75 76try: 77 GREENLET_COUNTER = itertools.count(start=1).next 78except AttributeError: 79 GREENLET_COUNTER = itertools.count(start=1).__next__ 80 81 82def _validate_sorttype(sort_type, list): 83 sort_type = sort_type.lower() 84 if sort_type not in list: 85 raise YappiError("Invalid SortType parameter: '%s'" % (sort_type)) 86 return sort_type 87 88 89def _validate_sortorder(sort_order): 90 sort_order = sort_order.lower() 91 if sort_order not in SORT_ORDERS: 92 raise YappiError("Invalid SortOrder parameter: '%s'" % (sort_order)) 93 return sort_order 94 95 96def _validate_columns(name, list): 97 name = name.lower() 98 if name not in list: 99 raise YappiError("Invalid Column name: '%s'" % (name)) 100 101 102def _ctx_name_callback(): 103 """ 104 We don't use threading.current_thread() because it will deadlock if 105 called when profiling threading._active_limbo_lock.acquire(). 106 See: #Issue48. 107 """ 108 try: 109 current_thread = threading._active[get_ident()] 110 return current_thread.__class__.__name__ 111 except KeyError: 112 # Threads may not be registered yet in first few profile callbacks. 113 return None 114 115 116def _profile_thread_callback(frame, event, arg): 117 """ 118 _profile_thread_callback will only be called once per-thread. _yappi will detect 119 the new thread and changes the profilefunc param of the ThreadState 120 structure. This is an internal function please don't mess with it. 121 """ 122 _yappi._profile_event(frame, event, arg) 123 124 125def _create_greenlet_callbacks(): 126 """ 127 Returns two functions: 128 - one that can identify unique greenlets. Identity of a greenlet 129 cannot be reused once a greenlet dies. 'id(greenlet)' cannot be used because 130 'id' returns an identifier that can be reused once a greenlet object is garbage 131 collected. 132 - one that can return the name of the greenlet class used to spawn the greenlet 133 """ 134 try: 135 from greenlet import getcurrent 136 except ImportError as exc: 137 raise YappiError("'greenlet' import failed with: %s" % repr(exc)) 138 139 def _get_greenlet_id(): 140 curr_greenlet = getcurrent() 141 id_ = getattr(curr_greenlet, "_yappi_tid", None) 142 if id_ is None: 143 id_ = GREENLET_COUNTER() 144 curr_greenlet._yappi_tid = id_ 145 return id_ 146 147 def _get_greenlet_name(): 148 return getcurrent().__class__.__name__ 149 150 return _get_greenlet_id, _get_greenlet_name 151 152 153def _fft(x, COL_SIZE=8): 154 """ 155 function to prettify time columns in stats. 156 """ 157 _rprecision = 6 158 while (_rprecision > 0): 159 _fmt = "%0." + "%d" % (_rprecision) + "f" 160 s = _fmt % (x) 161 if len(s) <= COL_SIZE: 162 break 163 _rprecision -= 1 164 return s 165 166 167def _func_fullname(builtin, module, lineno, name): 168 if builtin: 169 return "%s.%s" % (module, name) 170 else: 171 return "%s:%d %s" % (module, lineno, name) 172 173 174def module_matches(stat, modules): 175 176 if not isinstance(stat, YStat): 177 raise YappiError( 178 "Argument 'stat' shall be a YStat object. (%s)" % (stat) 179 ) 180 181 if not isinstance(modules, list): 182 raise YappiError( 183 "Argument 'modules' is not a list object. (%s)" % (modules) 184 ) 185 186 if not len(modules): 187 raise YappiError("Argument 'modules' cannot be empty.") 188 189 if stat.full_name not in _fn_descriptor_dict: 190 return False 191 192 modules = set(modules) 193 for module in modules: 194 if not isinstance(module, types.ModuleType): 195 raise YappiError("Non-module item in 'modules'. (%s)" % (module)) 196 return inspect.getmodule(_fn_descriptor_dict[stat.full_name]) in modules 197 198 199def func_matches(stat, funcs): 200 ''' 201 This function will not work with stats that are saved and loaded. That is 202 because current API of loading stats is as following: 203 yappi.get_func_stats(filter_callback=_filter).add('dummy.ys').print_all() 204 205 funcs: is an iterable that selects functions via method descriptor/bound method 206 or function object. selector type depends on the function object: If function 207 is a builtin method, you can use method_descriptor. If it is a builtin function 208 you can select it like e.g: `time.sleep`. For other cases you could use anything 209 that has a code object. 210 ''' 211 212 if not isinstance(stat, YStat): 213 raise YappiError( 214 "Argument 'stat' shall be a YStat object. (%s)" % (stat) 215 ) 216 217 if not isinstance(funcs, list): 218 raise YappiError( 219 "Argument 'funcs' is not a list object. (%s)" % (funcs) 220 ) 221 222 if not len(funcs): 223 raise YappiError("Argument 'funcs' cannot be empty.") 224 225 if stat.full_name not in _fn_descriptor_dict: 226 return False 227 228 funcs = set(funcs) 229 for func in funcs.copy(): 230 if not callable(func): 231 raise YappiError("Non-callable item in 'funcs'. (%s)" % (func)) 232 233 # If there is no CodeObject found, use func itself. It might be a 234 # method descriptor, builtin func..etc. 235 if getattr(func, "__code__", None): 236 funcs.add(func.__code__) 237 238 try: 239 return _fn_descriptor_dict[stat.full_name] in funcs 240 except TypeError: 241 # some builtion methods like <method 'get' of 'dict' objects> are not hashable 242 # thus we cannot search for them in funcs set. 243 return False 244 245 246""" 247Converts our internal yappi's YFuncStats (YSTAT type) to PSTAT. So there are 248some differences between the statistics parameters. The PSTAT format is as following: 249 250PSTAT expects a dict. entry as following: 251 252stats[("mod_name", line_no, "func_name")] = \ 253 ( total_call_count, actual_call_count, total_time, cumulative_time, 254 { 255 ("mod_name", line_no, "func_name") : 256 (total_call_count, --> total count caller called the callee 257 actual_call_count, --> total count caller called the callee - (recursive calls) 258 total_time, --> total time caller spent _only_ for this function (not further subcalls) 259 cumulative_time) --> total time caller spent for this function 260 } --> callers dict 261 ) 262 263Note that in PSTAT the total time spent in the function is called as cumulative_time and 264the time spent _only_ in the function as total_time. From Yappi's perspective, this means: 265 266total_time (inline time) = tsub 267cumulative_time (total time) = ttot 268 269Other than that we hold called functions in a profile entry as named 'children'. On the 270other hand, PSTAT expects to have a dict of callers of the function. So we also need to 271convert children to callers dict. 272From Python Docs: 273''' 274With cProfile, each caller is preceded by three numbers: 275the number of times this specific call was made, and the total 276and cumulative times spent in the current function while it was 277invoked by this specific caller. 278''' 279That means we only need to assign ChildFuncStat's ttot/tsub values to the caller 280properly. Docs indicate that when b() is called by a() pstat holds the total time 281of b() when called by a, just like yappi. 282 283PSTAT only expects to have the above dict to be saved. 284""" 285 286 287def convert2pstats(stats): 288 from collections import defaultdict 289 """ 290 Converts the internal stat type of yappi(which is returned by a call to YFuncStats.get()) 291 as pstats object. 292 """ 293 if not isinstance(stats, YFuncStats): 294 raise YappiError("Source stats must be derived from YFuncStats.") 295 296 import pstats 297 298 class _PStatHolder: 299 300 def __init__(self, d): 301 self.stats = d 302 303 def create_stats(self): 304 pass 305 306 def pstat_id(fs): 307 return (fs.module, fs.lineno, fs.name) 308 309 _pdict = {} 310 311 # convert callees to callers 312 _callers = defaultdict(dict) 313 for fs in stats: 314 for ct in fs.children: 315 _callers[ct][pstat_id(fs) 316 ] = (ct.ncall, ct.nactualcall, ct.tsub, ct.ttot) 317 318 # populate the pstat dict. 319 for fs in stats: 320 _pdict[pstat_id(fs)] = ( 321 fs.ncall, 322 fs.nactualcall, 323 fs.tsub, 324 fs.ttot, 325 _callers[fs], 326 ) 327 328 return pstats.Stats(_PStatHolder(_pdict)) 329 330 331def profile(clock_type="cpu", profile_builtins=False, return_callback=None): 332 """ 333 A profile decorator that can be used to profile a single call. 334 335 We need to clear_stats() on entry/exit of the function unfortunately. 336 As yappi is a per-interpreter resource, we cannot simply resume profiling 337 session upon exit of the function, that is because we _may_ simply change 338 start() params which may differ from the paused session that may cause instable 339 results. So, if you use a decorator, then global profiling may return bogus 340 results or no results at all. 341 """ 342 343 def _profile_dec(func): 344 345 def wrapper(*args, **kwargs): 346 if func._rec_level == 0: 347 clear_stats() 348 set_clock_type(clock_type) 349 start(profile_builtins, profile_threads=False) 350 func._rec_level += 1 351 try: 352 return func(*args, **kwargs) 353 finally: 354 func._rec_level -= 1 355 # only show profile information when recursion level of the 356 # function becomes 0. Otherwise, we are in the middle of a 357 # recursive call tree and not finished yet. 358 if func._rec_level == 0: 359 try: 360 stop() 361 if return_callback is None: 362 sys.stdout.write(LINESEP) 363 sys.stdout.write( 364 "Executed in %s %s clock seconds" % ( 365 _fft(get_thread_stats()[0].ttot 366 ), clock_type.upper() 367 ) 368 ) 369 sys.stdout.write(LINESEP) 370 get_func_stats().print_all() 371 else: 372 return_callback(func, get_func_stats()) 373 finally: 374 clear_stats() 375 376 func._rec_level = 0 377 return wrapper 378 379 return _profile_dec 380 381 382class StatString(object): 383 """ 384 Class to prettify/trim a profile result column. 385 """ 386 _TRAIL_DOT = ".." 387 _LEFT = 1 388 _RIGHT = 2 389 390 def __init__(self, s): 391 self._s = str(s) 392 393 def _trim(self, length, direction): 394 if (len(self._s) > length): 395 if direction == self._LEFT: 396 self._s = self._s[-length:] 397 return self._TRAIL_DOT + self._s[len(self._TRAIL_DOT):] 398 elif direction == self._RIGHT: 399 self._s = self._s[:length] 400 return self._s[:-len(self._TRAIL_DOT)] + self._TRAIL_DOT 401 return self._s + (" " * (length - len(self._s))) 402 403 def ltrim(self, length): 404 return self._trim(length, self._LEFT) 405 406 def rtrim(self, length): 407 return self._trim(length, self._RIGHT) 408 409 410class YStat(dict): 411 """ 412 Class to hold a profile result line in a dict object, which all items can also be accessed as 413 instance attributes where their attribute name is the given key. Mimicked NamedTuples. 414 """ 415 _KEYS = {} 416 417 def __init__(self, values): 418 super(YStat, self).__init__() 419 420 for key, i in self._KEYS.items(): 421 setattr(self, key, values[i]) 422 423 def __setattr__(self, name, value): 424 self[self._KEYS[name]] = value 425 super(YStat, self).__setattr__(name, value) 426 427 428class YFuncStat(YStat): 429 """ 430 Class holding information for function stats. 431 """ 432 _KEYS = { 433 'name': 0, 434 'module': 1, 435 'lineno': 2, 436 'ncall': 3, 437 'nactualcall': 4, 438 'builtin': 5, 439 'ttot': 6, 440 'tsub': 7, 441 'index': 8, 442 'children': 9, 443 'ctx_id': 10, 444 'ctx_name': 11, 445 'tag': 12, 446 'tavg': 14, 447 'full_name': 15 448 } 449 450 def __eq__(self, other): 451 if other is None: 452 return False 453 return self.full_name == other.full_name 454 455 def __ne__(self, other): 456 return not self == other 457 458 def __add__(self, other): 459 460 # do not merge if merging the same instance 461 if self is other: 462 return self 463 464 self.ncall += other.ncall 465 self.nactualcall += other.nactualcall 466 self.ttot += other.ttot 467 self.tsub += other.tsub 468 self.tavg = self.ttot / self.ncall 469 470 for other_child_stat in other.children: 471 # all children point to a valid entry, and we shall have merged previous entries by here. 472 self.children.append(other_child_stat) 473 return self 474 475 def __hash__(self): 476 return hash(self.full_name) 477 478 def is_recursive(self): 479 # we have a known bug where call_leave not called for some thread functions(run() especially) 480 # in that case ncalls will be updated in call_enter, however nactualcall will not. This is for 481 # checking that case. 482 if self.nactualcall == 0: 483 return False 484 return self.ncall != self.nactualcall 485 486 def strip_dirs(self): 487 self.module = os.path.basename(self.module) 488 self.full_name = _func_fullname( 489 self.builtin, self.module, self.lineno, self.name 490 ) 491 return self 492 493 def _print(self, out, columns): 494 for x in sorted(columns.keys()): 495 title, size = columns[x] 496 if title == "name": 497 out.write(StatString(self.full_name).ltrim(size)) 498 out.write(" " * COLUMN_GAP) 499 elif title == "ncall": 500 if self.is_recursive(): 501 out.write( 502 StatString("%d/%d" % (self.ncall, self.nactualcall) 503 ).rtrim(size) 504 ) 505 else: 506 out.write(StatString(self.ncall).rtrim(size)) 507 out.write(" " * COLUMN_GAP) 508 elif title == "tsub": 509 out.write(StatString(_fft(self.tsub, size)).rtrim(size)) 510 out.write(" " * COLUMN_GAP) 511 elif title == "ttot": 512 out.write(StatString(_fft(self.ttot, size)).rtrim(size)) 513 out.write(" " * COLUMN_GAP) 514 elif title == "tavg": 515 out.write(StatString(_fft(self.tavg, size)).rtrim(size)) 516 out.write(LINESEP) 517 518 519class YChildFuncStat(YFuncStat): 520 """ 521 Class holding information for children function stats. 522 """ 523 _KEYS = { 524 'index': 0, 525 'ncall': 1, 526 'nactualcall': 2, 527 'ttot': 3, 528 'tsub': 4, 529 'tavg': 5, 530 'builtin': 6, 531 'full_name': 7, 532 'module': 8, 533 'lineno': 9, 534 'name': 10 535 } 536 537 def __add__(self, other): 538 if other is None: 539 return self 540 self.nactualcall += other.nactualcall 541 self.ncall += other.ncall 542 self.ttot += other.ttot 543 self.tsub += other.tsub 544 self.tavg = self.ttot / self.ncall 545 return self 546 547 548class YThreadStat(YStat): 549 """ 550 Class holding information for thread stats. 551 """ 552 _KEYS = { 553 'name': 0, 554 'id': 1, 555 'tid': 2, 556 'ttot': 3, 557 'sched_count': 4, 558 } 559 560 def __eq__(self, other): 561 if other is None: 562 return False 563 return self.id == other.id 564 565 def __ne__(self, other): 566 return not self == other 567 568 def __hash__(self, *args, **kwargs): 569 return hash(self.id) 570 571 def _print(self, out, columns): 572 for x in sorted(columns.keys()): 573 title, size = columns[x] 574 if title == "name": 575 out.write(StatString(self.name).ltrim(size)) 576 out.write(" " * COLUMN_GAP) 577 elif title == "id": 578 out.write(StatString(self.id).rtrim(size)) 579 out.write(" " * COLUMN_GAP) 580 elif title == "tid": 581 out.write(StatString(self.tid).rtrim(size)) 582 out.write(" " * COLUMN_GAP) 583 elif title == "ttot": 584 out.write(StatString(_fft(self.ttot, size)).rtrim(size)) 585 out.write(" " * COLUMN_GAP) 586 elif title == "scnt": 587 out.write(StatString(self.sched_count).rtrim(size)) 588 out.write(LINESEP) 589 590 591class YGreenletStat(YStat): 592 """ 593 Class holding information for thread stats. 594 """ 595 _KEYS = { 596 'name': 0, 597 'id': 1, 598 'ttot': 3, 599 'sched_count': 4, 600 } 601 602 def __eq__(self, other): 603 if other is None: 604 return False 605 return self.id == other.id 606 607 def __ne__(self, other): 608 return not self == other 609 610 def __hash__(self, *args, **kwargs): 611 return hash(self.id) 612 613 def _print(self, out, columns): 614 for x in sorted(columns.keys()): 615 title, size = columns[x] 616 if title == "name": 617 out.write(StatString(self.name).ltrim(size)) 618 out.write(" " * COLUMN_GAP) 619 elif title == "id": 620 out.write(StatString(self.id).rtrim(size)) 621 out.write(" " * COLUMN_GAP) 622 elif title == "ttot": 623 out.write(StatString(_fft(self.ttot, size)).rtrim(size)) 624 out.write(" " * COLUMN_GAP) 625 elif title == "scnt": 626 out.write(StatString(self.sched_count).rtrim(size)) 627 out.write(LINESEP) 628 629 630class YStats(object): 631 """ 632 Main Stats class where we collect the information from _yappi and apply the user filters. 633 """ 634 635 def __init__(self): 636 self._clock_type = None 637 self._as_dict = {} 638 self._as_list = [] 639 640 def get(self): 641 self._clock_type = _yappi.get_clock_type() 642 self.sort(DEFAULT_SORT_TYPE, DEFAULT_SORT_ORDER) 643 return self 644 645 def sort(self, sort_type, sort_order): 646 # sort case insensitive for strings 647 self._as_list.sort( 648 key=lambda stat: stat[sort_type].lower() \ 649 if isinstance(stat[sort_type], str) else stat[sort_type], 650 reverse=(sort_order == SORT_ORDERS["desc"]) 651 ) 652 return self 653 654 def clear(self): 655 del self._as_list[:] 656 self._as_dict.clear() 657 658 def empty(self): 659 return (len(self._as_list) == 0) 660 661 def __getitem__(self, key): 662 try: 663 return self._as_list[key] 664 except IndexError: 665 return None 666 667 def count(self, item): 668 return self._as_list.count(item) 669 670 def __iter__(self): 671 return iter(self._as_list) 672 673 def __len__(self): 674 return len(self._as_list) 675 676 def pop(self): 677 item = self._as_list.pop() 678 del self._as_dict[item] 679 return item 680 681 def append(self, item): 682 # increment/update the stat if we already have it 683 684 existing = self._as_dict.get(item) 685 if existing: 686 existing += item 687 return 688 self._as_list.append(item) 689 self._as_dict[item] = item 690 691 def _print_header(self, out, columns): 692 for x in sorted(columns.keys()): 693 title, size = columns[x] 694 if len(title) > size: 695 raise YappiError("Column title exceeds available length[%s:%d]" % \ 696 (title, size)) 697 out.write(title) 698 out.write(" " * (COLUMN_GAP + size - len(title))) 699 out.write(LINESEP) 700 701 def _debug_check_sanity(self): 702 """ 703 Check for basic sanity errors in stats. e.g: Check for duplicate stats. 704 """ 705 for x in self: 706 if self.count(x) > 1: 707 return False 708 return True 709 710 711class YStatsIndexable(YStats): 712 713 def __init__(self): 714 super(YStatsIndexable, self).__init__() 715 self._additional_indexing = {} 716 717 def clear(self): 718 super(YStatsIndexable, self).clear() 719 self._additional_indexing.clear() 720 721 def pop(self): 722 item = super(YStatsIndexable, self).pop() 723 self._additional_indexing.pop(item.index, None) 724 self._additional_indexing.pop(item.full_name, None) 725 return item 726 727 def append(self, item): 728 super(YStatsIndexable, self).append(item) 729 # setdefault so that we don't replace them if they're already there. 730 self._additional_indexing.setdefault(item.index, item) 731 self._additional_indexing.setdefault(item.full_name, item) 732 733 def __getitem__(self, key): 734 if isinstance(key, int): 735 # search by item.index 736 return self._additional_indexing.get(key, None) 737 elif isinstance(key, str): 738 # search by item.full_name 739 return self._additional_indexing.get(key, None) 740 elif isinstance(key, YFuncStat) or isinstance(key, YChildFuncStat): 741 return self._additional_indexing.get(key.index, None) 742 743 return super(YStatsIndexable, self).__getitem__(key) 744 745 746class YChildFuncStats(YStatsIndexable): 747 748 def sort(self, sort_type, sort_order="desc"): 749 sort_type = _validate_sorttype(sort_type, SORT_TYPES_CHILDFUNCSTATS) 750 sort_order = _validate_sortorder(sort_order) 751 752 return super(YChildFuncStats, self).sort( 753 SORT_TYPES_CHILDFUNCSTATS[sort_type], SORT_ORDERS[sort_order] 754 ) 755 756 def print_all( 757 self, 758 out=sys.stdout, 759 columns={ 760 0: ("name", 36), 761 1: ("ncall", 5), 762 2: ("tsub", 8), 763 3: ("ttot", 8), 764 4: ("tavg", 8) 765 } 766 ): 767 """ 768 Prints all of the child function profiler results to a given file. (stdout by default) 769 """ 770 if self.empty() or len(columns) == 0: 771 return 772 773 for _, col in columns.items(): 774 _validate_columns(col[0], COLUMNS_FUNCSTATS) 775 776 out.write(LINESEP) 777 self._print_header(out, columns) 778 for stat in self: 779 stat._print(out, columns) 780 781 def strip_dirs(self): 782 for stat in self: 783 stat.strip_dirs() 784 return self 785 786 787class YFuncStats(YStatsIndexable): 788 789 _idx_max = 0 790 _sort_type = None 791 _sort_order = None 792 _SUPPORTED_LOAD_FORMATS = ['YSTAT'] 793 _SUPPORTED_SAVE_FORMATS = ['YSTAT', 'CALLGRIND', 'PSTAT'] 794 795 def __init__(self, files=[]): 796 super(YFuncStats, self).__init__() 797 self.add(files) 798 799 self._filter_callback = None 800 801 def strip_dirs(self): 802 for stat in self: 803 stat.strip_dirs() 804 stat.children.strip_dirs() 805 return self 806 807 def get(self, filter={}, filter_callback=None): 808 _yappi._pause() 809 self.clear() 810 try: 811 self._filter_callback = filter_callback 812 _yappi.enum_func_stats(self._enumerator, filter) 813 self._filter_callback = None 814 815 # convert the children info from tuple to YChildFuncStat 816 for stat in self: 817 _childs = YChildFuncStats() 818 for child_tpl in stat.children: 819 rstat = self[child_tpl[0]] 820 821 # sometimes even the profile results does not contain the result because of filtering 822 # or timing(call_leave called but call_enter is not), with this we ensure that the children 823 # index always point to a valid stat. 824 if rstat is None: 825 continue 826 827 tavg = rstat.ttot / rstat.ncall 828 cfstat = YChildFuncStat( 829 child_tpl + ( 830 tavg, 831 rstat.builtin, 832 rstat.full_name, 833 rstat.module, 834 rstat.lineno, 835 rstat.name, 836 ) 837 ) 838 _childs.append(cfstat) 839 stat.children = _childs 840 result = super(YFuncStats, self).get() 841 finally: 842 _yappi._resume() 843 return result 844 845 def _enumerator(self, stat_entry): 846 global _fn_descriptor_dict 847 fname, fmodule, flineno, fncall, fnactualcall, fbuiltin, fttot, ftsub, \ 848 findex, fchildren, fctxid, fctxname, ftag, ffn_descriptor = stat_entry 849 850 # builtin function? 851 ffull_name = _func_fullname(bool(fbuiltin), fmodule, flineno, fname) 852 ftavg = fttot / fncall 853 fstat = YFuncStat(stat_entry + (ftavg, ffull_name)) 854 _fn_descriptor_dict[ffull_name] = ffn_descriptor 855 856 # do not show profile stats of yappi itself. 857 if os.path.basename( 858 fstat.module 859 ) == "yappi.py" or fstat.module == "_yappi": 860 return 861 862 fstat.builtin = bool(fstat.builtin) 863 864 if self._filter_callback: 865 if not self._filter_callback(fstat): 866 return 867 868 self.append(fstat) 869 870 # hold the max idx number for merging new entries(for making the merging 871 # entries indexes unique) 872 if self._idx_max < fstat.index: 873 self._idx_max = fstat.index 874 875 def _add_from_YSTAT(self, file): 876 try: 877 saved_stats, saved_clock_type = pickle.load(file) 878 except: 879 raise YappiError( 880 "Unable to load the saved profile information from %s." % 881 (file.name) 882 ) 883 884 # check if we really have some stats to be merged? 885 if not self.empty(): 886 if self._clock_type != saved_clock_type and self._clock_type is not None: 887 raise YappiError("Clock type mismatch between current and saved profiler sessions.[%s,%s]" % \ 888 (self._clock_type, saved_clock_type)) 889 890 self._clock_type = saved_clock_type 891 892 # add 'not present' previous entries with unique indexes 893 for saved_stat in saved_stats: 894 if saved_stat not in self: 895 self._idx_max += 1 896 saved_stat.index = self._idx_max 897 self.append(saved_stat) 898 899 # fix children's index values 900 for saved_stat in saved_stats: 901 for saved_child_stat in saved_stat.children: 902 # we know for sure child's index is pointing to a valid stat in saved_stats 903 # so as saved_stat is already in sync. (in above loop), we can safely assume 904 # that we shall point to a valid stat in current_stats with the child's full_name 905 saved_child_stat.index = self[saved_child_stat.full_name].index 906 907 # merge stats 908 for saved_stat in saved_stats: 909 saved_stat_in_curr = self[saved_stat.full_name] 910 saved_stat_in_curr += saved_stat 911 912 def _save_as_YSTAT(self, path): 913 with open(path, "wb") as f: 914 pickle.dump((self, self._clock_type), f, YPICKLE_PROTOCOL) 915 916 def _save_as_PSTAT(self, path): 917 """ 918 Save the profiling information as PSTAT. 919 """ 920 _stats = convert2pstats(self) 921 _stats.dump_stats(path) 922 923 def _save_as_CALLGRIND(self, path): 924 """ 925 Writes all the function stats in a callgrind-style format to the given 926 file. (stdout by default) 927 """ 928 header = """version: 1\ncreator: %s\npid: %d\ncmd: %s\npart: 1\n\nevents: Ticks""" % \ 929 ('yappi', os.getpid(), ' '.join(sys.argv)) 930 931 lines = [header] 932 933 # add function definitions 934 file_ids = [''] 935 func_ids = [''] 936 for func_stat in self: 937 file_ids += ['fl=(%d) %s' % (func_stat.index, func_stat.module)] 938 func_ids += [ 939 'fn=(%d) %s %s:%s' % ( 940 func_stat.index, func_stat.name, func_stat.module, 941 func_stat.lineno 942 ) 943 ] 944 945 lines += file_ids + func_ids 946 947 # add stats for each function we have a record of 948 for func_stat in self: 949 func_stats = [ 950 '', 951 'fl=(%d)' % func_stat.index, 952 'fn=(%d)' % func_stat.index 953 ] 954 func_stats += [ 955 '%s %s' % (func_stat.lineno, int(func_stat.tsub * 1e6)) 956 ] 957 958 # children functions stats 959 for child in func_stat.children: 960 func_stats += [ 961 'cfl=(%d)' % child.index, 962 'cfn=(%d)' % child.index, 963 'calls=%d 0' % child.ncall, 964 '0 %d' % int(child.ttot * 1e6) 965 ] 966 lines += func_stats 967 968 with open(path, "w") as f: 969 f.write('\n'.join(lines)) 970 971 def add(self, files, type="ystat"): 972 type = type.upper() 973 if type not in self._SUPPORTED_LOAD_FORMATS: 974 raise NotImplementedError( 975 'Loading from (%s) format is not possible currently.' 976 ) 977 if isinstance(files, str): 978 files = [ 979 files, 980 ] 981 for fd in files: 982 with open(fd, "rb") as f: 983 add_func = getattr(self, "_add_from_%s" % (type)) 984 add_func(file=f) 985 986 return self.sort(DEFAULT_SORT_TYPE, DEFAULT_SORT_ORDER) 987 988 def save(self, path, type="ystat"): 989 type = type.upper() 990 if type not in self._SUPPORTED_SAVE_FORMATS: 991 raise NotImplementedError( 992 'Saving in "%s" format is not possible currently.' % (type) 993 ) 994 995 save_func = getattr(self, "_save_as_%s" % (type)) 996 save_func(path=path) 997 998 def print_all( 999 self, 1000 out=sys.stdout, 1001 columns={ 1002 0: ("name", 36), 1003 1: ("ncall", 5), 1004 2: ("tsub", 8), 1005 3: ("ttot", 8), 1006 4: ("tavg", 8) 1007 } 1008 ): 1009 """ 1010 Prints all of the function profiler results to a given file. (stdout by default) 1011 """ 1012 if self.empty(): 1013 return 1014 1015 for _, col in columns.items(): 1016 _validate_columns(col[0], COLUMNS_FUNCSTATS) 1017 1018 out.write(LINESEP) 1019 out.write("Clock type: %s" % (self._clock_type.upper())) 1020 out.write(LINESEP) 1021 out.write("Ordered by: %s, %s" % (self._sort_type, self._sort_order)) 1022 out.write(LINESEP) 1023 out.write(LINESEP) 1024 1025 self._print_header(out, columns) 1026 for stat in self: 1027 stat._print(out, columns) 1028 1029 def sort(self, sort_type, sort_order="desc"): 1030 sort_type = _validate_sorttype(sort_type, SORT_TYPES_FUNCSTATS) 1031 sort_order = _validate_sortorder(sort_order) 1032 1033 self._sort_type = sort_type 1034 self._sort_order = sort_order 1035 1036 return super(YFuncStats, self).sort( 1037 SORT_TYPES_FUNCSTATS[sort_type], SORT_ORDERS[sort_order] 1038 ) 1039 1040 def debug_print(self): 1041 if self.empty(): 1042 return 1043 1044 console = sys.stdout 1045 CHILD_STATS_LEFT_MARGIN = 5 1046 for stat in self: 1047 console.write("index: %d" % stat.index) 1048 console.write(LINESEP) 1049 console.write("full_name: %s" % stat.full_name) 1050 console.write(LINESEP) 1051 console.write("ncall: %d/%d" % (stat.ncall, stat.nactualcall)) 1052 console.write(LINESEP) 1053 console.write("ttot: %s" % _fft(stat.ttot)) 1054 console.write(LINESEP) 1055 console.write("tsub: %s" % _fft(stat.tsub)) 1056 console.write(LINESEP) 1057 console.write("children: ") 1058 console.write(LINESEP) 1059 for child_stat in stat.children: 1060 console.write(LINESEP) 1061 console.write(" " * CHILD_STATS_LEFT_MARGIN) 1062 console.write("index: %d" % child_stat.index) 1063 console.write(LINESEP) 1064 console.write(" " * CHILD_STATS_LEFT_MARGIN) 1065 console.write("child_full_name: %s" % child_stat.full_name) 1066 console.write(LINESEP) 1067 console.write(" " * CHILD_STATS_LEFT_MARGIN) 1068 console.write( 1069 "ncall: %d/%d" % (child_stat.ncall, child_stat.nactualcall) 1070 ) 1071 console.write(LINESEP) 1072 console.write(" " * CHILD_STATS_LEFT_MARGIN) 1073 console.write("ttot: %s" % _fft(child_stat.ttot)) 1074 console.write(LINESEP) 1075 console.write(" " * CHILD_STATS_LEFT_MARGIN) 1076 console.write("tsub: %s" % _fft(child_stat.tsub)) 1077 console.write(LINESEP) 1078 console.write(LINESEP) 1079 1080 1081class _YContextStats(YStats): 1082 1083 _BACKEND = None 1084 _STAT_CLASS = None 1085 _SORT_TYPES = None 1086 _DEFAULT_PRINT_COLUMNS = None 1087 _ALL_COLUMNS = None 1088 1089 def get(self): 1090 1091 backend = _yappi.get_context_backend() 1092 if self._BACKEND != backend: 1093 raise YappiError( 1094 "Cannot retrieve stats for '%s' when backend is set as '%s'" % 1095 (self._BACKEND.lower(), backend.lower()) 1096 ) 1097 1098 _yappi._pause() 1099 self.clear() 1100 try: 1101 _yappi.enum_context_stats(self._enumerator) 1102 result = super(_YContextStats, self).get() 1103 finally: 1104 _yappi._resume() 1105 return result 1106 1107 def _enumerator(self, stat_entry): 1108 tstat = self._STAT_CLASS(stat_entry) 1109 self.append(tstat) 1110 1111 def sort(self, sort_type, sort_order="desc"): 1112 sort_type = _validate_sorttype(sort_type, self._SORT_TYPES) 1113 sort_order = _validate_sortorder(sort_order) 1114 1115 return super(_YContextStats, self).sort( 1116 self._SORT_TYPES[sort_type], SORT_ORDERS[sort_order] 1117 ) 1118 1119 def print_all(self, out=sys.stdout, columns=None): 1120 """ 1121 Prints all of the thread profiler results to a given file. (stdout by default) 1122 """ 1123 1124 if columns is None: 1125 columns = self._DEFAULT_PRINT_COLUMNS 1126 1127 if self.empty(): 1128 return 1129 1130 for _, col in columns.items(): 1131 _validate_columns(col[0], self._ALL_COLUMNS) 1132 1133 out.write(LINESEP) 1134 self._print_header(out, columns) 1135 for stat in self: 1136 stat._print(out, columns) 1137 1138 def strip_dirs(self): 1139 pass # do nothing 1140 1141 1142class YThreadStats(_YContextStats): 1143 _BACKEND = NATIVE_THREAD 1144 _STAT_CLASS = YThreadStat 1145 _SORT_TYPES = { 1146 "name": 0, 1147 "id": 1, 1148 "tid": 2, 1149 "totaltime": 3, 1150 "schedcount": 4, 1151 "ttot": 3, 1152 "scnt": 4 1153 } 1154 _DEFAULT_PRINT_COLUMNS = { 1155 0: ("name", 13), 1156 1: ("id", 5), 1157 2: ("tid", 15), 1158 3: ("ttot", 8), 1159 4: ("scnt", 10) 1160 } 1161 _ALL_COLUMNS = ["name", "id", "tid", "ttot", "scnt"] 1162 1163 1164class YGreenletStats(_YContextStats): 1165 _BACKEND = GREENLET 1166 _STAT_CLASS = YGreenletStat 1167 _SORT_TYPES = { 1168 "name": 0, 1169 "id": 1, 1170 "totaltime": 3, 1171 "schedcount": 4, 1172 "ttot": 3, 1173 "scnt": 4 1174 } 1175 _DEFAULT_PRINT_COLUMNS = { 1176 0: ("name", 13), 1177 1: ("id", 5), 1178 2: ("ttot", 8), 1179 3: ("scnt", 10) 1180 } 1181 _ALL_COLUMNS = ["name", "id", "ttot", "scnt"] 1182 1183 1184def is_running(): 1185 """ 1186 Returns true if the profiler is running, false otherwise. 1187 """ 1188 return bool(_yappi.is_running()) 1189 1190 1191def start(builtins=False, profile_threads=True, profile_greenlets=True): 1192 """ 1193 Start profiler. 1194 1195 profile_threads: Set to True to profile multiple threads. Set to false 1196 to profile only the invoking thread. This argument is only respected when 1197 context backend is 'native_thread' and ignored otherwise. 1198 1199 profile_greenlets: Set to True to to profile multiple greenlets. Set to 1200 False to profile only the invoking greenlet. This argument is only respected 1201 when context backend is 'greenlet' and ignored otherwise. 1202 """ 1203 backend = _yappi.get_context_backend() 1204 profile_contexts = ( 1205 (profile_threads and backend == NATIVE_THREAD) 1206 or (profile_greenlets and backend == GREENLET) 1207 ) 1208 if profile_contexts: 1209 threading.setprofile(_profile_thread_callback) 1210 _yappi.start(builtins, profile_contexts) 1211 1212 1213def get_func_stats(tag=None, ctx_id=None, filter=None, filter_callback=None): 1214 """ 1215 Gets the function profiler results with given filters and returns an iterable. 1216 1217 filter: is here mainly for backward compat. we will not document it anymore. 1218 tag, ctx_id: select given tag and ctx_id related stats in C side. 1219 filter_callback: we could do it like: get_func_stats().filter(). The problem 1220 with this approach is YFuncStats has an internal list which complicates: 1221 - delete() operation because list deletions are O(n) 1222 - sort() and pop() operations currently work on sorted list and they hold the 1223 list as sorted. 1224 To preserve above behaviour and have a delete() method, we can use an OrderedDict() 1225 maybe, but simply that is not worth the effort for an extra filter() call. Maybe 1226 in the future. 1227 """ 1228 if not filter: 1229 filter = {} 1230 1231 if tag: 1232 filter['tag'] = tag 1233 if ctx_id: 1234 filter['ctx_id'] = ctx_id 1235 1236 # multiple invocation pause/resume is allowed. This is needed because 1237 # not only get() is executed here. 1238 _yappi._pause() 1239 try: 1240 stats = YFuncStats().get(filter=filter, filter_callback=filter_callback) 1241 finally: 1242 _yappi._resume() 1243 return stats 1244 1245 1246def get_thread_stats(): 1247 """ 1248 Gets the thread profiler results with given filters and returns an iterable. 1249 """ 1250 return YThreadStats().get() 1251 1252 1253def get_greenlet_stats(): 1254 """ 1255 Gets the greenlet stats captured by the profiler 1256 """ 1257 return YGreenletStats().get() 1258 1259 1260def stop(): 1261 """ 1262 Stop profiler. 1263 """ 1264 _yappi.stop() 1265 threading.setprofile(None) 1266 1267 1268@contextmanager 1269def run(builtins=False, profile_threads=True, profile_greenlets=True): 1270 """ 1271 Context manger for profiling block of code. 1272 1273 Starts profiling before entering the context, and stop profilying when 1274 exiting from the context. 1275 1276 Usage: 1277 1278 with yappi.run(): 1279 print("this call is profiled") 1280 1281 Warning: don't use this recursively, the inner context will stop profiling 1282 when exited: 1283 1284 with yappi.run(): 1285 with yappi.run(): 1286 print("this call will be profiled") 1287 print("this call will *not* be profiled") 1288 """ 1289 start( 1290 builtins=builtins, 1291 profile_threads=profile_threads, 1292 profile_greenlets=profile_greenlets 1293 ) 1294 try: 1295 yield 1296 finally: 1297 stop() 1298 1299 1300def clear_stats(): 1301 """ 1302 Clears all of the profile results. 1303 """ 1304 _yappi._pause() 1305 try: 1306 _yappi.clear_stats() 1307 finally: 1308 _yappi._resume() 1309 1310 1311def get_clock_time(): 1312 """ 1313 Returns the current clock time with regard to current clock type. 1314 """ 1315 return _yappi.get_clock_time() 1316 1317 1318def get_clock_type(): 1319 """ 1320 Returns the underlying clock type 1321 """ 1322 return _yappi.get_clock_type() 1323 1324 1325def get_clock_info(): 1326 """ 1327 Returns a dict containing the OS API used for timing, the precision of the 1328 underlying clock. 1329 """ 1330 return _yappi.get_clock_info() 1331 1332 1333def set_clock_type(type): 1334 """ 1335 Sets the internal clock type for timing. Profiler shall not have any previous stats. 1336 Otherwise an exception is thrown. 1337 """ 1338 type = type.upper() 1339 if type not in CLOCK_TYPES: 1340 raise YappiError("Invalid clock type:%s" % (type)) 1341 1342 _yappi.set_clock_type(CLOCK_TYPES[type]) 1343 1344 1345def get_mem_usage(): 1346 """ 1347 Returns the internal memory usage of the profiler itself. 1348 """ 1349 return _yappi.get_mem_usage() 1350 1351 1352def set_tag_callback(cbk): 1353 """ 1354 Every stat. entry will have a specific tag field and users might be able 1355 to filter on stats via tag field. 1356 """ 1357 return _yappi.set_tag_callback(cbk) 1358 1359 1360def set_context_backend(type): 1361 """ 1362 Sets the internal context backend used to track execution context. 1363 1364 type must be one of 'greenlet' or 'native_thread'. For example: 1365 1366 >>> import greenlet, yappi 1367 >>> yappi.set_context_backend("greenlet") 1368 1369 Setting the context backend will reset any callbacks configured via: 1370 - set_context_id_callback 1371 - set_context_name_callback 1372 1373 The default callbacks for the backend provided will be installed instead. 1374 Configure the callbacks each time after setting context backend. 1375 """ 1376 type = type.upper() 1377 if type not in BACKEND_TYPES: 1378 raise YappiError("Invalid backend type: %s" % (type)) 1379 1380 if type == GREENLET: 1381 id_cbk, name_cbk = _create_greenlet_callbacks() 1382 _yappi.set_context_id_callback(id_cbk) 1383 set_context_name_callback(name_cbk) 1384 else: 1385 _yappi.set_context_id_callback(None) 1386 set_context_name_callback(None) 1387 1388 _yappi.set_context_backend(BACKEND_TYPES[type]) 1389 1390 1391def set_context_id_callback(callback): 1392 """ 1393 Use a number other than thread_id to determine the current context. 1394 1395 The callback must take no arguments and return an integer. For example: 1396 1397 >>> import greenlet, yappi 1398 >>> yappi.set_context_id_callback(lambda: id(greenlet.getcurrent())) 1399 """ 1400 return _yappi.set_context_id_callback(callback) 1401 1402 1403def set_context_name_callback(callback): 1404 """ 1405 Set the callback to retrieve current context's name. 1406 1407 The callback must take no arguments and return a string. For example: 1408 1409 >>> import greenlet, yappi 1410 >>> yappi.set_context_name_callback( 1411 ... lambda: greenlet.getcurrent().__class__.__name__) 1412 1413 If the callback cannot return the name at this time but may be able to 1414 return it later, it should return None. 1415 """ 1416 if callback is None: 1417 return _yappi.set_context_name_callback(_ctx_name_callback) 1418 return _yappi.set_context_name_callback(callback) 1419 1420 1421# set _ctx_name_callback by default at import time. 1422set_context_name_callback(None) 1423 1424 1425def main(): 1426 from optparse import OptionParser 1427 usage = "%s [-b] [-c clock_type] [-o output_file] [-f output_format] [-s] [scriptfile] args ..." % os.path.basename( 1428 sys.argv[0] 1429 ) 1430 parser = OptionParser(usage=usage) 1431 parser.allow_interspersed_args = False 1432 parser.add_option( 1433 "-c", 1434 "--clock-type", 1435 default="cpu", 1436 choices=sorted(c.lower() for c in CLOCK_TYPES), 1437 metavar="clock_type", 1438 help="Clock type to use during profiling" 1439 "(\"cpu\" or \"wall\", default is \"cpu\")." 1440 ) 1441 parser.add_option( 1442 "-b", 1443 "--builtins", 1444 action="store_true", 1445 dest="profile_builtins", 1446 default=False, 1447 help="Profiles builtin functions when set. [default: False]" 1448 ) 1449 parser.add_option( 1450 "-o", 1451 "--output-file", 1452 metavar="output_file", 1453 help="Write stats to output_file." 1454 ) 1455 parser.add_option( 1456 "-f", 1457 "--output-format", 1458 default="pstat", 1459 choices=("pstat", "callgrind", "ystat"), 1460 metavar="output_format", 1461 help="Write stats in the specified" 1462 "format (\"pstat\", \"callgrind\" or \"ystat\", default is " 1463 "\"pstat\")." 1464 ) 1465 parser.add_option( 1466 "-s", 1467 "--single_thread", 1468 action="store_true", 1469 dest="profile_single_thread", 1470 default=False, 1471 help="Profiles only the thread that calls start(). [default: False]" 1472 ) 1473 if not sys.argv[1:]: 1474 parser.print_usage() 1475 sys.exit(2) 1476 1477 (options, args) = parser.parse_args() 1478 sys.argv[:] = args 1479 1480 if (len(sys.argv) > 0): 1481 sys.path.insert(0, os.path.dirname(sys.argv[0])) 1482 set_clock_type(options.clock_type) 1483 start(options.profile_builtins, not options.profile_single_thread) 1484 try: 1485 if sys.version_info >= (3, 0): 1486 exec( 1487 compile(open(sys.argv[0]).read(), sys.argv[0], 'exec'), 1488 sys._getframe(1).f_globals, 1489 sys._getframe(1).f_locals 1490 ) 1491 else: 1492 execfile( 1493 sys.argv[0], 1494 sys._getframe(1).f_globals, 1495 sys._getframe(1).f_locals 1496 ) 1497 finally: 1498 stop() 1499 if options.output_file: 1500 stats = get_func_stats() 1501 stats.save(options.output_file, options.output_format) 1502 else: 1503 # we will currently use default params for these 1504 get_func_stats().print_all() 1505 get_thread_stats().print_all() 1506 else: 1507 parser.print_usage() 1508 1509 1510if __name__ == "__main__": 1511 main() 1512