1#!/usr/bin/env python 2# 3# scons-time - run SCons timings and collect statistics 4# 5# A script for running a configuration through SCons with a standard 6# set of invocations to collect timing and memory statistics and to 7# capture the results in a consistent set of output files for display 8# and analysis. 9# 10 11# 12# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 The SCons Foundation 13# 14# Permission is hereby granted, free of charge, to any person obtaining 15# a copy of this software and associated documentation files (the 16# "Software"), to deal in the Software without restriction, including 17# without limitation the rights to use, copy, modify, merge, publish, 18# distribute, sublicense, and/or sell copies of the Software, and to 19# permit persons to whom the Software is furnished to do so, subject to 20# the following conditions: 21# 22# The above copyright notice and this permission notice shall be included 23# in all copies or substantial portions of the Software. 24# 25# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 26# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 27# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 29# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 30# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 31# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 32# 33 34from __future__ import nested_scopes 35from __future__ import print_function 36 37__revision__ = "src/script/scons-time.py 4369 2009/09/19 15:58:29 scons" 38 39import getopt 40import glob 41import os 42import os.path 43import re 44import shutil 45import string 46import sys 47import tempfile 48import time 49 50 51def make_temp_file(**kw): 52 try: 53 result = tempfile.mktemp(**kw) 54 try: 55 result = os.path.realpath(result) 56 except AttributeError: 57 # Python 2.1 has no os.path.realpath() method. 58 pass 59 except TypeError: 60 try: 61 save_template = tempfile.template 62 prefix = kw['prefix'] 63 del kw['prefix'] 64 tempfile.template = prefix 65 result = tempfile.mktemp(**kw) 66 finally: 67 tempfile.template = save_template 68 return result 69 70def HACK_for_exec(cmd, *args): 71 ''' 72 For some reason, Python won't allow an exec() within a function 73 that also declares an internal function (including lambda functions). 74 This function is a hack that calls exec() in a function with no 75 internal functions. 76 ''' 77 if not args: exec(cmd) 78 elif len(args) == 1: exec(cmd, args[0]) 79 else: exec(cmd, args[0], args[1]) 80 81class Plotter: 82 def increment_size(self, largest): 83 """ 84 Return the size of each horizontal increment line for a specified 85 maximum value. This returns a value that will provide somewhere 86 between 5 and 9 horizontal lines on the graph, on some set of 87 boundaries that are multiples of 10/100/1000/etc. 88 """ 89 i = largest / 5 90 if not i: 91 return largest 92 multiplier = 1 93 while i >= 10: 94 i = i / 10 95 multiplier = multiplier * 10 96 return i * multiplier 97 98 def max_graph_value(self, largest): 99 # Round up to next integer. 100 largest = int(largest) + 1 101 increment = self.increment_size(largest) 102 return ((largest + increment - 1) / increment) * increment 103 104class Line: 105 def __init__(self, points, type, title, label, comment, fmt="%s %s"): 106 self.points = points 107 self.type = type 108 self.title = title 109 self.label = label 110 self.comment = comment 111 self.fmt = fmt 112 113 def print_label(self, inx, x, y): 114 if self.label: 115 print('set label %s "%s" at %s,%s right' % (inx, self.label, x, y)) 116 117 def plot_string(self): 118 if self.title: 119 title_string = 'title "%s"' % self.title 120 else: 121 title_string = 'notitle' 122 return "'-' %s with lines lt %s" % (title_string, self.type) 123 124 def print_points(self, fmt=None): 125 if fmt is None: 126 fmt = self.fmt 127 if self.comment: 128 print('# %s' % self.comment) 129 for x, y in self.points: 130 # If y is None, it usually represents some kind of break 131 # in the line's index number. We might want to represent 132 # this some way rather than just drawing the line straight 133 # between the two points on either side. 134 if not y is None: 135 print(fmt % (x, y)) 136 print('e') 137 138 def get_x_values(self): 139 return [ p[0] for p in self.points ] 140 141 def get_y_values(self): 142 return [ p[1] for p in self.points ] 143 144class Gnuplotter(Plotter): 145 146 def __init__(self, title, key_location): 147 self.lines = [] 148 self.title = title 149 self.key_location = key_location 150 151 def line(self, points, type, title=None, label=None, comment=None, fmt='%s %s'): 152 if points: 153 line = Line(points, type, title, label, comment, fmt) 154 self.lines.append(line) 155 156 def plot_string(self, line): 157 return line.plot_string() 158 159 def vertical_bar(self, x, type, label, comment): 160 if self.get_min_x() <= x and x <= self.get_max_x(): 161 points = [(x, 0), (x, self.max_graph_value(self.get_max_y()))] 162 self.line(points, type, label, comment) 163 164 def get_all_x_values(self): 165 result = [] 166 for line in self.lines: 167 result.extend(line.get_x_values()) 168 return filter(lambda r: not r is None, result) 169 170 def get_all_y_values(self): 171 result = [] 172 for line in self.lines: 173 result.extend(line.get_y_values()) 174 return filter(lambda r: not r is None, result) 175 176 def get_min_x(self): 177 try: 178 return self.min_x 179 except AttributeError: 180 try: 181 self.min_x = min(self.get_all_x_values()) 182 except ValueError: 183 self.min_x = 0 184 return self.min_x 185 186 def get_max_x(self): 187 try: 188 return self.max_x 189 except AttributeError: 190 try: 191 self.max_x = max(self.get_all_x_values()) 192 except ValueError: 193 self.max_x = 0 194 return self.max_x 195 196 def get_min_y(self): 197 try: 198 return self.min_y 199 except AttributeError: 200 try: 201 self.min_y = min(self.get_all_y_values()) 202 except ValueError: 203 self.min_y = 0 204 return self.min_y 205 206 def get_max_y(self): 207 try: 208 return self.max_y 209 except AttributeError: 210 try: 211 self.max_y = max(self.get_all_y_values()) 212 except ValueError: 213 self.max_y = 0 214 return self.max_y 215 216 def draw(self): 217 218 if not self.lines: 219 return 220 221 if self.title: 222 print('set title "%s"' % self.title) 223 print('set key %s' % self.key_location) 224 225 min_y = self.get_min_y() 226 max_y = self.max_graph_value(self.get_max_y()) 227 range = max_y - min_y 228 incr = range / 10.0 229 start = min_y + (max_y / 2.0) + (2.0 * incr) 230 position = [ start - (i * incr) for i in xrange(5) ] 231 232 inx = 1 233 for line in self.lines: 234 line.print_label(inx, line.points[0][0]-1, 235 position[(inx-1) % len(position)]) 236 inx += 1 237 238 plot_strings = [ self.plot_string(l) for l in self.lines ] 239 print('plot ' + ', \\\n '.join(plot_strings)) 240 241 for line in self.lines: 242 line.print_points() 243 244 245 246def untar(fname): 247 import tarfile 248 tar = tarfile.open(name=fname, mode='r') 249 for tarinfo in tar: 250 tar.extract(tarinfo) 251 tar.close() 252 253def unzip(fname): 254 import zipfile 255 zf = zipfile.ZipFile(fname, 'r') 256 for name in zf.namelist(): 257 dir = os.path.dirname(name) 258 try: 259 os.makedirs(dir) 260 except: 261 pass 262 open(name, 'w').write(zf.read(name)) 263 264def read_tree(dir): 265 def read_files(arg, dirname, fnames): 266 for fn in fnames: 267 fn = os.path.join(dirname, fn) 268 if os.path.isfile(fn): 269 open(fn, 'rb').read() 270 os.path.walk('.', read_files, None) 271 272def redirect_to_file(command, log): 273 return '%s > %s 2>&1' % (command, log) 274 275def tee_to_file(command, log): 276 return '%s 2>&1 | tee %s' % (command, log) 277 278 279 280class SConsTimer: 281 """ 282 Usage: scons-time SUBCOMMAND [ARGUMENTS] 283 Type "scons-time help SUBCOMMAND" for help on a specific subcommand. 284 285 Available subcommands: 286 func Extract test-run data for a function 287 help Provides help 288 mem Extract --debug=memory data from test runs 289 obj Extract --debug=count data from test runs 290 time Extract --debug=time data from test runs 291 run Runs a test configuration 292 """ 293 294 name = 'scons-time' 295 name_spaces = ' '*len(name) 296 297 def makedict(**kw): 298 return kw 299 300 default_settings = makedict( 301 aegis = 'aegis', 302 aegis_project = None, 303 chdir = None, 304 config_file = None, 305 initial_commands = [], 306 key_location = 'bottom left', 307 orig_cwd = os.getcwd(), 308 outdir = None, 309 prefix = '', 310 python = '"%s"' % sys.executable, 311 redirect = redirect_to_file, 312 scons = None, 313 scons_flags = '--debug=count --debug=memory --debug=time --debug=memoizer', 314 scons_lib_dir = None, 315 scons_wrapper = None, 316 startup_targets = '--help', 317 subdir = None, 318 subversion_url = None, 319 svn = 'svn', 320 svn_co_flag = '-q', 321 tar = 'tar', 322 targets = '', 323 targets0 = None, 324 targets1 = None, 325 targets2 = None, 326 title = None, 327 unzip = 'unzip', 328 verbose = False, 329 vertical_bars = [], 330 331 unpack_map = { 332 '.tar.gz' : (untar, '%(tar)s xzf %%s'), 333 '.tgz' : (untar, '%(tar)s xzf %%s'), 334 '.tar' : (untar, '%(tar)s xf %%s'), 335 '.zip' : (unzip, '%(unzip)s %%s'), 336 }, 337 ) 338 339 run_titles = [ 340 'Startup', 341 'Full build', 342 'Up-to-date build', 343 ] 344 345 run_commands = [ 346 '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof0)s %(targets0)s', 347 '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof1)s %(targets1)s', 348 '%(python)s %(scons_wrapper)s %(scons_flags)s --profile=%(prof2)s %(targets2)s', 349 ] 350 351 stages = [ 352 'pre-read', 353 'post-read', 354 'pre-build', 355 'post-build', 356 ] 357 358 stage_strings = { 359 'pre-read' : 'Memory before reading SConscript files:', 360 'post-read' : 'Memory after reading SConscript files:', 361 'pre-build' : 'Memory before building targets:', 362 'post-build' : 'Memory after building targets:', 363 } 364 365 memory_string_all = 'Memory ' 366 367 default_stage = stages[-1] 368 369 time_strings = { 370 'total' : 'Total build time', 371 'SConscripts' : 'Total SConscript file execution time', 372 'SCons' : 'Total SCons execution time', 373 'commands' : 'Total command execution time', 374 } 375 376 time_string_all = 'Total .* time' 377 378 # 379 380 def __init__(self): 381 self.__dict__.update(self.default_settings) 382 383 # Functions for displaying and executing commands. 384 385 def subst(self, x, dictionary): 386 try: 387 return x % dictionary 388 except TypeError: 389 # x isn't a string (it's probably a Python function), 390 # so just return it. 391 return x 392 393 def subst_variables(self, command, dictionary): 394 """ 395 Substitutes (via the format operator) the values in the specified 396 dictionary into the specified command. 397 398 The command can be an (action, string) tuple. In all cases, we 399 perform substitution on strings and don't worry if something isn't 400 a string. (It's probably a Python function to be executed.) 401 """ 402 try: 403 command + '' 404 except TypeError: 405 action = command[0] 406 string = command[1] 407 args = command[2:] 408 else: 409 action = command 410 string = action 411 args = (()) 412 action = self.subst(action, dictionary) 413 string = self.subst(string, dictionary) 414 return (action, string, args) 415 416 def _do_not_display(self, msg, *args): 417 pass 418 419 def display(self, msg, *args): 420 """ 421 Displays the specified message. 422 423 Each message is prepended with a standard prefix of our name 424 plus the time. 425 """ 426 if callable(msg): 427 msg = msg(*args) 428 else: 429 msg = msg % args 430 if msg is None: 431 return 432 fmt = '%s[%s]: %s\n' 433 sys.stdout.write(fmt % (self.name, time.strftime('%H:%M:%S'), msg)) 434 435 def _do_not_execute(self, action, *args): 436 pass 437 438 def execute(self, action, *args): 439 """ 440 Executes the specified action. 441 442 The action is called if it's a callable Python function, and 443 otherwise passed to os.system(). 444 """ 445 if callable(action): 446 action(*args) 447 else: 448 os.system(action % args) 449 450 def run_command_list(self, commands, dict): 451 """ 452 Executes a list of commands, substituting values from the 453 specified dictionary. 454 """ 455 commands = [ self.subst_variables(c, dict) for c in commands ] 456 for action, string, args in commands: 457 self.display(string, *args) 458 sys.stdout.flush() 459 status = self.execute(action, *args) 460 if status: 461 sys.exit(status) 462 463 def log_display(self, command, log): 464 command = self.subst(command, self.__dict__) 465 if log: 466 command = self.redirect(command, log) 467 return command 468 469 def log_execute(self, command, log): 470 command = self.subst(command, self.__dict__) 471 output = os.popen(command).read() 472 if self.verbose: 473 sys.stdout.write(output) 474 open(log, 'wb').write(output) 475 476 # 477 478 def archive_splitext(self, path): 479 """ 480 Splits an archive name into a filename base and extension. 481 482 This is like os.path.splitext() (which it calls) except that it 483 also looks for '.tar.gz' and treats it as an atomic extensions. 484 """ 485 if path.endswith('.tar.gz'): 486 return path[:-7], path[-7:] 487 else: 488 return os.path.splitext(path) 489 490 def args_to_files(self, args, tail=None): 491 """ 492 Takes a list of arguments, expands any glob patterns, and 493 returns the last "tail" files from the list. 494 """ 495 files = [] 496 for a in args: 497 g = sorted(glob.glob(a)) 498 files.extend(g) 499 500 if tail: 501 files = files[-tail:] 502 503 return files 504 505 def ascii_table(self, files, columns, 506 line_function, file_function=lambda x: x, 507 *args, **kw): 508 509 header_fmt = ' '.join(['%12s'] * len(columns)) 510 line_fmt = header_fmt + ' %s' 511 512 print(header_fmt % columns) 513 514 for file in files: 515 t = line_function(file, *args, **kw) 516 if t is None: 517 t = [] 518 diff = len(columns) - len(t) 519 if diff > 0: 520 t += [''] * diff 521 t.append(file_function(file)) 522 print(line_fmt % tuple(t)) 523 524 def collect_results(self, files, function, *args, **kw): 525 results = {} 526 527 for file in files: 528 base = os.path.splitext(file)[0] 529 run, index = string.split(base, '-')[-2:] 530 531 run = int(run) 532 index = int(index) 533 534 value = function(file, *args, **kw) 535 536 try: 537 r = results[index] 538 except KeyError: 539 r = [] 540 results[index] = r 541 r.append((run, value)) 542 543 return results 544 545 def doc_to_help(self, obj): 546 """ 547 Translates an object's __doc__ string into help text. 548 549 This strips a consistent number of spaces from each line in the 550 help text, essentially "outdenting" the text to the left-most 551 column. 552 """ 553 doc = obj.__doc__ 554 if doc is None: 555 return '' 556 return self.outdent(doc) 557 558 def find_next_run_number(self, dir, prefix): 559 """ 560 Returns the next run number in a directory for the specified prefix. 561 562 Examines the contents the specified directory for files with the 563 specified prefix, extracts the run numbers from each file name, 564 and returns the next run number after the largest it finds. 565 """ 566 x = re.compile(re.escape(prefix) + '-([0-9]+).*') 567 matches = map(lambda e, x=x: x.match(e), os.listdir(dir)) 568 matches = filter(None, matches) 569 if not matches: 570 return 0 571 run_numbers = map(lambda m: int(m.group(1)), matches) 572 return int(max(run_numbers)) + 1 573 574 def gnuplot_results(self, results, fmt='%s %.3f'): 575 """ 576 Prints out a set of results in Gnuplot format. 577 """ 578 gp = Gnuplotter(self.title, self.key_location) 579 580 indices = sorted(results.keys()) 581 582 for i in indices: 583 try: 584 t = self.run_titles[i] 585 except IndexError: 586 t = '??? %s ???' % i 587 results[i].sort() 588 gp.line(results[i], i+1, t, None, t, fmt=fmt) 589 590 for bar_tuple in self.vertical_bars: 591 try: 592 x, type, label, comment = bar_tuple 593 except ValueError: 594 x, type, label = bar_tuple 595 comment = label 596 gp.vertical_bar(x, type, label, comment) 597 598 gp.draw() 599 600 def logfile_name(self, invocation): 601 """ 602 Returns the absolute path of a log file for the specified 603 invocation number. 604 """ 605 name = self.prefix_run + '-%d.log' % invocation 606 return os.path.join(self.outdir, name) 607 608 def outdent(self, s): 609 """ 610 Strip as many spaces from each line as are found at the beginning 611 of the first line in the list. 612 """ 613 lines = s.split('\n') 614 if lines[0] == '': 615 lines = lines[1:] 616 spaces = re.match(' *', lines[0]).group(0) 617 def strip_initial_spaces(l, s=spaces): 618 if l.startswith(spaces): 619 l = l[len(spaces):] 620 return l 621 return '\n'.join([ strip_initial_spaces(l) for l in lines ]) + '\n' 622 623 def profile_name(self, invocation): 624 """ 625 Returns the absolute path of a profile file for the specified 626 invocation number. 627 """ 628 name = self.prefix_run + '-%d.prof' % invocation 629 return os.path.join(self.outdir, name) 630 631 def set_env(self, key, value): 632 os.environ[key] = value 633 634 # 635 636 def get_debug_times(self, file, time_string=None): 637 """ 638 Fetch times from the --debug=time strings in the specified file. 639 """ 640 if time_string is None: 641 search_string = self.time_string_all 642 else: 643 search_string = time_string 644 contents = open(file).read() 645 if not contents: 646 sys.stderr.write('file %s has no contents!\n' % repr(file)) 647 return None 648 result = re.findall(r'%s: ([\d\.]*)' % search_string, contents)[-4:] 649 result = [ float(r) for r in result ] 650 if not time_string is None: 651 try: 652 result = result[0] 653 except IndexError: 654 sys.stderr.write('file %s has no results!\n' % repr(file)) 655 return None 656 return result 657 658 def get_function_profile(self, file, function): 659 """ 660 Returns the file, line number, function name, and cumulative time. 661 """ 662 try: 663 import pstats 664 except ImportError as e: 665 sys.stderr.write('%s: func: %s\n' % (self.name, e)) 666 sys.stderr.write('%s This version of Python is missing the profiler.\n' % self.name_spaces) 667 sys.stderr.write('%s Cannot use the "func" subcommand.\n' % self.name_spaces) 668 sys.exit(1) 669 statistics = pstats.Stats(file).stats 670 matches = [ e for e in statistics.items() if e[0][2] == function ] 671 r = matches[0] 672 return r[0][0], r[0][1], r[0][2], r[1][3] 673 674 def get_function_time(self, file, function): 675 """ 676 Returns just the cumulative time for the specified function. 677 """ 678 return self.get_function_profile(file, function)[3] 679 680 def get_memory(self, file, memory_string=None): 681 """ 682 Returns a list of integers of the amount of memory used. The 683 default behavior is to return all the stages. 684 """ 685 if memory_string is None: 686 search_string = self.memory_string_all 687 else: 688 search_string = memory_string 689 lines = open(file).readlines() 690 lines = [ l for l in lines if l.startswith(search_string) ][-4:] 691 result = [ int(l.split()[-1]) for l in lines[-4:] ] 692 if len(result) == 1: 693 result = result[0] 694 return result 695 696 def get_object_counts(self, file, object_name, index=None): 697 """ 698 Returns the counts of the specified object_name. 699 """ 700 object_string = ' ' + object_name + '\n' 701 lines = open(file).readlines() 702 line = [ l for l in lines if l.endswith(object_string) ][0] 703 result = [ int(field) for field in line.split()[:4] ] 704 if index is not None: 705 result = result[index] 706 return result 707 708 # 709 710 command_alias = {} 711 712 def execute_subcommand(self, argv): 713 """ 714 Executes the do_*() function for the specified subcommand (argv[0]). 715 """ 716 if not argv: 717 return 718 cmdName = self.command_alias.get(argv[0], argv[0]) 719 try: 720 func = getattr(self, 'do_' + cmdName) 721 except AttributeError: 722 return self.default(argv) 723 try: 724 return func(argv) 725 except TypeError as e: 726 sys.stderr.write("%s %s: %s\n" % (self.name, cmdName, e)) 727 import traceback 728 traceback.print_exc(file=sys.stderr) 729 sys.stderr.write("Try '%s help %s'\n" % (self.name, cmdName)) 730 731 def default(self, argv): 732 """ 733 The default behavior for an unknown subcommand. Prints an 734 error message and exits. 735 """ 736 sys.stderr.write('%s: Unknown subcommand "%s".\n' % (self.name, argv[0])) 737 sys.stderr.write('Type "%s help" for usage.\n' % self.name) 738 sys.exit(1) 739 740 # 741 742 def do_help(self, argv): 743 """ 744 """ 745 if argv[1:]: 746 for arg in argv[1:]: 747 try: 748 func = getattr(self, 'do_' + arg) 749 except AttributeError: 750 sys.stderr.write('%s: No help for "%s"\n' % (self.name, arg)) 751 else: 752 try: 753 help = getattr(self, 'help_' + arg) 754 except AttributeError: 755 sys.stdout.write(self.doc_to_help(func)) 756 sys.stdout.flush() 757 else: 758 help() 759 else: 760 doc = self.doc_to_help(self.__class__) 761 if doc: 762 sys.stdout.write(doc) 763 sys.stdout.flush() 764 return None 765 766 # 767 768 def help_func(self): 769 help = """\ 770 Usage: scons-time func [OPTIONS] FILE [...] 771 772 -C DIR, --chdir=DIR Change to DIR before looking for files 773 -f FILE, --file=FILE Read configuration from specified FILE 774 --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT 775 --func=NAME, --function=NAME Report time for function NAME 776 -h, --help Print this help and exit 777 -p STRING, --prefix=STRING Use STRING as log file/profile prefix 778 -t NUMBER, --tail=NUMBER Only report the last NUMBER files 779 --title=TITLE Specify the output plot TITLE 780 """ 781 sys.stdout.write(self.outdent(help)) 782 sys.stdout.flush() 783 784 def do_func(self, argv): 785 """ 786 """ 787 format = 'ascii' 788 function_name = '_main' 789 tail = None 790 791 short_opts = '?C:f:hp:t:' 792 793 long_opts = [ 794 'chdir=', 795 'file=', 796 'fmt=', 797 'format=', 798 'func=', 799 'function=', 800 'help', 801 'prefix=', 802 'tail=', 803 'title=', 804 ] 805 806 opts, args = getopt.getopt(argv[1:], short_opts, long_opts) 807 808 for o, a in opts: 809 if o in ('-C', '--chdir'): 810 self.chdir = a 811 elif o in ('-f', '--file'): 812 self.config_file = a 813 elif o in ('--fmt', '--format'): 814 format = a 815 elif o in ('--func', '--function'): 816 function_name = a 817 elif o in ('-?', '-h', '--help'): 818 self.do_help(['help', 'func']) 819 sys.exit(0) 820 elif o in ('--max',): 821 max_time = int(a) 822 elif o in ('-p', '--prefix'): 823 self.prefix = a 824 elif o in ('-t', '--tail'): 825 tail = int(a) 826 elif o in ('--title',): 827 self.title = a 828 829 if self.config_file: 830 exec(open(self.config_file, 'rU').read(), self.__dict__) 831 832 if self.chdir: 833 os.chdir(self.chdir) 834 835 if not args: 836 837 pattern = '%s*.prof' % self.prefix 838 args = self.args_to_files([pattern], tail) 839 840 if not args: 841 if self.chdir: 842 directory = self.chdir 843 else: 844 directory = os.getcwd() 845 846 sys.stderr.write('%s: func: No arguments specified.\n' % self.name) 847 sys.stderr.write('%s No %s*.prof files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) 848 sys.stderr.write('%s Type "%s help func" for help.\n' % (self.name_spaces, self.name)) 849 sys.exit(1) 850 851 else: 852 853 args = self.args_to_files(args, tail) 854 855 cwd_ = os.getcwd() + os.sep 856 857 if format == 'ascii': 858 859 for file in args: 860 try: 861 f, line, func, time = \ 862 self.get_function_profile(file, function_name) 863 except ValueError as e: 864 sys.stderr.write("%s: func: %s: %s\n" % 865 (self.name, file, e)) 866 else: 867 if f.startswith(cwd_): 868 f = f[len(cwd_):] 869 print("%.3f %s:%d(%s)" % (time, f, line, func)) 870 871 elif format == 'gnuplot': 872 873 results = self.collect_results(args, self.get_function_time, 874 function_name) 875 876 self.gnuplot_results(results) 877 878 else: 879 880 sys.stderr.write('%s: func: Unknown format "%s".\n' % (self.name, format)) 881 sys.exit(1) 882 883 # 884 885 def help_mem(self): 886 help = """\ 887 Usage: scons-time mem [OPTIONS] FILE [...] 888 889 -C DIR, --chdir=DIR Change to DIR before looking for files 890 -f FILE, --file=FILE Read configuration from specified FILE 891 --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT 892 -h, --help Print this help and exit 893 -p STRING, --prefix=STRING Use STRING as log file/profile prefix 894 --stage=STAGE Plot memory at the specified stage: 895 pre-read, post-read, pre-build, 896 post-build (default: post-build) 897 -t NUMBER, --tail=NUMBER Only report the last NUMBER files 898 --title=TITLE Specify the output plot TITLE 899 """ 900 sys.stdout.write(self.outdent(help)) 901 sys.stdout.flush() 902 903 def do_mem(self, argv): 904 905 format = 'ascii' 906 logfile_path = lambda x: x 907 stage = self.default_stage 908 tail = None 909 910 short_opts = '?C:f:hp:t:' 911 912 long_opts = [ 913 'chdir=', 914 'file=', 915 'fmt=', 916 'format=', 917 'help', 918 'prefix=', 919 'stage=', 920 'tail=', 921 'title=', 922 ] 923 924 opts, args = getopt.getopt(argv[1:], short_opts, long_opts) 925 926 for o, a in opts: 927 if o in ('-C', '--chdir'): 928 self.chdir = a 929 elif o in ('-f', '--file'): 930 self.config_file = a 931 elif o in ('--fmt', '--format'): 932 format = a 933 elif o in ('-?', '-h', '--help'): 934 self.do_help(['help', 'mem']) 935 sys.exit(0) 936 elif o in ('-p', '--prefix'): 937 self.prefix = a 938 elif o in ('--stage',): 939 if not a in self.stages: 940 sys.stderr.write('%s: mem: Unrecognized stage "%s".\n' % (self.name, a)) 941 sys.exit(1) 942 stage = a 943 elif o in ('-t', '--tail'): 944 tail = int(a) 945 elif o in ('--title',): 946 self.title = a 947 948 if self.config_file: 949 HACK_for_exec(open(self.config_file, 'rU').read(), self.__dict__) 950 951 if self.chdir: 952 os.chdir(self.chdir) 953 logfile_path = lambda x, c=self.chdir: os.path.join(c, x) 954 955 if not args: 956 957 pattern = '%s*.log' % self.prefix 958 args = self.args_to_files([pattern], tail) 959 960 if not args: 961 if self.chdir: 962 directory = self.chdir 963 else: 964 directory = os.getcwd() 965 966 sys.stderr.write('%s: mem: No arguments specified.\n' % self.name) 967 sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) 968 sys.stderr.write('%s Type "%s help mem" for help.\n' % (self.name_spaces, self.name)) 969 sys.exit(1) 970 971 else: 972 973 args = self.args_to_files(args, tail) 974 975 cwd_ = os.getcwd() + os.sep 976 977 if format == 'ascii': 978 979 self.ascii_table(args, tuple(self.stages), self.get_memory, logfile_path) 980 981 elif format == 'gnuplot': 982 983 results = self.collect_results(args, self.get_memory, 984 self.stage_strings[stage]) 985 986 self.gnuplot_results(results) 987 988 else: 989 990 sys.stderr.write('%s: mem: Unknown format "%s".\n' % (self.name, format)) 991 sys.exit(1) 992 993 return 0 994 995 # 996 997 def help_obj(self): 998 help = """\ 999 Usage: scons-time obj [OPTIONS] OBJECT FILE [...] 1000 1001 -C DIR, --chdir=DIR Change to DIR before looking for files 1002 -f FILE, --file=FILE Read configuration from specified FILE 1003 --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT 1004 -h, --help Print this help and exit 1005 -p STRING, --prefix=STRING Use STRING as log file/profile prefix 1006 --stage=STAGE Plot memory at the specified stage: 1007 pre-read, post-read, pre-build, 1008 post-build (default: post-build) 1009 -t NUMBER, --tail=NUMBER Only report the last NUMBER files 1010 --title=TITLE Specify the output plot TITLE 1011 """ 1012 sys.stdout.write(self.outdent(help)) 1013 sys.stdout.flush() 1014 1015 def do_obj(self, argv): 1016 1017 format = 'ascii' 1018 logfile_path = lambda x: x 1019 stage = self.default_stage 1020 tail = None 1021 1022 short_opts = '?C:f:hp:t:' 1023 1024 long_opts = [ 1025 'chdir=', 1026 'file=', 1027 'fmt=', 1028 'format=', 1029 'help', 1030 'prefix=', 1031 'stage=', 1032 'tail=', 1033 'title=', 1034 ] 1035 1036 opts, args = getopt.getopt(argv[1:], short_opts, long_opts) 1037 1038 for o, a in opts: 1039 if o in ('-C', '--chdir'): 1040 self.chdir = a 1041 elif o in ('-f', '--file'): 1042 self.config_file = a 1043 elif o in ('--fmt', '--format'): 1044 format = a 1045 elif o in ('-?', '-h', '--help'): 1046 self.do_help(['help', 'obj']) 1047 sys.exit(0) 1048 elif o in ('-p', '--prefix'): 1049 self.prefix = a 1050 elif o in ('--stage',): 1051 if not a in self.stages: 1052 sys.stderr.write('%s: obj: Unrecognized stage "%s".\n' % (self.name, a)) 1053 sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) 1054 sys.exit(1) 1055 stage = a 1056 elif o in ('-t', '--tail'): 1057 tail = int(a) 1058 elif o in ('--title',): 1059 self.title = a 1060 1061 if not args: 1062 sys.stderr.write('%s: obj: Must specify an object name.\n' % self.name) 1063 sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) 1064 sys.exit(1) 1065 1066 object_name = args.pop(0) 1067 1068 if self.config_file: 1069 HACK_for_exec(open(self.config_file, 'rU').read(), self.__dict__) 1070 1071 if self.chdir: 1072 os.chdir(self.chdir) 1073 logfile_path = lambda x, c=self.chdir: os.path.join(c, x) 1074 1075 if not args: 1076 1077 pattern = '%s*.log' % self.prefix 1078 args = self.args_to_files([pattern], tail) 1079 1080 if not args: 1081 if self.chdir: 1082 directory = self.chdir 1083 else: 1084 directory = os.getcwd() 1085 1086 sys.stderr.write('%s: obj: No arguments specified.\n' % self.name) 1087 sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) 1088 sys.stderr.write('%s Type "%s help obj" for help.\n' % (self.name_spaces, self.name)) 1089 sys.exit(1) 1090 1091 else: 1092 1093 args = self.args_to_files(args, tail) 1094 1095 cwd_ = os.getcwd() + os.sep 1096 1097 if format == 'ascii': 1098 1099 self.ascii_table(args, tuple(self.stages), self.get_object_counts, logfile_path, object_name) 1100 1101 elif format == 'gnuplot': 1102 1103 stage_index = 0 1104 for s in self.stages: 1105 if stage == s: 1106 break 1107 stage_index = stage_index + 1 1108 1109 results = self.collect_results(args, self.get_object_counts, 1110 object_name, stage_index) 1111 1112 self.gnuplot_results(results) 1113 1114 else: 1115 1116 sys.stderr.write('%s: obj: Unknown format "%s".\n' % (self.name, format)) 1117 sys.exit(1) 1118 1119 return 0 1120 1121 # 1122 1123 def help_run(self): 1124 help = """\ 1125 Usage: scons-time run [OPTIONS] [FILE ...] 1126 1127 --aegis=PROJECT Use SCons from the Aegis PROJECT 1128 --chdir=DIR Name of unpacked directory for chdir 1129 -f FILE, --file=FILE Read configuration from specified FILE 1130 -h, --help Print this help and exit 1131 -n, --no-exec No execute, just print command lines 1132 --number=NUMBER Put output in files for run NUMBER 1133 --outdir=OUTDIR Put output files in OUTDIR 1134 -p STRING, --prefix=STRING Use STRING as log file/profile prefix 1135 --python=PYTHON Time using the specified PYTHON 1136 -q, --quiet Don't print command lines 1137 --scons=SCONS Time using the specified SCONS 1138 --svn=URL, --subversion=URL Use SCons from Subversion URL 1139 -v, --verbose Display output of commands 1140 """ 1141 sys.stdout.write(self.outdent(help)) 1142 sys.stdout.flush() 1143 1144 def do_run(self, argv): 1145 """ 1146 """ 1147 run_number_list = [None] 1148 1149 short_opts = '?f:hnp:qs:v' 1150 1151 long_opts = [ 1152 'aegis=', 1153 'file=', 1154 'help', 1155 'no-exec', 1156 'number=', 1157 'outdir=', 1158 'prefix=', 1159 'python=', 1160 'quiet', 1161 'scons=', 1162 'svn=', 1163 'subdir=', 1164 'subversion=', 1165 'verbose', 1166 ] 1167 1168 opts, args = getopt.getopt(argv[1:], short_opts, long_opts) 1169 1170 for o, a in opts: 1171 if o in ('--aegis',): 1172 self.aegis_project = a 1173 elif o in ('-f', '--file'): 1174 self.config_file = a 1175 elif o in ('-?', '-h', '--help'): 1176 self.do_help(['help', 'run']) 1177 sys.exit(0) 1178 elif o in ('-n', '--no-exec'): 1179 self.execute = self._do_not_execute 1180 elif o in ('--number',): 1181 run_number_list = self.split_run_numbers(a) 1182 elif o in ('--outdir',): 1183 self.outdir = a 1184 elif o in ('-p', '--prefix'): 1185 self.prefix = a 1186 elif o in ('--python',): 1187 self.python = a 1188 elif o in ('-q', '--quiet'): 1189 self.display = self._do_not_display 1190 elif o in ('-s', '--subdir'): 1191 self.subdir = a 1192 elif o in ('--scons',): 1193 self.scons = a 1194 elif o in ('--svn', '--subversion'): 1195 self.subversion_url = a 1196 elif o in ('-v', '--verbose'): 1197 self.redirect = tee_to_file 1198 self.verbose = True 1199 self.svn_co_flag = '' 1200 1201 if not args and not self.config_file: 1202 sys.stderr.write('%s: run: No arguments or -f config file specified.\n' % self.name) 1203 sys.stderr.write('%s Type "%s help run" for help.\n' % (self.name_spaces, self.name)) 1204 sys.exit(1) 1205 1206 if self.config_file: 1207 exec(open(self.config_file, 'rU').read(), self.__dict__) 1208 1209 if args: 1210 self.archive_list = args 1211 1212 archive_file_name = os.path.split(self.archive_list[0])[1] 1213 1214 if not self.subdir: 1215 self.subdir = self.archive_splitext(archive_file_name)[0] 1216 1217 if not self.prefix: 1218 self.prefix = self.archive_splitext(archive_file_name)[0] 1219 1220 prepare = None 1221 if self.subversion_url: 1222 prepare = self.prep_subversion_run 1223 elif self.aegis_project: 1224 prepare = self.prep_aegis_run 1225 1226 for run_number in run_number_list: 1227 self.individual_run(run_number, self.archive_list, prepare) 1228 1229 def split_run_numbers(self, s): 1230 result = [] 1231 for n in s.split(','): 1232 try: 1233 x, y = n.split('-') 1234 except ValueError: 1235 result.append(int(n)) 1236 else: 1237 result.extend(range(int(x), int(y)+1)) 1238 return result 1239 1240 def scons_path(self, dir): 1241 return os.path.join(dir, 'src', 'script', 'scons.py') 1242 1243 def scons_lib_dir_path(self, dir): 1244 return os.path.join(dir, 'src', 'engine') 1245 1246 def prep_aegis_run(self, commands, removals): 1247 self.aegis_tmpdir = make_temp_file(prefix = self.name + '-aegis-') 1248 removals.append((shutil.rmtree, 'rm -rf %%s', self.aegis_tmpdir)) 1249 1250 self.aegis_parent_project = os.path.splitext(self.aegis_project)[0] 1251 self.scons = self.scons_path(self.aegis_tmpdir) 1252 self.scons_lib_dir = self.scons_lib_dir_path(self.aegis_tmpdir) 1253 1254 commands.extend([ 1255 'mkdir %(aegis_tmpdir)s', 1256 (lambda: os.chdir(self.aegis_tmpdir), 'cd %(aegis_tmpdir)s'), 1257 '%(aegis)s -cp -ind -p %(aegis_parent_project)s .', 1258 '%(aegis)s -cp -ind -p %(aegis_project)s -delta %(run_number)s .', 1259 ]) 1260 1261 def prep_subversion_run(self, commands, removals): 1262 self.svn_tmpdir = make_temp_file(prefix = self.name + '-svn-') 1263 removals.append((shutil.rmtree, 'rm -rf %%s', self.svn_tmpdir)) 1264 1265 self.scons = self.scons_path(self.svn_tmpdir) 1266 self.scons_lib_dir = self.scons_lib_dir_path(self.svn_tmpdir) 1267 1268 commands.extend([ 1269 'mkdir %(svn_tmpdir)s', 1270 '%(svn)s co %(svn_co_flag)s -r %(run_number)s %(subversion_url)s %(svn_tmpdir)s', 1271 ]) 1272 1273 def individual_run(self, run_number, archive_list, prepare=None): 1274 """ 1275 Performs an individual run of the default SCons invocations. 1276 """ 1277 1278 commands = [] 1279 removals = [] 1280 1281 if prepare: 1282 prepare(commands, removals) 1283 1284 save_scons = self.scons 1285 save_scons_wrapper = self.scons_wrapper 1286 save_scons_lib_dir = self.scons_lib_dir 1287 1288 if self.outdir is None: 1289 self.outdir = self.orig_cwd 1290 elif not os.path.isabs(self.outdir): 1291 self.outdir = os.path.join(self.orig_cwd, self.outdir) 1292 1293 if self.scons is None: 1294 self.scons = self.scons_path(self.orig_cwd) 1295 1296 if self.scons_lib_dir is None: 1297 self.scons_lib_dir = self.scons_lib_dir_path(self.orig_cwd) 1298 1299 if self.scons_wrapper is None: 1300 self.scons_wrapper = self.scons 1301 1302 if not run_number: 1303 run_number = self.find_next_run_number(self.outdir, self.prefix) 1304 1305 self.run_number = str(run_number) 1306 1307 self.prefix_run = self.prefix + '-%03d' % run_number 1308 1309 if self.targets0 is None: 1310 self.targets0 = self.startup_targets 1311 if self.targets1 is None: 1312 self.targets1 = self.targets 1313 if self.targets2 is None: 1314 self.targets2 = self.targets 1315 1316 self.tmpdir = make_temp_file(prefix = self.name + '-') 1317 1318 commands.extend([ 1319 'mkdir %(tmpdir)s', 1320 1321 (os.chdir, 'cd %%s', self.tmpdir), 1322 ]) 1323 1324 for archive in archive_list: 1325 if not os.path.isabs(archive): 1326 archive = os.path.join(self.orig_cwd, archive) 1327 if os.path.isdir(archive): 1328 dest = os.path.split(archive)[1] 1329 commands.append((shutil.copytree, 'cp -r %%s %%s', archive, dest)) 1330 else: 1331 suffix = self.archive_splitext(archive)[1] 1332 unpack_command = self.unpack_map.get(suffix) 1333 if not unpack_command: 1334 dest = os.path.split(archive)[1] 1335 commands.append((shutil.copyfile, 'cp %%s %%s', archive, dest)) 1336 else: 1337 commands.append(unpack_command + (archive,)) 1338 1339 commands.extend([ 1340 (os.chdir, 'cd %%s', self.subdir), 1341 ]) 1342 1343 commands.extend(self.initial_commands) 1344 1345 commands.extend([ 1346 (lambda: read_tree('.'), 1347 'find * -type f | xargs cat > /dev/null'), 1348 1349 (self.set_env, 'export %%s=%%s', 1350 'SCONS_LIB_DIR', self.scons_lib_dir), 1351 1352 '%(python)s %(scons_wrapper)s --version', 1353 ]) 1354 1355 index = 0 1356 for run_command in self.run_commands: 1357 setattr(self, 'prof%d' % index, self.profile_name(index)) 1358 c = ( 1359 self.log_execute, 1360 self.log_display, 1361 run_command, 1362 self.logfile_name(index), 1363 ) 1364 commands.append(c) 1365 index = index + 1 1366 1367 commands.extend([ 1368 (os.chdir, 'cd %%s', self.orig_cwd), 1369 ]) 1370 1371 if not os.environ.get('PRESERVE'): 1372 commands.extend(removals) 1373 1374 commands.append((shutil.rmtree, 'rm -rf %%s', self.tmpdir)) 1375 1376 self.run_command_list(commands, self.__dict__) 1377 1378 self.scons = save_scons 1379 self.scons_lib_dir = save_scons_lib_dir 1380 self.scons_wrapper = save_scons_wrapper 1381 1382 # 1383 1384 def help_time(self): 1385 help = """\ 1386 Usage: scons-time time [OPTIONS] FILE [...] 1387 1388 -C DIR, --chdir=DIR Change to DIR before looking for files 1389 -f FILE, --file=FILE Read configuration from specified FILE 1390 --fmt=FORMAT, --format=FORMAT Print data in specified FORMAT 1391 -h, --help Print this help and exit 1392 -p STRING, --prefix=STRING Use STRING as log file/profile prefix 1393 -t NUMBER, --tail=NUMBER Only report the last NUMBER files 1394 --which=TIMER Plot timings for TIMER: total, 1395 SConscripts, SCons, commands. 1396 """ 1397 sys.stdout.write(self.outdent(help)) 1398 sys.stdout.flush() 1399 1400 def do_time(self, argv): 1401 1402 format = 'ascii' 1403 logfile_path = lambda x: x 1404 tail = None 1405 which = 'total' 1406 1407 short_opts = '?C:f:hp:t:' 1408 1409 long_opts = [ 1410 'chdir=', 1411 'file=', 1412 'fmt=', 1413 'format=', 1414 'help', 1415 'prefix=', 1416 'tail=', 1417 'title=', 1418 'which=', 1419 ] 1420 1421 opts, args = getopt.getopt(argv[1:], short_opts, long_opts) 1422 1423 for o, a in opts: 1424 if o in ('-C', '--chdir'): 1425 self.chdir = a 1426 elif o in ('-f', '--file'): 1427 self.config_file = a 1428 elif o in ('--fmt', '--format'): 1429 format = a 1430 elif o in ('-?', '-h', '--help'): 1431 self.do_help(['help', 'time']) 1432 sys.exit(0) 1433 elif o in ('-p', '--prefix'): 1434 self.prefix = a 1435 elif o in ('-t', '--tail'): 1436 tail = int(a) 1437 elif o in ('--title',): 1438 self.title = a 1439 elif o in ('--which',): 1440 if not a in self.time_strings.keys(): 1441 sys.stderr.write('%s: time: Unrecognized timer "%s".\n' % (self.name, a)) 1442 sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) 1443 sys.exit(1) 1444 which = a 1445 1446 if self.config_file: 1447 HACK_for_exec(open(self.config_file, 'rU').read(), self.__dict__) 1448 1449 if self.chdir: 1450 os.chdir(self.chdir) 1451 logfile_path = lambda x, c=self.chdir: os.path.join(c, x) 1452 1453 if not args: 1454 1455 pattern = '%s*.log' % self.prefix 1456 args = self.args_to_files([pattern], tail) 1457 1458 if not args: 1459 if self.chdir: 1460 directory = self.chdir 1461 else: 1462 directory = os.getcwd() 1463 1464 sys.stderr.write('%s: time: No arguments specified.\n' % self.name) 1465 sys.stderr.write('%s No %s*.log files found in "%s".\n' % (self.name_spaces, self.prefix, directory)) 1466 sys.stderr.write('%s Type "%s help time" for help.\n' % (self.name_spaces, self.name)) 1467 sys.exit(1) 1468 1469 else: 1470 1471 args = self.args_to_files(args, tail) 1472 1473 cwd_ = os.getcwd() + os.sep 1474 1475 if format == 'ascii': 1476 1477 columns = ("Total", "SConscripts", "SCons", "commands") 1478 self.ascii_table(args, columns, self.get_debug_times, logfile_path) 1479 1480 elif format == 'gnuplot': 1481 1482 results = self.collect_results(args, self.get_debug_times, 1483 self.time_strings[which]) 1484 1485 self.gnuplot_results(results, fmt='%s %.6f') 1486 1487 else: 1488 1489 sys.stderr.write('%s: time: Unknown format "%s".\n' % (self.name, format)) 1490 sys.exit(1) 1491 1492if __name__ == '__main__': 1493 opts, args = getopt.getopt(sys.argv[1:], 'h?V', ['help', 'version']) 1494 1495 ST = SConsTimer() 1496 1497 for o, a in opts: 1498 if o in ('-?', '-h', '--help'): 1499 ST.do_help(['help']) 1500 sys.exit(0) 1501 elif o in ('-V', '--version'): 1502 sys.stdout.write('scons-time version\n') 1503 sys.exit(0) 1504 1505 if not args: 1506 sys.stderr.write('Type "%s help" for usage.\n' % ST.name) 1507 sys.exit(1) 1508 1509 ST.execute_subcommand(args) 1510 1511# Local Variables: 1512# tab-width:4 1513# indent-tabs-mode:nil 1514# End: 1515# vim: set expandtab tabstop=4 shiftwidth=4: 1516