1# Copyright 2014 the V8 project authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5""" 6Performance runner for d8. 7 8Call e.g. with tools/run-perf.py --arch ia32 some_suite.json 9 10The suite json format is expected to be: 11{ 12 "path": <relative path chunks to perf resources and main file>, 13 "owners": [<list of email addresses of benchmark owners (required)>], 14 "name": <optional suite name, file name is default>, 15 "archs": [<architecture name for which this suite is run>, ...], 16 "binary": <name of binary to run, default "d8">, 17 "flags": [<flag to d8>, ...], 18 "test_flags": [<flag to the test file>, ...], 19 "run_count": <how often will this suite run (optional)>, 20 "run_count_XXX": <how often will this suite run for arch XXX (optional)>, 21 "timeout": <how long test is allowed to run>, 22 "timeout_XXX": <how long test is allowed run run for arch XXX>, 23 "retry_count": <how many times to retry failures (in addition to first try)", 24 "retry_count_XXX": <how many times to retry failures for arch XXX> 25 "resources": [<js file to be moved to android device>, ...] 26 "main": <main js perf runner file>, 27 "results_regexp": <optional regexp>, 28 "results_processor": <optional python results processor script>, 29 "units": <the unit specification for the performance dashboard>, 30 "process_size": <flag - collect maximum memory used by the process>, 31 "tests": [ 32 { 33 "name": <name of the trace>, 34 "results_regexp": <optional more specific regexp>, 35 "results_processor": <optional python results processor script>, 36 "units": <the unit specification for the performance dashboard>, 37 "process_size": <flag - collect maximum memory used by the process>, 38 }, ... 39 ] 40} 41 42The tests field can also nest other suites in arbitrary depth. A suite 43with a "main" file is a leaf suite that can contain one more level of 44tests. 45 46A suite's results_regexp is expected to have one string place holder 47"%s" for the trace name. A trace's results_regexp overwrites suite 48defaults. 49 50A suite's results_processor may point to an optional python script. If 51specified, it is called after running the tests (with a path relative to the 52suite level's path). It is expected to read the measurement's output text 53on stdin and print the processed output to stdout. 54 55The results_regexp will be applied to the processed output. 56 57A suite without "tests" is considered a performance test itself. 58 59Full example (suite with one runner): 60{ 61 "path": ["."], 62 "owners": ["username@chromium.org"], 63 "flags": ["--expose-gc"], 64 "test_flags": ["5"], 65 "archs": ["ia32", "x64"], 66 "run_count": 5, 67 "run_count_ia32": 3, 68 "main": "run.js", 69 "results_regexp": "^%s: (.+)$", 70 "units": "score", 71 "tests": [ 72 {"name": "Richards"}, 73 {"name": "DeltaBlue"}, 74 {"name": "NavierStokes", 75 "results_regexp": "^NavierStokes: (.+)$"} 76 ] 77} 78 79Full example (suite with several runners): 80{ 81 "path": ["."], 82 "owners": ["username@chromium.org", "otherowner@google.com"], 83 "flags": ["--expose-gc"], 84 "archs": ["ia32", "x64"], 85 "run_count": 5, 86 "units": "score", 87 "tests": [ 88 {"name": "Richards", 89 "path": ["richards"], 90 "main": "run.js", 91 "run_count": 3, 92 "results_regexp": "^Richards: (.+)$"}, 93 {"name": "NavierStokes", 94 "path": ["navier_stokes"], 95 "main": "run.js", 96 "results_regexp": "^NavierStokes: (.+)$"} 97 ] 98} 99 100Path pieces are concatenated. D8 is always run with the suite's path as cwd. 101 102The test flags are passed to the js test file after '--'. 103""" 104 105# for py2/py3 compatibility 106from __future__ import print_function 107from functools import reduce 108 109from collections import OrderedDict 110import copy 111import json 112import logging 113import math 114import argparse 115import os 116import re 117import subprocess 118import sys 119import time 120import traceback 121 122import numpy 123 124from testrunner.local import android 125from testrunner.local import command 126from testrunner.local import utils 127from testrunner.objects.output import Output, NULL_OUTPUT 128 129try: 130 basestring # Python 2 131except NameError: # Python 3 132 basestring = str 133 134SUPPORTED_ARCHS = ['arm', 135 'ia32', 136 'mips', 137 'mipsel', 138 'x64', 139 'arm64'] 140 141GENERIC_RESULTS_RE = re.compile(r'^RESULT ([^:]+): ([^=]+)= ([^ ]+) ([^ ]*)$') 142RESULT_STDDEV_RE = re.compile(r'^\{([^\}]+)\}$') 143RESULT_LIST_RE = re.compile(r'^\[([^\]]+)\]$') 144TOOLS_BASE = os.path.abspath(os.path.dirname(__file__)) 145INFRA_FAILURE_RETCODE = 87 146MIN_RUNS_FOR_CONFIDENCE = 10 147 148 149def GeometricMean(values): 150 """Returns the geometric mean of a list of values. 151 152 The mean is calculated using log to avoid overflow. 153 """ 154 values = map(float, values) 155 return math.exp(sum(map(math.log, values)) / len(values)) 156 157 158class ResultTracker(object): 159 """Class that tracks trace/runnable results and produces script output. 160 161 The output is structured like this: 162 { 163 "traces": [ 164 { 165 "graphs": ["path", "to", "trace", "config"], 166 "units": <string describing units, e.g. "ms" or "KB">, 167 "results": [<list of values measured over several runs>], 168 "stddev": <stddev of the value if measure by script or ''> 169 }, 170 ... 171 ], 172 "runnables": [ 173 { 174 "graphs": ["path", "to", "runnable", "config"], 175 "durations": [<list of durations of each runnable run in seconds>], 176 "timeout": <timeout configured for runnable in seconds>, 177 }, 178 ... 179 ], 180 "errors": [<list of strings describing errors>], 181 } 182 """ 183 def __init__(self): 184 self.traces = {} 185 self.errors = [] 186 self.runnables = {} 187 188 def AddTraceResult(self, trace, result, stddev): 189 if trace.name not in self.traces: 190 self.traces[trace.name] = { 191 'graphs': trace.graphs, 192 'units': trace.units, 193 'results': [result], 194 'stddev': stddev or '', 195 } 196 else: 197 existing_entry = self.traces[trace.name] 198 assert trace.graphs == existing_entry['graphs'] 199 assert trace.units == existing_entry['units'] 200 if stddev: 201 existing_entry['stddev'] = stddev 202 existing_entry['results'].append(result) 203 204 def TraceHasStdDev(self, trace): 205 return trace.name in self.traces and self.traces[trace.name]['stddev'] != '' 206 207 def AddError(self, error): 208 self.errors.append(error) 209 210 def AddRunnableDuration(self, runnable, duration): 211 """Records a duration of a specific run of the runnable.""" 212 if runnable.name not in self.runnables: 213 self.runnables[runnable.name] = { 214 'graphs': runnable.graphs, 215 'durations': [duration], 216 'timeout': runnable.timeout, 217 } 218 else: 219 existing_entry = self.runnables[runnable.name] 220 assert runnable.timeout == existing_entry['timeout'] 221 assert runnable.graphs == existing_entry['graphs'] 222 existing_entry['durations'].append(duration) 223 224 def ToDict(self): 225 return { 226 'traces': self.traces.values(), 227 'errors': self.errors, 228 'runnables': self.runnables.values(), 229 } 230 231 def WriteToFile(self, file_name): 232 with open(file_name, 'w') as f: 233 f.write(json.dumps(self.ToDict())) 234 235 def HasEnoughRuns(self, graph_config, confidence_level): 236 """Checks if the mean of the results for a given trace config is within 237 0.1% of the true value with the specified confidence level. 238 239 This assumes Gaussian distribution of the noise and based on 240 https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule. 241 242 Args: 243 graph_config: An instance of GraphConfig. 244 confidence_level: Number of standard deviations from the mean that all 245 values must lie within. Typical values are 1, 2 and 3 and correspond 246 to 68%, 95% and 99.7% probability that the measured value is within 247 0.1% of the true value. 248 249 Returns: 250 True if specified confidence level have been achieved. 251 """ 252 if not isinstance(graph_config, TraceConfig): 253 return all(self.HasEnoughRuns(child, confidence_level) 254 for child in graph_config.children) 255 256 trace = self.traces.get(graph_config.name, {}) 257 results = trace.get('results', []) 258 logging.debug('HasEnoughRuns for %s', graph_config.name) 259 260 if len(results) < MIN_RUNS_FOR_CONFIDENCE: 261 logging.debug(' Ran %d times, need at least %d', 262 len(results), MIN_RUNS_FOR_CONFIDENCE) 263 return False 264 265 logging.debug(' Results: %d entries', len(results)) 266 mean = numpy.mean(results) 267 mean_stderr = numpy.std(results) / numpy.sqrt(len(results)) 268 logging.debug(' Mean: %.2f, mean_stderr: %.2f', mean, mean_stderr) 269 logging.info('>>> Confidence level is %.2f', mean / (1000.0 * mean_stderr)) 270 return confidence_level * mean_stderr < mean / 1000.0 271 272 def __str__(self): # pragma: no cover 273 return json.dumps(self.ToDict(), indent=2, separators=(',', ': ')) 274 275 276def RunResultsProcessor(results_processor, output, count): 277 # Dummy pass through for null-runs. 278 if output.stdout is None: 279 return output 280 281 # We assume the results processor is relative to the suite. 282 assert os.path.exists(results_processor) 283 p = subprocess.Popen( 284 [sys.executable, results_processor], 285 stdin=subprocess.PIPE, 286 stdout=subprocess.PIPE, 287 stderr=subprocess.PIPE, 288 ) 289 new_output = copy.copy(output) 290 new_output.stdout, _ = p.communicate(input=output.stdout) 291 logging.info('>>> Processed stdout (#%d):\n%s', count, output.stdout) 292 return new_output 293 294 295class Node(object): 296 """Represents a node in the suite tree structure.""" 297 def __init__(self, *args): 298 self._children = [] 299 300 def AppendChild(self, child): 301 self._children.append(child) 302 303 @property 304 def children(self): 305 return self._children 306 307 308class DefaultSentinel(Node): 309 """Fake parent node with all default values.""" 310 def __init__(self, binary = 'd8'): 311 super(DefaultSentinel, self).__init__() 312 self.binary = binary 313 self.run_count = 10 314 self.timeout = 60 315 self.retry_count = 4 316 self.path = [] 317 self.graphs = [] 318 self.flags = [] 319 self.test_flags = [] 320 self.process_size = False 321 self.resources = [] 322 self.results_processor = None 323 self.results_regexp = None 324 self.stddev_regexp = None 325 self.units = 'score' 326 self.total = False 327 self.owners = [] 328 329 330class GraphConfig(Node): 331 """Represents a suite definition. 332 333 Can either be a leaf or an inner node that provides default values. 334 """ 335 def __init__(self, suite, parent, arch): 336 super(GraphConfig, self).__init__() 337 self._suite = suite 338 339 assert isinstance(suite.get('path', []), list) 340 assert isinstance(suite.get('owners', []), list) 341 assert isinstance(suite['name'], basestring) 342 assert isinstance(suite.get('flags', []), list) 343 assert isinstance(suite.get('test_flags', []), list) 344 assert isinstance(suite.get('resources', []), list) 345 346 # Accumulated values. 347 self.path = parent.path[:] + suite.get('path', []) 348 self.graphs = parent.graphs[:] + [suite['name']] 349 self.flags = parent.flags[:] + suite.get('flags', []) 350 self.test_flags = parent.test_flags[:] + suite.get('test_flags', []) 351 self.owners = parent.owners[:] + suite.get('owners', []) 352 353 # Values independent of parent node. 354 self.resources = suite.get('resources', []) 355 356 # Descrete values (with parent defaults). 357 self.binary = suite.get('binary', parent.binary) 358 self.run_count = suite.get('run_count', parent.run_count) 359 self.run_count = suite.get('run_count_%s' % arch, self.run_count) 360 self.retry_count = suite.get('retry_count', parent.retry_count) 361 self.retry_count = suite.get('retry_count_%s' % arch, self.retry_count) 362 self.timeout = suite.get('timeout', parent.timeout) 363 self.timeout = suite.get('timeout_%s' % arch, self.timeout) 364 self.units = suite.get('units', parent.units) 365 self.total = suite.get('total', parent.total) 366 self.results_processor = suite.get( 367 'results_processor', parent.results_processor) 368 self.process_size = suite.get('process_size', parent.process_size) 369 370 # A regular expression for results. If the parent graph provides a 371 # regexp and the current suite has none, a string place holder for the 372 # suite name is expected. 373 # TODO(machenbach): Currently that makes only sense for the leaf level. 374 # Multiple place holders for multiple levels are not supported. 375 if parent.results_regexp: 376 regexp_default = parent.results_regexp % re.escape(suite['name']) 377 else: 378 regexp_default = None 379 self.results_regexp = suite.get('results_regexp', regexp_default) 380 381 # A similar regular expression for the standard deviation (optional). 382 if parent.stddev_regexp: 383 stddev_default = parent.stddev_regexp % re.escape(suite['name']) 384 else: 385 stddev_default = None 386 self.stddev_regexp = suite.get('stddev_regexp', stddev_default) 387 388 @property 389 def name(self): 390 return '/'.join(self.graphs) 391 392 393class TraceConfig(GraphConfig): 394 """Represents a leaf in the suite tree structure.""" 395 def __init__(self, suite, parent, arch): 396 super(TraceConfig, self).__init__(suite, parent, arch) 397 assert self.results_regexp 398 assert self.owners 399 400 def ConsumeOutput(self, output, result_tracker): 401 """Extracts trace results from the output. 402 403 Args: 404 output: Output object from the test run. 405 result_tracker: Result tracker to be updated. 406 407 Returns: 408 The raw extracted result value or None if an error occurred. 409 """ 410 result = None 411 stddev = None 412 413 try: 414 result = float( 415 re.search(self.results_regexp, output.stdout, re.M).group(1)) 416 except ValueError: 417 result_tracker.AddError( 418 'Regexp "%s" returned a non-numeric for test %s.' % 419 (self.results_regexp, self.name)) 420 except: 421 result_tracker.AddError( 422 'Regexp "%s" did not match for test %s.' % 423 (self.results_regexp, self.name)) 424 425 try: 426 if self.stddev_regexp: 427 if result_tracker.TraceHasStdDev(self): 428 result_tracker.AddError( 429 'Test %s should only run once since a stddev is provided by the ' 430 'test.' % self.name) 431 stddev = re.search(self.stddev_regexp, output.stdout, re.M).group(1) 432 except: 433 result_tracker.AddError( 434 'Regexp "%s" did not match for test %s.' % 435 (self.stddev_regexp, self.name)) 436 437 if result: 438 result_tracker.AddTraceResult(self, result, stddev) 439 return result 440 441 442class RunnableConfig(GraphConfig): 443 """Represents a runnable suite definition (i.e. has a main file). 444 """ 445 def __init__(self, suite, parent, arch): 446 super(RunnableConfig, self).__init__(suite, parent, arch) 447 self.arch = arch 448 449 @property 450 def main(self): 451 return self._suite.get('main', '') 452 453 def ChangeCWD(self, suite_path): 454 """Changes the cwd to to path defined in the current graph. 455 456 The tests are supposed to be relative to the suite configuration. 457 """ 458 suite_dir = os.path.abspath(os.path.dirname(suite_path)) 459 bench_dir = os.path.normpath(os.path.join(*self.path)) 460 os.chdir(os.path.join(suite_dir, bench_dir)) 461 462 def GetCommandFlags(self, extra_flags=None): 463 suffix = ['--'] + self.test_flags if self.test_flags else [] 464 return self.flags + (extra_flags or []) + [self.main] + suffix 465 466 def GetCommand(self, cmd_prefix, shell_dir, extra_flags=None): 467 # TODO(machenbach): This requires +.exe if run on windows. 468 extra_flags = extra_flags or [] 469 if self.binary != 'd8' and '--prof' in extra_flags: 470 logging.info('Profiler supported only on a benchmark run with d8') 471 472 if self.process_size: 473 cmd_prefix = ['/usr/bin/time', '--format=MaxMemory: %MKB'] + cmd_prefix 474 if self.binary.endswith('.py'): 475 # Copy cmd_prefix instead of update (+=). 476 cmd_prefix = cmd_prefix + [sys.executable] 477 478 return command.Command( 479 cmd_prefix=cmd_prefix, 480 shell=os.path.join(shell_dir, self.binary), 481 args=self.GetCommandFlags(extra_flags=extra_flags), 482 timeout=self.timeout or 60, 483 handle_sigterm=True) 484 485 def ProcessOutput(self, output, result_tracker, count): 486 """Processes test run output and updates result tracker. 487 488 Args: 489 output: Output object from the test run. 490 result_tracker: ResultTracker object to be updated. 491 count: Index of the test run (used for better logging). 492 """ 493 if self.results_processor: 494 output = RunResultsProcessor(self.results_processor, output, count) 495 496 results_for_total = [] 497 for trace in self.children: 498 result = trace.ConsumeOutput(output, result_tracker) 499 if result: 500 results_for_total.append(result) 501 502 if self.total: 503 # Produce total metric only when all traces have produced results. 504 if len(self.children) != len(results_for_total): 505 result_tracker.AddError( 506 'Not all traces have produced results. Can not compute total for ' 507 '%s.' % self.name) 508 return 509 510 # Calculate total as a the geometric mean for results from all traces. 511 total_trace = TraceConfig( 512 {'name': 'Total', 'units': self.children[0].units}, self, self.arch) 513 result_tracker.AddTraceResult( 514 total_trace, GeometricMean(results_for_total), '') 515 516 517class RunnableTraceConfig(TraceConfig, RunnableConfig): 518 """Represents a runnable suite definition that is a leaf.""" 519 def __init__(self, suite, parent, arch): 520 super(RunnableTraceConfig, self).__init__(suite, parent, arch) 521 522 def ProcessOutput(self, output, result_tracker, count): 523 result_tracker.AddRunnableDuration(self, output.duration) 524 self.ConsumeOutput(output, result_tracker) 525 526 527def MakeGraphConfig(suite, arch, parent): 528 """Factory method for making graph configuration objects.""" 529 if isinstance(parent, RunnableConfig): 530 # Below a runnable can only be traces. 531 return TraceConfig(suite, parent, arch) 532 elif suite.get('main') is not None: 533 # A main file makes this graph runnable. Empty strings are accepted. 534 if suite.get('tests'): 535 # This graph has subgraphs (traces). 536 return RunnableConfig(suite, parent, arch) 537 else: 538 # This graph has no subgraphs, it's a leaf. 539 return RunnableTraceConfig(suite, parent, arch) 540 elif suite.get('tests'): 541 # This is neither a leaf nor a runnable. 542 return GraphConfig(suite, parent, arch) 543 else: # pragma: no cover 544 raise Exception('Invalid suite configuration.') 545 546 547def BuildGraphConfigs(suite, arch, parent): 548 """Builds a tree structure of graph objects that corresponds to the suite 549 configuration. 550 """ 551 552 # TODO(machenbach): Implement notion of cpu type? 553 if arch not in suite.get('archs', SUPPORTED_ARCHS): 554 return None 555 556 graph = MakeGraphConfig(suite, arch, parent) 557 for subsuite in suite.get('tests', []): 558 BuildGraphConfigs(subsuite, arch, graph) 559 parent.AppendChild(graph) 560 return graph 561 562 563def FlattenRunnables(node, node_cb): 564 """Generator that traverses the tree structure and iterates over all 565 runnables. 566 """ 567 node_cb(node) 568 if isinstance(node, RunnableConfig): 569 yield node 570 elif isinstance(node, Node): 571 for child in node._children: 572 for result in FlattenRunnables(child, node_cb): 573 yield result 574 else: # pragma: no cover 575 raise Exception('Invalid suite configuration.') 576 577 578class Platform(object): 579 def __init__(self, args): 580 self.shell_dir = args.shell_dir 581 self.shell_dir_secondary = args.shell_dir_secondary 582 self.extra_flags = args.extra_flags.split() 583 self.args = args 584 585 @staticmethod 586 def ReadBuildConfig(args): 587 config_path = os.path.join(args.shell_dir, 'v8_build_config.json') 588 if not os.path.isfile(config_path): 589 return {} 590 with open(config_path) as f: 591 return json.load(f) 592 593 @staticmethod 594 def GetPlatform(args): 595 if Platform.ReadBuildConfig(args).get('is_android', False): 596 return AndroidPlatform(args) 597 else: 598 return DesktopPlatform(args) 599 600 def _Run(self, runnable, count, secondary=False): 601 raise NotImplementedError() # pragma: no cover 602 603 def _LoggedRun(self, runnable, count, secondary=False): 604 suffix = ' - secondary' if secondary else '' 605 title = '>>> %%s (#%d)%s:' % ((count + 1), suffix) 606 try: 607 output = self._Run(runnable, count, secondary) 608 except OSError: 609 logging.exception(title % 'OSError') 610 raise 611 if output.stdout: 612 logging.info(title % 'Stdout' + '\n%s', output.stdout) 613 if output.stderr: # pragma: no cover 614 # Print stderr for debugging. 615 logging.info(title % 'Stderr' + '\n%s', output.stderr) 616 logging.warning('>>> Test timed out after %ss.', runnable.timeout) 617 if output.exit_code != 0: 618 logging.warning('>>> Test crashed with exit code %d.', output.exit_code) 619 return output 620 621 def Run(self, runnable, count, secondary): 622 """Execute the benchmark's main file. 623 624 Args: 625 runnable: A Runnable benchmark instance. 626 count: The number of this (repeated) run. 627 secondary: True if secondary run should be executed. 628 629 Returns: 630 A tuple with the two benchmark outputs. The latter will be NULL_OUTPUT if 631 secondary is False. 632 """ 633 output = self._LoggedRun(runnable, count, secondary=False) 634 if secondary: 635 return output, self._LoggedRun(runnable, count, secondary=True) 636 else: 637 return output, NULL_OUTPUT 638 639 640class DesktopPlatform(Platform): 641 def __init__(self, args): 642 super(DesktopPlatform, self).__init__(args) 643 self.command_prefix = [] 644 645 # Setup command class to OS specific version. 646 command.setup(utils.GuessOS(), args.device) 647 648 if args.prioritize or args.affinitize != None: 649 self.command_prefix = ['schedtool'] 650 if args.prioritize: 651 self.command_prefix += ['-n', '-20'] 652 if args.affinitize != None: 653 # schedtool expects a bit pattern when setting affinity, where each 654 # bit set to '1' corresponds to a core where the process may run on. 655 # First bit corresponds to CPU 0. Since the 'affinitize' parameter is 656 # a core number, we need to map to said bit pattern. 657 cpu = int(args.affinitize) 658 core = 1 << cpu 659 self.command_prefix += ['-a', ('0x%x' % core)] 660 self.command_prefix += ['-e'] 661 662 def PreExecution(self): 663 pass 664 665 def PostExecution(self): 666 pass 667 668 def PreTests(self, node, path): 669 if isinstance(node, RunnableConfig): 670 node.ChangeCWD(path) 671 672 def _Run(self, runnable, count, secondary=False): 673 shell_dir = self.shell_dir_secondary if secondary else self.shell_dir 674 cmd = runnable.GetCommand(self.command_prefix, shell_dir, self.extra_flags) 675 output = cmd.execute() 676 677 if output.IsSuccess() and '--prof' in self.extra_flags: 678 os_prefix = {'linux': 'linux', 'macos': 'mac'}.get(utils.GuessOS()) 679 if os_prefix: 680 tick_tools = os.path.join(TOOLS_BASE, '%s-tick-processor' % os_prefix) 681 subprocess.check_call(tick_tools + ' --only-summary', shell=True) 682 else: # pragma: no cover 683 logging.warning( 684 'Profiler option currently supported on Linux and Mac OS.') 685 686 # /usr/bin/time outputs to stderr 687 if runnable.process_size: 688 output.stdout += output.stderr 689 return output 690 691 692class AndroidPlatform(Platform): # pragma: no cover 693 694 def __init__(self, args): 695 super(AndroidPlatform, self).__init__(args) 696 self.driver = android.android_driver(args.device) 697 698 def PreExecution(self): 699 self.driver.set_high_perf_mode() 700 701 def PostExecution(self): 702 self.driver.set_default_perf_mode() 703 self.driver.tear_down() 704 705 def PreTests(self, node, path): 706 if isinstance(node, RunnableConfig): 707 node.ChangeCWD(path) 708 suite_dir = os.path.abspath(os.path.dirname(path)) 709 if node.path: 710 bench_rel = os.path.normpath(os.path.join(*node.path)) 711 bench_abs = os.path.join(suite_dir, bench_rel) 712 else: 713 bench_rel = '.' 714 bench_abs = suite_dir 715 716 self.driver.push_executable(self.shell_dir, 'bin', node.binary) 717 if self.shell_dir_secondary: 718 self.driver.push_executable( 719 self.shell_dir_secondary, 'bin_secondary', node.binary) 720 721 if isinstance(node, RunnableConfig): 722 self.driver.push_file(bench_abs, node.main, bench_rel) 723 for resource in node.resources: 724 self.driver.push_file(bench_abs, resource, bench_rel) 725 726 def _Run(self, runnable, count, secondary=False): 727 target_dir = 'bin_secondary' if secondary else 'bin' 728 self.driver.drop_ram_caches() 729 730 # Relative path to benchmark directory. 731 if runnable.path: 732 bench_rel = os.path.normpath(os.path.join(*runnable.path)) 733 else: 734 bench_rel = '.' 735 736 logcat_file = None 737 if self.args.dump_logcats_to: 738 runnable_name = '-'.join(runnable.graphs) 739 logcat_file = os.path.join( 740 self.args.dump_logcats_to, 'logcat-%s-#%d%s.log' % ( 741 runnable_name, count + 1, '-secondary' if secondary else '')) 742 logging.debug('Dumping logcat into %s', logcat_file) 743 744 output = Output() 745 start = time.time() 746 try: 747 output.stdout = self.driver.run( 748 target_dir=target_dir, 749 binary=runnable.binary, 750 args=runnable.GetCommandFlags(self.extra_flags), 751 rel_path=bench_rel, 752 timeout=runnable.timeout, 753 logcat_file=logcat_file, 754 ) 755 except android.CommandFailedException as e: 756 output.stdout = e.output 757 output.exit_code = e.status 758 except android.TimeoutException as e: 759 output.stdout = e.output 760 output.timed_out = True 761 if runnable.process_size: 762 output.stdout += 'MaxMemory: Unsupported' 763 output.duration = time.time() - start 764 return output 765 766 767class CustomMachineConfiguration: 768 def __init__(self, disable_aslr = False, governor = None): 769 self.aslr_backup = None 770 self.governor_backup = None 771 self.disable_aslr = disable_aslr 772 self.governor = governor 773 774 def __enter__(self): 775 if self.disable_aslr: 776 self.aslr_backup = CustomMachineConfiguration.GetASLR() 777 CustomMachineConfiguration.SetASLR(0) 778 if self.governor != None: 779 self.governor_backup = CustomMachineConfiguration.GetCPUGovernor() 780 CustomMachineConfiguration.SetCPUGovernor(self.governor) 781 return self 782 783 def __exit__(self, type, value, traceback): 784 if self.aslr_backup != None: 785 CustomMachineConfiguration.SetASLR(self.aslr_backup) 786 if self.governor_backup != None: 787 CustomMachineConfiguration.SetCPUGovernor(self.governor_backup) 788 789 @staticmethod 790 def GetASLR(): 791 try: 792 with open('/proc/sys/kernel/randomize_va_space', 'r') as f: 793 return int(f.readline().strip()) 794 except Exception: 795 logging.exception('Failed to get current ASLR settings.') 796 raise 797 798 @staticmethod 799 def SetASLR(value): 800 try: 801 with open('/proc/sys/kernel/randomize_va_space', 'w') as f: 802 f.write(str(value)) 803 except Exception: 804 logging.exception( 805 'Failed to update ASLR to %s. Are we running under sudo?', value) 806 raise 807 808 new_value = CustomMachineConfiguration.GetASLR() 809 if value != new_value: 810 raise Exception('Present value is %s' % new_value) 811 812 @staticmethod 813 def GetCPUCoresRange(): 814 try: 815 with open('/sys/devices/system/cpu/present', 'r') as f: 816 indexes = f.readline() 817 r = map(int, indexes.split('-')) 818 if len(r) == 1: 819 return range(r[0], r[0] + 1) 820 return range(r[0], r[1] + 1) 821 except Exception: 822 logging.exception('Failed to retrieve number of CPUs.') 823 raise 824 825 @staticmethod 826 def GetCPUPathForId(cpu_index): 827 ret = '/sys/devices/system/cpu/cpu' 828 ret += str(cpu_index) 829 ret += '/cpufreq/scaling_governor' 830 return ret 831 832 @staticmethod 833 def GetCPUGovernor(): 834 try: 835 cpu_indices = CustomMachineConfiguration.GetCPUCoresRange() 836 ret = None 837 for cpu_index in cpu_indices: 838 cpu_device = CustomMachineConfiguration.GetCPUPathForId(cpu_index) 839 with open(cpu_device, 'r') as f: 840 # We assume the governors of all CPUs are set to the same value 841 val = f.readline().strip() 842 if ret == None: 843 ret = val 844 elif ret != val: 845 raise Exception('CPU cores have differing governor settings') 846 return ret 847 except Exception: 848 logging.exception('Failed to get the current CPU governor. Is the CPU ' 849 'governor disabled? Check BIOS.') 850 raise 851 852 @staticmethod 853 def SetCPUGovernor(value): 854 try: 855 cpu_indices = CustomMachineConfiguration.GetCPUCoresRange() 856 for cpu_index in cpu_indices: 857 cpu_device = CustomMachineConfiguration.GetCPUPathForId(cpu_index) 858 with open(cpu_device, 'w') as f: 859 f.write(value) 860 861 except Exception: 862 logging.exception('Failed to change CPU governor to %s. Are we ' 863 'running under sudo?', value) 864 raise 865 866 cur_value = CustomMachineConfiguration.GetCPUGovernor() 867 if cur_value != value: 868 raise Exception('Could not set CPU governor. Present value is %s' 869 % cur_value ) 870 871 872class MaxTotalDurationReachedError(Exception): 873 """Exception used to stop running tests when max total duration is reached.""" 874 pass 875 876 877def Main(argv): 878 parser = argparse.ArgumentParser() 879 parser.add_argument('--arch', 880 help='The architecture to run tests for. Pass "auto" ' 881 'to auto-detect.', default='x64', 882 choices=SUPPORTED_ARCHS + ['auto']) 883 parser.add_argument('--buildbot', 884 help='Adapt to path structure used on buildbots and adds ' 885 'timestamps/level to all logged status messages', 886 default=False, action='store_true') 887 parser.add_argument('-d', '--device', 888 help='The device ID to run Android tests on. If not ' 889 'given it will be autodetected.') 890 parser.add_argument('--extra-flags', 891 help='Additional flags to pass to the test executable', 892 default='') 893 parser.add_argument('--json-test-results', 894 help='Path to a file for storing json results.') 895 parser.add_argument('--json-test-results-secondary', 896 help='Path to a file for storing json results from run ' 897 'without patch or for reference build run.') 898 parser.add_argument('--outdir', help='Base directory with compile output', 899 default='out') 900 parser.add_argument('--outdir-secondary', 901 help='Base directory with compile output without patch ' 902 'or for reference build') 903 parser.add_argument('--binary-override-path', 904 help='JavaScript engine binary. By default, d8 under ' 905 'architecture-specific build dir. ' 906 'Not supported in conjunction with outdir-secondary.') 907 parser.add_argument('--prioritize', 908 help='Raise the priority to nice -20 for the ' 909 'benchmarking process.Requires Linux, schedtool, and ' 910 'sudo privileges.', default=False, action='store_true') 911 parser.add_argument('--affinitize', 912 help='Run benchmarking process on the specified core. ' 913 'For example: --affinitize=0 will run the benchmark ' 914 'process on core 0. --affinitize=3 will run the ' 915 'benchmark process on core 3. Requires Linux, schedtool, ' 916 'and sudo privileges.', default=None) 917 parser.add_argument('--noaslr', 918 help='Disable ASLR for the duration of the benchmarked ' 919 'process. Requires Linux and sudo privileges.', 920 default=False, action='store_true') 921 parser.add_argument('--cpu-governor', 922 help='Set cpu governor to specified policy for the ' 923 'duration of the benchmarked process. Typical options: ' 924 '"powersave" for more stable results, or "performance" ' 925 'for shorter completion time of suite, with potentially ' 926 'more noise in results.') 927 parser.add_argument('--filter', 928 help='Only run the benchmarks beginning with this ' 929 'string. For example: ' 930 '--filter=JSTests/TypedArrays/ will run only TypedArray ' 931 'benchmarks from the JSTests suite.', 932 default='') 933 parser.add_argument('--confidence-level', type=float, 934 help='Repeatedly runs each benchmark until specified ' 935 'confidence level is reached. The value is interpreted ' 936 'as the number of standard deviations from the mean that ' 937 'all values must lie within. Typical values are 1, 2 and ' 938 '3 and correspond to 68%%, 95%% and 99.7%% probability ' 939 'that the measured value is within 0.1%% of the true ' 940 'value. Larger values result in more retries and thus ' 941 'longer runtime, but also provide more reliable results. ' 942 'Also see --max-total-duration flag.') 943 parser.add_argument('--max-total-duration', type=int, default=7140, # 1h 59m 944 help='Max total duration in seconds allowed for retries ' 945 'across all tests. This is especially useful in ' 946 'combination with the --confidence-level flag.') 947 parser.add_argument('--dump-logcats-to', 948 help='Writes logcat output from each test into specified ' 949 'directory. Only supported for android targets.') 950 parser.add_argument('--run-count', type=int, default=0, 951 help='Override the run count specified by the test ' 952 'suite. The default 0 uses the suite\'s config.') 953 parser.add_argument('-v', '--verbose', default=False, action='store_true', 954 help='Be verbose and print debug output.') 955 parser.add_argument('suite', nargs='+', help='Path to the suite config file.') 956 957 try: 958 args = parser.parse_args(argv) 959 except SystemExit: 960 return INFRA_FAILURE_RETCODE 961 962 logging.basicConfig( 963 level=logging.DEBUG if args.verbose else logging.INFO, 964 format='%(asctime)s %(levelname)-8s %(message)s') 965 966 if args.arch == 'auto': # pragma: no cover 967 args.arch = utils.DefaultArch() 968 if args.arch not in SUPPORTED_ARCHS: 969 logging.error( 970 'Auto-detected architecture "%s" is not supported.', args.arch) 971 return INFRA_FAILURE_RETCODE 972 973 if (args.json_test_results_secondary and 974 not args.outdir_secondary): # pragma: no cover 975 logging.error('For writing secondary json test results, a secondary outdir ' 976 'patch must be specified.') 977 return INFRA_FAILURE_RETCODE 978 979 workspace = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) 980 981 if args.buildbot: 982 build_config = 'Release' 983 else: 984 build_config = '%s.release' % args.arch 985 986 if args.binary_override_path == None: 987 args.shell_dir = os.path.join(workspace, args.outdir, build_config) 988 default_binary_name = 'd8' 989 else: 990 if not os.path.isfile(args.binary_override_path): 991 logging.error('binary-override-path must be a file name') 992 return INFRA_FAILURE_RETCODE 993 if args.outdir_secondary: 994 logging.error('specify either binary-override-path or outdir-secondary') 995 return INFRA_FAILURE_RETCODE 996 args.shell_dir = os.path.abspath( 997 os.path.dirname(args.binary_override_path)) 998 default_binary_name = os.path.basename(args.binary_override_path) 999 1000 if args.outdir_secondary: 1001 args.shell_dir_secondary = os.path.join( 1002 workspace, args.outdir_secondary, build_config) 1003 else: 1004 args.shell_dir_secondary = None 1005 1006 if args.json_test_results: 1007 args.json_test_results = os.path.abspath(args.json_test_results) 1008 1009 if args.json_test_results_secondary: 1010 args.json_test_results_secondary = os.path.abspath( 1011 args.json_test_results_secondary) 1012 1013 # Ensure all arguments have absolute path before we start changing current 1014 # directory. 1015 args.suite = map(os.path.abspath, args.suite) 1016 1017 prev_aslr = None 1018 prev_cpu_gov = None 1019 platform = Platform.GetPlatform(args) 1020 1021 result_tracker = ResultTracker() 1022 result_tracker_secondary = ResultTracker() 1023 have_failed_tests = False 1024 with CustomMachineConfiguration(governor = args.cpu_governor, 1025 disable_aslr = args.noaslr) as conf: 1026 for path in args.suite: 1027 if not os.path.exists(path): # pragma: no cover 1028 result_tracker.AddError('Configuration file %s does not exist.' % path) 1029 continue 1030 1031 with open(path) as f: 1032 suite = json.loads(f.read()) 1033 1034 # If no name is given, default to the file name without .json. 1035 suite.setdefault('name', os.path.splitext(os.path.basename(path))[0]) 1036 1037 # Setup things common to one test suite. 1038 platform.PreExecution() 1039 1040 # Build the graph/trace tree structure. 1041 default_parent = DefaultSentinel(default_binary_name) 1042 root = BuildGraphConfigs(suite, args.arch, default_parent) 1043 1044 # Callback to be called on each node on traversal. 1045 def NodeCB(node): 1046 platform.PreTests(node, path) 1047 1048 # Traverse graph/trace tree and iterate over all runnables. 1049 start = time.time() 1050 try: 1051 for runnable in FlattenRunnables(root, NodeCB): 1052 runnable_name = '/'.join(runnable.graphs) 1053 if (not runnable_name.startswith(args.filter) and 1054 runnable_name + '/' != args.filter): 1055 continue 1056 logging.info('>>> Running suite: %s', runnable_name) 1057 1058 def RunGenerator(runnable): 1059 if args.confidence_level: 1060 counter = 0 1061 while not result_tracker.HasEnoughRuns( 1062 runnable, args.confidence_level): 1063 yield counter 1064 counter += 1 1065 else: 1066 for i in range(0, max(1, args.run_count or runnable.run_count)): 1067 yield i 1068 1069 for i in RunGenerator(runnable): 1070 attempts_left = runnable.retry_count + 1 1071 while attempts_left: 1072 total_duration = time.time() - start 1073 if total_duration > args.max_total_duration: 1074 logging.info( 1075 '>>> Stopping now since running for too long (%ds > %ds)', 1076 total_duration, args.max_total_duration) 1077 raise MaxTotalDurationReachedError() 1078 1079 output, output_secondary = platform.Run( 1080 runnable, i, secondary=args.shell_dir_secondary) 1081 result_tracker.AddRunnableDuration(runnable, output.duration) 1082 result_tracker_secondary.AddRunnableDuration( 1083 runnable, output_secondary.duration) 1084 1085 if output.IsSuccess() and output_secondary.IsSuccess(): 1086 runnable.ProcessOutput(output, result_tracker, i) 1087 if output_secondary is not NULL_OUTPUT: 1088 runnable.ProcessOutput( 1089 output_secondary, result_tracker_secondary, i) 1090 break 1091 1092 attempts_left -= 1 1093 if not attempts_left: 1094 logging.info('>>> Suite %s failed after %d retries', 1095 runnable_name, runnable.retry_count + 1) 1096 have_failed_tests = True 1097 else: 1098 logging.info('>>> Retrying suite: %s', runnable_name) 1099 except MaxTotalDurationReachedError: 1100 have_failed_tests = True 1101 1102 platform.PostExecution() 1103 1104 if args.json_test_results: 1105 result_tracker.WriteToFile(args.json_test_results) 1106 else: # pragma: no cover 1107 print('Primary results:', result_tracker) 1108 1109 if args.shell_dir_secondary: 1110 if args.json_test_results_secondary: 1111 result_tracker_secondary.WriteToFile(args.json_test_results_secondary) 1112 else: # pragma: no cover 1113 print('Secondary results:', result_tracker_secondary) 1114 1115 if (result_tracker.errors or result_tracker_secondary.errors or 1116 have_failed_tests): 1117 return 1 1118 1119 return 0 1120 1121 1122def MainWrapper(): 1123 try: 1124 return Main(sys.argv[1:]) 1125 except: 1126 # Log uncaptured exceptions and report infra failure to the caller. 1127 traceback.print_exc() 1128 return INFRA_FAILURE_RETCODE 1129 1130 1131if __name__ == '__main__': # pragma: no cover 1132 sys.exit(MainWrapper()) 1133