1#!/usr/bin/env vpython 2# Copyright (c) 2013 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6""" A utility to generate an up-to-date orderfile. 7 8The orderfile is used by the linker to order text sections such that the 9sections are placed consecutively in the order specified. This allows us 10to page in less code during start-up. 11 12Example usage: 13 tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \ 14 --target-arch=arm 15""" 16 17from __future__ import print_function 18 19import argparse 20import hashlib 21import json 22import glob 23import logging 24import os 25import shutil 26import subprocess 27import sys 28import tempfile 29import time 30 31import cluster 32import cyglog_to_orderfile 33import patch_orderfile 34import process_profiles 35import profile_android_startup 36import symbol_extractor 37 38_SRC_PATH = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) 39sys.path.append(os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil')) 40from devil.android import device_utils 41from devil.android.sdk import version_codes 42 43 44_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 45 os.pardir, os.pardir) 46sys.path.append(os.path.join(_SRC_PATH, 'build', 'android')) 47import devil_chromium 48from pylib import constants 49 50 51# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly 52constants.SetBuildType('Release') 53 54 55# Architecture specific GN args. Trying to build an orderfile for an 56# architecture not listed here will eventually throw. 57_ARCH_GN_ARGS = { 58 'arm': [ 'target_cpu = "arm"' ], 59 'arm64': [ 'target_cpu = "arm64"', 60 'android_64bit_browser = true'], 61} 62 63class CommandError(Exception): 64 """Indicates that a dispatched shell command exited with a non-zero status.""" 65 66 def __init__(self, value): 67 super(CommandError, self).__init__() 68 self.value = value 69 70 def __str__(self): 71 return repr(self.value) 72 73 74def _GenerateHash(file_path): 75 """Calculates and returns the hash of the file at file_path.""" 76 sha1 = hashlib.sha1() 77 with open(file_path, 'rb') as f: 78 while True: 79 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. 80 chunk = f.read(1024 * 1024) 81 if not chunk: 82 break 83 sha1.update(chunk) 84 return sha1.hexdigest() 85 86 87def _GetFileExtension(file_name): 88 """Calculates the file extension from a file name. 89 90 Args: 91 file_name: The source file name. 92 Returns: 93 The part of file_name after the dot (.) or None if the file has no 94 extension. 95 Examples: /home/user/foo.bar -> bar 96 /home/user.name/foo -> None 97 /home/user/.foo -> None 98 /home/user/foo.bar.baz -> baz 99 """ 100 file_name_parts = os.path.basename(file_name).split('.') 101 if len(file_name_parts) > 1: 102 return file_name_parts[-1] 103 else: 104 return None 105 106 107def _StashOutputDirectory(buildpath): 108 """Takes the output directory and stashes it in the default output directory. 109 110 This allows it to be used for incremental builds next time (after unstashing) 111 by keeping it in a place that isn't deleted normally, while also ensuring 112 that it is properly clobbered when appropriate. 113 114 This is a dirty hack to deal with the needs of clobbering while also handling 115 incremental builds and the hardcoded relative paths used in some of the 116 project files. 117 118 Args: 119 buildpath: The path where the building happens. If this corresponds to the 120 default output directory, no action is taken. 121 """ 122 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname( 123 constants.GetOutDirectory())): 124 return 125 name = os.path.basename(buildpath) 126 stashpath = os.path.join(constants.GetOutDirectory(), name) 127 if not os.path.exists(buildpath): 128 return 129 if os.path.exists(stashpath): 130 shutil.rmtree(stashpath, ignore_errors=True) 131 shutil.move(buildpath, stashpath) 132 133 134def _UnstashOutputDirectory(buildpath): 135 """Inverse of _StashOutputDirectory. 136 137 Moves the output directory stashed within the default output directory 138 (out/Release) to the position where the builds can actually happen. 139 140 This is a dirty hack to deal with the needs of clobbering while also handling 141 incremental builds and the hardcoded relative paths used in some of the 142 project files. 143 144 Args: 145 buildpath: The path where the building happens. If this corresponds to the 146 default output directory, no action is taken. 147 """ 148 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname( 149 constants.GetOutDirectory())): 150 return 151 name = os.path.basename(buildpath) 152 stashpath = os.path.join(constants.GetOutDirectory(), name) 153 if not os.path.exists(stashpath): 154 return 155 if os.path.exists(buildpath): 156 shutil.rmtree(buildpath, ignore_errors=True) 157 shutil.move(stashpath, buildpath) 158 159 160class StepRecorder(object): 161 """Records steps and timings.""" 162 163 def __init__(self, buildbot): 164 self.timings = [] 165 self._previous_step = ('', 0.0) 166 self._buildbot = buildbot 167 self._error_recorded = False 168 169 def BeginStep(self, name): 170 """Marks a beginning of the next step in the script. 171 172 On buildbot, this prints a specially formatted name that will show up 173 in the waterfall. Otherwise, just prints the step name. 174 175 Args: 176 name: The name of the step. 177 """ 178 self.EndStep() 179 self._previous_step = (name, time.time()) 180 print('Running step: ', name) 181 182 def EndStep(self): 183 """Records successful completion of the current step. 184 185 This is optional if the step is immediately followed by another BeginStep. 186 """ 187 if self._previous_step[0]: 188 elapsed = time.time() - self._previous_step[1] 189 print('Step %s took %f seconds' % (self._previous_step[0], elapsed)) 190 self.timings.append((self._previous_step[0], elapsed)) 191 192 self._previous_step = ('', 0.0) 193 194 def FailStep(self, message=None): 195 """Marks that a particular step has failed. 196 197 On buildbot, this will mark the current step as failed on the waterfall. 198 Otherwise we will just print an optional failure message. 199 200 Args: 201 message: An optional explanation as to why the step failed. 202 """ 203 print('STEP FAILED!!') 204 if message: 205 print(message) 206 self._error_recorded = True 207 self.EndStep() 208 209 def ErrorRecorded(self): 210 """True if FailStep has been called.""" 211 return self._error_recorded 212 213 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True, 214 stdout=None): 215 """Execute a shell command. 216 217 Args: 218 cmd: A list of command strings. 219 cwd: Directory in which the command should be executed, defaults to build 220 root of script's location if not specified. 221 raise_on_error: If true will raise a CommandError if the call doesn't 222 succeed and mark the step as failed. 223 stdout: A file to redirect stdout for the command to. 224 225 Returns: 226 The process's return code. 227 228 Raises: 229 CommandError: An error executing the specified command. 230 """ 231 print('Executing %s in %s' % (' '.join(cmd), cwd)) 232 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ) 233 process.wait() 234 if raise_on_error and process.returncode != 0: 235 self.FailStep() 236 raise CommandError('Exception executing command %s' % ' '.join(cmd)) 237 return process.returncode 238 239 240class ClankCompiler(object): 241 """Handles compilation of clank.""" 242 243 def __init__(self, out_dir, step_recorder, arch, use_goma, goma_dir, 244 system_health_profiling, monochrome, public, orderfile_location): 245 self._out_dir = out_dir 246 self._step_recorder = step_recorder 247 self._arch = arch 248 self._use_goma = use_goma 249 self._goma_dir = goma_dir 250 self._system_health_profiling = system_health_profiling 251 self._public = public 252 self._orderfile_location = orderfile_location 253 if monochrome: 254 self._apk = 'Monochrome.apk' 255 self._apk_target = 'monochrome_apk' 256 self._libname = 'libmonochrome' 257 self._libchrome_target = 'libmonochrome' 258 else: 259 self._apk = 'Chrome.apk' 260 self._apk_target = 'chrome_apk' 261 self._libname = 'libchrome' 262 self._libchrome_target = 'libchrome' 263 if public: 264 self._apk = self._apk.replace('.apk', 'Public.apk') 265 self._apk_target = self._apk_target.replace('_apk', '_public_apk') 266 267 self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj') 268 self.lib_chrome_so = os.path.join( 269 self._out_dir, 'Release', 'lib.unstripped', 270 '{}.so'.format(self._libname)) 271 self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk) 272 273 def Build(self, instrumented, use_call_graph, target): 274 """Builds the provided ninja target with or without order_profiling on. 275 276 Args: 277 instrumented: (bool) Whether we want to build an instrumented binary. 278 use_call_graph: (bool) Whether to use the call graph instrumentation. 279 target: (str) The name of the ninja target to build. 280 """ 281 self._step_recorder.BeginStep('Compile %s' % target) 282 assert not use_call_graph or instrumented, ('You can not enable call graph ' 283 'without instrumentation!') 284 285 # Set the "Release Official" flavor, the parts affecting performance. 286 args = [ 287 'enable_resource_whitelist_generation=false', 288 'is_chrome_branded=' + str(not self._public).lower(), 289 'is_debug=false', 290 'is_official_build=true', 291 'symbol_level=1', # to fit 30 GiB RAM on the bot when LLD is running 292 'target_os="android"', 293 'use_goma=' + str(self._use_goma).lower(), 294 'use_order_profiling=' + str(instrumented).lower(), 295 'use_call_graph=' + str(use_call_graph).lower(), 296 ] 297 args += _ARCH_GN_ARGS[self._arch] 298 if self._goma_dir: 299 args += ['goma_dir="%s"' % self._goma_dir] 300 if self._system_health_profiling: 301 args += ['devtools_instrumentation_dumping = ' + 302 str(instrumented).lower()] 303 304 if self._public and os.path.exists(self._orderfile_location): 305 # GN needs the orderfile path to be source-absolute. 306 src_abs_orderfile = os.path.relpath(self._orderfile_location, 307 constants.DIR_SOURCE_ROOT) 308 args += ['chrome_orderfile="//{}"'.format(src_abs_orderfile)] 309 310 self._step_recorder.RunCommand( 311 ['gn', 'gen', os.path.join(self._out_dir, 'Release'), 312 '--args=' + ' '.join(args)]) 313 314 self._step_recorder.RunCommand( 315 ['autoninja', '-C', 316 os.path.join(self._out_dir, 'Release'), target]) 317 318 def ForceRelink(self): 319 """Forces libchrome.so or libmonochrome.so to be re-linked. 320 321 With partitioned libraries enabled, deleting these library files does not 322 guarantee they'll be recreated by the linker (they may simply be 323 re-extracted from a combined library). To be safe, touch a source file 324 instead. See http://crbug.com/972701 for more explanation. 325 """ 326 file_to_touch = os.path.join(constants.DIR_SOURCE_ROOT, 'chrome', 'browser', 327 'chrome_browser_main_android.cc') 328 assert os.path.exists(file_to_touch) 329 self._step_recorder.RunCommand(['touch', file_to_touch]) 330 331 def CompileChromeApk(self, instrumented, use_call_graph, force_relink=False): 332 """Builds a Chrome.apk either with or without order_profiling on. 333 334 Args: 335 instrumented: (bool) Whether to build an instrumented apk. 336 use_call_graph: (bool) Whether to use the call graph instrumentation. 337 force_relink: Whether libchromeview.so should be re-created. 338 """ 339 if force_relink: 340 self.ForceRelink() 341 self.Build(instrumented, use_call_graph, self._apk_target) 342 343 def CompileLibchrome(self, instrumented, use_call_graph, force_relink=False): 344 """Builds a libchrome.so either with or without order_profiling on. 345 346 Args: 347 instrumented: (bool) Whether to build an instrumented apk. 348 use_call_graph: (bool) Whether to use the call graph instrumentation. 349 force_relink: (bool) Whether libchrome.so should be re-created. 350 """ 351 if force_relink: 352 self.ForceRelink() 353 self.Build(instrumented, use_call_graph, self._libchrome_target) 354 355 356class OrderfileUpdater(object): 357 """Handles uploading and committing a new orderfile in the repository. 358 359 Only used for testing or on a bot. 360 """ 361 362 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None 363 _CLOUD_STORAGE_BUCKET = None 364 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py' 365 366 def __init__(self, repository_root, step_recorder): 367 """Constructor. 368 369 Args: 370 repository_root: (str) Root of the target repository. 371 step_recorder: (StepRecorder) Step recorder, for logging. 372 """ 373 self._repository_root = repository_root 374 self._step_recorder = step_recorder 375 376 def CommitStashedFileHashes(self, files): 377 """Commits unpatched and patched orderfiles hashes if changed. 378 379 The files are committed only if their associated sha1 hash files match, and 380 are modified in git. In normal operations the hash files are changed only 381 when a file is uploaded to cloud storage. If the hash file is not modified 382 in git, the file is skipped. 383 384 Args: 385 files: [str or None] specifies file paths. None items are ignored. 386 387 Raises: 388 Exception if the hash file does not match the file. 389 NotImplementedError when the commit logic hasn't been overridden. 390 """ 391 files_to_commit = list(filter(None, files)) 392 if files_to_commit: 393 self._CommitStashedFiles(files_to_commit) 394 395 def UploadToCloudStorage(self, filename, use_debug_location): 396 """Uploads a file to cloud storage. 397 398 Args: 399 filename: (str) File to upload. 400 use_debug_location: (bool) Whether to use the debug location. 401 """ 402 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location 403 else self._CLOUD_STORAGE_BUCKET) 404 extension = _GetFileExtension(filename) 405 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket] 406 if extension: 407 cmd.extend(['-z', extension]) 408 cmd.append(filename) 409 self._step_recorder.RunCommand(cmd) 410 print('Download: https://sandbox.google.com/storage/%s/%s' % 411 (bucket, _GenerateHash(filename))) 412 413 def _GetHashFilePathAndContents(self, filename): 414 """Gets the name and content of the hash file created from uploading the 415 given file. 416 417 Args: 418 filename: (str) The file that was uploaded to cloud storage. 419 420 Returns: 421 A tuple of the hash file name, relative to the reository root, and the 422 content, which should be the sha1 hash of the file 423 ('base_file.sha1', hash) 424 """ 425 abs_hash_filename = filename + '.sha1' 426 rel_hash_filename = os.path.relpath( 427 abs_hash_filename, self._repository_root) 428 with open(abs_hash_filename, 'r') as f: 429 return (rel_hash_filename, f.read()) 430 431 def _CommitFiles(self, files_to_commit, commit_message_lines): 432 """Commits a list of files, with a given message.""" 433 raise NotImplementedError 434 435 def _GitStash(self): 436 """Git stash the current clank tree. 437 438 Raises: 439 NotImplementedError when the stash logic hasn't been overridden. 440 """ 441 raise NotImplementedError 442 443 def _CommitStashedFiles(self, expected_files_in_stash): 444 """Commits stashed files. 445 446 The local repository is updated and then the files to commit are taken from 447 modified files from the git stash. The modified files should be a subset of 448 |expected_files_in_stash|. If there are unexpected modified files, this 449 function may raise. This is meant to be paired with _GitStash(). 450 451 Args: 452 expected_files_in_stash: [str] paths to a possible superset of files 453 expected to be stashed & committed. 454 455 Raises: 456 NotImplementedError when the commit logic hasn't been overridden. 457 """ 458 raise NotImplementedError 459 460 461class OrderfileNoopUpdater(OrderfileUpdater): 462 def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename): 463 # pylint: disable=unused-argument 464 return 465 466 def UploadToCloudStorage(self, filename, use_debug_location): 467 # pylint: disable=unused-argument 468 return 469 470 def _CommitFiles(self, files_to_commit, commit_message_lines): 471 raise NotImplementedError 472 473 474class OrderfileGenerator(object): 475 """A utility for generating a new orderfile for Clank. 476 477 Builds an instrumented binary, profiles a run of the application, and 478 generates an updated orderfile. 479 """ 480 _CHECK_ORDERFILE_SCRIPT = os.path.join( 481 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py') 482 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname( 483 constants.GetOutDirectory()))) # Normally /path/to/src 484 485 # Previous orderfile_generator debug files would be overwritten. 486 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files' 487 488 def _PrepareOrderfilePaths(self): 489 if self._options.public: 490 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT, 491 '') 492 if not os.path.exists(os.path.join(self._clank_dir, 'orderfiles')): 493 os.makedirs(os.path.join(self._clank_dir, 'orderfiles')) 494 else: 495 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT, 496 'clank') 497 498 self._unpatched_orderfile_filename = os.path.join( 499 self._clank_dir, 'orderfiles', 'unpatched_orderfile.%s') 500 self._path_to_orderfile = os.path.join( 501 self._clank_dir, 'orderfiles', 'orderfile.%s.out') 502 503 def _GetPathToOrderfile(self): 504 """Gets the path to the architecture-specific orderfile.""" 505 return self._path_to_orderfile % self._options.arch 506 507 def _GetUnpatchedOrderfileFilename(self): 508 """Gets the path to the architecture-specific unpatched orderfile.""" 509 return self._unpatched_orderfile_filename % self._options.arch 510 511 def _SetDevice(self): 512 """ Selects the device to be used by the script. 513 514 Returns: 515 (Device with given serial ID) : if the --device flag is set. 516 (Device running Android[K,L]) : if --use-legacy-chrome-apk flag is set or 517 no device running Android N+ was found. 518 (Device running Android N+) : Otherwise. 519 520 Raises Error: 521 If no device meeting the requirements has been found. 522 """ 523 devices = None 524 if self._options.device: 525 devices = [device_utils.DeviceUtils(self._options.device)] 526 else: 527 devices = device_utils.DeviceUtils.HealthyDevices() 528 529 assert devices, 'Expected at least one connected device' 530 531 if self._options.use_legacy_chrome_apk: 532 self._monochrome = False 533 for device in devices: 534 device_version = device.build_version_sdk 535 if (device_version >= version_codes.KITKAT 536 and device_version <= version_codes.LOLLIPOP_MR1): 537 return device 538 539 assert not self._options.use_legacy_chrome_apk, \ 540 'No device found running suitable android version for Chrome.apk.' 541 542 preferred_device = None 543 for device in devices: 544 if device.build_version_sdk >= version_codes.NOUGAT: 545 preferred_device = device 546 break 547 548 self._monochrome = preferred_device is not None 549 550 return preferred_device if preferred_device else devices[0] 551 552 553 def __init__(self, options, orderfile_updater_class): 554 self._options = options 555 self._instrumented_out_dir = os.path.join( 556 self._BUILD_ROOT, self._options.arch + '_instrumented_out') 557 if self._options.use_call_graph: 558 self._instrumented_out_dir += '_call_graph' 559 560 self._uninstrumented_out_dir = os.path.join( 561 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out') 562 self._no_orderfile_out_dir = os.path.join( 563 self._BUILD_ROOT, self._options.arch + '_no_orderfile_out') 564 565 self._PrepareOrderfilePaths() 566 567 if options.profile: 568 output_directory = os.path.join(self._instrumented_out_dir, 'Release') 569 host_profile_dir = os.path.join(output_directory, 'profile_data') 570 urls = [profile_android_startup.AndroidProfileTool.TEST_URL] 571 use_wpr = True 572 simulate_user = False 573 urls = options.urls 574 use_wpr = not options.no_wpr 575 simulate_user = options.simulate_user 576 device = self._SetDevice() 577 self._profiler = profile_android_startup.AndroidProfileTool( 578 output_directory, host_profile_dir, use_wpr, urls, simulate_user, 579 device, debug=self._options.streamline_for_debugging) 580 if options.pregenerated_profiles: 581 self._profiler.SetPregeneratedProfiles( 582 glob.glob(options.pregenerated_profiles)) 583 else: 584 assert not options.pregenerated_profiles, ( 585 '--pregenerated-profiles cannot be used with --skip-profile') 586 assert not options.profile_save_dir, ( 587 '--profile-save-dir cannot be used with --skip-profile') 588 self._monochrome = not self._options.use_legacy_chrome_apk 589 590 # Outlined function handling enabled by default for all architectures. 591 self._order_outlined_functions = not options.noorder_outlined_functions 592 593 self._output_data = {} 594 self._step_recorder = StepRecorder(options.buildbot) 595 self._compiler = None 596 if orderfile_updater_class is None: 597 if options.public: 598 orderfile_updater_class = OrderfileNoopUpdater 599 else: 600 orderfile_updater_class = OrderfileUpdater 601 assert issubclass(orderfile_updater_class, OrderfileUpdater) 602 self._orderfile_updater = orderfile_updater_class(self._clank_dir, 603 self._step_recorder) 604 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found' 605 symbol_extractor.SetArchitecture(options.arch) 606 607 @staticmethod 608 def _RemoveBlanks(src_file, dest_file): 609 """A utility to remove blank lines from a file. 610 611 Args: 612 src_file: The name of the file to remove the blanks from. 613 dest_file: The name of the file to write the output without blanks. 614 """ 615 assert src_file != dest_file, 'Source and destination need to be distinct' 616 617 try: 618 src = open(src_file, 'r') 619 dest = open(dest_file, 'w') 620 for line in src: 621 if line and not line.isspace(): 622 dest.write(line) 623 finally: 624 src.close() 625 dest.close() 626 627 def _GenerateAndProcessProfile(self): 628 """Invokes a script to merge the per-thread traces into one file. 629 630 The produced list of offsets is saved in 631 self._GetUnpatchedOrderfileFilename(). 632 """ 633 self._step_recorder.BeginStep('Generate Profile Data') 634 files = [] 635 logging.getLogger().setLevel(logging.DEBUG) 636 637 if self._options.profile_save_dir: 638 # The directory must not preexist, to ensure purity of data. Check 639 # before profiling to save time. 640 if os.path.exists(self._options.profile_save_dir): 641 raise Exception('Profile save directory must not pre-exist') 642 os.makedirs(self._options.profile_save_dir) 643 644 if self._options.system_health_orderfile: 645 files = self._profiler.CollectSystemHealthProfile( 646 self._compiler.chrome_apk) 647 self._MaybeSaveProfile(files) 648 try: 649 self._ProcessPhasedOrderfile(files) 650 except Exception: 651 for f in files: 652 self._SaveForDebugging(f) 653 self._SaveForDebugging(self._compiler.lib_chrome_so) 654 raise 655 finally: 656 self._profiler.Cleanup() 657 else: 658 self._CollectLegacyProfile() 659 logging.getLogger().setLevel(logging.INFO) 660 661 def _ProcessPhasedOrderfile(self, files): 662 """Process the phased orderfiles produced by system health benchmarks. 663 664 The offsets will be placed in _GetUnpatchedOrderfileFilename(). 665 666 Args: 667 file: Profile files pulled locally. 668 """ 669 self._step_recorder.BeginStep('Process Phased Orderfile') 670 profiles = process_profiles.ProfileManager(files) 671 processor = process_profiles.SymbolOffsetProcessor( 672 self._compiler.lib_chrome_so) 673 ordered_symbols = cluster.ClusterOffsets(profiles, processor, 674 call_graph=self._options.use_call_graph) 675 if not ordered_symbols: 676 raise Exception('Failed to get ordered symbols') 677 for sym in ordered_symbols: 678 assert not sym.startswith('OUTLINED_FUNCTION_'), ( 679 'Outlined function found in instrumented function, very likely ' 680 'something has gone very wrong!') 681 self._output_data['offsets_kib'] = processor.SymbolsSize( 682 ordered_symbols) / 1024 683 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile: 684 orderfile.write('\n'.join(ordered_symbols)) 685 686 def _CollectLegacyProfile(self): 687 files = [] 688 try: 689 files = self._profiler.CollectProfile( 690 self._compiler.chrome_apk, 691 constants.PACKAGE_INFO['chrome']) 692 self._MaybeSaveProfile(files) 693 self._step_recorder.BeginStep('Process profile') 694 assert os.path.exists(self._compiler.lib_chrome_so) 695 offsets = process_profiles.GetReachedOffsetsFromDumpFiles( 696 files, self._compiler.lib_chrome_so) 697 if not offsets: 698 raise Exception('No profiler offsets found in {}'.format( 699 '\n'.join(files))) 700 processor = process_profiles.SymbolOffsetProcessor( 701 self._compiler.lib_chrome_so) 702 ordered_symbols = processor.GetOrderedSymbols(offsets) 703 if not ordered_symbols: 704 raise Exception('No symbol names from offsets found in {}'.format( 705 '\n'.join(files))) 706 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile: 707 orderfile.write('\n'.join(ordered_symbols)) 708 except Exception: 709 for f in files: 710 self._SaveForDebugging(f) 711 raise 712 finally: 713 self._profiler.Cleanup() 714 715 def _MaybeSaveProfile(self, files): 716 if self._options.profile_save_dir: 717 logging.info('Saving profiles to %s', self._options.profile_save_dir) 718 for f in files: 719 shutil.copy(f, self._options.profile_save_dir) 720 logging.info('Saved profile %s', f) 721 722 def _PatchOrderfile(self): 723 """Patches the orderfile using clean version of libchrome.so.""" 724 self._step_recorder.BeginStep('Patch Orderfile') 725 patch_orderfile.GeneratePatchedOrderfile( 726 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so, 727 self._GetPathToOrderfile(), self._order_outlined_functions) 728 729 def _VerifySymbolOrder(self): 730 self._step_recorder.BeginStep('Verify Symbol Order') 731 return_code = self._step_recorder.RunCommand( 732 [self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so, 733 self._GetPathToOrderfile(), 734 '--target-arch=' + self._options.arch], 735 constants.DIR_SOURCE_ROOT, 736 raise_on_error=False) 737 if return_code: 738 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code) 739 740 def _RecordHash(self, file_name): 741 """Records the hash of the file into the output_data dictionary.""" 742 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash( 743 file_name) 744 745 def _SaveFileLocally(self, file_name, file_sha1): 746 """Saves the file to a temporary location and prints the sha1sum.""" 747 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES): 748 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES) 749 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES) 750 print('File: %s, saved in: %s, sha1sum: %s' % 751 (file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)) 752 753 def _SaveForDebugging(self, filename): 754 """Uploads the file to cloud storage or saves to a temporary location.""" 755 file_sha1 = _GenerateHash(filename) 756 if not self._options.buildbot: 757 self._SaveFileLocally(filename, file_sha1) 758 else: 759 print('Uploading file for debugging: ' + filename) 760 self._orderfile_updater.UploadToCloudStorage( 761 filename, use_debug_location=True) 762 763 def _SaveForDebuggingWithOverwrite(self, file_name): 764 """Uploads and overwrites the file in cloud storage or copies locally. 765 766 Should be used for large binaries like lib_chrome_so. 767 768 Args: 769 file_name: (str) File to upload. 770 """ 771 file_sha1 = _GenerateHash(file_name) 772 if not self._options.buildbot: 773 self._SaveFileLocally(file_name, file_sha1) 774 else: 775 print('Uploading file for debugging: %s, sha1sum: %s' % (file_name, 776 file_sha1)) 777 upload_location = '%s/%s' % ( 778 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name)) 779 self._step_recorder.RunCommand([ 780 'gsutil.py', 'cp', file_name, 'gs://' + upload_location]) 781 print('Uploaded to: https://sandbox.google.com/storage/' + 782 upload_location) 783 784 def _MaybeArchiveOrderfile(self, filename): 785 """In buildbot configuration, uploads the generated orderfile to 786 Google Cloud Storage. 787 788 Args: 789 filename: (str) Orderfile to upload. 790 """ 791 # First compute hashes so that we can download them later if we need to. 792 self._step_recorder.BeginStep('Compute hash for ' + filename) 793 self._RecordHash(filename) 794 if self._options.buildbot: 795 self._step_recorder.BeginStep('Archive ' + filename) 796 self._orderfile_updater.UploadToCloudStorage( 797 filename, use_debug_location=False) 798 799 def UploadReadyOrderfiles(self): 800 self._step_recorder.BeginStep('Upload Ready Orderfiles') 801 for file_name in [self._GetUnpatchedOrderfileFilename(), 802 self._GetPathToOrderfile()]: 803 self._orderfile_updater.UploadToCloudStorage( 804 file_name, use_debug_location=False) 805 806 def _NativeCodeMemoryBenchmark(self, apk): 807 """Runs system_health.memory_mobile to assess native code memory footprint. 808 809 Args: 810 apk: (str) Path to the apk. 811 812 Returns: 813 results: ([int]) Values of native code memory footprint in bytes from the 814 benchmark results. 815 """ 816 self._step_recorder.BeginStep("Running orderfile.memory_mobile") 817 try: 818 out_dir = tempfile.mkdtemp() 819 self._profiler._RunCommand(['tools/perf/run_benchmark', 820 '--device={}'.format( 821 self._profiler._device.serial), 822 '--browser=exact', 823 '--output-format=csv', 824 '--output-dir={}'.format(out_dir), 825 '--reset-results', 826 '--browser-executable={}'.format(apk), 827 'orderfile.memory_mobile']) 828 829 out_file_path = os.path.join(out_dir, 'results.csv') 830 if not os.path.exists(out_file_path): 831 raise Exception('Results file not found!') 832 833 results = {} 834 with open(out_file_path, 'r') as f: 835 reader = csv.DictReader(f) 836 for row in reader: 837 if not row['name'].endswith('NativeCodeResidentMemory'): 838 continue 839 # Note: NativeCodeResidentMemory records a single sample from each 840 # story run, so this average (reported as 'avg') is exactly the value 841 # of that one sample. Each story is run multiple times, so this loop 842 # will accumulate into a list all values for all runs of each story. 843 results.setdefault(row['name'], {}).setdefault( 844 row['stories'], []).append(row['avg']) 845 846 if not results: 847 raise Exception('Could not find relevant results') 848 849 return results 850 851 except Exception as e: 852 return 'Error: ' + str(e) 853 854 finally: 855 shutil.rmtree(out_dir) 856 857 858 def _PerformanceBenchmark(self, apk): 859 """Runs Speedometer2.0 to assess performance. 860 861 Args: 862 apk: (str) Path to the apk. 863 864 Returns: 865 results: ([float]) Speedometer2.0 results samples in milliseconds. 866 """ 867 self._step_recorder.BeginStep("Running Speedometer2.0.") 868 try: 869 out_dir = tempfile.mkdtemp() 870 self._profiler._RunCommand(['tools/perf/run_benchmark', 871 '--device={}'.format( 872 self._profiler._device.serial), 873 '--browser=exact', 874 '--output-format=histograms', 875 '--output-dir={}'.format(out_dir), 876 '--reset-results', 877 '--browser-executable={}'.format(apk), 878 'speedometer2']) 879 880 out_file_path = os.path.join(out_dir, 'histograms.json') 881 if not os.path.exists(out_file_path): 882 raise Exception('Results file not found!') 883 884 with open(out_file_path, 'r') as f: 885 results = json.load(f) 886 887 if not results: 888 raise Exception('Results file is empty.') 889 890 for el in results: 891 if 'name' in el and el['name'] == 'Total' and 'sampleValues' in el: 892 return el['sampleValues'] 893 894 raise Exception('Unexpected results format.') 895 896 except Exception as e: 897 return 'Error: ' + str(e) 898 899 finally: 900 shutil.rmtree(out_dir) 901 902 903 def RunBenchmark(self, out_directory, no_orderfile=False): 904 """Builds chrome apk and runs performance and memory benchmarks. 905 906 Builds a non-instrumented version of chrome. 907 Installs chrome apk on the device. 908 Runs Speedometer2.0 benchmark to assess performance. 909 Runs system_health.memory_mobile to evaluate memory footprint. 910 911 Args: 912 out_directory: (str) Path to out directory for this build. 913 no_orderfile: (bool) True if chrome to be built without orderfile. 914 915 Returns: 916 benchmark_results: (dict) Results extracted from benchmarks. 917 """ 918 try: 919 _UnstashOutputDirectory(out_directory) 920 self._compiler = ClankCompiler(out_directory, self._step_recorder, 921 self._options.arch, self._options.use_goma, 922 self._options.goma_dir, 923 self._options.system_health_orderfile, 924 self._monochrome, self._options.public, 925 self._GetPathToOrderfile()) 926 927 if no_orderfile: 928 orderfile_path = self._GetPathToOrderfile() 929 backup_orderfile = orderfile_path + '.backup' 930 shutil.move(orderfile_path, backup_orderfile) 931 open(orderfile_path, 'w').close() 932 933 # Build APK to be installed on the device. 934 self._compiler.CompileChromeApk(instrumented=False, 935 use_call_graph=False, 936 force_relink=True) 937 benchmark_results = dict() 938 benchmark_results['Speedometer2.0'] = self._PerformanceBenchmark( 939 self._compiler.chrome_apk) 940 benchmark_results['orderfile.memory_mobile'] = ( 941 self._NativeCodeMemoryBenchmark(self._compiler.chrome_apk)) 942 943 except Exception as e: 944 benchmark_results['Error'] = str(e) 945 946 finally: 947 if no_orderfile and os.path.exists(backup_orderfile): 948 shutil.move(backup_orderfile, orderfile_path) 949 _StashOutputDirectory(out_directory) 950 951 return benchmark_results 952 953 def Generate(self): 954 """Generates and maybe upload an order.""" 955 assert (bool(self._options.profile) ^ 956 bool(self._options.manual_symbol_offsets)) 957 if self._options.system_health_orderfile and not self._options.profile: 958 raise AssertionError('--system_health_orderfile must be not be used ' 959 'with --skip-profile') 960 if (self._options.manual_symbol_offsets and 961 not self._options.system_health_orderfile): 962 raise AssertionError('--manual-symbol-offsets must be used with ' 963 '--system_health_orderfile.') 964 965 if self._options.profile: 966 try: 967 _UnstashOutputDirectory(self._instrumented_out_dir) 968 self._compiler = ClankCompiler( 969 self._instrumented_out_dir, self._step_recorder, self._options.arch, 970 self._options.use_goma, self._options.goma_dir, 971 self._options.system_health_orderfile, self._monochrome, 972 self._options.public, self._GetPathToOrderfile()) 973 if not self._options.pregenerated_profiles: 974 # If there are pregenerated profiles, the instrumented build should 975 # not be changed to avoid invalidating the pregenerated profile 976 # offsets. 977 self._compiler.CompileChromeApk(instrumented=True, 978 use_call_graph= 979 self._options.use_call_graph) 980 self._GenerateAndProcessProfile() 981 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename()) 982 finally: 983 _StashOutputDirectory(self._instrumented_out_dir) 984 elif self._options.manual_symbol_offsets: 985 assert self._options.manual_libname 986 assert self._options.manual_objdir 987 with file(self._options.manual_symbol_offsets) as f: 988 symbol_offsets = [int(x) for x in f.xreadlines()] 989 processor = process_profiles.SymbolOffsetProcessor( 990 self._compiler.manual_libname) 991 generator = cyglog_to_orderfile.OffsetOrderfileGenerator( 992 processor, cyglog_to_orderfile.ObjectFileProcessor( 993 self._options.manual_objdir)) 994 ordered_sections = generator.GetOrderedSections(symbol_offsets) 995 if not ordered_sections: # Either None or empty is a problem. 996 raise Exception('Failed to get ordered sections') 997 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile: 998 orderfile.write('\n'.join(ordered_sections)) 999 1000 if self._options.patch: 1001 if self._options.profile: 1002 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(), 1003 self._GetPathToOrderfile()) 1004 try: 1005 _UnstashOutputDirectory(self._uninstrumented_out_dir) 1006 self._compiler = ClankCompiler( 1007 self._uninstrumented_out_dir, self._step_recorder, 1008 self._options.arch, self._options.use_goma, self._options.goma_dir, 1009 self._options.system_health_orderfile, self._monochrome, 1010 self._options.public, self._GetPathToOrderfile()) 1011 1012 self._compiler.CompileLibchrome(instrumented=False, 1013 use_call_graph=False) 1014 self._PatchOrderfile() 1015 # Because identical code folding is a bit different with and without 1016 # the orderfile build, we need to re-patch the orderfile with code 1017 # folding as close to the final version as possible. 1018 self._compiler.CompileLibchrome(instrumented=False, 1019 use_call_graph=False, force_relink=True) 1020 self._PatchOrderfile() 1021 self._compiler.CompileLibchrome(instrumented=False, 1022 use_call_graph=False, force_relink=True) 1023 self._VerifySymbolOrder() 1024 self._MaybeArchiveOrderfile(self._GetPathToOrderfile()) 1025 finally: 1026 _StashOutputDirectory(self._uninstrumented_out_dir) 1027 1028 if self._options.benchmark: 1029 self._output_data['orderfile_benchmark_results'] = self.RunBenchmark( 1030 self._uninstrumented_out_dir) 1031 self._output_data['no_orderfile_benchmark_results'] = self.RunBenchmark( 1032 self._no_orderfile_out_dir, no_orderfile=True) 1033 1034 self._orderfile_updater._GitStash() 1035 self._step_recorder.EndStep() 1036 return not self._step_recorder.ErrorRecorded() 1037 1038 def GetReportingData(self): 1039 """Get a dictionary of reporting data (timings, output hashes)""" 1040 self._output_data['timings'] = self._step_recorder.timings 1041 return self._output_data 1042 1043 def CommitStashedOrderfileHashes(self): 1044 """Commit any orderfile hash files in the current checkout. 1045 1046 Only possible if running on the buildbot. 1047 1048 Returns: true on success. 1049 """ 1050 if not self._options.buildbot: 1051 logging.error('Trying to commit when not running on the buildbot') 1052 return False 1053 self._orderfile_updater._CommitStashedFiles([ 1054 filename + '.sha1' 1055 for filename in (self._GetUnpatchedOrderfileFilename(), 1056 self._GetPathToOrderfile())]) 1057 return True 1058 1059 1060def CreateArgumentParser(): 1061 """Creates and returns the argument parser.""" 1062 parser = argparse.ArgumentParser() 1063 parser.add_argument('--no-benchmark', action='store_false', dest='benchmark', 1064 default=True, help='Disables running benchmarks.') 1065 parser.add_argument( 1066 '--buildbot', action='store_true', 1067 help='If true, the script expects to be run on a buildbot') 1068 parser.add_argument( 1069 '--device', default=None, type=str, 1070 help='Device serial number on which to run profiling.') 1071 parser.add_argument( 1072 '--verify', action='store_true', 1073 help='If true, the script only verifies the current orderfile') 1074 parser.add_argument('--target-arch', action='store', dest='arch', 1075 default='arm', 1076 choices=['arm', 'arm64'], 1077 help='The target architecture for which to build.') 1078 parser.add_argument('--output-json', action='store', dest='json_file', 1079 help='Location to save stats in json format') 1080 parser.add_argument( 1081 '--skip-profile', action='store_false', dest='profile', default=True, 1082 help='Don\'t generate a profile on the device. Only patch from the ' 1083 'existing profile.') 1084 parser.add_argument( 1085 '--skip-patch', action='store_false', dest='patch', default=True, 1086 help='Only generate the raw (unpatched) orderfile, don\'t patch it.') 1087 parser.add_argument('--goma-dir', help='GOMA directory.') 1088 parser.add_argument( 1089 '--use-goma', action='store_true', help='Enable GOMA.', default=False) 1090 parser.add_argument('--adb-path', help='Path to the adb binary.') 1091 1092 parser.add_argument('--public', action='store_true', 1093 help='Required if your checkout is non-internal.', 1094 default=False) 1095 parser.add_argument('--nosystem-health-orderfile', action='store_false', 1096 dest='system_health_orderfile', default=True, 1097 help=('Create an orderfile based on an about:blank ' 1098 'startup benchmark instead of system health ' 1099 'benchmarks.')) 1100 parser.add_argument( 1101 '--use-legacy-chrome-apk', action='store_true', default=False, 1102 help=('Compile and instrument chrome for [L, K] devices.')) 1103 parser.add_argument('--manual-symbol-offsets', default=None, type=str, 1104 help=('File of list of ordered symbol offsets generated ' 1105 'by manual profiling. Must set other --manual* ' 1106 'flags if this is used, and must --skip-profile.')) 1107 parser.add_argument('--manual-libname', default=None, type=str, 1108 help=('Library filename corresponding to ' 1109 '--manual-symbol-offsets.')) 1110 parser.add_argument('--manual-objdir', default=None, type=str, 1111 help=('Root of object file directory corresponding to ' 1112 '--manual-symbol-offsets.')) 1113 parser.add_argument('--noorder-outlined-functions', action='store_true', 1114 help='Disable outlined functions in the orderfile.') 1115 parser.add_argument('--pregenerated-profiles', default=None, type=str, 1116 help=('Pregenerated profiles to use instead of running ' 1117 'profile step. Cannot be used with ' 1118 '--skip-profiles.')) 1119 parser.add_argument('--profile-save-dir', default=None, type=str, 1120 help=('Directory to save any profiles created. These can ' 1121 'be used with --pregenerated-profiles. Cannot be ' 1122 'used with --skip-profiles.')) 1123 parser.add_argument('--upload-ready-orderfiles', action='store_true', 1124 help=('Skip orderfile generation and manually upload ' 1125 'orderfiles (both patched and unpatched) from ' 1126 'their normal location in the tree to the cloud ' 1127 'storage. DANGEROUS! USE WITH CARE!')) 1128 parser.add_argument('--streamline-for-debugging', action='store_true', 1129 help=('Streamline where possible the run for faster ' 1130 'iteration while debugging. The orderfile ' 1131 'generated will be valid and nontrivial, but ' 1132 'may not be based on a representative profile ' 1133 'or other such considerations. Use with caution.')) 1134 parser.add_argument('--commit-hashes', action='store_true', 1135 help=('Commit any orderfile hash files in the current ' 1136 'checkout; performs no other action')) 1137 parser.add_argument('--use-call-graph', action='store_true', default=False, 1138 help='Use call graph instrumentation.') 1139 profile_android_startup.AddProfileCollectionArguments(parser) 1140 return parser 1141 1142 1143def CreateOrderfile(options, orderfile_updater_class=None): 1144 """Creates an orderfile. 1145 1146 Args: 1147 options: As returned from optparse.OptionParser.parse_args() 1148 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater. 1149 Use to explicitly set an OrderfileUpdater class, 1150 the defaults are OrderfileUpdater, or 1151 OrderfileNoopUpdater if --public is set. 1152 1153 Returns: 1154 True iff success. 1155 """ 1156 logging.basicConfig(level=logging.INFO) 1157 devil_chromium.Initialize(adb_path=options.adb_path) 1158 1159 generator = OrderfileGenerator(options, orderfile_updater_class) 1160 try: 1161 if options.verify: 1162 generator._VerifySymbolOrder() 1163 elif options.commit_hashes: 1164 return generator.CommitStashedOrderfileHashes() 1165 elif options.upload_ready_orderfiles: 1166 return generator.UploadReadyOrderfiles() 1167 else: 1168 return generator.Generate() 1169 finally: 1170 json_output = json.dumps(generator.GetReportingData(), 1171 indent=2) + '\n' 1172 if options.json_file: 1173 with open(options.json_file, 'w') as f: 1174 f.write(json_output) 1175 print(json_output) 1176 return False 1177 1178 1179def main(): 1180 parser = CreateArgumentParser() 1181 options = parser.parse_args() 1182 return 0 if CreateOrderfile(options) else 1 1183 1184 1185if __name__ == '__main__': 1186 sys.exit(main()) 1187