1# Copyright 2013 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Contains common helpers for GN action()s.""" 6 7import atexit 8import collections 9import contextlib 10import filecmp 11import fnmatch 12import json 13import logging 14import os 15import pipes 16import re 17import shutil 18import stat 19import subprocess 20import sys 21import tempfile 22import time 23import zipfile 24 25sys.path.append(os.path.join(os.path.dirname(__file__), 26 os.pardir, os.pardir, os.pardir)) 27import gn_helpers 28 29# Use relative paths to improved hermetic property of build scripts. 30DIR_SOURCE_ROOT = os.path.relpath( 31 os.environ.get( 32 'CHECKOUT_SOURCE_ROOT', 33 os.path.join( 34 os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 35 os.pardir))) 36JAVA_HOME = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'current') 37JAVAC_PATH = os.path.join(JAVA_HOME, 'bin', 'javac') 38JAVAP_PATH = os.path.join(JAVA_HOME, 'bin', 'javap') 39RT_JAR_PATH = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'extras', 40 'java_8', 'jre', 'lib', 'rt.jar') 41 42try: 43 string_types = basestring 44except NameError: 45 string_types = (str, bytes) 46 47 48def JavaCmd(verify=True, xmx='1G'): 49 ret = [os.path.join(JAVA_HOME, 'bin', 'java')] 50 # Limit heap to avoid Java not GC'ing when it should, and causing 51 # bots to OOM when many java commands are runnig at the same time 52 # https://crbug.com/1098333 53 ret += ['-Xmx' + xmx] 54 55 # Disable bytecode verification for local builds gives a ~2% speed-up. 56 if not verify: 57 ret += ['-noverify'] 58 59 return ret 60 61 62@contextlib.contextmanager 63def TempDir(**kwargs): 64 dirname = tempfile.mkdtemp(**kwargs) 65 try: 66 yield dirname 67 finally: 68 shutil.rmtree(dirname) 69 70 71def MakeDirectory(dir_path): 72 try: 73 os.makedirs(dir_path) 74 except OSError: 75 pass 76 77 78def DeleteDirectory(dir_path): 79 if os.path.exists(dir_path): 80 shutil.rmtree(dir_path) 81 82 83def Touch(path, fail_if_missing=False): 84 if fail_if_missing and not os.path.exists(path): 85 raise Exception(path + ' doesn\'t exist.') 86 87 MakeDirectory(os.path.dirname(path)) 88 with open(path, 'a'): 89 os.utime(path, None) 90 91 92def FindInDirectory(directory, filename_filter='*'): 93 files = [] 94 for root, _dirnames, filenames in os.walk(directory): 95 matched_files = fnmatch.filter(filenames, filename_filter) 96 files.extend((os.path.join(root, f) for f in matched_files)) 97 return files 98 99 100def ParseGnList(value): 101 """Converts a "GN-list" command-line parameter into a list. 102 103 Conversions handled: 104 * None -> [] 105 * '' -> [] 106 * 'asdf' -> ['asdf'] 107 * '["a", "b"]' -> ['a', 'b'] 108 * ['["a", "b"]', 'c'] -> ['a', 'b', 'c'] (flattened list) 109 110 The common use for this behavior is in the Android build where things can 111 take lists of @FileArg references that are expanded via ExpandFileArgs. 112 """ 113 # Convert None to []. 114 if not value: 115 return [] 116 # Convert a list of GN lists to a flattened list. 117 if isinstance(value, list): 118 ret = [] 119 for arg in value: 120 ret.extend(ParseGnList(arg)) 121 return ret 122 # Convert normal GN list. 123 if value.startswith('['): 124 return gn_helpers.GNValueParser(value).ParseList() 125 # Convert a single string value to a list. 126 return [value] 127 128 129def CheckOptions(options, parser, required=None): 130 if not required: 131 return 132 for option_name in required: 133 if getattr(options, option_name) is None: 134 parser.error('--%s is required' % option_name.replace('_', '-')) 135 136 137def WriteJson(obj, path, only_if_changed=False): 138 old_dump = None 139 if os.path.exists(path): 140 with open(path, 'r') as oldfile: 141 old_dump = oldfile.read() 142 143 new_dump = json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': ')) 144 145 if not only_if_changed or old_dump != new_dump: 146 with open(path, 'w') as outfile: 147 outfile.write(new_dump) 148 149 150@contextlib.contextmanager 151def AtomicOutput(path, only_if_changed=True, mode='w+b'): 152 """Helper to prevent half-written outputs. 153 154 Args: 155 path: Path to the final output file, which will be written atomically. 156 only_if_changed: If True (the default), do not touch the filesystem 157 if the content has not changed. 158 mode: The mode to open the file in (str). 159 Returns: 160 A python context manager that yelds a NamedTemporaryFile instance 161 that must be used by clients to write the data to. On exit, the 162 manager will try to replace the final output file with the 163 temporary one if necessary. The temporary file is always destroyed 164 on exit. 165 Example: 166 with build_utils.AtomicOutput(output_path) as tmp_file: 167 subprocess.check_call(['prog', '--output', tmp_file.name]) 168 """ 169 # Create in same directory to ensure same filesystem when moving. 170 dirname = os.path.dirname(path) 171 if not os.path.exists(dirname): 172 MakeDirectory(dirname) 173 with tempfile.NamedTemporaryFile( 174 mode, suffix=os.path.basename(path), dir=dirname, delete=False) as f: 175 try: 176 yield f 177 178 # file should be closed before comparison/move. 179 f.close() 180 if not (only_if_changed and os.path.exists(path) and 181 filecmp.cmp(f.name, path)): 182 shutil.move(f.name, path) 183 finally: 184 if os.path.exists(f.name): 185 os.unlink(f.name) 186 187 188class CalledProcessError(Exception): 189 """This exception is raised when the process run by CheckOutput 190 exits with a non-zero exit code.""" 191 192 def __init__(self, cwd, args, output): 193 super(CalledProcessError, self).__init__() 194 self.cwd = cwd 195 self.args = args 196 self.output = output 197 198 def __str__(self): 199 # A user should be able to simply copy and paste the command that failed 200 # into their shell. 201 copyable_command = '( cd {}; {} )'.format(os.path.abspath(self.cwd), 202 ' '.join(map(pipes.quote, self.args))) 203 return 'Command failed: {}\n{}'.format(copyable_command, self.output) 204 205 206def FilterLines(output, filter_string): 207 """Output filter from build_utils.CheckOutput. 208 209 Args: 210 output: Executable output as from build_utils.CheckOutput. 211 filter_string: An RE string that will filter (remove) matching 212 lines from |output|. 213 214 Returns: 215 The filtered output, as a single string. 216 """ 217 re_filter = re.compile(filter_string) 218 return '\n'.join( 219 line for line in output.split('\n') if not re_filter.search(line)) 220 221 222def FilterReflectiveAccessJavaWarnings(output): 223 """Filters out warnings about illegal reflective access operation. 224 225 These warnings were introduced in Java 9, and generally mean that dependencies 226 need to be updated. 227 """ 228 # WARNING: An illegal reflective access operation has occurred 229 # WARNING: Illegal reflective access by ... 230 # WARNING: Please consider reporting this to the maintainers of ... 231 # WARNING: Use --illegal-access=warn to enable warnings of further ... 232 # WARNING: All illegal access operations will be denied in a future release 233 return FilterLines( 234 output, r'WARNING: (' 235 'An illegal reflective|' 236 'Illegal reflective access|' 237 'Please consider reporting this to|' 238 'Use --illegal-access=warn|' 239 'All illegal access operations)') 240 241 242# This can be used in most cases like subprocess.check_output(). The output, 243# particularly when the command fails, better highlights the command's failure. 244# If the command fails, raises a build_utils.CalledProcessError. 245def CheckOutput(args, 246 cwd=None, 247 env=None, 248 print_stdout=False, 249 print_stderr=True, 250 stdout_filter=None, 251 stderr_filter=None, 252 fail_on_output=True, 253 fail_func=lambda returncode, stderr: returncode != 0): 254 if not cwd: 255 cwd = os.getcwd() 256 257 logging.info('CheckOutput: %s', ' '.join(args)) 258 child = subprocess.Popen(args, 259 stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env) 260 stdout, stderr = child.communicate() 261 262 # For Python3 only: 263 if isinstance(stdout, bytes) and sys.version_info >= (3, ): 264 stdout = stdout.decode('utf-8') 265 stderr = stderr.decode('utf-8') 266 267 if stdout_filter is not None: 268 stdout = stdout_filter(stdout) 269 270 if stderr_filter is not None: 271 stderr = stderr_filter(stderr) 272 273 if fail_func and fail_func(child.returncode, stderr): 274 raise CalledProcessError(cwd, args, stdout + stderr) 275 276 if print_stdout: 277 sys.stdout.write(stdout) 278 if print_stderr: 279 sys.stderr.write(stderr) 280 281 has_stdout = print_stdout and stdout 282 has_stderr = print_stderr and stderr 283 if fail_on_output and (has_stdout or has_stderr): 284 MSG = """\ 285Command failed because it wrote to {}. 286You can often set treat_warnings_as_errors=false to not treat output as \ 287failure (useful when developing locally).""" 288 if has_stdout and has_stderr: 289 stream_string = 'stdout and stderr' 290 elif has_stdout: 291 stream_string = 'stdout' 292 else: 293 stream_string = 'stderr' 294 raise CalledProcessError(cwd, args, MSG.format(stream_string)) 295 296 return stdout 297 298 299def GetModifiedTime(path): 300 # For a symlink, the modified time should be the greater of the link's 301 # modified time and the modified time of the target. 302 return max(os.lstat(path).st_mtime, os.stat(path).st_mtime) 303 304 305def IsTimeStale(output, inputs): 306 if not os.path.exists(output): 307 return True 308 309 output_time = GetModifiedTime(output) 310 for i in inputs: 311 if GetModifiedTime(i) > output_time: 312 return True 313 return False 314 315 316def _CheckZipPath(name): 317 if os.path.normpath(name) != name: 318 raise Exception('Non-canonical zip path: %s' % name) 319 if os.path.isabs(name): 320 raise Exception('Absolute zip path: %s' % name) 321 322 323def _IsSymlink(zip_file, name): 324 zi = zip_file.getinfo(name) 325 326 # The two high-order bytes of ZipInfo.external_attr represent 327 # UNIX permissions and file type bits. 328 return stat.S_ISLNK(zi.external_attr >> 16) 329 330 331def ExtractAll(zip_path, path=None, no_clobber=True, pattern=None, 332 predicate=None): 333 if path is None: 334 path = os.getcwd() 335 elif not os.path.exists(path): 336 MakeDirectory(path) 337 338 if not zipfile.is_zipfile(zip_path): 339 raise Exception('Invalid zip file: %s' % zip_path) 340 341 extracted = [] 342 with zipfile.ZipFile(zip_path) as z: 343 for name in z.namelist(): 344 if name.endswith('/'): 345 MakeDirectory(os.path.join(path, name)) 346 continue 347 if pattern is not None: 348 if not fnmatch.fnmatch(name, pattern): 349 continue 350 if predicate and not predicate(name): 351 continue 352 _CheckZipPath(name) 353 if no_clobber: 354 output_path = os.path.join(path, name) 355 if os.path.exists(output_path): 356 raise Exception( 357 'Path already exists from zip: %s %s %s' 358 % (zip_path, name, output_path)) 359 if _IsSymlink(z, name): 360 dest = os.path.join(path, name) 361 MakeDirectory(os.path.dirname(dest)) 362 os.symlink(z.read(name), dest) 363 extracted.append(dest) 364 else: 365 z.extract(name, path) 366 extracted.append(os.path.join(path, name)) 367 368 return extracted 369 370 371def HermeticDateTime(timestamp=None): 372 """Returns a constant ZipInfo.date_time tuple. 373 374 Args: 375 timestamp: Unix timestamp to use for files in the archive. 376 377 Returns: 378 A ZipInfo.date_time tuple for Jan 1, 2001, or the given timestamp. 379 """ 380 if not timestamp: 381 return (2001, 1, 1, 0, 0, 0) 382 utc_time = time.gmtime(timestamp) 383 return (utc_time.tm_year, utc_time.tm_mon, utc_time.tm_mday, utc_time.tm_hour, 384 utc_time.tm_min, utc_time.tm_sec) 385 386 387def HermeticZipInfo(*args, **kwargs): 388 """Creates a zipfile.ZipInfo with a constant timestamp and external_attr. 389 390 If a date_time value is not provided in the positional or keyword arguments, 391 the default value from HermeticDateTime is used. 392 393 Args: 394 See zipfile.ZipInfo. 395 396 Returns: 397 A zipfile.ZipInfo. 398 """ 399 # The caller may have provided a date_time either as a positional parameter 400 # (args[1]) or as a keyword parameter. Use the default hermetic date_time if 401 # none was provided. 402 date_time = None 403 if len(args) >= 2: 404 date_time = args[1] 405 elif 'date_time' in kwargs: 406 date_time = kwargs['date_time'] 407 if not date_time: 408 kwargs['date_time'] = HermeticDateTime() 409 ret = zipfile.ZipInfo(*args, **kwargs) 410 ret.external_attr = (0o644 << 16) 411 return ret 412 413 414def AddToZipHermetic(zip_file, 415 zip_path, 416 src_path=None, 417 data=None, 418 compress=None, 419 date_time=None): 420 """Adds a file to the given ZipFile with a hard-coded modified time. 421 422 Args: 423 zip_file: ZipFile instance to add the file to. 424 zip_path: Destination path within the zip file (or ZipInfo instance). 425 src_path: Path of the source file. Mutually exclusive with |data|. 426 data: File data as a string. 427 compress: Whether to enable compression. Default is taken from ZipFile 428 constructor. 429 date_time: The last modification date and time for the archive member. 430 """ 431 assert (src_path is None) != (data is None), ( 432 '|src_path| and |data| are mutually exclusive.') 433 if isinstance(zip_path, zipfile.ZipInfo): 434 zipinfo = zip_path 435 zip_path = zipinfo.filename 436 else: 437 zipinfo = HermeticZipInfo(filename=zip_path, date_time=date_time) 438 439 _CheckZipPath(zip_path) 440 441 if src_path and os.path.islink(src_path): 442 zipinfo.filename = zip_path 443 zipinfo.external_attr |= stat.S_IFLNK << 16 # mark as a symlink 444 zip_file.writestr(zipinfo, os.readlink(src_path)) 445 return 446 447 # zipfile.write() does 448 # external_attr = (os.stat(src_path)[0] & 0xFFFF) << 16 449 # but we want to use _HERMETIC_FILE_ATTR, so manually set 450 # the few attr bits we care about. 451 if src_path: 452 st = os.stat(src_path) 453 for mode in (stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH): 454 if st.st_mode & mode: 455 zipinfo.external_attr |= mode << 16 456 457 if src_path: 458 with open(src_path, 'rb') as f: 459 data = f.read() 460 461 # zipfile will deflate even when it makes the file bigger. To avoid 462 # growing files, disable compression at an arbitrary cut off point. 463 if len(data) < 16: 464 compress = False 465 466 # None converts to ZIP_STORED, when passed explicitly rather than the 467 # default passed to the ZipFile constructor. 468 compress_type = zip_file.compression 469 if compress is not None: 470 compress_type = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED 471 zip_file.writestr(zipinfo, data, compress_type) 472 473 474def DoZip(inputs, 475 output, 476 base_dir=None, 477 compress_fn=None, 478 zip_prefix_path=None, 479 timestamp=None): 480 """Creates a zip file from a list of files. 481 482 Args: 483 inputs: A list of paths to zip, or a list of (zip_path, fs_path) tuples. 484 output: Path, fileobj, or ZipFile instance to add files to. 485 base_dir: Prefix to strip from inputs. 486 compress_fn: Applied to each input to determine whether or not to compress. 487 By default, items will be |zipfile.ZIP_STORED|. 488 zip_prefix_path: Path prepended to file path in zip file. 489 timestamp: Unix timestamp to use for files in the archive. 490 """ 491 if base_dir is None: 492 base_dir = '.' 493 input_tuples = [] 494 for tup in inputs: 495 if isinstance(tup, string_types): 496 tup = (os.path.relpath(tup, base_dir), tup) 497 if tup[0].startswith('..'): 498 raise Exception('Invalid zip_path: ' + tup[0]) 499 input_tuples.append(tup) 500 501 # Sort by zip path to ensure stable zip ordering. 502 input_tuples.sort(key=lambda tup: tup[0]) 503 504 out_zip = output 505 if not isinstance(output, zipfile.ZipFile): 506 out_zip = zipfile.ZipFile(output, 'w') 507 508 date_time = HermeticDateTime(timestamp) 509 try: 510 for zip_path, fs_path in input_tuples: 511 if zip_prefix_path: 512 zip_path = os.path.join(zip_prefix_path, zip_path) 513 compress = compress_fn(zip_path) if compress_fn else None 514 AddToZipHermetic(out_zip, 515 zip_path, 516 src_path=fs_path, 517 compress=compress, 518 date_time=date_time) 519 finally: 520 if output is not out_zip: 521 out_zip.close() 522 523 524def ZipDir(output, base_dir, compress_fn=None, zip_prefix_path=None): 525 """Creates a zip file from a directory.""" 526 inputs = [] 527 for root, _, files in os.walk(base_dir): 528 for f in files: 529 inputs.append(os.path.join(root, f)) 530 531 if isinstance(output, zipfile.ZipFile): 532 DoZip( 533 inputs, 534 output, 535 base_dir, 536 compress_fn=compress_fn, 537 zip_prefix_path=zip_prefix_path) 538 else: 539 with AtomicOutput(output) as f: 540 DoZip( 541 inputs, 542 f, 543 base_dir, 544 compress_fn=compress_fn, 545 zip_prefix_path=zip_prefix_path) 546 547 548def MatchesGlob(path, filters): 549 """Returns whether the given path matches any of the given glob patterns.""" 550 return filters and any(fnmatch.fnmatch(path, f) for f in filters) 551 552 553def MergeZips(output, input_zips, path_transform=None, compress=None): 554 """Combines all files from |input_zips| into |output|. 555 556 Args: 557 output: Path, fileobj, or ZipFile instance to add files to. 558 input_zips: Iterable of paths to zip files to merge. 559 path_transform: Called for each entry path. Returns a new path, or None to 560 skip the file. 561 compress: Overrides compression setting from origin zip entries. 562 """ 563 path_transform = path_transform or (lambda p: p) 564 added_names = set() 565 566 out_zip = output 567 if not isinstance(output, zipfile.ZipFile): 568 out_zip = zipfile.ZipFile(output, 'w') 569 570 try: 571 for in_file in input_zips: 572 with zipfile.ZipFile(in_file, 'r') as in_zip: 573 for info in in_zip.infolist(): 574 # Ignore directories. 575 if info.filename[-1] == '/': 576 continue 577 dst_name = path_transform(info.filename) 578 if not dst_name: 579 continue 580 already_added = dst_name in added_names 581 if not already_added: 582 if compress is not None: 583 compress_entry = compress 584 else: 585 compress_entry = info.compress_type != zipfile.ZIP_STORED 586 AddToZipHermetic( 587 out_zip, 588 dst_name, 589 data=in_zip.read(info), 590 compress=compress_entry) 591 added_names.add(dst_name) 592 finally: 593 if output is not out_zip: 594 out_zip.close() 595 596 597def GetSortedTransitiveDependencies(top, deps_func): 598 """Gets the list of all transitive dependencies in sorted order. 599 600 There should be no cycles in the dependency graph (crashes if cycles exist). 601 602 Args: 603 top: A list of the top level nodes 604 deps_func: A function that takes a node and returns a list of its direct 605 dependencies. 606 Returns: 607 A list of all transitive dependencies of nodes in top, in order (a node will 608 appear in the list at a higher index than all of its dependencies). 609 """ 610 # Find all deps depth-first, maintaining original order in the case of ties. 611 deps_map = collections.OrderedDict() 612 def discover(nodes): 613 for node in nodes: 614 if node in deps_map: 615 continue 616 deps = deps_func(node) 617 discover(deps) 618 deps_map[node] = deps 619 620 discover(top) 621 return list(deps_map) 622 623 624def InitLogging(enabling_env): 625 logging.basicConfig( 626 level=logging.DEBUG if os.environ.get(enabling_env) else logging.WARNING, 627 format='%(levelname).1s %(process)d %(relativeCreated)6d %(message)s') 628 script_name = os.path.basename(sys.argv[0]) 629 logging.info('Started (%s)', script_name) 630 631 my_pid = os.getpid() 632 633 def log_exit(): 634 # Do not log for fork'ed processes. 635 if os.getpid() == my_pid: 636 logging.info("Job's done (%s)", script_name) 637 638 atexit.register(log_exit) 639 640 641def AddDepfileOption(parser): 642 # TODO(agrieve): Get rid of this once we've moved to argparse. 643 if hasattr(parser, 'add_option'): 644 func = parser.add_option 645 else: 646 func = parser.add_argument 647 func('--depfile', 648 help='Path to depfile (refer to `gn help depfile`)') 649 650 651def WriteDepfile(depfile_path, first_gn_output, inputs=None): 652 assert depfile_path != first_gn_output # http://crbug.com/646165 653 assert not isinstance(inputs, string_types) # Easy mistake to make 654 inputs = inputs or [] 655 MakeDirectory(os.path.dirname(depfile_path)) 656 # Ninja does not support multiple outputs in depfiles. 657 with open(depfile_path, 'w') as depfile: 658 depfile.write(first_gn_output.replace(' ', '\\ ')) 659 depfile.write(': \\\n ') 660 depfile.write(' \\\n '.join(i.replace(' ', '\\ ') for i in inputs)) 661 depfile.write('\n') 662 663 664def ExpandFileArgs(args): 665 """Replaces file-arg placeholders in args. 666 667 These placeholders have the form: 668 @FileArg(filename:key1:key2:...:keyn) 669 670 The value of such a placeholder is calculated by reading 'filename' as json. 671 And then extracting the value at [key1][key2]...[keyn]. If a key has a '[]' 672 suffix the (intermediate) value will be interpreted as a single item list and 673 the single item will be returned or used for further traversal. 674 675 Note: This intentionally does not return the list of files that appear in such 676 placeholders. An action that uses file-args *must* know the paths of those 677 files prior to the parsing of the arguments (typically by explicitly listing 678 them in the action's inputs in build files). 679 """ 680 new_args = list(args) 681 file_jsons = dict() 682 r = re.compile('@FileArg\((.*?)\)') 683 for i, arg in enumerate(args): 684 match = r.search(arg) 685 if not match: 686 continue 687 688 def get_key(key): 689 if key.endswith('[]'): 690 return key[:-2], True 691 return key, False 692 693 lookup_path = match.group(1).split(':') 694 file_path, _ = get_key(lookup_path[0]) 695 if not file_path in file_jsons: 696 with open(file_path) as f: 697 file_jsons[file_path] = json.load(f) 698 699 expansion = file_jsons 700 for k in lookup_path: 701 k, flatten = get_key(k) 702 expansion = expansion[k] 703 if flatten: 704 if not isinstance(expansion, list) or not len(expansion) == 1: 705 raise Exception('Expected single item list but got %s' % expansion) 706 expansion = expansion[0] 707 708 # This should match ParseGnList. The output is either a GN-formatted list 709 # or a literal (with no quotes). 710 if isinstance(expansion, list): 711 new_args[i] = (arg[:match.start()] + gn_helpers.ToGNString(expansion) + 712 arg[match.end():]) 713 else: 714 new_args[i] = arg[:match.start()] + str(expansion) + arg[match.end():] 715 716 return new_args 717 718 719def ReadSourcesList(sources_list_file_name): 720 """Reads a GN-written file containing list of file names and returns a list. 721 722 Note that this function should not be used to parse response files. 723 """ 724 with open(sources_list_file_name) as f: 725 return [file_name.strip() for file_name in f] 726