1# Copyright 2013 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5"""Contains common helpers for GN action()s.""" 6 7import atexit 8import collections 9import contextlib 10import filecmp 11import fnmatch 12import json 13import logging 14import os 15import pipes 16import re 17import shutil 18import stat 19import subprocess 20import sys 21import tempfile 22import zipfile 23 24sys.path.append(os.path.join(os.path.dirname(__file__), 25 os.pardir, os.pardir, os.pardir)) 26import gn_helpers 27 28# Use relative paths to improved hermetic property of build scripts. 29DIR_SOURCE_ROOT = os.path.relpath( 30 os.environ.get( 31 'CHECKOUT_SOURCE_ROOT', 32 os.path.join( 33 os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 34 os.pardir))) 35JAVA_HOME = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'current') 36JAVAC_PATH = os.path.join(JAVA_HOME, 'bin', 'javac') 37JAVAP_PATH = os.path.join(JAVA_HOME, 'bin', 'javap') 38RT_JAR_PATH = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'extras', 39 'java_8', 'jre', 'lib', 'rt.jar') 40 41try: 42 string_types = basestring 43except NameError: 44 string_types = (str, bytes) 45 46 47def JavaCmd(verify=True, xmx='1G'): 48 ret = [os.path.join(JAVA_HOME, 'bin', 'java')] 49 # Limit heap to avoid Java not GC'ing when it should, and causing 50 # bots to OOM when many java commands are runnig at the same time 51 # https://crbug.com/1098333 52 ret += ['-Xmx' + xmx] 53 54 # Disable bytecode verification for local builds gives a ~2% speed-up. 55 if not verify: 56 ret += ['-noverify'] 57 58 return ret 59 60 61@contextlib.contextmanager 62def TempDir(**kwargs): 63 dirname = tempfile.mkdtemp(**kwargs) 64 try: 65 yield dirname 66 finally: 67 shutil.rmtree(dirname) 68 69 70def MakeDirectory(dir_path): 71 try: 72 os.makedirs(dir_path) 73 except OSError: 74 pass 75 76 77def DeleteDirectory(dir_path): 78 if os.path.exists(dir_path): 79 shutil.rmtree(dir_path) 80 81 82def Touch(path, fail_if_missing=False): 83 if fail_if_missing and not os.path.exists(path): 84 raise Exception(path + ' doesn\'t exist.') 85 86 MakeDirectory(os.path.dirname(path)) 87 with open(path, 'a'): 88 os.utime(path, None) 89 90 91def FindInDirectory(directory, filename_filter='*'): 92 files = [] 93 for root, _dirnames, filenames in os.walk(directory): 94 matched_files = fnmatch.filter(filenames, filename_filter) 95 files.extend((os.path.join(root, f) for f in matched_files)) 96 return files 97 98 99def ParseGnList(value): 100 """Converts a "GN-list" command-line parameter into a list. 101 102 Conversions handled: 103 * None -> [] 104 * '' -> [] 105 * 'asdf' -> ['asdf'] 106 * '["a", "b"]' -> ['a', 'b'] 107 * ['["a", "b"]', 'c'] -> ['a', 'b', 'c'] (flattened list) 108 109 The common use for this behavior is in the Android build where things can 110 take lists of @FileArg references that are expanded via ExpandFileArgs. 111 """ 112 # Convert None to []. 113 if not value: 114 return [] 115 # Convert a list of GN lists to a flattened list. 116 if isinstance(value, list): 117 ret = [] 118 for arg in value: 119 ret.extend(ParseGnList(arg)) 120 return ret 121 # Convert normal GN list. 122 if value.startswith('['): 123 return gn_helpers.GNValueParser(value).ParseList() 124 # Convert a single string value to a list. 125 return [value] 126 127 128def CheckOptions(options, parser, required=None): 129 if not required: 130 return 131 for option_name in required: 132 if getattr(options, option_name) is None: 133 parser.error('--%s is required' % option_name.replace('_', '-')) 134 135 136def WriteJson(obj, path, only_if_changed=False): 137 old_dump = None 138 if os.path.exists(path): 139 with open(path, 'r') as oldfile: 140 old_dump = oldfile.read() 141 142 new_dump = json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': ')) 143 144 if not only_if_changed or old_dump != new_dump: 145 with open(path, 'w') as outfile: 146 outfile.write(new_dump) 147 148 149@contextlib.contextmanager 150def AtomicOutput(path, only_if_changed=True, mode='w+b'): 151 """Helper to prevent half-written outputs. 152 153 Args: 154 path: Path to the final output file, which will be written atomically. 155 only_if_changed: If True (the default), do not touch the filesystem 156 if the content has not changed. 157 mode: The mode to open the file in (str). 158 Returns: 159 A python context manager that yelds a NamedTemporaryFile instance 160 that must be used by clients to write the data to. On exit, the 161 manager will try to replace the final output file with the 162 temporary one if necessary. The temporary file is always destroyed 163 on exit. 164 Example: 165 with build_utils.AtomicOutput(output_path) as tmp_file: 166 subprocess.check_call(['prog', '--output', tmp_file.name]) 167 """ 168 # Create in same directory to ensure same filesystem when moving. 169 dirname = os.path.dirname(path) 170 if not os.path.exists(dirname): 171 MakeDirectory(dirname) 172 with tempfile.NamedTemporaryFile( 173 mode, suffix=os.path.basename(path), dir=dirname, delete=False) as f: 174 try: 175 yield f 176 177 # file should be closed before comparison/move. 178 f.close() 179 if not (only_if_changed and os.path.exists(path) and 180 filecmp.cmp(f.name, path)): 181 shutil.move(f.name, path) 182 finally: 183 if os.path.exists(f.name): 184 os.unlink(f.name) 185 186 187class CalledProcessError(Exception): 188 """This exception is raised when the process run by CheckOutput 189 exits with a non-zero exit code.""" 190 191 def __init__(self, cwd, args, output): 192 super(CalledProcessError, self).__init__() 193 self.cwd = cwd 194 self.args = args 195 self.output = output 196 197 def __str__(self): 198 # A user should be able to simply copy and paste the command that failed 199 # into their shell. 200 copyable_command = '( cd {}; {} )'.format(os.path.abspath(self.cwd), 201 ' '.join(map(pipes.quote, self.args))) 202 return 'Command failed: {}\n{}'.format(copyable_command, self.output) 203 204 205def FilterLines(output, filter_string): 206 """Output filter from build_utils.CheckOutput. 207 208 Args: 209 output: Executable output as from build_utils.CheckOutput. 210 filter_string: An RE string that will filter (remove) matching 211 lines from |output|. 212 213 Returns: 214 The filtered output, as a single string. 215 """ 216 re_filter = re.compile(filter_string) 217 return '\n'.join( 218 line for line in output.split('\n') if not re_filter.search(line)) 219 220 221def FilterReflectiveAccessJavaWarnings(output): 222 """Filters out warnings about illegal reflective access operation. 223 224 These warnings were introduced in Java 9, and generally mean that dependencies 225 need to be updated. 226 """ 227 # WARNING: An illegal reflective access operation has occurred 228 # WARNING: Illegal reflective access by ... 229 # WARNING: Please consider reporting this to the maintainers of ... 230 # WARNING: Use --illegal-access=warn to enable warnings of further ... 231 # WARNING: All illegal access operations will be denied in a future release 232 return FilterLines( 233 output, r'WARNING: (' 234 'An illegal reflective|' 235 'Illegal reflective access|' 236 'Please consider reporting this to|' 237 'Use --illegal-access=warn|' 238 'All illegal access operations)') 239 240 241# This can be used in most cases like subprocess.check_output(). The output, 242# particularly when the command fails, better highlights the command's failure. 243# If the command fails, raises a build_utils.CalledProcessError. 244def CheckOutput(args, 245 cwd=None, 246 env=None, 247 print_stdout=False, 248 print_stderr=True, 249 stdout_filter=None, 250 stderr_filter=None, 251 fail_on_output=True, 252 fail_func=lambda returncode, stderr: returncode != 0): 253 if not cwd: 254 cwd = os.getcwd() 255 256 child = subprocess.Popen(args, 257 stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env) 258 stdout, stderr = child.communicate() 259 260 # For Python3 only: 261 if isinstance(stdout, bytes) and sys.version_info >= (3, ): 262 stdout = stdout.decode('utf-8') 263 stderr = stderr.decode('utf-8') 264 265 if stdout_filter is not None: 266 stdout = stdout_filter(stdout) 267 268 if stderr_filter is not None: 269 stderr = stderr_filter(stderr) 270 271 if fail_func and fail_func(child.returncode, stderr): 272 raise CalledProcessError(cwd, args, stdout + stderr) 273 274 if print_stdout: 275 sys.stdout.write(stdout) 276 if print_stderr: 277 sys.stderr.write(stderr) 278 279 has_stdout = print_stdout and stdout 280 has_stderr = print_stderr and stderr 281 if fail_on_output and (has_stdout or has_stderr): 282 MSG = """\ 283Command failed because it wrote to {}. 284You can often set treat_warnings_as_errors=false to not treat output as \ 285failure (useful when developing locally).""" 286 if has_stdout and has_stderr: 287 stream_string = 'stdout and stderr' 288 elif has_stdout: 289 stream_string = 'stdout' 290 else: 291 stream_string = 'stderr' 292 raise CalledProcessError(cwd, args, MSG.format(stream_string)) 293 294 return stdout 295 296 297def GetModifiedTime(path): 298 # For a symlink, the modified time should be the greater of the link's 299 # modified time and the modified time of the target. 300 return max(os.lstat(path).st_mtime, os.stat(path).st_mtime) 301 302 303def IsTimeStale(output, inputs): 304 if not os.path.exists(output): 305 return True 306 307 output_time = GetModifiedTime(output) 308 for i in inputs: 309 if GetModifiedTime(i) > output_time: 310 return True 311 return False 312 313 314def _CheckZipPath(name): 315 if os.path.normpath(name) != name: 316 raise Exception('Non-canonical zip path: %s' % name) 317 if os.path.isabs(name): 318 raise Exception('Absolute zip path: %s' % name) 319 320 321def _IsSymlink(zip_file, name): 322 zi = zip_file.getinfo(name) 323 324 # The two high-order bytes of ZipInfo.external_attr represent 325 # UNIX permissions and file type bits. 326 return stat.S_ISLNK(zi.external_attr >> 16) 327 328 329def ExtractAll(zip_path, path=None, no_clobber=True, pattern=None, 330 predicate=None): 331 if path is None: 332 path = os.getcwd() 333 elif not os.path.exists(path): 334 MakeDirectory(path) 335 336 if not zipfile.is_zipfile(zip_path): 337 raise Exception('Invalid zip file: %s' % zip_path) 338 339 extracted = [] 340 with zipfile.ZipFile(zip_path) as z: 341 for name in z.namelist(): 342 if name.endswith('/'): 343 MakeDirectory(os.path.join(path, name)) 344 continue 345 if pattern is not None: 346 if not fnmatch.fnmatch(name, pattern): 347 continue 348 if predicate and not predicate(name): 349 continue 350 _CheckZipPath(name) 351 if no_clobber: 352 output_path = os.path.join(path, name) 353 if os.path.exists(output_path): 354 raise Exception( 355 'Path already exists from zip: %s %s %s' 356 % (zip_path, name, output_path)) 357 if _IsSymlink(z, name): 358 dest = os.path.join(path, name) 359 MakeDirectory(os.path.dirname(dest)) 360 os.symlink(z.read(name), dest) 361 extracted.append(dest) 362 else: 363 z.extract(name, path) 364 extracted.append(os.path.join(path, name)) 365 366 return extracted 367 368 369def HermeticZipInfo(*args, **kwargs): 370 """Creates a ZipInfo with a constant timestamp and external_attr.""" 371 ret = zipfile.ZipInfo(*args, **kwargs) 372 ret.date_time = (2001, 1, 1, 0, 0, 0) 373 ret.external_attr = (0o644 << 16) 374 return ret 375 376 377def AddToZipHermetic(zip_file, 378 zip_path, 379 src_path=None, 380 data=None, 381 compress=None): 382 """Adds a file to the given ZipFile with a hard-coded modified time. 383 384 Args: 385 zip_file: ZipFile instance to add the file to. 386 zip_path: Destination path within the zip file (or ZipInfo instance). 387 src_path: Path of the source file. Mutually exclusive with |data|. 388 data: File data as a string. 389 compress: Whether to enable compression. Default is taken from ZipFile 390 constructor. 391 """ 392 assert (src_path is None) != (data is None), ( 393 '|src_path| and |data| are mutually exclusive.') 394 if isinstance(zip_path, zipfile.ZipInfo): 395 zipinfo = zip_path 396 zip_path = zipinfo.filename 397 else: 398 zipinfo = HermeticZipInfo(filename=zip_path) 399 400 _CheckZipPath(zip_path) 401 402 if src_path and os.path.islink(src_path): 403 zipinfo.filename = zip_path 404 zipinfo.external_attr |= stat.S_IFLNK << 16 # mark as a symlink 405 zip_file.writestr(zipinfo, os.readlink(src_path)) 406 return 407 408 # zipfile.write() does 409 # external_attr = (os.stat(src_path)[0] & 0xFFFF) << 16 410 # but we want to use _HERMETIC_FILE_ATTR, so manually set 411 # the few attr bits we care about. 412 if src_path: 413 st = os.stat(src_path) 414 for mode in (stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH): 415 if st.st_mode & mode: 416 zipinfo.external_attr |= mode << 16 417 418 if src_path: 419 with open(src_path, 'rb') as f: 420 data = f.read() 421 422 # zipfile will deflate even when it makes the file bigger. To avoid 423 # growing files, disable compression at an arbitrary cut off point. 424 if len(data) < 16: 425 compress = False 426 427 # None converts to ZIP_STORED, when passed explicitly rather than the 428 # default passed to the ZipFile constructor. 429 compress_type = zip_file.compression 430 if compress is not None: 431 compress_type = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED 432 zip_file.writestr(zipinfo, data, compress_type) 433 434 435def DoZip(inputs, output, base_dir=None, compress_fn=None, 436 zip_prefix_path=None): 437 """Creates a zip file from a list of files. 438 439 Args: 440 inputs: A list of paths to zip, or a list of (zip_path, fs_path) tuples. 441 output: Path, fileobj, or ZipFile instance to add files to. 442 base_dir: Prefix to strip from inputs. 443 compress_fn: Applied to each input to determine whether or not to compress. 444 By default, items will be |zipfile.ZIP_STORED|. 445 zip_prefix_path: Path prepended to file path in zip file. 446 """ 447 if base_dir is None: 448 base_dir = '.' 449 input_tuples = [] 450 for tup in inputs: 451 if isinstance(tup, string_types): 452 tup = (os.path.relpath(tup, base_dir), tup) 453 if tup[0].startswith('..'): 454 raise Exception('Invalid zip_path: ' + tup[0]) 455 input_tuples.append(tup) 456 457 # Sort by zip path to ensure stable zip ordering. 458 input_tuples.sort(key=lambda tup: tup[0]) 459 460 out_zip = output 461 if not isinstance(output, zipfile.ZipFile): 462 out_zip = zipfile.ZipFile(output, 'w') 463 464 try: 465 for zip_path, fs_path in input_tuples: 466 if zip_prefix_path: 467 zip_path = os.path.join(zip_prefix_path, zip_path) 468 compress = compress_fn(zip_path) if compress_fn else None 469 AddToZipHermetic(out_zip, zip_path, src_path=fs_path, compress=compress) 470 finally: 471 if output is not out_zip: 472 out_zip.close() 473 474 475def ZipDir(output, base_dir, compress_fn=None, zip_prefix_path=None): 476 """Creates a zip file from a directory.""" 477 inputs = [] 478 for root, _, files in os.walk(base_dir): 479 for f in files: 480 inputs.append(os.path.join(root, f)) 481 482 if isinstance(output, zipfile.ZipFile): 483 DoZip( 484 inputs, 485 output, 486 base_dir, 487 compress_fn=compress_fn, 488 zip_prefix_path=zip_prefix_path) 489 else: 490 with AtomicOutput(output) as f: 491 DoZip( 492 inputs, 493 f, 494 base_dir, 495 compress_fn=compress_fn, 496 zip_prefix_path=zip_prefix_path) 497 498 499def MatchesGlob(path, filters): 500 """Returns whether the given path matches any of the given glob patterns.""" 501 return filters and any(fnmatch.fnmatch(path, f) for f in filters) 502 503 504def MergeZips(output, input_zips, path_transform=None, compress=None): 505 """Combines all files from |input_zips| into |output|. 506 507 Args: 508 output: Path, fileobj, or ZipFile instance to add files to. 509 input_zips: Iterable of paths to zip files to merge. 510 path_transform: Called for each entry path. Returns a new path, or None to 511 skip the file. 512 compress: Overrides compression setting from origin zip entries. 513 """ 514 path_transform = path_transform or (lambda p: p) 515 added_names = set() 516 517 out_zip = output 518 if not isinstance(output, zipfile.ZipFile): 519 out_zip = zipfile.ZipFile(output, 'w') 520 521 try: 522 for in_file in input_zips: 523 with zipfile.ZipFile(in_file, 'r') as in_zip: 524 for info in in_zip.infolist(): 525 # Ignore directories. 526 if info.filename[-1] == '/': 527 continue 528 dst_name = path_transform(info.filename) 529 if not dst_name: 530 continue 531 already_added = dst_name in added_names 532 if not already_added: 533 if compress is not None: 534 compress_entry = compress 535 else: 536 compress_entry = info.compress_type != zipfile.ZIP_STORED 537 AddToZipHermetic( 538 out_zip, 539 dst_name, 540 data=in_zip.read(info), 541 compress=compress_entry) 542 added_names.add(dst_name) 543 finally: 544 if output is not out_zip: 545 out_zip.close() 546 547 548def GetSortedTransitiveDependencies(top, deps_func): 549 """Gets the list of all transitive dependencies in sorted order. 550 551 There should be no cycles in the dependency graph (crashes if cycles exist). 552 553 Args: 554 top: A list of the top level nodes 555 deps_func: A function that takes a node and returns a list of its direct 556 dependencies. 557 Returns: 558 A list of all transitive dependencies of nodes in top, in order (a node will 559 appear in the list at a higher index than all of its dependencies). 560 """ 561 # Find all deps depth-first, maintaining original order in the case of ties. 562 deps_map = collections.OrderedDict() 563 def discover(nodes): 564 for node in nodes: 565 if node in deps_map: 566 continue 567 deps = deps_func(node) 568 discover(deps) 569 deps_map[node] = deps 570 571 discover(top) 572 return list(deps_map) 573 574 575def InitLogging(enabling_env): 576 logging.basicConfig( 577 level=logging.DEBUG if os.environ.get(enabling_env) else logging.WARNING, 578 format='%(levelname).1s %(process)d %(relativeCreated)6d %(message)s') 579 script_name = os.path.basename(sys.argv[0]) 580 logging.info('Started (%s)', script_name) 581 582 my_pid = os.getpid() 583 584 def log_exit(): 585 # Do not log for fork'ed processes. 586 if os.getpid() == my_pid: 587 logging.info("Job's done (%s)", script_name) 588 589 atexit.register(log_exit) 590 591 592def AddDepfileOption(parser): 593 # TODO(agrieve): Get rid of this once we've moved to argparse. 594 if hasattr(parser, 'add_option'): 595 func = parser.add_option 596 else: 597 func = parser.add_argument 598 func('--depfile', 599 help='Path to depfile (refer to `gn help depfile`)') 600 601 602def WriteDepfile(depfile_path, first_gn_output, inputs=None): 603 assert depfile_path != first_gn_output # http://crbug.com/646165 604 assert not isinstance(inputs, string_types) # Easy mistake to make 605 inputs = inputs or [] 606 MakeDirectory(os.path.dirname(depfile_path)) 607 # Ninja does not support multiple outputs in depfiles. 608 with open(depfile_path, 'w') as depfile: 609 depfile.write(first_gn_output.replace(' ', '\\ ')) 610 depfile.write(': ') 611 depfile.write(' '.join(i.replace(' ', '\\ ') for i in inputs)) 612 depfile.write('\n') 613 614 615def ExpandFileArgs(args): 616 """Replaces file-arg placeholders in args. 617 618 These placeholders have the form: 619 @FileArg(filename:key1:key2:...:keyn) 620 621 The value of such a placeholder is calculated by reading 'filename' as json. 622 And then extracting the value at [key1][key2]...[keyn]. If a key has a '[]' 623 suffix the (intermediate) value will be interpreted as a single item list and 624 the single item will be returned or used for further traversal. 625 626 Note: This intentionally does not return the list of files that appear in such 627 placeholders. An action that uses file-args *must* know the paths of those 628 files prior to the parsing of the arguments (typically by explicitly listing 629 them in the action's inputs in build files). 630 """ 631 new_args = list(args) 632 file_jsons = dict() 633 r = re.compile('@FileArg\((.*?)\)') 634 for i, arg in enumerate(args): 635 match = r.search(arg) 636 if not match: 637 continue 638 639 def get_key(key): 640 if key.endswith('[]'): 641 return key[:-2], True 642 return key, False 643 644 lookup_path = match.group(1).split(':') 645 file_path, _ = get_key(lookup_path[0]) 646 if not file_path in file_jsons: 647 with open(file_path) as f: 648 file_jsons[file_path] = json.load(f) 649 650 expansion = file_jsons 651 for k in lookup_path: 652 k, flatten = get_key(k) 653 expansion = expansion[k] 654 if flatten: 655 if not isinstance(expansion, list) or not len(expansion) == 1: 656 raise Exception('Expected single item list but got %s' % expansion) 657 expansion = expansion[0] 658 659 # This should match ParseGnList. The output is either a GN-formatted list 660 # or a literal (with no quotes). 661 if isinstance(expansion, list): 662 new_args[i] = (arg[:match.start()] + gn_helpers.ToGNString(expansion) + 663 arg[match.end():]) 664 else: 665 new_args[i] = arg[:match.start()] + str(expansion) + arg[match.end():] 666 667 return new_args 668 669 670def ReadSourcesList(sources_list_file_name): 671 """Reads a GN-written file containing list of file names and returns a list. 672 673 Note that this function should not be used to parse response files. 674 """ 675 with open(sources_list_file_name) as f: 676 return [file_name.strip() for file_name in f] 677