1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Contains common helpers for GN action()s."""
6
7import atexit
8import collections
9import contextlib
10import filecmp
11import fnmatch
12import json
13import logging
14import os
15import pipes
16import re
17import shutil
18import stat
19import subprocess
20import sys
21import tempfile
22import zipfile
23
24sys.path.append(os.path.join(os.path.dirname(__file__),
25                             os.pardir, os.pardir, os.pardir))
26import gn_helpers
27
28# Use relative paths to improved hermetic property of build scripts.
29DIR_SOURCE_ROOT = os.path.relpath(
30    os.environ.get(
31        'CHECKOUT_SOURCE_ROOT',
32        os.path.join(
33            os.path.dirname(__file__), os.pardir, os.pardir, os.pardir,
34            os.pardir)))
35JAVA_HOME = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'current')
36JAVAC_PATH = os.path.join(JAVA_HOME, 'bin', 'javac')
37JAVAP_PATH = os.path.join(JAVA_HOME, 'bin', 'javap')
38RT_JAR_PATH = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'extras',
39                           'java_8', 'jre', 'lib', 'rt.jar')
40
41try:
42  string_types = basestring
43except NameError:
44  string_types = (str, bytes)
45
46
47def JavaCmd(verify=True, xmx='1G'):
48  ret = [os.path.join(JAVA_HOME, 'bin', 'java')]
49  # Limit heap to avoid Java not GC'ing when it should, and causing
50  # bots to OOM when many java commands are runnig at the same time
51  # https://crbug.com/1098333
52  ret += ['-Xmx' + xmx]
53
54  # Disable bytecode verification for local builds gives a ~2% speed-up.
55  if not verify:
56    ret += ['-noverify']
57
58  return ret
59
60
61@contextlib.contextmanager
62def TempDir(**kwargs):
63  dirname = tempfile.mkdtemp(**kwargs)
64  try:
65    yield dirname
66  finally:
67    shutil.rmtree(dirname)
68
69
70def MakeDirectory(dir_path):
71  try:
72    os.makedirs(dir_path)
73  except OSError:
74    pass
75
76
77def DeleteDirectory(dir_path):
78  if os.path.exists(dir_path):
79    shutil.rmtree(dir_path)
80
81
82def Touch(path, fail_if_missing=False):
83  if fail_if_missing and not os.path.exists(path):
84    raise Exception(path + ' doesn\'t exist.')
85
86  MakeDirectory(os.path.dirname(path))
87  with open(path, 'a'):
88    os.utime(path, None)
89
90
91def FindInDirectory(directory, filename_filter='*'):
92  files = []
93  for root, _dirnames, filenames in os.walk(directory):
94    matched_files = fnmatch.filter(filenames, filename_filter)
95    files.extend((os.path.join(root, f) for f in matched_files))
96  return files
97
98
99def ParseGnList(value):
100  """Converts a "GN-list" command-line parameter into a list.
101
102  Conversions handled:
103    * None -> []
104    * '' -> []
105    * 'asdf' -> ['asdf']
106    * '["a", "b"]' -> ['a', 'b']
107    * ['["a", "b"]', 'c'] -> ['a', 'b', 'c']  (flattened list)
108
109  The common use for this behavior is in the Android build where things can
110  take lists of @FileArg references that are expanded via ExpandFileArgs.
111  """
112  # Convert None to [].
113  if not value:
114    return []
115  # Convert a list of GN lists to a flattened list.
116  if isinstance(value, list):
117    ret = []
118    for arg in value:
119      ret.extend(ParseGnList(arg))
120    return ret
121  # Convert normal GN list.
122  if value.startswith('['):
123    return gn_helpers.GNValueParser(value).ParseList()
124  # Convert a single string value to a list.
125  return [value]
126
127
128def CheckOptions(options, parser, required=None):
129  if not required:
130    return
131  for option_name in required:
132    if getattr(options, option_name) is None:
133      parser.error('--%s is required' % option_name.replace('_', '-'))
134
135
136def WriteJson(obj, path, only_if_changed=False):
137  old_dump = None
138  if os.path.exists(path):
139    with open(path, 'r') as oldfile:
140      old_dump = oldfile.read()
141
142  new_dump = json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '))
143
144  if not only_if_changed or old_dump != new_dump:
145    with open(path, 'w') as outfile:
146      outfile.write(new_dump)
147
148
149@contextlib.contextmanager
150def AtomicOutput(path, only_if_changed=True, mode='w+b'):
151  """Helper to prevent half-written outputs.
152
153  Args:
154    path: Path to the final output file, which will be written atomically.
155    only_if_changed: If True (the default), do not touch the filesystem
156      if the content has not changed.
157    mode: The mode to open the file in (str).
158  Returns:
159    A python context manager that yelds a NamedTemporaryFile instance
160    that must be used by clients to write the data to. On exit, the
161    manager will try to replace the final output file with the
162    temporary one if necessary. The temporary file is always destroyed
163    on exit.
164  Example:
165    with build_utils.AtomicOutput(output_path) as tmp_file:
166      subprocess.check_call(['prog', '--output', tmp_file.name])
167  """
168  # Create in same directory to ensure same filesystem when moving.
169  dirname = os.path.dirname(path)
170  if not os.path.exists(dirname):
171    MakeDirectory(dirname)
172  with tempfile.NamedTemporaryFile(
173      mode, suffix=os.path.basename(path), dir=dirname, delete=False) as f:
174    try:
175      yield f
176
177      # file should be closed before comparison/move.
178      f.close()
179      if not (only_if_changed and os.path.exists(path) and
180              filecmp.cmp(f.name, path)):
181        shutil.move(f.name, path)
182    finally:
183      if os.path.exists(f.name):
184        os.unlink(f.name)
185
186
187class CalledProcessError(Exception):
188  """This exception is raised when the process run by CheckOutput
189  exits with a non-zero exit code."""
190
191  def __init__(self, cwd, args, output):
192    super(CalledProcessError, self).__init__()
193    self.cwd = cwd
194    self.args = args
195    self.output = output
196
197  def __str__(self):
198    # A user should be able to simply copy and paste the command that failed
199    # into their shell.
200    copyable_command = '( cd {}; {} )'.format(os.path.abspath(self.cwd),
201        ' '.join(map(pipes.quote, self.args)))
202    return 'Command failed: {}\n{}'.format(copyable_command, self.output)
203
204
205def FilterLines(output, filter_string):
206  """Output filter from build_utils.CheckOutput.
207
208  Args:
209    output: Executable output as from build_utils.CheckOutput.
210    filter_string: An RE string that will filter (remove) matching
211        lines from |output|.
212
213  Returns:
214    The filtered output, as a single string.
215  """
216  re_filter = re.compile(filter_string)
217  return '\n'.join(
218      line for line in output.split('\n') if not re_filter.search(line))
219
220
221def FilterReflectiveAccessJavaWarnings(output):
222  """Filters out warnings about illegal reflective access operation.
223
224  These warnings were introduced in Java 9, and generally mean that dependencies
225  need to be updated.
226  """
227  #  WARNING: An illegal reflective access operation has occurred
228  #  WARNING: Illegal reflective access by ...
229  #  WARNING: Please consider reporting this to the maintainers of ...
230  #  WARNING: Use --illegal-access=warn to enable warnings of further ...
231  #  WARNING: All illegal access operations will be denied in a future release
232  return FilterLines(
233      output, r'WARNING: ('
234      'An illegal reflective|'
235      'Illegal reflective access|'
236      'Please consider reporting this to|'
237      'Use --illegal-access=warn|'
238      'All illegal access operations)')
239
240
241# This can be used in most cases like subprocess.check_output(). The output,
242# particularly when the command fails, better highlights the command's failure.
243# If the command fails, raises a build_utils.CalledProcessError.
244def CheckOutput(args,
245                cwd=None,
246                env=None,
247                print_stdout=False,
248                print_stderr=True,
249                stdout_filter=None,
250                stderr_filter=None,
251                fail_on_output=True,
252                fail_func=lambda returncode, stderr: returncode != 0):
253  if not cwd:
254    cwd = os.getcwd()
255
256  child = subprocess.Popen(args,
257      stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env)
258  stdout, stderr = child.communicate()
259
260  # For Python3 only:
261  if isinstance(stdout, bytes) and sys.version_info >= (3, ):
262    stdout = stdout.decode('utf-8')
263    stderr = stderr.decode('utf-8')
264
265  if stdout_filter is not None:
266    stdout = stdout_filter(stdout)
267
268  if stderr_filter is not None:
269    stderr = stderr_filter(stderr)
270
271  if fail_func and fail_func(child.returncode, stderr):
272    raise CalledProcessError(cwd, args, stdout + stderr)
273
274  if print_stdout:
275    sys.stdout.write(stdout)
276  if print_stderr:
277    sys.stderr.write(stderr)
278
279  has_stdout = print_stdout and stdout
280  has_stderr = print_stderr and stderr
281  if fail_on_output and (has_stdout or has_stderr):
282    MSG = """\
283Command failed because it wrote to {}.
284You can often set treat_warnings_as_errors=false to not treat output as \
285failure (useful when developing locally)."""
286    if has_stdout and has_stderr:
287      stream_string = 'stdout and stderr'
288    elif has_stdout:
289      stream_string = 'stdout'
290    else:
291      stream_string = 'stderr'
292    raise CalledProcessError(cwd, args, MSG.format(stream_string))
293
294  return stdout
295
296
297def GetModifiedTime(path):
298  # For a symlink, the modified time should be the greater of the link's
299  # modified time and the modified time of the target.
300  return max(os.lstat(path).st_mtime, os.stat(path).st_mtime)
301
302
303def IsTimeStale(output, inputs):
304  if not os.path.exists(output):
305    return True
306
307  output_time = GetModifiedTime(output)
308  for i in inputs:
309    if GetModifiedTime(i) > output_time:
310      return True
311  return False
312
313
314def _CheckZipPath(name):
315  if os.path.normpath(name) != name:
316    raise Exception('Non-canonical zip path: %s' % name)
317  if os.path.isabs(name):
318    raise Exception('Absolute zip path: %s' % name)
319
320
321def _IsSymlink(zip_file, name):
322  zi = zip_file.getinfo(name)
323
324  # The two high-order bytes of ZipInfo.external_attr represent
325  # UNIX permissions and file type bits.
326  return stat.S_ISLNK(zi.external_attr >> 16)
327
328
329def ExtractAll(zip_path, path=None, no_clobber=True, pattern=None,
330               predicate=None):
331  if path is None:
332    path = os.getcwd()
333  elif not os.path.exists(path):
334    MakeDirectory(path)
335
336  if not zipfile.is_zipfile(zip_path):
337    raise Exception('Invalid zip file: %s' % zip_path)
338
339  extracted = []
340  with zipfile.ZipFile(zip_path) as z:
341    for name in z.namelist():
342      if name.endswith('/'):
343        MakeDirectory(os.path.join(path, name))
344        continue
345      if pattern is not None:
346        if not fnmatch.fnmatch(name, pattern):
347          continue
348      if predicate and not predicate(name):
349        continue
350      _CheckZipPath(name)
351      if no_clobber:
352        output_path = os.path.join(path, name)
353        if os.path.exists(output_path):
354          raise Exception(
355              'Path already exists from zip: %s %s %s'
356              % (zip_path, name, output_path))
357      if _IsSymlink(z, name):
358        dest = os.path.join(path, name)
359        MakeDirectory(os.path.dirname(dest))
360        os.symlink(z.read(name), dest)
361        extracted.append(dest)
362      else:
363        z.extract(name, path)
364        extracted.append(os.path.join(path, name))
365
366  return extracted
367
368
369def HermeticZipInfo(*args, **kwargs):
370  """Creates a ZipInfo with a constant timestamp and external_attr."""
371  ret = zipfile.ZipInfo(*args, **kwargs)
372  ret.date_time = (2001, 1, 1, 0, 0, 0)
373  ret.external_attr = (0o644 << 16)
374  return ret
375
376
377def AddToZipHermetic(zip_file,
378                     zip_path,
379                     src_path=None,
380                     data=None,
381                     compress=None):
382  """Adds a file to the given ZipFile with a hard-coded modified time.
383
384  Args:
385    zip_file: ZipFile instance to add the file to.
386    zip_path: Destination path within the zip file (or ZipInfo instance).
387    src_path: Path of the source file. Mutually exclusive with |data|.
388    data: File data as a string.
389    compress: Whether to enable compression. Default is taken from ZipFile
390        constructor.
391  """
392  assert (src_path is None) != (data is None), (
393      '|src_path| and |data| are mutually exclusive.')
394  if isinstance(zip_path, zipfile.ZipInfo):
395    zipinfo = zip_path
396    zip_path = zipinfo.filename
397  else:
398    zipinfo = HermeticZipInfo(filename=zip_path)
399
400  _CheckZipPath(zip_path)
401
402  if src_path and os.path.islink(src_path):
403    zipinfo.filename = zip_path
404    zipinfo.external_attr |= stat.S_IFLNK << 16  # mark as a symlink
405    zip_file.writestr(zipinfo, os.readlink(src_path))
406    return
407
408  # zipfile.write() does
409  #     external_attr = (os.stat(src_path)[0] & 0xFFFF) << 16
410  # but we want to use _HERMETIC_FILE_ATTR, so manually set
411  # the few attr bits we care about.
412  if src_path:
413    st = os.stat(src_path)
414    for mode in (stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH):
415      if st.st_mode & mode:
416        zipinfo.external_attr |= mode << 16
417
418  if src_path:
419    with open(src_path, 'rb') as f:
420      data = f.read()
421
422  # zipfile will deflate even when it makes the file bigger. To avoid
423  # growing files, disable compression at an arbitrary cut off point.
424  if len(data) < 16:
425    compress = False
426
427  # None converts to ZIP_STORED, when passed explicitly rather than the
428  # default passed to the ZipFile constructor.
429  compress_type = zip_file.compression
430  if compress is not None:
431    compress_type = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
432  zip_file.writestr(zipinfo, data, compress_type)
433
434
435def DoZip(inputs, output, base_dir=None, compress_fn=None,
436          zip_prefix_path=None):
437  """Creates a zip file from a list of files.
438
439  Args:
440    inputs: A list of paths to zip, or a list of (zip_path, fs_path) tuples.
441    output: Path, fileobj, or ZipFile instance to add files to.
442    base_dir: Prefix to strip from inputs.
443    compress_fn: Applied to each input to determine whether or not to compress.
444        By default, items will be |zipfile.ZIP_STORED|.
445    zip_prefix_path: Path prepended to file path in zip file.
446  """
447  if base_dir is None:
448    base_dir = '.'
449  input_tuples = []
450  for tup in inputs:
451    if isinstance(tup, string_types):
452      tup = (os.path.relpath(tup, base_dir), tup)
453      if tup[0].startswith('..'):
454        raise Exception('Invalid zip_path: ' + tup[0])
455    input_tuples.append(tup)
456
457  # Sort by zip path to ensure stable zip ordering.
458  input_tuples.sort(key=lambda tup: tup[0])
459
460  out_zip = output
461  if not isinstance(output, zipfile.ZipFile):
462    out_zip = zipfile.ZipFile(output, 'w')
463
464  try:
465    for zip_path, fs_path in input_tuples:
466      if zip_prefix_path:
467        zip_path = os.path.join(zip_prefix_path, zip_path)
468      compress = compress_fn(zip_path) if compress_fn else None
469      AddToZipHermetic(out_zip, zip_path, src_path=fs_path, compress=compress)
470  finally:
471    if output is not out_zip:
472      out_zip.close()
473
474
475def ZipDir(output, base_dir, compress_fn=None, zip_prefix_path=None):
476  """Creates a zip file from a directory."""
477  inputs = []
478  for root, _, files in os.walk(base_dir):
479    for f in files:
480      inputs.append(os.path.join(root, f))
481
482  if isinstance(output, zipfile.ZipFile):
483    DoZip(
484        inputs,
485        output,
486        base_dir,
487        compress_fn=compress_fn,
488        zip_prefix_path=zip_prefix_path)
489  else:
490    with AtomicOutput(output) as f:
491      DoZip(
492          inputs,
493          f,
494          base_dir,
495          compress_fn=compress_fn,
496          zip_prefix_path=zip_prefix_path)
497
498
499def MatchesGlob(path, filters):
500  """Returns whether the given path matches any of the given glob patterns."""
501  return filters and any(fnmatch.fnmatch(path, f) for f in filters)
502
503
504def MergeZips(output, input_zips, path_transform=None, compress=None):
505  """Combines all files from |input_zips| into |output|.
506
507  Args:
508    output: Path, fileobj, or ZipFile instance to add files to.
509    input_zips: Iterable of paths to zip files to merge.
510    path_transform: Called for each entry path. Returns a new path, or None to
511        skip the file.
512    compress: Overrides compression setting from origin zip entries.
513  """
514  path_transform = path_transform or (lambda p: p)
515  added_names = set()
516
517  out_zip = output
518  if not isinstance(output, zipfile.ZipFile):
519    out_zip = zipfile.ZipFile(output, 'w')
520
521  try:
522    for in_file in input_zips:
523      with zipfile.ZipFile(in_file, 'r') as in_zip:
524        for info in in_zip.infolist():
525          # Ignore directories.
526          if info.filename[-1] == '/':
527            continue
528          dst_name = path_transform(info.filename)
529          if not dst_name:
530            continue
531          already_added = dst_name in added_names
532          if not already_added:
533            if compress is not None:
534              compress_entry = compress
535            else:
536              compress_entry = info.compress_type != zipfile.ZIP_STORED
537            AddToZipHermetic(
538                out_zip,
539                dst_name,
540                data=in_zip.read(info),
541                compress=compress_entry)
542            added_names.add(dst_name)
543  finally:
544    if output is not out_zip:
545      out_zip.close()
546
547
548def GetSortedTransitiveDependencies(top, deps_func):
549  """Gets the list of all transitive dependencies in sorted order.
550
551  There should be no cycles in the dependency graph (crashes if cycles exist).
552
553  Args:
554    top: A list of the top level nodes
555    deps_func: A function that takes a node and returns a list of its direct
556        dependencies.
557  Returns:
558    A list of all transitive dependencies of nodes in top, in order (a node will
559    appear in the list at a higher index than all of its dependencies).
560  """
561  # Find all deps depth-first, maintaining original order in the case of ties.
562  deps_map = collections.OrderedDict()
563  def discover(nodes):
564    for node in nodes:
565      if node in deps_map:
566        continue
567      deps = deps_func(node)
568      discover(deps)
569      deps_map[node] = deps
570
571  discover(top)
572  return list(deps_map)
573
574
575def InitLogging(enabling_env):
576  logging.basicConfig(
577      level=logging.DEBUG if os.environ.get(enabling_env) else logging.WARNING,
578      format='%(levelname).1s %(process)d %(relativeCreated)6d %(message)s')
579  script_name = os.path.basename(sys.argv[0])
580  logging.info('Started (%s)', script_name)
581
582  my_pid = os.getpid()
583
584  def log_exit():
585    # Do not log for fork'ed processes.
586    if os.getpid() == my_pid:
587      logging.info("Job's done (%s)", script_name)
588
589  atexit.register(log_exit)
590
591
592def AddDepfileOption(parser):
593  # TODO(agrieve): Get rid of this once we've moved to argparse.
594  if hasattr(parser, 'add_option'):
595    func = parser.add_option
596  else:
597    func = parser.add_argument
598  func('--depfile',
599       help='Path to depfile (refer to `gn help depfile`)')
600
601
602def WriteDepfile(depfile_path, first_gn_output, inputs=None):
603  assert depfile_path != first_gn_output  # http://crbug.com/646165
604  assert not isinstance(inputs, string_types)  # Easy mistake to make
605  inputs = inputs or []
606  MakeDirectory(os.path.dirname(depfile_path))
607  # Ninja does not support multiple outputs in depfiles.
608  with open(depfile_path, 'w') as depfile:
609    depfile.write(first_gn_output.replace(' ', '\\ '))
610    depfile.write(': ')
611    depfile.write(' '.join(i.replace(' ', '\\ ') for i in inputs))
612    depfile.write('\n')
613
614
615def ExpandFileArgs(args):
616  """Replaces file-arg placeholders in args.
617
618  These placeholders have the form:
619    @FileArg(filename:key1:key2:...:keyn)
620
621  The value of such a placeholder is calculated by reading 'filename' as json.
622  And then extracting the value at [key1][key2]...[keyn]. If a key has a '[]'
623  suffix the (intermediate) value will be interpreted as a single item list and
624  the single item will be returned or used for further traversal.
625
626  Note: This intentionally does not return the list of files that appear in such
627  placeholders. An action that uses file-args *must* know the paths of those
628  files prior to the parsing of the arguments (typically by explicitly listing
629  them in the action's inputs in build files).
630  """
631  new_args = list(args)
632  file_jsons = dict()
633  r = re.compile('@FileArg\((.*?)\)')
634  for i, arg in enumerate(args):
635    match = r.search(arg)
636    if not match:
637      continue
638
639    def get_key(key):
640      if key.endswith('[]'):
641        return key[:-2], True
642      return key, False
643
644    lookup_path = match.group(1).split(':')
645    file_path, _ = get_key(lookup_path[0])
646    if not file_path in file_jsons:
647      with open(file_path) as f:
648        file_jsons[file_path] = json.load(f)
649
650    expansion = file_jsons
651    for k in lookup_path:
652      k, flatten = get_key(k)
653      expansion = expansion[k]
654      if flatten:
655        if not isinstance(expansion, list) or not len(expansion) == 1:
656          raise Exception('Expected single item list but got %s' % expansion)
657        expansion = expansion[0]
658
659    # This should match ParseGnList. The output is either a GN-formatted list
660    # or a literal (with no quotes).
661    if isinstance(expansion, list):
662      new_args[i] = (arg[:match.start()] + gn_helpers.ToGNString(expansion) +
663                     arg[match.end():])
664    else:
665      new_args[i] = arg[:match.start()] + str(expansion) + arg[match.end():]
666
667  return new_args
668
669
670def ReadSourcesList(sources_list_file_name):
671  """Reads a GN-written file containing list of file names and returns a list.
672
673  Note that this function should not be used to parse response files.
674  """
675  with open(sources_list_file_name) as f:
676    return [file_name.strip() for file_name in f]
677