1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Contains common helpers for GN action()s."""
6
7import atexit
8import collections
9import contextlib
10import filecmp
11import fnmatch
12import json
13import logging
14import os
15import pipes
16import re
17import shutil
18import stat
19import subprocess
20import sys
21import tempfile
22import time
23import zipfile
24
25sys.path.append(os.path.join(os.path.dirname(__file__),
26                             os.pardir, os.pardir, os.pardir))
27import gn_helpers
28
29# Use relative paths to improved hermetic property of build scripts.
30DIR_SOURCE_ROOT = os.path.relpath(
31    os.environ.get(
32        'CHECKOUT_SOURCE_ROOT',
33        os.path.join(
34            os.path.dirname(__file__), os.pardir, os.pardir, os.pardir,
35            os.pardir)))
36JAVA_HOME = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'current')
37JAVAC_PATH = os.path.join(JAVA_HOME, 'bin', 'javac')
38JAVAP_PATH = os.path.join(JAVA_HOME, 'bin', 'javap')
39RT_JAR_PATH = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'jdk', 'extras',
40                           'java_8', 'jre', 'lib', 'rt.jar')
41
42try:
43  string_types = basestring
44except NameError:
45  string_types = (str, bytes)
46
47
48def JavaCmd(verify=True, xmx='1G'):
49  ret = [os.path.join(JAVA_HOME, 'bin', 'java')]
50  # Limit heap to avoid Java not GC'ing when it should, and causing
51  # bots to OOM when many java commands are runnig at the same time
52  # https://crbug.com/1098333
53  ret += ['-Xmx' + xmx]
54
55  # Disable bytecode verification for local builds gives a ~2% speed-up.
56  if not verify:
57    ret += ['-noverify']
58
59  return ret
60
61
62@contextlib.contextmanager
63def TempDir(**kwargs):
64  dirname = tempfile.mkdtemp(**kwargs)
65  try:
66    yield dirname
67  finally:
68    shutil.rmtree(dirname)
69
70
71def MakeDirectory(dir_path):
72  try:
73    os.makedirs(dir_path)
74  except OSError:
75    pass
76
77
78def DeleteDirectory(dir_path):
79  if os.path.exists(dir_path):
80    shutil.rmtree(dir_path)
81
82
83def Touch(path, fail_if_missing=False):
84  if fail_if_missing and not os.path.exists(path):
85    raise Exception(path + ' doesn\'t exist.')
86
87  MakeDirectory(os.path.dirname(path))
88  with open(path, 'a'):
89    os.utime(path, None)
90
91
92def FindInDirectory(directory, filename_filter='*'):
93  files = []
94  for root, _dirnames, filenames in os.walk(directory):
95    matched_files = fnmatch.filter(filenames, filename_filter)
96    files.extend((os.path.join(root, f) for f in matched_files))
97  return files
98
99
100def ParseGnList(value):
101  """Converts a "GN-list" command-line parameter into a list.
102
103  Conversions handled:
104    * None -> []
105    * '' -> []
106    * 'asdf' -> ['asdf']
107    * '["a", "b"]' -> ['a', 'b']
108    * ['["a", "b"]', 'c'] -> ['a', 'b', 'c']  (flattened list)
109
110  The common use for this behavior is in the Android build where things can
111  take lists of @FileArg references that are expanded via ExpandFileArgs.
112  """
113  # Convert None to [].
114  if not value:
115    return []
116  # Convert a list of GN lists to a flattened list.
117  if isinstance(value, list):
118    ret = []
119    for arg in value:
120      ret.extend(ParseGnList(arg))
121    return ret
122  # Convert normal GN list.
123  if value.startswith('['):
124    return gn_helpers.GNValueParser(value).ParseList()
125  # Convert a single string value to a list.
126  return [value]
127
128
129def CheckOptions(options, parser, required=None):
130  if not required:
131    return
132  for option_name in required:
133    if getattr(options, option_name) is None:
134      parser.error('--%s is required' % option_name.replace('_', '-'))
135
136
137def WriteJson(obj, path, only_if_changed=False):
138  old_dump = None
139  if os.path.exists(path):
140    with open(path, 'r') as oldfile:
141      old_dump = oldfile.read()
142
143  new_dump = json.dumps(obj, sort_keys=True, indent=2, separators=(',', ': '))
144
145  if not only_if_changed or old_dump != new_dump:
146    with open(path, 'w') as outfile:
147      outfile.write(new_dump)
148
149
150@contextlib.contextmanager
151def AtomicOutput(path, only_if_changed=True, mode='w+b'):
152  """Helper to prevent half-written outputs.
153
154  Args:
155    path: Path to the final output file, which will be written atomically.
156    only_if_changed: If True (the default), do not touch the filesystem
157      if the content has not changed.
158    mode: The mode to open the file in (str).
159  Returns:
160    A python context manager that yelds a NamedTemporaryFile instance
161    that must be used by clients to write the data to. On exit, the
162    manager will try to replace the final output file with the
163    temporary one if necessary. The temporary file is always destroyed
164    on exit.
165  Example:
166    with build_utils.AtomicOutput(output_path) as tmp_file:
167      subprocess.check_call(['prog', '--output', tmp_file.name])
168  """
169  # Create in same directory to ensure same filesystem when moving.
170  dirname = os.path.dirname(path)
171  if not os.path.exists(dirname):
172    MakeDirectory(dirname)
173  with tempfile.NamedTemporaryFile(
174      mode, suffix=os.path.basename(path), dir=dirname, delete=False) as f:
175    try:
176      yield f
177
178      # file should be closed before comparison/move.
179      f.close()
180      if not (only_if_changed and os.path.exists(path) and
181              filecmp.cmp(f.name, path)):
182        shutil.move(f.name, path)
183    finally:
184      if os.path.exists(f.name):
185        os.unlink(f.name)
186
187
188class CalledProcessError(Exception):
189  """This exception is raised when the process run by CheckOutput
190  exits with a non-zero exit code."""
191
192  def __init__(self, cwd, args, output):
193    super(CalledProcessError, self).__init__()
194    self.cwd = cwd
195    self.args = args
196    self.output = output
197
198  def __str__(self):
199    # A user should be able to simply copy and paste the command that failed
200    # into their shell.
201    copyable_command = '( cd {}; {} )'.format(os.path.abspath(self.cwd),
202        ' '.join(map(pipes.quote, self.args)))
203    return 'Command failed: {}\n{}'.format(copyable_command, self.output)
204
205
206def FilterLines(output, filter_string):
207  """Output filter from build_utils.CheckOutput.
208
209  Args:
210    output: Executable output as from build_utils.CheckOutput.
211    filter_string: An RE string that will filter (remove) matching
212        lines from |output|.
213
214  Returns:
215    The filtered output, as a single string.
216  """
217  re_filter = re.compile(filter_string)
218  return '\n'.join(
219      line for line in output.split('\n') if not re_filter.search(line))
220
221
222def FilterReflectiveAccessJavaWarnings(output):
223  """Filters out warnings about illegal reflective access operation.
224
225  These warnings were introduced in Java 9, and generally mean that dependencies
226  need to be updated.
227  """
228  #  WARNING: An illegal reflective access operation has occurred
229  #  WARNING: Illegal reflective access by ...
230  #  WARNING: Please consider reporting this to the maintainers of ...
231  #  WARNING: Use --illegal-access=warn to enable warnings of further ...
232  #  WARNING: All illegal access operations will be denied in a future release
233  return FilterLines(
234      output, r'WARNING: ('
235      'An illegal reflective|'
236      'Illegal reflective access|'
237      'Please consider reporting this to|'
238      'Use --illegal-access=warn|'
239      'All illegal access operations)')
240
241
242# This can be used in most cases like subprocess.check_output(). The output,
243# particularly when the command fails, better highlights the command's failure.
244# If the command fails, raises a build_utils.CalledProcessError.
245def CheckOutput(args,
246                cwd=None,
247                env=None,
248                print_stdout=False,
249                print_stderr=True,
250                stdout_filter=None,
251                stderr_filter=None,
252                fail_on_output=True,
253                fail_func=lambda returncode, stderr: returncode != 0):
254  if not cwd:
255    cwd = os.getcwd()
256
257  logging.info('CheckOutput: %s', ' '.join(args))
258  child = subprocess.Popen(args,
259      stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env)
260  stdout, stderr = child.communicate()
261
262  # For Python3 only:
263  if isinstance(stdout, bytes) and sys.version_info >= (3, ):
264    stdout = stdout.decode('utf-8')
265    stderr = stderr.decode('utf-8')
266
267  if stdout_filter is not None:
268    stdout = stdout_filter(stdout)
269
270  if stderr_filter is not None:
271    stderr = stderr_filter(stderr)
272
273  if fail_func and fail_func(child.returncode, stderr):
274    raise CalledProcessError(cwd, args, stdout + stderr)
275
276  if print_stdout:
277    sys.stdout.write(stdout)
278  if print_stderr:
279    sys.stderr.write(stderr)
280
281  has_stdout = print_stdout and stdout
282  has_stderr = print_stderr and stderr
283  if fail_on_output and (has_stdout or has_stderr):
284    MSG = """\
285Command failed because it wrote to {}.
286You can often set treat_warnings_as_errors=false to not treat output as \
287failure (useful when developing locally)."""
288    if has_stdout and has_stderr:
289      stream_string = 'stdout and stderr'
290    elif has_stdout:
291      stream_string = 'stdout'
292    else:
293      stream_string = 'stderr'
294    raise CalledProcessError(cwd, args, MSG.format(stream_string))
295
296  return stdout
297
298
299def GetModifiedTime(path):
300  # For a symlink, the modified time should be the greater of the link's
301  # modified time and the modified time of the target.
302  return max(os.lstat(path).st_mtime, os.stat(path).st_mtime)
303
304
305def IsTimeStale(output, inputs):
306  if not os.path.exists(output):
307    return True
308
309  output_time = GetModifiedTime(output)
310  for i in inputs:
311    if GetModifiedTime(i) > output_time:
312      return True
313  return False
314
315
316def _CheckZipPath(name):
317  if os.path.normpath(name) != name:
318    raise Exception('Non-canonical zip path: %s' % name)
319  if os.path.isabs(name):
320    raise Exception('Absolute zip path: %s' % name)
321
322
323def _IsSymlink(zip_file, name):
324  zi = zip_file.getinfo(name)
325
326  # The two high-order bytes of ZipInfo.external_attr represent
327  # UNIX permissions and file type bits.
328  return stat.S_ISLNK(zi.external_attr >> 16)
329
330
331def ExtractAll(zip_path, path=None, no_clobber=True, pattern=None,
332               predicate=None):
333  if path is None:
334    path = os.getcwd()
335  elif not os.path.exists(path):
336    MakeDirectory(path)
337
338  if not zipfile.is_zipfile(zip_path):
339    raise Exception('Invalid zip file: %s' % zip_path)
340
341  extracted = []
342  with zipfile.ZipFile(zip_path) as z:
343    for name in z.namelist():
344      if name.endswith('/'):
345        MakeDirectory(os.path.join(path, name))
346        continue
347      if pattern is not None:
348        if not fnmatch.fnmatch(name, pattern):
349          continue
350      if predicate and not predicate(name):
351        continue
352      _CheckZipPath(name)
353      if no_clobber:
354        output_path = os.path.join(path, name)
355        if os.path.exists(output_path):
356          raise Exception(
357              'Path already exists from zip: %s %s %s'
358              % (zip_path, name, output_path))
359      if _IsSymlink(z, name):
360        dest = os.path.join(path, name)
361        MakeDirectory(os.path.dirname(dest))
362        os.symlink(z.read(name), dest)
363        extracted.append(dest)
364      else:
365        z.extract(name, path)
366        extracted.append(os.path.join(path, name))
367
368  return extracted
369
370
371def HermeticDateTime(timestamp=None):
372  """Returns a constant ZipInfo.date_time tuple.
373
374  Args:
375    timestamp: Unix timestamp to use for files in the archive.
376
377  Returns:
378    A ZipInfo.date_time tuple for Jan 1, 2001, or the given timestamp.
379  """
380  if not timestamp:
381    return (2001, 1, 1, 0, 0, 0)
382  utc_time = time.gmtime(timestamp)
383  return (utc_time.tm_year, utc_time.tm_mon, utc_time.tm_mday, utc_time.tm_hour,
384          utc_time.tm_min, utc_time.tm_sec)
385
386
387def HermeticZipInfo(*args, **kwargs):
388  """Creates a zipfile.ZipInfo with a constant timestamp and external_attr.
389
390  If a date_time value is not provided in the positional or keyword arguments,
391  the default value from HermeticDateTime is used.
392
393  Args:
394    See zipfile.ZipInfo.
395
396  Returns:
397    A zipfile.ZipInfo.
398  """
399  # The caller may have provided a date_time either as a positional parameter
400  # (args[1]) or as a keyword parameter. Use the default hermetic date_time if
401  # none was provided.
402  date_time = None
403  if len(args) >= 2:
404    date_time = args[1]
405  elif 'date_time' in kwargs:
406    date_time = kwargs['date_time']
407  if not date_time:
408    kwargs['date_time'] = HermeticDateTime()
409  ret = zipfile.ZipInfo(*args, **kwargs)
410  ret.external_attr = (0o644 << 16)
411  return ret
412
413
414def AddToZipHermetic(zip_file,
415                     zip_path,
416                     src_path=None,
417                     data=None,
418                     compress=None,
419                     date_time=None):
420  """Adds a file to the given ZipFile with a hard-coded modified time.
421
422  Args:
423    zip_file: ZipFile instance to add the file to.
424    zip_path: Destination path within the zip file (or ZipInfo instance).
425    src_path: Path of the source file. Mutually exclusive with |data|.
426    data: File data as a string.
427    compress: Whether to enable compression. Default is taken from ZipFile
428        constructor.
429    date_time: The last modification date and time for the archive member.
430  """
431  assert (src_path is None) != (data is None), (
432      '|src_path| and |data| are mutually exclusive.')
433  if isinstance(zip_path, zipfile.ZipInfo):
434    zipinfo = zip_path
435    zip_path = zipinfo.filename
436  else:
437    zipinfo = HermeticZipInfo(filename=zip_path, date_time=date_time)
438
439  _CheckZipPath(zip_path)
440
441  if src_path and os.path.islink(src_path):
442    zipinfo.filename = zip_path
443    zipinfo.external_attr |= stat.S_IFLNK << 16  # mark as a symlink
444    zip_file.writestr(zipinfo, os.readlink(src_path))
445    return
446
447  # zipfile.write() does
448  #     external_attr = (os.stat(src_path)[0] & 0xFFFF) << 16
449  # but we want to use _HERMETIC_FILE_ATTR, so manually set
450  # the few attr bits we care about.
451  if src_path:
452    st = os.stat(src_path)
453    for mode in (stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH):
454      if st.st_mode & mode:
455        zipinfo.external_attr |= mode << 16
456
457  if src_path:
458    with open(src_path, 'rb') as f:
459      data = f.read()
460
461  # zipfile will deflate even when it makes the file bigger. To avoid
462  # growing files, disable compression at an arbitrary cut off point.
463  if len(data) < 16:
464    compress = False
465
466  # None converts to ZIP_STORED, when passed explicitly rather than the
467  # default passed to the ZipFile constructor.
468  compress_type = zip_file.compression
469  if compress is not None:
470    compress_type = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
471  zip_file.writestr(zipinfo, data, compress_type)
472
473
474def DoZip(inputs,
475          output,
476          base_dir=None,
477          compress_fn=None,
478          zip_prefix_path=None,
479          timestamp=None):
480  """Creates a zip file from a list of files.
481
482  Args:
483    inputs: A list of paths to zip, or a list of (zip_path, fs_path) tuples.
484    output: Path, fileobj, or ZipFile instance to add files to.
485    base_dir: Prefix to strip from inputs.
486    compress_fn: Applied to each input to determine whether or not to compress.
487        By default, items will be |zipfile.ZIP_STORED|.
488    zip_prefix_path: Path prepended to file path in zip file.
489    timestamp: Unix timestamp to use for files in the archive.
490  """
491  if base_dir is None:
492    base_dir = '.'
493  input_tuples = []
494  for tup in inputs:
495    if isinstance(tup, string_types):
496      tup = (os.path.relpath(tup, base_dir), tup)
497      if tup[0].startswith('..'):
498        raise Exception('Invalid zip_path: ' + tup[0])
499    input_tuples.append(tup)
500
501  # Sort by zip path to ensure stable zip ordering.
502  input_tuples.sort(key=lambda tup: tup[0])
503
504  out_zip = output
505  if not isinstance(output, zipfile.ZipFile):
506    out_zip = zipfile.ZipFile(output, 'w')
507
508  date_time = HermeticDateTime(timestamp)
509  try:
510    for zip_path, fs_path in input_tuples:
511      if zip_prefix_path:
512        zip_path = os.path.join(zip_prefix_path, zip_path)
513      compress = compress_fn(zip_path) if compress_fn else None
514      AddToZipHermetic(out_zip,
515                       zip_path,
516                       src_path=fs_path,
517                       compress=compress,
518                       date_time=date_time)
519  finally:
520    if output is not out_zip:
521      out_zip.close()
522
523
524def ZipDir(output, base_dir, compress_fn=None, zip_prefix_path=None):
525  """Creates a zip file from a directory."""
526  inputs = []
527  for root, _, files in os.walk(base_dir):
528    for f in files:
529      inputs.append(os.path.join(root, f))
530
531  if isinstance(output, zipfile.ZipFile):
532    DoZip(
533        inputs,
534        output,
535        base_dir,
536        compress_fn=compress_fn,
537        zip_prefix_path=zip_prefix_path)
538  else:
539    with AtomicOutput(output) as f:
540      DoZip(
541          inputs,
542          f,
543          base_dir,
544          compress_fn=compress_fn,
545          zip_prefix_path=zip_prefix_path)
546
547
548def MatchesGlob(path, filters):
549  """Returns whether the given path matches any of the given glob patterns."""
550  return filters and any(fnmatch.fnmatch(path, f) for f in filters)
551
552
553def MergeZips(output, input_zips, path_transform=None, compress=None):
554  """Combines all files from |input_zips| into |output|.
555
556  Args:
557    output: Path, fileobj, or ZipFile instance to add files to.
558    input_zips: Iterable of paths to zip files to merge.
559    path_transform: Called for each entry path. Returns a new path, or None to
560        skip the file.
561    compress: Overrides compression setting from origin zip entries.
562  """
563  path_transform = path_transform or (lambda p: p)
564  added_names = set()
565
566  out_zip = output
567  if not isinstance(output, zipfile.ZipFile):
568    out_zip = zipfile.ZipFile(output, 'w')
569
570  try:
571    for in_file in input_zips:
572      with zipfile.ZipFile(in_file, 'r') as in_zip:
573        for info in in_zip.infolist():
574          # Ignore directories.
575          if info.filename[-1] == '/':
576            continue
577          dst_name = path_transform(info.filename)
578          if not dst_name:
579            continue
580          already_added = dst_name in added_names
581          if not already_added:
582            if compress is not None:
583              compress_entry = compress
584            else:
585              compress_entry = info.compress_type != zipfile.ZIP_STORED
586            AddToZipHermetic(
587                out_zip,
588                dst_name,
589                data=in_zip.read(info),
590                compress=compress_entry)
591            added_names.add(dst_name)
592  finally:
593    if output is not out_zip:
594      out_zip.close()
595
596
597def GetSortedTransitiveDependencies(top, deps_func):
598  """Gets the list of all transitive dependencies in sorted order.
599
600  There should be no cycles in the dependency graph (crashes if cycles exist).
601
602  Args:
603    top: A list of the top level nodes
604    deps_func: A function that takes a node and returns a list of its direct
605        dependencies.
606  Returns:
607    A list of all transitive dependencies of nodes in top, in order (a node will
608    appear in the list at a higher index than all of its dependencies).
609  """
610  # Find all deps depth-first, maintaining original order in the case of ties.
611  deps_map = collections.OrderedDict()
612  def discover(nodes):
613    for node in nodes:
614      if node in deps_map:
615        continue
616      deps = deps_func(node)
617      discover(deps)
618      deps_map[node] = deps
619
620  discover(top)
621  return list(deps_map)
622
623
624def InitLogging(enabling_env):
625  logging.basicConfig(
626      level=logging.DEBUG if os.environ.get(enabling_env) else logging.WARNING,
627      format='%(levelname).1s %(process)d %(relativeCreated)6d %(message)s')
628  script_name = os.path.basename(sys.argv[0])
629  logging.info('Started (%s)', script_name)
630
631  my_pid = os.getpid()
632
633  def log_exit():
634    # Do not log for fork'ed processes.
635    if os.getpid() == my_pid:
636      logging.info("Job's done (%s)", script_name)
637
638  atexit.register(log_exit)
639
640
641def AddDepfileOption(parser):
642  # TODO(agrieve): Get rid of this once we've moved to argparse.
643  if hasattr(parser, 'add_option'):
644    func = parser.add_option
645  else:
646    func = parser.add_argument
647  func('--depfile',
648       help='Path to depfile (refer to `gn help depfile`)')
649
650
651def WriteDepfile(depfile_path, first_gn_output, inputs=None):
652  assert depfile_path != first_gn_output  # http://crbug.com/646165
653  assert not isinstance(inputs, string_types)  # Easy mistake to make
654  inputs = inputs or []
655  MakeDirectory(os.path.dirname(depfile_path))
656  # Ninja does not support multiple outputs in depfiles.
657  with open(depfile_path, 'w') as depfile:
658    depfile.write(first_gn_output.replace(' ', '\\ '))
659    depfile.write(': \\\n ')
660    depfile.write(' \\\n '.join(i.replace(' ', '\\ ') for i in inputs))
661    depfile.write('\n')
662
663
664def ExpandFileArgs(args):
665  """Replaces file-arg placeholders in args.
666
667  These placeholders have the form:
668    @FileArg(filename:key1:key2:...:keyn)
669
670  The value of such a placeholder is calculated by reading 'filename' as json.
671  And then extracting the value at [key1][key2]...[keyn]. If a key has a '[]'
672  suffix the (intermediate) value will be interpreted as a single item list and
673  the single item will be returned or used for further traversal.
674
675  Note: This intentionally does not return the list of files that appear in such
676  placeholders. An action that uses file-args *must* know the paths of those
677  files prior to the parsing of the arguments (typically by explicitly listing
678  them in the action's inputs in build files).
679  """
680  new_args = list(args)
681  file_jsons = dict()
682  r = re.compile('@FileArg\((.*?)\)')
683  for i, arg in enumerate(args):
684    match = r.search(arg)
685    if not match:
686      continue
687
688    def get_key(key):
689      if key.endswith('[]'):
690        return key[:-2], True
691      return key, False
692
693    lookup_path = match.group(1).split(':')
694    file_path, _ = get_key(lookup_path[0])
695    if not file_path in file_jsons:
696      with open(file_path) as f:
697        file_jsons[file_path] = json.load(f)
698
699    expansion = file_jsons
700    for k in lookup_path:
701      k, flatten = get_key(k)
702      expansion = expansion[k]
703      if flatten:
704        if not isinstance(expansion, list) or not len(expansion) == 1:
705          raise Exception('Expected single item list but got %s' % expansion)
706        expansion = expansion[0]
707
708    # This should match ParseGnList. The output is either a GN-formatted list
709    # or a literal (with no quotes).
710    if isinstance(expansion, list):
711      new_args[i] = (arg[:match.start()] + gn_helpers.ToGNString(expansion) +
712                     arg[match.end():])
713    else:
714      new_args[i] = arg[:match.start()] + str(expansion) + arg[match.end():]
715
716  return new_args
717
718
719def ReadSourcesList(sources_list_file_name):
720  """Reads a GN-written file containing list of file names and returns a list.
721
722  Note that this function should not be used to parse response files.
723  """
724  with open(sources_list_file_name) as f:
725    return [file_name.strip() for file_name in f]
726