grit/tool/build.py

# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

'''The 'grit build' tool.
'''

from __future__ import print_function

import codecs
import filecmp
import getopt
import gzip
import os
import shutil
import sys

import six

from grit import grd_reader
from grit import shortcuts
from grit import util
from grit.format import minifier
from grit.node import brotli_util
from grit.node import include
from grit.node import message
from grit.node import structure
from grit.tool import interface


# It would be cleaner to have each module register itself, but that would
# require importing all of them on every run of GRIT.
'''Map from <output> node types to modules under grit.format.'''
_format_modules = {
  'android': 'android_xml',
  'c_format': 'c_format',
  'chrome_messages_json': 'chrome_messages_json',
  'chrome_messages_json_gzip': 'chrome_messages_json',
  'data_package': 'data_pack',
  'policy_templates': 'policy_templates_json',
  'rc_all': 'rc',
  'rc_header': 'rc_header',
  'rc_nontranslateable': 'rc',
  'rc_translateable': 'rc',
  'resource_file_map_source': 'resource_map',
  'resource_map_header': 'resource_map',
  'resource_map_source': 'resource_map',
}

def GetFormatter(type):
  modulename = 'grit.format.' + _format_modules[type]
  __import__(modulename)
  module = sys.modules[modulename]
  try:
    return module.Format
  except AttributeError:
    return module.GetFormatter(type)


class RcBuilder(interface.Tool):
  '''A tool that builds RC files and resource header files for compilation.

Usage:  grit build [-o OUTPUTDIR] [-D NAME[=VAL]]*

All output options for this tool are specified in the input file (see
'grit help' for details on how to specify the input file - it is a global
option).

Options:

  -a FILE           Assert that the given file is an output. There can be
                    multiple "-a" flags listed for multiple outputs. If a "-a"
                    or "--assert-file-list" argument is present, then the list
                    of asserted files must match the output files or the tool
                    will fail. The use-case is for the build system to maintain
                    separate lists of output files and to catch errors if the
                    build system's list and the grit list are out-of-sync.

  --assert-file-list  Provide a file listing multiple asserted output files.
                    There is one file name per line. This acts like specifying
                    each file with "-a" on the command line, but without the
                    possibility of running into OS line-length limits for very
                    long lists.

  -o OUTPUTDIR      Specify what directory output paths are relative to.
                    Defaults to the current directory.

  -p FILE           Specify a file containing a pre-determined mapping from
                    resource names to resource ids which will be used to assign
                    resource ids to those resources. Resources not found in this
                    file will be assigned ids normally. The motivation is to run
                    your app's startup and have it dump the resources it loads,
                    and then pass these via this flag. This will pack startup
                    resources together, thus reducing paging while all other
                    resources are unperturbed. The file should have the format:
                      RESOURCE_ONE_NAME 123
                      RESOURCE_TWO_NAME 124

  -D NAME[=VAL]     Specify a C-preprocessor-like define NAME with optional
                    value VAL (defaults to 1) which will be used to control
                    conditional inclusion of resources.

  -E NAME=VALUE     Set environment variable NAME to VALUE (within grit).

  -f FIRSTIDSFILE   Path to a python file that specifies the first id of
                    value to use for resources.  A non-empty value here will
                    override the value specified in the <grit> node's
                    first_ids_file.

  -w ALLOWLISTFILE  Path to a file containing the string names of the
                    resources to include.  Anything not listed is dropped.

  -t PLATFORM       Specifies the platform the build is targeting; defaults
                    to the value of sys.platform. The value provided via this
                    flag should match what sys.platform would report for your
                    target platform; see grit.node.base.EvaluateCondition.

  --allowlist-support
                    Generate code to support extracting a resource allowlist
                    from executables.

  --write-only-new flag
                    If flag is non-0, write output files to a temporary file
                    first, and copy it to the real output only if the new file
                    is different from the old file.  This allows some build
                    systems to realize that dependent build steps might be
                    unnecessary, at the cost of comparing the output data at
                    grit time.

  --depend-on-stamp
                    If specified along with --depfile and --depdir, the depfile
                    generated will depend on a stampfile instead of the first
                    output in the input .grd file.

  --js-minifier     A command to run the Javascript minifier. If not set then
                    Javascript won't be minified. The command should read the
                    original Javascript from standard input, and output the
                    minified Javascript to standard output. A non-zero exit
                    status will be taken as indicating failure.

  --css-minifier    A command to run the CSS minifier. If not set then CSS won't
                    be minified. The command should read the original CSS from
                    standard input, and output the minified CSS to standard
                    output. A non-zero exit status will be taken as indicating
                    failure.

  --brotli          The full path to the brotli executable generated by
                    third_party/brotli/BUILD.gn, required if any entries use
                    compress="brotli".

Conditional inclusion of resources only affects the output of files which
control which resources get linked into a binary, e.g. it affects .rc files
meant for compilation but it does not affect resource header files (that define
IDs).  This helps ensure that values of IDs stay the same, that all messages
are exported to translation interchange files (e.g. XMB files), etc.
'''

  def ShortDescription(self):
    return 'A tool that builds RC files for compilation.'

  def Run(self, opts, args):
    brotli_util.SetBrotliCommand(None)
    os.environ['cwd'] = os.getcwd()
    self.output_directory = '.'
    first_ids_file = None
    predetermined_ids_file = None
    allowlist_filenames = []
    assert_output_files = []
    target_platform = None
    depfile = None
    depdir = None
    allowlist_support = False
    write_only_new = False
    depend_on_stamp = False
    js_minifier = None
    css_minifier = None
    replace_ellipsis = True
    (own_opts, args) = getopt.getopt(
        args, 'a:p:o:D:E:f:w:t:',
        ('depdir=', 'depfile=', 'assert-file-list=', 'help',
         'output-all-resource-defines', 'no-output-all-resource-defines',
         'no-replace-ellipsis', 'depend-on-stamp', 'js-minifier=',
         'css-minifier=', 'write-only-new=', 'allowlist-support', 'brotli='))
    for (key, val) in own_opts:
      if key == '-a':
        assert_output_files.append(val)
      elif key == '--assert-file-list':
        with open(val) as f:
          assert_output_files += f.read().splitlines()
      elif key == '-o':
        self.output_directory = val
      elif key == '-D':
        name, val = util.ParseDefine(val)
        self.defines[name] = val
      elif key == '-E':
        (env_name, env_value) = val.split('=', 1)
        os.environ[env_name] = env_value
      elif key == '-f':
        # TODO(joi@chromium.org): Remove this override once change
        # lands in WebKit.grd to specify the first_ids_file in the
        # .grd itself.
        first_ids_file = val
      elif key == '-w':
        allowlist_filenames.append(val)
      elif key == '--no-replace-ellipsis':
        replace_ellipsis = False
      elif key == '-p':
        predetermined_ids_file = val
      elif key == '-t':
        target_platform = val
      elif key == '--depdir':
        depdir = val
      elif key == '--depfile':
        depfile = val
      elif key == '--write-only-new':
        write_only_new = val != '0'
      elif key == '--depend-on-stamp':
        depend_on_stamp = True
      elif key == '--js-minifier':
        js_minifier = val
      elif key == '--css-minifier':
        css_minifier = val
      elif key == '--allowlist-support':
        allowlist_support = True
      elif key == '--brotli':
        brotli_util.SetBrotliCommand([os.path.abspath(val)])
      elif key == '--help':
        self.ShowUsage()
        sys.exit(0)

    if len(args):
      print('This tool takes no tool-specific arguments.')
      return 2
    self.SetOptions(opts)
    self.VerboseOut('Output directory: %s (absolute path: %s)\n' %
                    (self.output_directory,
                     os.path.abspath(self.output_directory)))

    if allowlist_filenames:
      self.allowlist_names = set()
      for allowlist_filename in allowlist_filenames:
        self.VerboseOut('Using allowlist: %s\n' % allowlist_filename)
        allowlist_contents = util.ReadFile(allowlist_filename, 'utf-8')
        self.allowlist_names.update(allowlist_contents.strip().split('\n'))

    if js_minifier:
      minifier.SetJsMinifier(js_minifier)

    if css_minifier:
      minifier.SetCssMinifier(css_minifier)

    self.write_only_new = write_only_new

    self.res = grd_reader.Parse(opts.input,
                                debug=opts.extra_verbose,
                                first_ids_file=first_ids_file,
                                predetermined_ids_file=predetermined_ids_file,
                                defines=self.defines,
                                target_platform=target_platform)

    # Set an output context so that conditionals can use defines during the
    # gathering stage; we use a dummy language here since we are not outputting
    # a specific language.
    self.res.SetOutputLanguage('en')
    self.res.SetAllowlistSupportEnabled(allowlist_support)
    self.res.RunGatherers()

    # Replace ... with the single-character version. http://crbug.com/621772
    if replace_ellipsis:
      for node in self.res:
        if isinstance(node, message.MessageNode):
          node.SetReplaceEllipsis(True)

    self.Process()

    if assert_output_files:
      if not self.CheckAssertedOutputFiles(assert_output_files):
        return 2

    if depfile and depdir:
      self.GenerateDepfile(depfile, depdir, first_ids_file, depend_on_stamp)

    return 0

  def __init__(self, defines=None):
    # Default file-creation function is codecs.open().  Only done to allow
    # overriding by unit test.
    self.fo_create = codecs.open

    # key/value pairs of C-preprocessor like defines that are used for
    # conditional output of resources
    self.defines = defines or {}

    # self.res is a fully-populated resource tree if Run()
    # has been called, otherwise None.
    self.res = None

    # The set of names that are allowlisted to actually be included in the
    # output.
    self.allowlist_names = None

    # Whether to compare outputs to their old contents before writing.
    self.write_only_new = False

  @staticmethod
  def AddAllowlistTags(start_node, allowlist_names):
    # Walk the tree of nodes added attributes for the nodes that shouldn't
    # be written into the target files (skip markers).
    for node in start_node:
      # Same trick data_pack.py uses to see what nodes actually result in
      # real items.
      if (isinstance(node, include.IncludeNode) or
          isinstance(node, message.MessageNode) or
          isinstance(node, structure.StructureNode)):
        text_ids = node.GetTextualIds()
        # Mark the item to be skipped if it wasn't in the allowlist.
        if text_ids and text_ids[0] not in allowlist_names:
          node.SetAllowlistMarkedAsSkip(True)

  @staticmethod
  def ProcessNode(node, output_node, outfile):
    '''Processes a node in-order, calling its formatter before and after
    recursing to its children.

    Args:
      node: grit.node.base.Node subclass
      output_node: grit.node.io.OutputNode
      outfile: open filehandle
    '''
    base_dir = util.dirname(output_node.GetOutputFilename())

    formatter = GetFormatter(output_node.GetType())
    formatted = formatter(node, output_node.GetLanguage(), output_dir=base_dir)
    # NB: Formatters may be generators or return lists.  The writelines API
    # accepts iterables as a shortcut to calling write directly.  That means
    # you can pass strings (iteration yields characters), but not bytes (as
    # iteration yields integers).  Python 2 worked due to its quirks with
    # bytes/string implementation, but Python 3 fails.  It's also a bit more
    # inefficient to call write once per character/byte.  Handle all of this
    # ourselves by calling write directly on strings/bytes before falling back
    # to writelines.
    if isinstance(formatted, (six.string_types, six.binary_type)):
      outfile.write(formatted)
    else:
      outfile.writelines(formatted)
    if output_node.GetType() == 'data_package':
      with open(output_node.GetOutputFilename() + '.info', 'w') as infofile:
        if node.info:
          # We terminate with a newline so that when these files are
          # concatenated later we consistently terminate with a newline so
          # consumers can account for terminating newlines.
          infofile.writelines(['\n'.join(node.info), '\n'])

  @staticmethod
  def _EncodingForOutputType(output_type):
    # Microsoft's RC compiler can only deal with single-byte or double-byte
    # files (no UTF-8), so we make all RC files UTF-16 to support all
    # character sets.
    if output_type in ('rc_header', 'resource_file_map_source',
                       'resource_map_header', 'resource_map_source'):
      return 'cp1252'
    if output_type in ('android', 'c_format',  'plist', 'plist_strings', 'doc',
                       'json', 'android_policy', 'chrome_messages_json',
                       'chrome_messages_json_gzip', 'policy_templates'):
      return 'utf_8'
    # TODO(gfeher) modify here to set utf-8 encoding for admx/adml
    return 'utf_16'

  def Process(self):
    for output in self.res.GetOutputFiles():
      output.output_filename = os.path.abspath(os.path.join(
        self.output_directory, output.GetOutputFilename()))

    # If there are allowlisted names, tag the tree once up front, this way
    # while looping through the actual output, it is just an attribute check.
    if self.allowlist_names:
      self.AddAllowlistTags(self.res, self.allowlist_names)

    for output in self.res.GetOutputFiles():
      self.VerboseOut('Creating %s...' % output.GetOutputFilename())

      # Set the context, for conditional inclusion of resources
      self.res.SetOutputLanguage(output.GetLanguage())
      self.res.SetOutputContext(output.GetContext())
      self.res.SetFallbackToDefaultLayout(output.GetFallbackToDefaultLayout())
      self.res.SetDefines(self.defines)

      # Assign IDs only once to ensure that all outputs use the same IDs.
      if self.res.GetIdMap() is None:
        self.res.InitializeIds()

      # Make the output directory if it doesn't exist.
      self.MakeDirectoriesTo(output.GetOutputFilename())

      # Write the results to a temporary file and only overwrite the original
      # if the file changed.  This avoids unnecessary rebuilds.
      out_filename = output.GetOutputFilename()
      tmp_filename = out_filename + '.tmp'
      tmpfile = self.fo_create(tmp_filename, 'wb')

      output_type = output.GetType()
      if output_type != 'data_package':
        encoding = self._EncodingForOutputType(output_type)
        tmpfile = util.WrapOutputStream(tmpfile, encoding)

      # Iterate in-order through entire resource tree, calling formatters on
      # the entry into a node and on exit out of it.
      with tmpfile:
        self.ProcessNode(self.res, output, tmpfile)

      if output_type == 'chrome_messages_json_gzip':
        gz_filename = tmp_filename + '.gz'
        with open(tmp_filename, 'rb') as tmpfile, open(gz_filename, 'wb') as f:
          with gzip.GzipFile(filename='', mode='wb', fileobj=f, mtime=0) as fgz:
            shutil.copyfileobj(tmpfile, fgz)
        os.remove(tmp_filename)
        tmp_filename = gz_filename

      # Now copy from the temp file back to the real output, but on Windows,
      # only if the real output doesn't exist or the contents of the file
      # changed.  This prevents identical headers from being written and .cc
      # files from recompiling (which is painful on Windows).
      if not os.path.exists(out_filename):
        os.rename(tmp_filename, out_filename)
      else:
        # CHROMIUM SPECIFIC CHANGE.
        # This clashes with gyp + vstudio, which expect the output timestamp
        # to change on a rebuild, even if nothing has changed, so only do
        # it when opted in.
        if not self.write_only_new:
          write_file = True
        else:
          files_match = filecmp.cmp(out_filename, tmp_filename)
          write_file = not files_match
        if write_file:
          shutil.copy2(tmp_filename, out_filename)
        os.remove(tmp_filename)

      self.VerboseOut(' done.\n')

    # Print warnings if there are any duplicate shortcuts.
    warnings = shortcuts.GenerateDuplicateShortcutsWarnings(
        self.res.UberClique(), self.res.GetTcProject())
    if warnings:
      print('\n'.join(warnings))

    # Print out any fallback warnings, and missing translation errors, and
    # exit with an error code if there are missing translations in a non-pseudo
    # and non-official build.
    warnings = (self.res.UberClique().MissingTranslationsReport().
        encode('ascii', 'replace'))
    if warnings:
      self.VerboseOut(warnings)
    if self.res.UberClique().HasMissingTranslations():
      print(self.res.UberClique().missing_translations_)
      sys.exit(-1)


  def CheckAssertedOutputFiles(self, assert_output_files):
    '''Checks that the asserted output files are specified in the given list.

    Returns true if the asserted files are present. If they are not, returns
    False and prints the failure.
    '''
    # Compare the absolute path names, sorted.
    asserted = sorted([os.path.abspath(i) for i in assert_output_files])
    actual = sorted([
        os.path.abspath(os.path.join(self.output_directory,
                                     i.GetOutputFilename()))
        for i in self.res.GetOutputFiles()])

    if asserted != actual:
      missing = list(set(asserted) - set(actual))
      extra = list(set(actual) - set(asserted))
      error = '''Asserted file list does not match.

Expected output files:
%s
Actual output files:
%s
Missing output files:
%s
Extra output files:
%s
'''
      print(error % ('\n'.join(asserted), '\n'.join(actual), '\n'.join(missing),
                     ' \n'.join(extra)))
      return False
    return True


  def GenerateDepfile(self, depfile, depdir, first_ids_file, depend_on_stamp):
    '''Generate a depfile that contains the imlicit dependencies of the input
    grd. The depfile will be in the same format as a makefile, and will contain
    references to files relative to |depdir|. It will be put in |depfile|.

    For example, supposing we have three files in a directory src/

    src/
      blah.grd    <- depends on input{1,2}.xtb
      input1.xtb
      input2.xtb

    and we run

      grit -i blah.grd -o ../out/gen \
           --depdir ../out \
           --depfile ../out/gen/blah.rd.d

    from the directory src/ we will generate a depfile ../out/gen/blah.grd.d
    that has the contents

      gen/blah.h: ../src/input1.xtb ../src/input2.xtb

    Where "gen/blah.h" is the first output (Ninja expects the .d file to list
    the first output in cases where there is more than one). If the flag
    --depend-on-stamp is specified, "gen/blah.rd.d.stamp" will be used that is
    'touched' whenever a new depfile is generated.

    Note that all paths in the depfile are relative to ../out, the depdir.
    '''
    depfile = os.path.abspath(depfile)
    depdir = os.path.abspath(depdir)
    infiles = self.res.GetInputFiles()

    # We want to trigger a rebuild if the first ids change.
    if first_ids_file is not None:
      infiles.append(first_ids_file)

    if (depend_on_stamp):
      output_file = depfile + ".stamp"
      # Touch the stamp file before generating the depfile.
      with open(output_file, 'a'):
        os.utime(output_file, None)
    else:
      # Get the first output file relative to the depdir.
      outputs = self.res.GetOutputFiles()
      output_file = os.path.join(self.output_directory,
                                 outputs[0].GetOutputFilename())

    output_file = os.path.relpath(output_file, depdir)
    # The path prefix to prepend to dependencies in the depfile.
    prefix = os.path.relpath(os.getcwd(), depdir)
    deps_text = ' '.join([os.path.join(prefix, i) for i in infiles])

    depfile_contents = output_file + ': ' + deps_text
    self.MakeDirectoriesTo(depfile)
    outfile = self.fo_create(depfile, 'w', encoding='utf-8')
    outfile.write(depfile_contents)

  @staticmethod
  def MakeDirectoriesTo(file):
    '''Creates directories necessary to contain |file|.'''
    dir = os.path.split(file)[0]
    if not os.path.exists(dir):
      os.makedirs(dir)