1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import argparse
7import multiprocessing
8import os
9import posixpath
10import sys
11import urllib2
12
13import buildbot_common
14import build_version
15import generate_make
16import parse_dsc
17
18from build_paths import SDK_SRC_DIR, OUT_DIR, SDK_RESOURCE_DIR
19from build_paths import GSTORE
20from generate_index import LandingPage
21
22sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
23import getos
24
25
26MAKE = 'nacl_sdk/make_3.99.90-26-gf80222c/make.exe'
27LIB_DICT = {
28  'linux': [],
29  'mac': [],
30  'win': ['x86_32']
31}
32VALID_TOOLCHAINS = [
33  'clang-newlib',
34  'glibc',
35  'pnacl',
36  'win',
37  'linux',
38  'mac',
39]
40
41# Global verbosity setting.
42# If set to True (normally via a command line arg) then build_projects will
43# add V=1 to all calls to 'make'
44verbose = False
45
46
47def Trace(msg):
48  if verbose:
49    sys.stderr.write(str(msg) + '\n')
50
51
52def CopyFilesFromTo(filelist, srcdir, dstdir):
53  for filename in filelist:
54    srcpath = os.path.join(srcdir, filename)
55    dstpath = os.path.join(dstdir, filename)
56    buildbot_common.CopyFile(srcpath, dstpath)
57
58
59def UpdateHelpers(pepperdir, clobber=False):
60  tools_dir = os.path.join(pepperdir, 'tools')
61  if not os.path.exists(tools_dir):
62    buildbot_common.ErrorExit('SDK tools dir is missing: %s' % tools_dir)
63
64  exampledir = os.path.join(pepperdir, 'examples')
65  if clobber:
66    buildbot_common.RemoveDir(exampledir)
67  buildbot_common.MakeDir(exampledir)
68
69  # Copy files for individual build and landing page
70  files = ['favicon.ico', 'httpd.cmd', 'index.css', 'index.js',
71      'button_close.png', 'button_close_hover.png']
72  CopyFilesFromTo(files, SDK_RESOURCE_DIR, exampledir)
73
74  # Copy tools scripts and make includes
75  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
76      tools_dir)
77  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.mk'),
78      tools_dir)
79
80  # Copy tools/lib scripts
81  tools_lib_dir = os.path.join(pepperdir, 'tools', 'lib')
82  buildbot_common.MakeDir(tools_lib_dir)
83  buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', 'lib', '*.py'),
84      tools_lib_dir)
85
86  # On Windows add a prebuilt make
87  if getos.GetPlatform() == 'win':
88    buildbot_common.BuildStep('Add MAKE')
89    make_url = posixpath.join(GSTORE, MAKE)
90    make_exe = os.path.join(tools_dir, 'make.exe')
91    with open(make_exe, 'wb') as f:
92      f.write(urllib2.urlopen(make_url).read())
93
94
95def ValidateToolchains(toolchains):
96  invalid_toolchains = set(toolchains) - set(VALID_TOOLCHAINS)
97  if invalid_toolchains:
98    buildbot_common.ErrorExit('Invalid toolchain(s): %s' % (
99        ', '.join(invalid_toolchains)))
100
101
102def GetDeps(projects):
103  out = {}
104
105  # Build list of all project names
106  localtargets = [proj['NAME'] for proj in projects]
107
108  # For each project
109  for proj in projects:
110    deplist = []
111    # generate a list of dependencies
112    for targ in proj.get('TARGETS', []):
113      deplist.extend(targ.get('DEPS', []) + targ.get('LIBS', []))
114
115    # and add dependencies to targets built in this subtree
116    localdeps = [dep for dep in deplist if dep in localtargets]
117    if localdeps:
118      out[proj['NAME']] = localdeps
119
120  return out
121
122
123def UpdateProjects(pepperdir, project_tree, toolchains,
124                   clobber=False, configs=None, first_toolchain=False):
125  if configs is None:
126    configs = ['Debug', 'Release']
127  if not os.path.exists(os.path.join(pepperdir, 'tools')):
128    buildbot_common.ErrorExit('Examples depend on missing tools.')
129  if not os.path.exists(os.path.join(pepperdir, 'toolchain')):
130    buildbot_common.ErrorExit('Examples depend on missing toolchains.')
131
132  ValidateToolchains(toolchains)
133
134  # Create the library output directories
135  libdir = os.path.join(pepperdir, 'lib')
136  platform = getos.GetPlatform()
137  for config in configs:
138    for arch in LIB_DICT[platform]:
139      dirpath = os.path.join(libdir, '%s_%s_host' % (platform, arch), config)
140      if clobber:
141        buildbot_common.RemoveDir(dirpath)
142      buildbot_common.MakeDir(dirpath)
143
144  landing_page = None
145  for branch, projects in project_tree.iteritems():
146    dirpath = os.path.join(pepperdir, branch)
147    if clobber:
148      buildbot_common.RemoveDir(dirpath)
149    buildbot_common.MakeDir(dirpath)
150    targets = [desc['NAME'] for desc in projects if 'TARGETS' in desc]
151    deps = GetDeps(projects)
152
153    # Generate master make for this branch of projects
154    generate_make.GenerateMasterMakefile(pepperdir,
155                                         os.path.join(pepperdir, branch),
156                                         targets, deps)
157
158    if branch.startswith('examples') and not landing_page:
159      landing_page = LandingPage()
160
161    # Generate individual projects
162    for desc in projects:
163      srcroot = os.path.dirname(desc['FILEPATH'])
164      generate_make.ProcessProject(pepperdir, srcroot, pepperdir, desc,
165                                   toolchains, configs=configs,
166                                   first_toolchain=first_toolchain)
167
168      if branch.startswith('examples'):
169        landing_page.AddDesc(desc)
170
171  if landing_page:
172    # Generate the landing page text file.
173    index_html = os.path.join(pepperdir, 'examples', 'index.html')
174    index_template = os.path.join(SDK_RESOURCE_DIR, 'index.html.template')
175    with open(index_html, 'w') as fh:
176      out = landing_page.GeneratePage(index_template)
177      fh.write(out)
178
179  # Generate top Make for examples
180  targets = ['api', 'demo', 'getting_started', 'tutorial']
181  targets = [x for x in targets if 'examples/'+x in project_tree]
182  branch_name = 'examples'
183  generate_make.GenerateMasterMakefile(pepperdir,
184                                       os.path.join(pepperdir, branch_name),
185                                       targets, {})
186
187
188def BuildProjectsBranch(pepperdir, branch, deps, clean, config, args=None):
189  make_dir = os.path.join(pepperdir, branch)
190  print "\nMake: " + make_dir
191
192  if getos.GetPlatform() == 'win':
193    # We need to modify the environment to build host on Windows.
194    make = os.path.join(make_dir, 'make.bat')
195  else:
196    make = 'make'
197
198  env = None
199  if os.environ.get('USE_GOMA') == '1':
200    env = dict(os.environ)
201    env['NACL_COMPILER_PREFIX'] = 'gomacc'
202    # Add -m32 to the CFLAGS when building using i686-nacl-gcc
203    # otherwise goma won't recognise it as different to the x86_64
204    # build.
205    env['X86_32_CFLAGS'] = '-m32'
206    env['X86_32_CXXFLAGS'] = '-m32'
207    jobs = '50'
208  else:
209    jobs = str(multiprocessing.cpu_count())
210
211  make_cmd = [make, '-j', jobs]
212
213  make_cmd.append('CONFIG='+config)
214  if not deps:
215    make_cmd.append('IGNORE_DEPS=1')
216
217  if verbose:
218    make_cmd.append('V=1')
219
220  if args:
221    make_cmd += args
222  else:
223    make_cmd.append('TOOLCHAIN=all')
224
225  buildbot_common.Run(make_cmd, cwd=make_dir, env=env)
226  if clean:
227    # Clean to remove temporary files but keep the built
228    buildbot_common.Run(make_cmd + ['clean'], cwd=make_dir, env=env)
229
230
231def BuildProjects(pepperdir, project_tree, deps=True,
232                  clean=False, config='Debug'):
233  # Make sure we build libraries (which live in 'src') before
234  # any of the examples.
235  build_first = [p for p in project_tree if p != 'src']
236  build_second = [p for p in project_tree if p == 'src']
237
238  for branch in build_first + build_second:
239    BuildProjectsBranch(pepperdir, branch, deps, clean, config)
240
241
242def main(args):
243  parser = argparse.ArgumentParser(description=__doc__)
244  parser.add_argument('-c', '--clobber',
245      help='Clobber project directories before copying new files',
246      action='store_true', default=False)
247  parser.add_argument('-b', '--build',
248      help='Build the projects. Otherwise the projects are only copied.',
249      action='store_true')
250  parser.add_argument('--config',
251      help='Choose configuration to build (Debug or Release).  Builds both '
252           'by default')
253  parser.add_argument('-x', '--experimental',
254      help='Build experimental projects', action='store_true')
255  parser.add_argument('-t', '--toolchain',
256      help='Build using toolchain. Can be passed more than once.',
257      action='append', default=[])
258  parser.add_argument('-d', '--dest',
259      help='Select which build destinations (project types) are valid.',
260      action='append')
261  parser.add_argument('projects', nargs='*',
262      help='Select which projects to build.')
263  parser.add_argument('-v', '--verbose', action='store_true')
264
265  # To setup bash completion for this command first install optcomplete
266  # and then add this line to your .bashrc:
267  #  complete -F _optcomplete build_projects.py
268  try:
269    import optcomplete
270    optcomplete.autocomplete(parser)
271  except ImportError:
272    pass
273
274  options = parser.parse_args(args)
275
276  global verbose
277  if options.verbose:
278    verbose = True
279
280  buildbot_common.verbose = verbose
281
282  if 'NACL_SDK_ROOT' in os.environ:
283    # We don't want the currently configured NACL_SDK_ROOT to have any effect
284    # on the build.
285    del os.environ['NACL_SDK_ROOT']
286
287  pepper_ver = str(int(build_version.ChromeMajorVersion()))
288  pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
289
290  if not options.toolchain:
291    # Order matters here: the default toolchain for an example's Makefile will
292    # be the first toolchain in this list that is available in the example.
293    # e.g. If an example supports clang-newlib and glibc, then the default will
294    # be clang-newlib.
295    options.toolchain = ['pnacl', 'clang-newlib', 'glibc', 'host']
296
297  if 'host' in options.toolchain:
298    options.toolchain.remove('host')
299    options.toolchain.append(getos.GetPlatform())
300    Trace('Adding platform: ' + getos.GetPlatform())
301
302  ValidateToolchains(options.toolchain)
303
304  filters = {}
305  if options.toolchain:
306    filters['TOOLS'] = options.toolchain
307    Trace('Filter by toolchain: ' + str(options.toolchain))
308  if not options.experimental:
309    filters['EXPERIMENTAL'] = False
310  if options.dest:
311    filters['DEST'] = options.dest
312    Trace('Filter by type: ' + str(options.dest))
313  if options.projects:
314    filters['NAME'] = options.projects
315    Trace('Filter by name: ' + str(options.projects))
316
317  try:
318    project_tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters)
319  except parse_dsc.ValidationError as e:
320    buildbot_common.ErrorExit(str(e))
321
322  if verbose:
323    parse_dsc.PrintProjectTree(project_tree)
324
325  UpdateHelpers(pepperdir, clobber=options.clobber)
326  UpdateProjects(pepperdir, project_tree, options.toolchain,
327                 clobber=options.clobber)
328
329  if options.build:
330    if options.config:
331      configs = [options.config]
332    else:
333      configs = ['Debug', 'Release']
334    for config in configs:
335      BuildProjects(pepperdir, project_tree, config=config, deps=False)
336
337  return 0
338
339
340if __name__ == '__main__':
341  script_name = os.path.basename(sys.argv[0])
342  try:
343    sys.exit(main(sys.argv[1:]))
344  except parse_dsc.ValidationError as e:
345    buildbot_common.ErrorExit('%s: %s' % (script_name, e))
346  except KeyboardInterrupt:
347    buildbot_common.ErrorExit('%s: interrupted' % script_name)
348