1#!/usr/bin/env python 2# Copyright (c) 2013 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import argparse 7import multiprocessing 8import os 9import posixpath 10import sys 11import urllib2 12 13import buildbot_common 14import build_version 15import generate_make 16import parse_dsc 17 18from build_paths import SDK_SRC_DIR, OUT_DIR, SDK_RESOURCE_DIR 19from build_paths import GSTORE 20from generate_index import LandingPage 21 22sys.path.append(os.path.join(SDK_SRC_DIR, 'tools')) 23import getos 24 25 26MAKE = 'nacl_sdk/make_3.99.90-26-gf80222c/make.exe' 27LIB_DICT = { 28 'linux': [], 29 'mac': [], 30 'win': ['x86_32'] 31} 32VALID_TOOLCHAINS = [ 33 'clang-newlib', 34 'glibc', 35 'pnacl', 36 'win', 37 'linux', 38 'mac', 39] 40 41# Global verbosity setting. 42# If set to True (normally via a command line arg) then build_projects will 43# add V=1 to all calls to 'make' 44verbose = False 45 46 47def Trace(msg): 48 if verbose: 49 sys.stderr.write(str(msg) + '\n') 50 51 52def CopyFilesFromTo(filelist, srcdir, dstdir): 53 for filename in filelist: 54 srcpath = os.path.join(srcdir, filename) 55 dstpath = os.path.join(dstdir, filename) 56 buildbot_common.CopyFile(srcpath, dstpath) 57 58 59def UpdateHelpers(pepperdir, clobber=False): 60 tools_dir = os.path.join(pepperdir, 'tools') 61 if not os.path.exists(tools_dir): 62 buildbot_common.ErrorExit('SDK tools dir is missing: %s' % tools_dir) 63 64 exampledir = os.path.join(pepperdir, 'examples') 65 if clobber: 66 buildbot_common.RemoveDir(exampledir) 67 buildbot_common.MakeDir(exampledir) 68 69 # Copy files for individual build and landing page 70 files = ['favicon.ico', 'httpd.cmd', 'index.css', 'index.js', 71 'button_close.png', 'button_close_hover.png'] 72 CopyFilesFromTo(files, SDK_RESOURCE_DIR, exampledir) 73 74 # Copy tools scripts and make includes 75 buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'), 76 tools_dir) 77 buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.mk'), 78 tools_dir) 79 80 # Copy tools/lib scripts 81 tools_lib_dir = os.path.join(pepperdir, 'tools', 'lib') 82 buildbot_common.MakeDir(tools_lib_dir) 83 buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', 'lib', '*.py'), 84 tools_lib_dir) 85 86 # On Windows add a prebuilt make 87 if getos.GetPlatform() == 'win': 88 buildbot_common.BuildStep('Add MAKE') 89 make_url = posixpath.join(GSTORE, MAKE) 90 make_exe = os.path.join(tools_dir, 'make.exe') 91 with open(make_exe, 'wb') as f: 92 f.write(urllib2.urlopen(make_url).read()) 93 94 95def ValidateToolchains(toolchains): 96 invalid_toolchains = set(toolchains) - set(VALID_TOOLCHAINS) 97 if invalid_toolchains: 98 buildbot_common.ErrorExit('Invalid toolchain(s): %s' % ( 99 ', '.join(invalid_toolchains))) 100 101 102def GetDeps(projects): 103 out = {} 104 105 # Build list of all project names 106 localtargets = [proj['NAME'] for proj in projects] 107 108 # For each project 109 for proj in projects: 110 deplist = [] 111 # generate a list of dependencies 112 for targ in proj.get('TARGETS', []): 113 deplist.extend(targ.get('DEPS', []) + targ.get('LIBS', [])) 114 115 # and add dependencies to targets built in this subtree 116 localdeps = [dep for dep in deplist if dep in localtargets] 117 if localdeps: 118 out[proj['NAME']] = localdeps 119 120 return out 121 122 123def UpdateProjects(pepperdir, project_tree, toolchains, 124 clobber=False, configs=None, first_toolchain=False): 125 if configs is None: 126 configs = ['Debug', 'Release'] 127 if not os.path.exists(os.path.join(pepperdir, 'tools')): 128 buildbot_common.ErrorExit('Examples depend on missing tools.') 129 if not os.path.exists(os.path.join(pepperdir, 'toolchain')): 130 buildbot_common.ErrorExit('Examples depend on missing toolchains.') 131 132 ValidateToolchains(toolchains) 133 134 # Create the library output directories 135 libdir = os.path.join(pepperdir, 'lib') 136 platform = getos.GetPlatform() 137 for config in configs: 138 for arch in LIB_DICT[platform]: 139 dirpath = os.path.join(libdir, '%s_%s_host' % (platform, arch), config) 140 if clobber: 141 buildbot_common.RemoveDir(dirpath) 142 buildbot_common.MakeDir(dirpath) 143 144 landing_page = None 145 for branch, projects in project_tree.iteritems(): 146 dirpath = os.path.join(pepperdir, branch) 147 if clobber: 148 buildbot_common.RemoveDir(dirpath) 149 buildbot_common.MakeDir(dirpath) 150 targets = [desc['NAME'] for desc in projects if 'TARGETS' in desc] 151 deps = GetDeps(projects) 152 153 # Generate master make for this branch of projects 154 generate_make.GenerateMasterMakefile(pepperdir, 155 os.path.join(pepperdir, branch), 156 targets, deps) 157 158 if branch.startswith('examples') and not landing_page: 159 landing_page = LandingPage() 160 161 # Generate individual projects 162 for desc in projects: 163 srcroot = os.path.dirname(desc['FILEPATH']) 164 generate_make.ProcessProject(pepperdir, srcroot, pepperdir, desc, 165 toolchains, configs=configs, 166 first_toolchain=first_toolchain) 167 168 if branch.startswith('examples'): 169 landing_page.AddDesc(desc) 170 171 if landing_page: 172 # Generate the landing page text file. 173 index_html = os.path.join(pepperdir, 'examples', 'index.html') 174 index_template = os.path.join(SDK_RESOURCE_DIR, 'index.html.template') 175 with open(index_html, 'w') as fh: 176 out = landing_page.GeneratePage(index_template) 177 fh.write(out) 178 179 # Generate top Make for examples 180 targets = ['api', 'demo', 'getting_started', 'tutorial'] 181 targets = [x for x in targets if 'examples/'+x in project_tree] 182 branch_name = 'examples' 183 generate_make.GenerateMasterMakefile(pepperdir, 184 os.path.join(pepperdir, branch_name), 185 targets, {}) 186 187 188def BuildProjectsBranch(pepperdir, branch, deps, clean, config, args=None): 189 make_dir = os.path.join(pepperdir, branch) 190 print "\nMake: " + make_dir 191 192 if getos.GetPlatform() == 'win': 193 # We need to modify the environment to build host on Windows. 194 make = os.path.join(make_dir, 'make.bat') 195 else: 196 make = 'make' 197 198 env = None 199 if os.environ.get('USE_GOMA') == '1': 200 env = dict(os.environ) 201 env['NACL_COMPILER_PREFIX'] = 'gomacc' 202 # Add -m32 to the CFLAGS when building using i686-nacl-gcc 203 # otherwise goma won't recognise it as different to the x86_64 204 # build. 205 env['X86_32_CFLAGS'] = '-m32' 206 env['X86_32_CXXFLAGS'] = '-m32' 207 jobs = '50' 208 else: 209 jobs = str(multiprocessing.cpu_count()) 210 211 make_cmd = [make, '-j', jobs] 212 213 make_cmd.append('CONFIG='+config) 214 if not deps: 215 make_cmd.append('IGNORE_DEPS=1') 216 217 if verbose: 218 make_cmd.append('V=1') 219 220 if args: 221 make_cmd += args 222 else: 223 make_cmd.append('TOOLCHAIN=all') 224 225 buildbot_common.Run(make_cmd, cwd=make_dir, env=env) 226 if clean: 227 # Clean to remove temporary files but keep the built 228 buildbot_common.Run(make_cmd + ['clean'], cwd=make_dir, env=env) 229 230 231def BuildProjects(pepperdir, project_tree, deps=True, 232 clean=False, config='Debug'): 233 # Make sure we build libraries (which live in 'src') before 234 # any of the examples. 235 build_first = [p for p in project_tree if p != 'src'] 236 build_second = [p for p in project_tree if p == 'src'] 237 238 for branch in build_first + build_second: 239 BuildProjectsBranch(pepperdir, branch, deps, clean, config) 240 241 242def main(args): 243 parser = argparse.ArgumentParser(description=__doc__) 244 parser.add_argument('-c', '--clobber', 245 help='Clobber project directories before copying new files', 246 action='store_true', default=False) 247 parser.add_argument('-b', '--build', 248 help='Build the projects. Otherwise the projects are only copied.', 249 action='store_true') 250 parser.add_argument('--config', 251 help='Choose configuration to build (Debug or Release). Builds both ' 252 'by default') 253 parser.add_argument('-x', '--experimental', 254 help='Build experimental projects', action='store_true') 255 parser.add_argument('-t', '--toolchain', 256 help='Build using toolchain. Can be passed more than once.', 257 action='append', default=[]) 258 parser.add_argument('-d', '--dest', 259 help='Select which build destinations (project types) are valid.', 260 action='append') 261 parser.add_argument('projects', nargs='*', 262 help='Select which projects to build.') 263 parser.add_argument('-v', '--verbose', action='store_true') 264 265 # To setup bash completion for this command first install optcomplete 266 # and then add this line to your .bashrc: 267 # complete -F _optcomplete build_projects.py 268 try: 269 import optcomplete 270 optcomplete.autocomplete(parser) 271 except ImportError: 272 pass 273 274 options = parser.parse_args(args) 275 276 global verbose 277 if options.verbose: 278 verbose = True 279 280 buildbot_common.verbose = verbose 281 282 if 'NACL_SDK_ROOT' in os.environ: 283 # We don't want the currently configured NACL_SDK_ROOT to have any effect 284 # on the build. 285 del os.environ['NACL_SDK_ROOT'] 286 287 pepper_ver = str(int(build_version.ChromeMajorVersion())) 288 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver) 289 290 if not options.toolchain: 291 # Order matters here: the default toolchain for an example's Makefile will 292 # be the first toolchain in this list that is available in the example. 293 # e.g. If an example supports clang-newlib and glibc, then the default will 294 # be clang-newlib. 295 options.toolchain = ['pnacl', 'clang-newlib', 'glibc', 'host'] 296 297 if 'host' in options.toolchain: 298 options.toolchain.remove('host') 299 options.toolchain.append(getos.GetPlatform()) 300 Trace('Adding platform: ' + getos.GetPlatform()) 301 302 ValidateToolchains(options.toolchain) 303 304 filters = {} 305 if options.toolchain: 306 filters['TOOLS'] = options.toolchain 307 Trace('Filter by toolchain: ' + str(options.toolchain)) 308 if not options.experimental: 309 filters['EXPERIMENTAL'] = False 310 if options.dest: 311 filters['DEST'] = options.dest 312 Trace('Filter by type: ' + str(options.dest)) 313 if options.projects: 314 filters['NAME'] = options.projects 315 Trace('Filter by name: ' + str(options.projects)) 316 317 try: 318 project_tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters) 319 except parse_dsc.ValidationError as e: 320 buildbot_common.ErrorExit(str(e)) 321 322 if verbose: 323 parse_dsc.PrintProjectTree(project_tree) 324 325 UpdateHelpers(pepperdir, clobber=options.clobber) 326 UpdateProjects(pepperdir, project_tree, options.toolchain, 327 clobber=options.clobber) 328 329 if options.build: 330 if options.config: 331 configs = [options.config] 332 else: 333 configs = ['Debug', 'Release'] 334 for config in configs: 335 BuildProjects(pepperdir, project_tree, config=config, deps=False) 336 337 return 0 338 339 340if __name__ == '__main__': 341 script_name = os.path.basename(sys.argv[0]) 342 try: 343 sys.exit(main(sys.argv[1:])) 344 except parse_dsc.ValidationError as e: 345 buildbot_common.ErrorExit('%s: %s' % (script_name, e)) 346 except KeyboardInterrupt: 347 buildbot_common.ErrorExit('%s: interrupted' % script_name) 348