1#!/usr/bin/python
2# Copyright (c) 2012 The Native Client Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6from __future__ import print_function
7
8import optparse
9import os.path
10import shutil
11import subprocess
12import stat
13import sys
14import time
15import traceback
16
17
18ARCH_MAP = {
19    '32': {
20        'gn_arch': 'x86',
21        'scons_platform': 'x86-32',
22        },
23    '64': {
24        'gn_arch': 'x64',
25        'scons_platform': 'x86-64',
26        },
27    'arm': {
28        'gn_arch': 'arm',
29        'scons_platform': 'arm',
30        },
31    'mips32': {
32        'gn_arch': 'mipsel',
33        'scons_platform': 'mips32',
34        },
35    }
36
37
38def GNArch(arch):
39  return ARCH_MAP[arch]['gn_arch']
40
41
42def RunningOnBuildbot():
43  return os.environ.get('BUILDBOT_SLAVE_TYPE') is not None
44
45
46def GetHostPlatform():
47  sys_platform = sys.platform.lower()
48  if sys_platform.startswith('linux'):
49    return 'linux'
50  elif sys_platform in ('win', 'win32', 'windows', 'cygwin'):
51    return 'win'
52  elif sys_platform in ('darwin', 'mac'):
53    return 'mac'
54  else:
55    raise Exception('Can not determine the platform!')
56
57
58def SetDefaultContextAttributes(context):
59  """
60  Set default values for the attributes needed by the SCons function, so that
61  SCons can be run without needing ParseStandardCommandLine
62  """
63  platform = GetHostPlatform()
64  context['platform'] = platform
65  context['mode'] = 'opt'
66  context['default_scons_mode'] = ['opt-host', 'nacl']
67  context['default_scons_platform'] = ('x86-64' if platform == 'win'
68                                       else 'x86-32')
69  context['android'] = False
70  context['clang'] = True
71  context['asan'] = False
72  context['pnacl'] = False
73  context['use_glibc'] = False
74  context['use_breakpad_tools'] = False
75  context['max_jobs'] = 8
76  context['scons_args'] = []
77
78
79# Windows-specific environment manipulation
80def SetupWindowsEnvironment(context):
81  # Poke around looking for MSVC.  We should do something more principled in
82  # the future.
83
84  # NOTE!  This affects only SCons.  The GN build figures out MSVC on its own
85  # and will wind up with a different version than this one (the same version
86  # being used for Chromium builds).
87
88  # The name of Program Files can differ, depending on the bittage of Windows.
89  program_files = r'c:\Program Files (x86)'
90  if not os.path.exists(program_files):
91    program_files = r'c:\Program Files'
92    if not os.path.exists(program_files):
93      raise Exception('Cannot find the Program Files directory!')
94
95  # The location of MSVC can differ depending on the version.
96  msvc_locs = [
97      ('Microsoft Visual Studio 12.0', 'VS120COMNTOOLS', '2013'),
98      ('Microsoft Visual Studio 10.0', 'VS100COMNTOOLS', '2010'),
99      ('Microsoft Visual Studio 9.0', 'VS90COMNTOOLS', '2008'),
100      ('Microsoft Visual Studio 8.0', 'VS80COMNTOOLS', '2005'),
101  ]
102
103  for dirname, comntools_var, gyp_msvs_version in msvc_locs:
104    msvc = os.path.join(program_files, dirname)
105    if os.path.exists(msvc):
106      break
107  else:
108    # The break statement did not execute.
109    raise Exception('Cannot find MSVC!')
110
111  # Put MSVC in the path.
112  vc = os.path.join(msvc, 'VC')
113  comntools = os.path.join(msvc, 'Common7', 'Tools')
114  perf = os.path.join(msvc, 'Team Tools', 'Performance Tools')
115  context.SetEnv('PATH', os.pathsep.join([
116      context.GetEnv('PATH'),
117      vc,
118      comntools,
119      perf]))
120
121  # SCons needs this variable to find vsvars.bat.
122  # The end slash is needed because the batch files expect it.
123  context.SetEnv(comntools_var, comntools + '\\')
124
125  # This environment variable will SCons to print debug info while it searches
126  # for MSVC.
127  context.SetEnv('SCONS_MSCOMMON_DEBUG', '-')
128
129  # Needed for finding devenv.
130  context['msvc'] = msvc
131
132
133def SetupLinuxEnvironment(context):
134  if context['arch'] == 'mips32':
135    # Override the trusted compiler for mips, clang does not support mips
136    context['clang'] = False
137    # Ensure the trusted mips toolchain is installed.
138    cmd = ['build/package_version/package_version.py', '--packages',
139           'linux_x86/mips_trusted', 'sync', '-x']
140    Command(context, cmd)
141
142
143def ParseStandardCommandLine(context):
144  """
145  The standard buildbot scripts require 3 arguments to run.  The first
146  argument (dbg/opt) controls if the build is a debug or a release build.  The
147  second argument (32/64) controls the machine architecture being targeted.
148  The third argument (newlib/glibc) controls which c library we're using for
149  the nexes.  Different buildbots may have different sets of arguments.
150  """
151
152  parser = optparse.OptionParser()
153  parser.add_option('-n', '--dry-run', dest='dry_run', default=False,
154                    action='store_true', help='Do not execute any commands.')
155  parser.add_option('--inside-toolchain', dest='inside_toolchain',
156                    default=bool(os.environ.get('INSIDE_TOOLCHAIN')),
157                    action='store_true', help='Inside toolchain build.')
158  parser.add_option('--android', dest='android', default=False,
159                    action='store_true', help='Build for Android.')
160  parser.add_option('--no-clang', dest='clang', default=True,
161                    action='store_false',
162                    help="Don't build trusted code with clang.")
163  parser.add_option('--coverage', dest='coverage', default=False,
164                    action='store_true',
165                    help='Build and test for code coverage.')
166  parser.add_option('--validator', dest='validator', default=False,
167                    action='store_true',
168                    help='Only run validator regression test')
169  parser.add_option('--asan', dest='asan', default=False,
170                    action='store_true', help='Build trusted code with ASan.')
171  parser.add_option('--scons-args', dest='scons_args', default =[],
172                    action='append', help='Extra scons arguments.')
173  parser.add_option('--step-suffix', metavar='SUFFIX', default='',
174                    help='Append SUFFIX to buildbot step names.')
175  parser.add_option('--no-gn', dest='no_gn', default=False,
176                    action='store_true', help='Do not run the GN build')
177  parser.add_option('--no-scons', dest='no_scons', default=False,
178                    action='store_true', help='Do not run the SCons build '
179                    '(SCons will still be used to run tests)')
180  parser.add_option('--no-goma', dest='no_goma', default=False,
181                    action='store_true', help='Do not run with goma')
182  parser.add_option('--use-breakpad-tools', dest='use_breakpad_tools',
183                    default=False, action='store_true',
184                    help='Use breakpad tools for testing')
185  parser.add_option('--skip-build', dest='skip_build', default=False,
186                    action='store_true',
187                    help='Skip building steps in buildbot_pnacl')
188  parser.add_option('--skip-run', dest='skip_run', default=False,
189                    action='store_true',
190                    help='Skip test-running steps in buildbot_pnacl')
191
192  options, args = parser.parse_args()
193
194  if len(args) != 3:
195    parser.error('Expected 3 arguments: mode arch toolchain')
196
197  # script + 3 args == 4
198  mode, arch, toolchain = args
199  if mode not in ('dbg', 'opt', 'coverage'):
200    parser.error('Invalid mode %r' % mode)
201
202  if arch not in ARCH_MAP:
203    parser.error('Invalid arch %r' % arch)
204
205  if toolchain not in ('newlib', 'glibc', 'pnacl', 'nacl_clang'):
206    parser.error('Invalid toolchain %r' % toolchain)
207
208  # TODO(ncbray) allow a command-line override
209  platform = GetHostPlatform()
210
211  context['platform'] = platform
212  context['mode'] = mode
213  context['arch'] = arch
214  context['android'] = options.android
215  # ASan is Clang, so set the flag to simplify other checks.
216  context['clang'] = options.clang or options.asan
217  context['validator'] = options.validator
218  context['asan'] = options.asan
219  # TODO(ncbray) turn derived values into methods.
220  context['gn_is_debug'] = {
221      'opt': 'false',
222      'dbg': 'true',
223      'coverage': 'true'}[mode]
224  context['gn_arch'] = GNArch(arch)
225  context['default_scons_platform'] = ARCH_MAP[arch]['scons_platform']
226  context['default_scons_mode'] = ['nacl']
227  # Only Linux can build trusted code on ARM.
228  # TODO(mcgrathr): clean this up somehow
229  # SCons on Windows doesn't know how to find the native SDK/compiler anymore
230  # See https://bugs.chromium.org/p/nativeclient/issues/detail?id=4408
231  # TODO(dschuff): list which tests no longer run on Windows because of this
232  if (arch != 'arm' or platform == 'linux') and platform != 'win':
233    context['default_scons_mode'] += [mode + '-host']
234  context['use_glibc'] = toolchain == 'glibc'
235  context['pnacl'] = toolchain == 'pnacl'
236  context['nacl_clang'] = toolchain == 'nacl_clang'
237  context['max_jobs'] = 8
238  context['dry_run'] = options.dry_run
239  context['inside_toolchain'] = options.inside_toolchain
240  context['step_suffix'] = options.step_suffix
241  context['no_gn'] = options.no_gn
242  context['no_scons'] = options.no_scons
243  context['no_goma'] = options.no_goma
244  context['coverage'] = options.coverage
245  context['use_breakpad_tools'] = options.use_breakpad_tools
246  context['scons_args'] = options.scons_args
247  context['skip_build'] = options.skip_build
248  context['skip_run'] = options.skip_run
249
250  for key, value in sorted(context.config.items()):
251    print('%s=%s' % (key, value))
252
253
254def EnsureDirectoryExists(path):
255  """
256  Create a directory if it does not already exist.
257  Does not mask failures, but there really shouldn't be any.
258  """
259  if not os.path.exists(path):
260    os.makedirs(path)
261
262
263def TryToCleanContents(path, file_name_filter=lambda fn: True):
264  """
265  Remove the contents of a directory without touching the directory itself.
266  Ignores all failures.
267  """
268  if os.path.exists(path):
269    for fn in os.listdir(path):
270      TryToCleanPath(os.path.join(path, fn), file_name_filter)
271
272
273def TryToCleanPath(path, file_name_filter=lambda fn: True):
274  """
275  Removes a file or directory.
276  Ignores all failures.
277  """
278  if os.path.exists(path):
279    if file_name_filter(path):
280      print('Trying to remove %s' % path)
281      try:
282        RemovePath(path)
283      except Exception:
284        print('Failed to remove %s' % path)
285    else:
286      print('Skipping %s' % path)
287
288
289def Retry(op, *args):
290  # Windows seems to be prone to having commands that delete files or
291  # directories fail.  We currently do not have a complete understanding why,
292  # and as a workaround we simply retry the command a few times.
293  # It appears that file locks are hanging around longer than they should.  This
294  # may be a secondary effect of processes hanging around longer than they
295  # should.  This may be because when we kill a browser sel_ldr does not exit
296  # immediately, etc.
297  # Virus checkers can also accidently prevent files from being deleted, but
298  # that shouldn't be a problem on the bots.
299  if GetHostPlatform() == 'win':
300    count = 0
301    while True:
302      try:
303        op(*args)
304        break
305      except Exception:
306        print("FAILED: %s %s" % (op.__name__, repr(args)))
307        count += 1
308        if count < 5:
309          print("RETRY: %s %s" % (op.__name__, repr(args)))
310          time.sleep(pow(2, count))
311        else:
312          # Don't mask the exception.
313          raise
314  else:
315    op(*args)
316
317
318def PermissionsFixOnError(func, path, exc_info):
319  if not os.access(path, os.W_OK):
320    os.chmod(path, stat.S_IWUSR)
321    func(path)
322  else:
323    raise
324
325
326def _RemoveDirectory(path):
327  print('Removing %s' % path)
328  if os.path.exists(path):
329    shutil.rmtree(path, onerror=PermissionsFixOnError)
330    print('    Succeeded.')
331  else:
332    print('    Path does not exist, nothing to do.')
333
334
335def RemoveDirectory(path):
336  """
337  Remove a directory if it exists.
338  Does not mask failures, although it does retry a few times on Windows.
339  """
340  Retry(_RemoveDirectory, path)
341
342
343def RemovePath(path):
344  """Remove a path, file or directory."""
345  if os.path.isdir(path):
346    RemoveDirectory(path)
347  else:
348    if os.path.isfile(path) and not os.access(path, os.W_OK):
349      os.chmod(path, stat.S_IWUSR)
350    os.remove(path)
351
352
353# This is a sanity check so Command can print out better error information.
354def FileCanBeFound(name, paths):
355  # CWD
356  if os.path.exists(name):
357    return True
358  # Paths with directories are not resolved using the PATH variable.
359  if os.path.dirname(name):
360    return False
361  # In path
362  for path in paths.split(os.pathsep):
363    full = os.path.join(path, name)
364    if os.path.exists(full):
365      return True
366  return False
367
368
369def RemoveGypBuildDirectories():
370  # Remove all directories on all platforms.  Overkill, but it allows for
371  # straight-line code.
372  # Windows
373  RemoveDirectory('build/Debug')
374  RemoveDirectory('build/Release')
375  RemoveDirectory('build/Debug-Win32')
376  RemoveDirectory('build/Release-Win32')
377  RemoveDirectory('build/Debug-x64')
378  RemoveDirectory('build/Release-x64')
379  RemoveDirectory('../out_32')
380  RemoveDirectory('../out_32_clang')
381  RemoveDirectory('../out_64')
382  RemoveDirectory('../out_64_clang')
383
384  # Linux and Mac
385  RemoveDirectory('../xcodebuild')
386  RemoveDirectory('../out')
387  RemoveDirectory('src/third_party/nacl_sdk/arm-newlib')
388
389
390def RemoveSconsBuildDirectories():
391  RemoveDirectory('scons-out')
392  RemoveDirectory('breakpad-out')
393
394
395# Execute a command using Python's subprocess module.
396def Command(context, cmd, cwd=None):
397  print('Running command: %s' % ' '.join(cmd))
398
399  # Python's subprocess has a quirk.  A subprocess can execute with an
400  # arbitrary, user-defined environment.  The first argument of the command,
401  # however, is located using the PATH variable of the Python script that is
402  # launching the subprocess.  Modifying the PATH in the environment passed to
403  # the subprocess does not affect Python's search for the first argument of
404  # the command (the executable file.)  This is a little counter intuitive,
405  # so we're forcing the search to use the same PATH variable as is seen by
406  # the subprocess.
407  env = context.MakeCommandEnv()
408  script_path = os.environ['PATH']
409  os.environ['PATH'] = env['PATH']
410
411  try:
412    if FileCanBeFound(cmd[0], env['PATH']) or context['dry_run']:
413      # Make sure that print statements before the subprocess call have been
414      # flushed, otherwise the output of the subprocess call may appear before
415      # the print statements.
416      sys.stdout.flush()
417      if context['dry_run']:
418        retcode = 0
419      else:
420        retcode = subprocess.call(cmd, cwd=cwd, env=env)
421    else:
422      # Provide a nicer failure message.
423      # If subprocess cannot find the executable, it will throw a cryptic
424      # exception.
425      print('Executable %r cannot be found.' % cmd[0])
426      retcode = 1
427  finally:
428    os.environ['PATH'] = script_path
429
430  print('Command return code: %d' % retcode)
431  if retcode != 0:
432    raise StepFailed()
433  return retcode
434
435
436# A specialized version of CommandStep.
437def SCons(context, mode=None, platform=None, parallel=False, browser_test=False,
438          args=(), cwd=None):
439  python = sys.executable
440  if mode is None: mode = context['default_scons_mode']
441  if platform is None: platform = context['default_scons_platform']
442  if parallel:
443    jobs = context['max_jobs']
444  else:
445    jobs = 1
446  cmd = []
447  if browser_test and context.Linux():
448    # Although we could use the "browser_headless=1" Scons option, it runs
449    # xvfb-run once per Chromium invocation.  This is good for isolating
450    # the tests, but xvfb-run has a stupid fixed-period sleep, which would
451    # slow down the tests unnecessarily.
452    cmd.extend(['xvfb-run', '--auto-servernum'])
453  cmd.extend([
454      python, 'scons.py',
455      '--verbose',
456      '-k',
457      '-j%d' % jobs,
458      '--mode='+','.join(mode),
459      'platform='+platform,
460      ])
461  cmd.extend(context['scons_args'])
462  if not context['clang']: cmd.append('--no-clang')
463  if context['asan']: cmd.append('--asan')
464  if context['use_glibc']: cmd.append('--nacl_glibc')
465  if context['pnacl']: cmd.append('bitcode=1')
466  if context['nacl_clang']: cmd.append('nacl_clang=1')
467  if context['use_breakpad_tools']:
468    cmd.append('breakpad_tools_dir=breakpad-out')
469  if context['android']:
470    cmd.append('android=1')
471  # Append used-specified arguments.
472  cmd.extend(args)
473  Command(context, cmd, cwd)
474
475
476class StepFailed(Exception):
477  """
478  Thrown when the step has failed.
479  """
480
481
482class StopBuild(Exception):
483  """
484  Thrown when the entire build should stop.  This does not indicate a failure,
485  in of itself.
486  """
487
488
489class Step(object):
490  """
491  This class is used in conjunction with a Python "with" statement to ensure
492  that the preamble and postamble of each build step gets printed and failures
493  get logged.  This class also ensures that exceptions thrown inside a "with"
494  statement don't take down the entire build.
495  """
496
497  def __init__(self, name, status, halt_on_fail=True):
498    self.status = status
499
500    if 'step_suffix' in status.context:
501      suffix = status.context['step_suffix']
502    else:
503      suffix = ''
504    self.name = name + suffix
505    self.halt_on_fail = halt_on_fail
506    self.step_failed = False
507
508  # Called on entry to a 'with' block.
509  def __enter__(self):
510    sys.stdout.flush()
511    print()
512    print('@@@BUILD_STEP %s@@@' % self.name)
513    self.status.ReportBegin(self.name)
514
515  # The method is called on exit from a 'with' block - even for non-local
516  # control flow, i.e. exceptions, breaks, continues, returns, etc.
517  # If an exception is thrown inside a block wrapped with a 'with' statement,
518  # the __exit__ handler can suppress the exception by returning True.  This is
519  # used to isolate each step in the build - if an exception occurs in a given
520  # step, the step is treated as a failure.  This allows the postamble for each
521  # step to be printed and also allows the build to continue of the failure of
522  # a given step doesn't halt the build.
523  def __exit__(self, type, exception, trace):
524    sys.stdout.flush()
525    if exception is None:
526      # If exception is None, no exception occurred.
527      step_failed = False
528    elif isinstance(exception, StepFailed):
529      step_failed = True
530      print()
531      print('Halting build step because of failure.')
532      print()
533    else:
534      step_failed = True
535      print()
536      print('The build step threw an exception...')
537      print()
538      traceback.print_exception(type, exception, trace, file=sys.stdout)
539      print()
540
541    if step_failed:
542      self.status.ReportFail(self.name)
543      print('@@@STEP_FAILURE@@@')
544      if self.halt_on_fail:
545        print()
546        print('Entire build halted because %s failed.' % self.name)
547        sys.stdout.flush()
548        raise StopBuild()
549    else:
550      self.status.ReportPass(self.name)
551
552    sys.stdout.flush()
553    # Suppress any exception that occurred.
554    return True
555
556
557# Adds an arbitrary link inside the build stage on the waterfall.
558def StepLink(text, link):
559  print('@@@STEP_LINK@%s@%s@@@' % (text, link))
560
561
562# Adds arbitrary text inside the build stage on the waterfall.
563def StepText(text):
564  print('@@@STEP_TEXT@%s@@@' % (text))
565
566
567class BuildStatus(object):
568  """
569  Keeps track of the overall status of the build.
570  """
571
572  def __init__(self, context):
573    self.context = context
574    self.ever_failed = False
575    self.steps = []
576
577  def ReportBegin(self, name):
578    pass
579
580  def ReportPass(self, name):
581    self.steps.append((name, 'passed'))
582
583  def ReportFail(self, name):
584    self.steps.append((name, 'failed'))
585    self.ever_failed = True
586
587  # Handy info when this script is run outside of the buildbot.
588  def DisplayBuildStatus(self):
589    print()
590    for step, status in self.steps:
591      print('%-40s[%s]' % (step, status))
592    print()
593
594    if self.ever_failed:
595      print('Build failed.')
596    else:
597      print('Build succeeded.')
598
599  def ReturnValue(self):
600    return int(self.ever_failed)
601
602
603class BuildContext(object):
604  """
605  Encapsulates the information needed for running a build command.  This
606  includes environment variables and default arguments for SCons invocations.
607  """
608
609  # Only allow these attributes on objects of this type.
610  __slots__ = ['status', 'global_env', 'config']
611
612  def __init__(self):
613    # The contents of global_env override os.environ for any commands run via
614    # self.Command(...)
615    self.global_env = {}
616    # PATH is a special case. See: Command.
617    self.global_env['PATH'] = os.environ.get('PATH', '')
618
619    self.config = {}
620    self['dry_run'] = False
621
622  # Emulate dictionary subscripting.
623  def __getitem__(self, key):
624    return self.config[key]
625
626  # Emulate dictionary subscripting.
627  def __setitem__(self, key, value):
628    self.config[key] = value
629
630  # Emulate dictionary membership test
631  def __contains__(self, key):
632    return key in self.config
633
634  def Windows(self):
635    return self.config['platform'] == 'win'
636
637  def Linux(self):
638    return self.config['platform'] == 'linux'
639
640  def Mac(self):
641    return self.config['platform'] == 'mac'
642
643  def GetEnv(self, name, default=None):
644    return self.global_env.get(name, default)
645
646  def SetEnv(self, name, value):
647    self.global_env[name] = str(value)
648
649  def MakeCommandEnv(self):
650    # The external environment is not sanitized.
651    e = dict(os.environ)
652    # Arbitrary variables can be overridden.
653    e.update(self.global_env)
654    return e
655
656
657def RunBuild(script, status):
658  try:
659    script(status, status.context)
660  except StopBuild:
661    pass
662
663  # Emit a summary step for three reasons:
664  # - The annotator will attribute non-zero exit status to the last build step.
665  #   This can misattribute failures to the last build step.
666  # - runtest.py wraps the builds to scrape perf data. It emits an annotator
667  #   tag on exit which misattributes perf results to the last build step.
668  # - Provide a label step in which to show summary result.
669  #   Otherwise these go back to the preamble.
670  with Step('summary', status):
671    if status.ever_failed:
672      print('There were failed stages.')
673    else:
674      print('Success.')
675    # Display a summary of the build.
676    status.DisplayBuildStatus()
677
678  sys.exit(status.ReturnValue())
679