1#!/usr/bin/python
2# Copyright (c) 2012 The Native Client Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6from __future__ import print_function
7
8import json
9import os
10import subprocess
11import sys
12import tempfile
13
14sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
15import pynacl.platform
16import pynacl.file_tools
17
18python = sys.executable
19bash = '/bin/bash'
20echo = 'echo'
21
22
23BOT_ASSIGNMENT = {
24    ######################################################################
25    # Buildbots.
26    ######################################################################
27    'xp-newlib-opt':
28        python + ' buildbot\\buildbot_standard.py opt 32 newlib',
29    'xp-glibc-opt':
30        python + ' buildbot\\buildbot_standard.py opt 32 glibc',
31
32    'xp-bare-newlib-opt':
33        python + ' buildbot\\buildbot_standard.py opt 32 newlib',
34    'xp-bare-glibc-opt':
35        python + ' buildbot\\buildbot_standard.py opt 32 glibc',
36
37    'linux-64-validator-opt':
38        python + ' buildbot/buildbot_standard.py opt 64 glibc --validator',
39
40    # ASan.
41    'linux_64-newlib-dbg-asan':
42        python + ' buildbot/buildbot_standard.py opt 64 newlib --asan',
43    'mac-newlib-dbg-asan':
44        python + ' buildbot/buildbot_standard.py opt 64 newlib --asan',
45
46    # Sanitizer Pnacl toolchain buildbot.
47    'asan':
48        python +
49        ' buildbot/buildbot_pnacl_toolchain.py --buildbot --tests-arch x86-64 '
50        ' --sanitize address --skip-tests',
51
52    # PNaCl.
53    'odroid_32-newlib-arm_hw-pnacl-dbg':
54        bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-hw-dbg',
55    'odroid_32-newlib-arm_hw-pnacl-opt':
56        bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-hw-opt',
57    'linux_64-newlib-arm_qemu-pnacl-dbg':
58        bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-dbg',
59    'linux_64-newlib-arm_qemu-pnacl-opt':
60        bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-opt',
61    'linux_64-newlib-x86_32-pnacl':
62        python + ' buildbot/buildbot_pnacl.py opt 32 pnacl',
63    'linux_64-newlib-x86_64-pnacl':
64        python + ' buildbot/buildbot_pnacl.py opt 64 pnacl',
65    'mac-newlib-opt-pnacl':
66        python + ' buildbot/buildbot_pnacl.py opt 64 pnacl',
67    'win7-64-newlib-opt-pnacl':
68        python + ' buildbot/buildbot_pnacl.py opt 64 pnacl --no-clang',
69    # TODO: Use buildbot_pnacl.py instead of buildbot_pnacl.sh once
70    # the gyp_build is moved to buildbot_pnacl.py.
71    'linux_64-newlib-mips-pnacl':
72        bash + ' buildbot/buildbot_pnacl.sh mode-trybot-qemu mips32',
73    # PNaCl Spec
74    'linux_64-newlib-arm_qemu-pnacl-buildonly-spec':
75        bash + ' buildbot/buildbot_spec2k.sh pnacl-arm-buildonly',
76    'odroid_32-newlib-arm_hw-pnacl-spec':
77        bash + ' buildbot/buildbot_spec2k.sh pnacl-arm-hw',
78    'linux_64-newlib-x86_32-pnacl-spec':
79        bash + ' buildbot/buildbot_spec2k.sh pnacl-x8632',
80    'linux_64-newlib-x86_64-pnacl-spec':
81        bash + ' buildbot/buildbot_spec2k.sh pnacl-x8664',
82    # NaCl Spec
83    'linux_64-newlib-x86_32-spec':
84        bash + ' buildbot/buildbot_spec2k.sh nacl-x8632',
85    'linux_64-newlib-x86_64-spec':
86        bash + ' buildbot/buildbot_spec2k.sh nacl-x8664',
87
88    # Valgrind bots.
89    'linux-64-newlib-dbg-valgrind':
90        echo + ' "Valgrind bots are disabled: see '
91            'https://code.google.com/p/nativeclient/issues/detail?id=3158"',
92    'linux-64-glibc-dbg-valgrind':
93        echo + ' "Valgrind bots are disabled: see '
94            'https://code.google.com/p/nativeclient/issues/detail?id=3158"',
95
96    ######################################################################
97    # Trybots.
98    ######################################################################
99    'nacl-precise64_validator_opt':
100        python + ' buildbot/buildbot_standard.py opt 64 glibc --validator',
101    # Android trybots.
102    'nacl-precise64-newlib-dbg-android':
103        echo + ' "Android bots are disabled and going away"',
104    'nacl-precise64-newlib-opt-android':
105        echo + ' "Android bots are disabled and going away"',
106    # ASan.
107    'nacl-precise_64-newlib-dbg-asan':
108        python + ' buildbot/buildbot_standard.py opt 64 newlib --asan',
109    'nacl-mac-newlib-dbg-asan':
110        python + ' buildbot/buildbot_standard.py opt 64 newlib --asan',
111    # Pnacl main trybots
112    'nacl-precise_64-newlib-arm_qemu-pnacl':
113        bash + ' buildbot/buildbot_pnacl.sh mode-trybot-qemu arm',
114    'nacl-precise_64-newlib-x86_32-pnacl':
115         python + ' buildbot/buildbot_pnacl.py opt 32 pnacl',
116    'nacl-precise_64-newlib-x86_64-pnacl':
117         python + ' buildbot/buildbot_pnacl.py opt 64 pnacl',
118    'nacl-precise_64-newlib-mips-pnacl':
119        bash + ' buildbot/buildbot_pnacl.sh mode-trybot-qemu mips32',
120    'nacl-arm_opt':
121        bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-try',
122    'nacl-arm_hw_opt':
123        bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-hw-try',
124    'nacl-mac_newlib_opt_pnacl':
125        python + ' buildbot/buildbot_pnacl.py opt 64 pnacl',
126    'nacl-win7_64_newlib_opt_pnacl':
127        python + ' buildbot/buildbot_pnacl.py opt 64 pnacl --no-clang',
128    # Pnacl spec2k trybots
129    'nacl-precise_64-newlib-x86_32-pnacl-spec':
130        bash + ' buildbot/buildbot_spec2k.sh pnacl-trybot-x8632',
131    'nacl-precise_64-newlib-x86_64-pnacl-spec':
132        bash + ' buildbot/buildbot_spec2k.sh pnacl-trybot-x8664',
133    'nacl-arm_perf':
134        bash + ' buildbot/buildbot_spec2k.sh pnacl-trybot-arm-buildonly',
135    'nacl-arm_hw_perf':
136        bash + ' buildbot/buildbot_spec2k.sh pnacl-trybot-arm-hw',
137    # Toolchain glibc.
138    'precise64-glibc': bash + ' buildbot/buildbot_linux-glibc-makefile.sh',
139    'mac-glibc': bash + ' buildbot/buildbot_mac-glibc-makefile.sh',
140    'win7-glibc': 'buildbot\\buildbot_windows-glibc-makefile.bat',
141    # Toolchain (glibc) ARM.
142    'win7-toolchain_arm':
143        python +
144        ' buildbot/buildbot_toolchain_build.py'
145        ' --buildbot'
146        ' toolchain_build',
147    'mac-toolchain_arm':
148        python +
149        ' buildbot/buildbot_toolchain_build.py'
150        ' --buildbot'
151        ' toolchain_build',
152    'linux64-toolchain_arm':
153        python +
154        ' buildbot/buildbot_toolchain_build.py'
155        ' --buildbot'
156        ' --test_toolchain nacl_arm_glibc_raw'
157        ' toolchain_build',
158
159    # Pnacl toolchain builders.
160    'linux-pnacl-x86_32':
161        python +
162        ' buildbot/buildbot_pnacl_toolchain.py --buildbot --tests-arch x86-32',
163    'linux-pnacl-x86_64':
164        python +
165        ' buildbot/buildbot_pnacl_toolchain.py --buildbot --tests-arch x86-64',
166    'mac-pnacl-x86_32':
167        python +
168        ' buildbot/buildbot_pnacl_toolchain.py --buildbot',
169    'win-pnacl-x86_32':
170        python +
171        ' buildbot/buildbot_pnacl_toolchain.py --buildbot',
172
173    # Pnacl toolchain testers
174    'linux-pnacl-x86_64-tests-x86_64':
175        bash + ' buildbot/buildbot_pnacl_toolchain_tests.sh tc-test-bot x86-64',
176    'linux-pnacl-x86_64-tests-x86_32':
177        bash + ' buildbot/buildbot_pnacl_toolchain_tests.sh tc-test-bot x86-32',
178    'linux-pnacl-x86_64-tests-arm':
179        bash + ' buildbot/buildbot_pnacl_toolchain_tests.sh tc-test-bot arm',
180
181    # MIPS toolchain buildbot.
182    'linux-pnacl-x86_32-tests-mips':
183        bash + ' buildbot/buildbot_pnacl.sh mode-trybot-qemu mips32',
184
185    # Toolchain trybots.
186    'nacl-toolchain-precise64-newlib-arm': # TODO(bradnelson): rename
187        python +
188        ' buildbot/buildbot_toolchain_build.py'
189        ' --trybot'
190        ' --test_toolchain nacl_arm_glibc_raw'
191        ' toolchain_build',
192    'nacl-toolchain-mac-newlib-arm': # TODO(bradnelson): rename
193        python +
194        ' buildbot/buildbot_toolchain_build.py'
195        ' --trybot'
196        ' toolchain_build',
197    'nacl-toolchain-win7-newlib-arm': # TODO(bradnelson): rename
198        python +
199        ' buildbot/buildbot_toolchain_build.py'
200        ' --trybot'
201        ' toolchain_build',
202    'nacl-toolchain-precise64-glibc':
203        bash + ' buildbot/buildbot_linux-glibc-makefile.sh',
204    'nacl-toolchain-mac-glibc':
205        bash + ' buildbot/buildbot_mac-glibc-makefile.sh',
206    'nacl-toolchain-win7-glibc':
207        'buildbot\\buildbot_windows-glibc-makefile.bat',
208
209    # Pnacl toolchain trybots.
210    'nacl-toolchain-linux-pnacl-x86_32':
211        python +
212        ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-32',
213    'nacl-toolchain-linux-pnacl-x86_64':
214        python +
215        ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-64',
216    'nacl-toolchain-mac-pnacl-x86_32':
217        python + ' buildbot/buildbot_pnacl_toolchain.py --trybot',
218    'nacl-toolchain-win7-pnacl-x86_64':
219        python + ' buildbot/buildbot_pnacl_toolchain.py --trybot',
220
221    # Sanitizer Pnacl toolchain trybots.
222    'nacl-toolchain-asan':
223        python +
224        ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-64 '
225        ' --sanitize address --skip-tests',
226    # TODO(kschimpf): Bot is currently broken: --sanitize memory not understood.
227    'nacl-toolchain-msan':
228        python +
229        ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-64 '
230        ' --sanitize memory --skip-tests',
231    # TODO(kschimpf): Bot is currently broken: --sanitize thread not understood.
232    'nacl-toolchain-tsan':
233        python +
234        ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-64 '
235        ' --sanitize thread --skip-tests',
236    # TODO(kschimpf): Bot is currently broken: --sanitize undefined not
237    # understood.
238    'nacl-toolchain-ubsan':
239        python +
240        ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-64 '
241        ' --sanitize undefined --skip-tests',
242
243}
244
245special_for_arm = [
246    'win7_64',
247    'win7-64',
248    'linux-64',
249    'linux64'
250]
251for platform in [
252    'win7', 'win8', 'win',
253    'mac', 'linux'] + special_for_arm:
254  if platform in special_for_arm:
255    arch_variants = ['arm']
256  else:
257    arch_variants = ['', '32', '64', 'arm']
258  for arch in arch_variants:
259    arch_flags = ''
260    real_arch = arch
261    arch_part = '-' + arch
262    if platform == 'win7' and arch == '32':
263      arch_flags += ' --no-goma'
264    if arch == '':
265      arch_part = ''
266      real_arch = '64'
267    # Test with Breakpad tools only on basic Linux builds.
268    if sys.platform.startswith('linux'):
269      arch_flags += ' --use-breakpad-tools'
270    if 'win' in platform:
271      arch_flags += ' --no-scons --no-clang'
272    for mode in ['dbg', 'opt']:
273      for libc in ['newlib', 'glibc']:
274        # Buildbots.
275        for bare in ['', '-bare']:
276          for test in ['', '-test']:
277            name = platform + arch_part + bare + '-' + libc + '-' + mode + test
278            assert name not in BOT_ASSIGNMENT, name
279            BOT_ASSIGNMENT[name] = (
280                python + ' buildbot/buildbot_standard.py ' +
281                mode + ' ' + real_arch + ' ' + libc + arch_flags)
282        # Trybots
283        for arch_sep in ['', '-', '_']:
284          p = platform.replace('linux', 'precise')
285          name = 'nacl-' + p + arch_sep + arch + '_' + libc + '_' + mode
286          assert name not in BOT_ASSIGNMENT, name
287          BOT_ASSIGNMENT[name] = (
288              python + ' buildbot/buildbot_standard.py ' +
289              mode + ' ' + real_arch + ' ' + libc + arch_flags)
290
291
292def EscapeJson(data):
293  return '"' + json.dumps(data).replace('"', r'\"') + '"'
294
295
296def HasNoPerfResults(builder):
297  if 'pnacl-buildonly-spec' in builder:
298    return True
299  if 'android' in builder:
300    return True
301  return builder in [
302      'asan',
303      'mac-toolchain_arm',
304      'win-pnacl-x86_32',
305      'linux-pnacl-x86_32-tests-mips',
306  ]
307
308
309def Main():
310  builder = os.environ.get('BUILDBOT_BUILDERNAME')
311  build_number = os.environ.get('BUILDBOT_BUILDNUMBER')
312  build_revision = os.environ.get('BUILDBOT_GOT_REVISION',
313                                  os.environ.get('BUILDBOT_REVISION'))
314  slave_type = os.environ.get('BUILDBOT_SLAVE_TYPE')
315  bot_type = os.environ.get('BOT_TYPE')
316  cmd = BOT_ASSIGNMENT.get(builder)
317  if not cmd:
318    sys.stderr.write('ERROR - unset/invalid builder name\n')
319    sys.exit(1)
320
321  env = os.environ.copy()
322
323  # Don't write out .pyc files because in cases in which files move around or
324  # the PYTHONPATH / sys.path change, old .pyc files can be mistakenly used.
325  # This avoids the need for admin changes on the bots in this case.
326  env['PYTHONDONTWRITEBYTECODE'] = '1'
327
328  if bot_type != 'arm_hw_bot':
329    env['GSUTIL'] = pynacl.file_tools.Which('gsutil.py',
330                                            require_executable=False)
331
332  # When running from cygwin, we sometimes want to use a native python.
333  # The native python will use the depot_tools version by invoking python.bat.
334  if pynacl.platform.IsWindows():
335    env['NATIVE_PYTHON'] = 'python.bat'
336  else:
337    env['NATIVE_PYTHON'] = 'python'
338
339  if sys.platform == 'win32':
340    # If the temp directory is not on the same drive as the working directory,
341    # there can be random failures when cleaning up temp directories, so use
342    # a directory on the current drive. Use __file__ here instead of os.getcwd()
343    # because toolchain_main picks its working directories relative to __file__
344    filedrive, _ = os.path.splitdrive(__file__)
345    tempdrive, _ = os.path.splitdrive(env['TEMP'])
346    if tempdrive != filedrive:
347      env['TEMP'] = filedrive + '\\temp'
348      if not os.path.exists(env['TEMP']):
349        os.mkdir(env['TEMP'])
350
351  # Ensure a temp directory exists.
352  if 'TEMP' not in env:
353    env['TEMP'] = tempfile.gettempdir()
354
355  # Isolate build's temp directory to a particular location so we can clobber
356  # the whole thing predictably and so we have a record of who's leaking
357  # temporary files.
358  nacl_tmp = os.path.join(env['TEMP'], 'nacl_tmp')
359  if not os.path.exists(nacl_tmp):
360    os.mkdir(nacl_tmp)
361  env['TEMP'] = os.path.join(nacl_tmp, builder)
362  if not os.path.exists(env['TEMP']):
363    os.mkdir(env['TEMP'])
364
365  # Set all temp directory variants to the same thing.
366  env['TMPDIR'] = env['TEMP']
367  env['TMP'] = env['TEMP']
368  print('TEMP=%s' % env['TEMP'])
369  sys.stdout.flush()
370
371  print("%s runs: %s\n" % (builder, cmd))
372  sys.stdout.flush()
373  retcode = subprocess.call(cmd, env=env, shell=True)
374  sys.exit(retcode)
375
376
377if __name__ == '__main__':
378  Main()
379