1#!/usr/bin/python 2# Copyright (c) 2012 The Native Client Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from __future__ import print_function 7 8import json 9import os 10import subprocess 11import sys 12import tempfile 13 14sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 15import pynacl.platform 16import pynacl.file_tools 17 18python = sys.executable 19bash = '/bin/bash' 20echo = 'echo' 21 22 23BOT_ASSIGNMENT = { 24 ###################################################################### 25 # Buildbots. 26 ###################################################################### 27 'xp-newlib-opt': 28 python + ' buildbot\\buildbot_standard.py opt 32 newlib', 29 'xp-glibc-opt': 30 python + ' buildbot\\buildbot_standard.py opt 32 glibc', 31 32 'xp-bare-newlib-opt': 33 python + ' buildbot\\buildbot_standard.py opt 32 newlib', 34 'xp-bare-glibc-opt': 35 python + ' buildbot\\buildbot_standard.py opt 32 glibc', 36 37 'linux-64-validator-opt': 38 python + ' buildbot/buildbot_standard.py opt 64 glibc --validator', 39 40 # ASan. 41 'linux_64-newlib-dbg-asan': 42 python + ' buildbot/buildbot_standard.py opt 64 newlib --asan', 43 'mac-newlib-dbg-asan': 44 python + ' buildbot/buildbot_standard.py opt 64 newlib --asan', 45 46 # Sanitizer Pnacl toolchain buildbot. 47 'asan': 48 python + 49 ' buildbot/buildbot_pnacl_toolchain.py --buildbot --tests-arch x86-64 ' 50 ' --sanitize address --skip-tests', 51 52 # PNaCl. 53 'odroid_32-newlib-arm_hw-pnacl-dbg': 54 bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-hw-dbg', 55 'odroid_32-newlib-arm_hw-pnacl-opt': 56 bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-hw-opt', 57 'linux_64-newlib-arm_qemu-pnacl-dbg': 58 bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-dbg', 59 'linux_64-newlib-arm_qemu-pnacl-opt': 60 bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-opt', 61 'linux_64-newlib-x86_32-pnacl': 62 python + ' buildbot/buildbot_pnacl.py opt 32 pnacl', 63 'linux_64-newlib-x86_64-pnacl': 64 python + ' buildbot/buildbot_pnacl.py opt 64 pnacl', 65 'mac-newlib-opt-pnacl': 66 python + ' buildbot/buildbot_pnacl.py opt 64 pnacl', 67 'win7-64-newlib-opt-pnacl': 68 python + ' buildbot/buildbot_pnacl.py opt 64 pnacl --no-clang', 69 # TODO: Use buildbot_pnacl.py instead of buildbot_pnacl.sh once 70 # the gyp_build is moved to buildbot_pnacl.py. 71 'linux_64-newlib-mips-pnacl': 72 bash + ' buildbot/buildbot_pnacl.sh mode-trybot-qemu mips32', 73 # PNaCl Spec 74 'linux_64-newlib-arm_qemu-pnacl-buildonly-spec': 75 bash + ' buildbot/buildbot_spec2k.sh pnacl-arm-buildonly', 76 'odroid_32-newlib-arm_hw-pnacl-spec': 77 bash + ' buildbot/buildbot_spec2k.sh pnacl-arm-hw', 78 'linux_64-newlib-x86_32-pnacl-spec': 79 bash + ' buildbot/buildbot_spec2k.sh pnacl-x8632', 80 'linux_64-newlib-x86_64-pnacl-spec': 81 bash + ' buildbot/buildbot_spec2k.sh pnacl-x8664', 82 # NaCl Spec 83 'linux_64-newlib-x86_32-spec': 84 bash + ' buildbot/buildbot_spec2k.sh nacl-x8632', 85 'linux_64-newlib-x86_64-spec': 86 bash + ' buildbot/buildbot_spec2k.sh nacl-x8664', 87 88 # Valgrind bots. 89 'linux-64-newlib-dbg-valgrind': 90 echo + ' "Valgrind bots are disabled: see ' 91 'https://code.google.com/p/nativeclient/issues/detail?id=3158"', 92 'linux-64-glibc-dbg-valgrind': 93 echo + ' "Valgrind bots are disabled: see ' 94 'https://code.google.com/p/nativeclient/issues/detail?id=3158"', 95 96 ###################################################################### 97 # Trybots. 98 ###################################################################### 99 'nacl-precise64_validator_opt': 100 python + ' buildbot/buildbot_standard.py opt 64 glibc --validator', 101 # Android trybots. 102 'nacl-precise64-newlib-dbg-android': 103 echo + ' "Android bots are disabled and going away"', 104 'nacl-precise64-newlib-opt-android': 105 echo + ' "Android bots are disabled and going away"', 106 # ASan. 107 'nacl-precise_64-newlib-dbg-asan': 108 python + ' buildbot/buildbot_standard.py opt 64 newlib --asan', 109 'nacl-mac-newlib-dbg-asan': 110 python + ' buildbot/buildbot_standard.py opt 64 newlib --asan', 111 # Pnacl main trybots 112 'nacl-precise_64-newlib-arm_qemu-pnacl': 113 bash + ' buildbot/buildbot_pnacl.sh mode-trybot-qemu arm', 114 'nacl-precise_64-newlib-x86_32-pnacl': 115 python + ' buildbot/buildbot_pnacl.py opt 32 pnacl', 116 'nacl-precise_64-newlib-x86_64-pnacl': 117 python + ' buildbot/buildbot_pnacl.py opt 64 pnacl', 118 'nacl-precise_64-newlib-mips-pnacl': 119 bash + ' buildbot/buildbot_pnacl.sh mode-trybot-qemu mips32', 120 'nacl-arm_opt': 121 bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-try', 122 'nacl-arm_hw_opt': 123 bash + ' buildbot/buildbot_pnacl.sh mode-buildbot-arm-hw-try', 124 'nacl-mac_newlib_opt_pnacl': 125 python + ' buildbot/buildbot_pnacl.py opt 64 pnacl', 126 'nacl-win7_64_newlib_opt_pnacl': 127 python + ' buildbot/buildbot_pnacl.py opt 64 pnacl --no-clang', 128 # Pnacl spec2k trybots 129 'nacl-precise_64-newlib-x86_32-pnacl-spec': 130 bash + ' buildbot/buildbot_spec2k.sh pnacl-trybot-x8632', 131 'nacl-precise_64-newlib-x86_64-pnacl-spec': 132 bash + ' buildbot/buildbot_spec2k.sh pnacl-trybot-x8664', 133 'nacl-arm_perf': 134 bash + ' buildbot/buildbot_spec2k.sh pnacl-trybot-arm-buildonly', 135 'nacl-arm_hw_perf': 136 bash + ' buildbot/buildbot_spec2k.sh pnacl-trybot-arm-hw', 137 # Toolchain glibc. 138 'precise64-glibc': bash + ' buildbot/buildbot_linux-glibc-makefile.sh', 139 'mac-glibc': bash + ' buildbot/buildbot_mac-glibc-makefile.sh', 140 'win7-glibc': 'buildbot\\buildbot_windows-glibc-makefile.bat', 141 # Toolchain (glibc) ARM. 142 'win7-toolchain_arm': 143 python + 144 ' buildbot/buildbot_toolchain_build.py' 145 ' --buildbot' 146 ' toolchain_build', 147 'mac-toolchain_arm': 148 python + 149 ' buildbot/buildbot_toolchain_build.py' 150 ' --buildbot' 151 ' toolchain_build', 152 'linux64-toolchain_arm': 153 python + 154 ' buildbot/buildbot_toolchain_build.py' 155 ' --buildbot' 156 ' --test_toolchain nacl_arm_glibc_raw' 157 ' toolchain_build', 158 159 # Pnacl toolchain builders. 160 'linux-pnacl-x86_32': 161 python + 162 ' buildbot/buildbot_pnacl_toolchain.py --buildbot --tests-arch x86-32', 163 'linux-pnacl-x86_64': 164 python + 165 ' buildbot/buildbot_pnacl_toolchain.py --buildbot --tests-arch x86-64', 166 'mac-pnacl-x86_32': 167 python + 168 ' buildbot/buildbot_pnacl_toolchain.py --buildbot', 169 'win-pnacl-x86_32': 170 python + 171 ' buildbot/buildbot_pnacl_toolchain.py --buildbot', 172 173 # Pnacl toolchain testers 174 'linux-pnacl-x86_64-tests-x86_64': 175 bash + ' buildbot/buildbot_pnacl_toolchain_tests.sh tc-test-bot x86-64', 176 'linux-pnacl-x86_64-tests-x86_32': 177 bash + ' buildbot/buildbot_pnacl_toolchain_tests.sh tc-test-bot x86-32', 178 'linux-pnacl-x86_64-tests-arm': 179 bash + ' buildbot/buildbot_pnacl_toolchain_tests.sh tc-test-bot arm', 180 181 # MIPS toolchain buildbot. 182 'linux-pnacl-x86_32-tests-mips': 183 bash + ' buildbot/buildbot_pnacl.sh mode-trybot-qemu mips32', 184 185 # Toolchain trybots. 186 'nacl-toolchain-precise64-newlib-arm': # TODO(bradnelson): rename 187 python + 188 ' buildbot/buildbot_toolchain_build.py' 189 ' --trybot' 190 ' --test_toolchain nacl_arm_glibc_raw' 191 ' toolchain_build', 192 'nacl-toolchain-mac-newlib-arm': # TODO(bradnelson): rename 193 python + 194 ' buildbot/buildbot_toolchain_build.py' 195 ' --trybot' 196 ' toolchain_build', 197 'nacl-toolchain-win7-newlib-arm': # TODO(bradnelson): rename 198 python + 199 ' buildbot/buildbot_toolchain_build.py' 200 ' --trybot' 201 ' toolchain_build', 202 'nacl-toolchain-precise64-glibc': 203 bash + ' buildbot/buildbot_linux-glibc-makefile.sh', 204 'nacl-toolchain-mac-glibc': 205 bash + ' buildbot/buildbot_mac-glibc-makefile.sh', 206 'nacl-toolchain-win7-glibc': 207 'buildbot\\buildbot_windows-glibc-makefile.bat', 208 209 # Pnacl toolchain trybots. 210 'nacl-toolchain-linux-pnacl-x86_32': 211 python + 212 ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-32', 213 'nacl-toolchain-linux-pnacl-x86_64': 214 python + 215 ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-64', 216 'nacl-toolchain-mac-pnacl-x86_32': 217 python + ' buildbot/buildbot_pnacl_toolchain.py --trybot', 218 'nacl-toolchain-win7-pnacl-x86_64': 219 python + ' buildbot/buildbot_pnacl_toolchain.py --trybot', 220 221 # Sanitizer Pnacl toolchain trybots. 222 'nacl-toolchain-asan': 223 python + 224 ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-64 ' 225 ' --sanitize address --skip-tests', 226 # TODO(kschimpf): Bot is currently broken: --sanitize memory not understood. 227 'nacl-toolchain-msan': 228 python + 229 ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-64 ' 230 ' --sanitize memory --skip-tests', 231 # TODO(kschimpf): Bot is currently broken: --sanitize thread not understood. 232 'nacl-toolchain-tsan': 233 python + 234 ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-64 ' 235 ' --sanitize thread --skip-tests', 236 # TODO(kschimpf): Bot is currently broken: --sanitize undefined not 237 # understood. 238 'nacl-toolchain-ubsan': 239 python + 240 ' buildbot/buildbot_pnacl_toolchain.py --trybot --tests-arch x86-64 ' 241 ' --sanitize undefined --skip-tests', 242 243} 244 245special_for_arm = [ 246 'win7_64', 247 'win7-64', 248 'linux-64', 249 'linux64' 250] 251for platform in [ 252 'win7', 'win8', 'win', 253 'mac', 'linux'] + special_for_arm: 254 if platform in special_for_arm: 255 arch_variants = ['arm'] 256 else: 257 arch_variants = ['', '32', '64', 'arm'] 258 for arch in arch_variants: 259 arch_flags = '' 260 real_arch = arch 261 arch_part = '-' + arch 262 if platform == 'win7' and arch == '32': 263 arch_flags += ' --no-goma' 264 if arch == '': 265 arch_part = '' 266 real_arch = '64' 267 # Test with Breakpad tools only on basic Linux builds. 268 if sys.platform.startswith('linux'): 269 arch_flags += ' --use-breakpad-tools' 270 if 'win' in platform: 271 arch_flags += ' --no-scons --no-clang' 272 for mode in ['dbg', 'opt']: 273 for libc in ['newlib', 'glibc']: 274 # Buildbots. 275 for bare in ['', '-bare']: 276 for test in ['', '-test']: 277 name = platform + arch_part + bare + '-' + libc + '-' + mode + test 278 assert name not in BOT_ASSIGNMENT, name 279 BOT_ASSIGNMENT[name] = ( 280 python + ' buildbot/buildbot_standard.py ' + 281 mode + ' ' + real_arch + ' ' + libc + arch_flags) 282 # Trybots 283 for arch_sep in ['', '-', '_']: 284 p = platform.replace('linux', 'precise') 285 name = 'nacl-' + p + arch_sep + arch + '_' + libc + '_' + mode 286 assert name not in BOT_ASSIGNMENT, name 287 BOT_ASSIGNMENT[name] = ( 288 python + ' buildbot/buildbot_standard.py ' + 289 mode + ' ' + real_arch + ' ' + libc + arch_flags) 290 291 292def EscapeJson(data): 293 return '"' + json.dumps(data).replace('"', r'\"') + '"' 294 295 296def HasNoPerfResults(builder): 297 if 'pnacl-buildonly-spec' in builder: 298 return True 299 if 'android' in builder: 300 return True 301 return builder in [ 302 'asan', 303 'mac-toolchain_arm', 304 'win-pnacl-x86_32', 305 'linux-pnacl-x86_32-tests-mips', 306 ] 307 308 309def Main(): 310 builder = os.environ.get('BUILDBOT_BUILDERNAME') 311 build_number = os.environ.get('BUILDBOT_BUILDNUMBER') 312 build_revision = os.environ.get('BUILDBOT_GOT_REVISION', 313 os.environ.get('BUILDBOT_REVISION')) 314 slave_type = os.environ.get('BUILDBOT_SLAVE_TYPE') 315 bot_type = os.environ.get('BOT_TYPE') 316 cmd = BOT_ASSIGNMENT.get(builder) 317 if not cmd: 318 sys.stderr.write('ERROR - unset/invalid builder name\n') 319 sys.exit(1) 320 321 env = os.environ.copy() 322 323 # Don't write out .pyc files because in cases in which files move around or 324 # the PYTHONPATH / sys.path change, old .pyc files can be mistakenly used. 325 # This avoids the need for admin changes on the bots in this case. 326 env['PYTHONDONTWRITEBYTECODE'] = '1' 327 328 if bot_type != 'arm_hw_bot': 329 env['GSUTIL'] = pynacl.file_tools.Which('gsutil.py', 330 require_executable=False) 331 332 # When running from cygwin, we sometimes want to use a native python. 333 # The native python will use the depot_tools version by invoking python.bat. 334 if pynacl.platform.IsWindows(): 335 env['NATIVE_PYTHON'] = 'python.bat' 336 else: 337 env['NATIVE_PYTHON'] = 'python' 338 339 if sys.platform == 'win32': 340 # If the temp directory is not on the same drive as the working directory, 341 # there can be random failures when cleaning up temp directories, so use 342 # a directory on the current drive. Use __file__ here instead of os.getcwd() 343 # because toolchain_main picks its working directories relative to __file__ 344 filedrive, _ = os.path.splitdrive(__file__) 345 tempdrive, _ = os.path.splitdrive(env['TEMP']) 346 if tempdrive != filedrive: 347 env['TEMP'] = filedrive + '\\temp' 348 if not os.path.exists(env['TEMP']): 349 os.mkdir(env['TEMP']) 350 351 # Ensure a temp directory exists. 352 if 'TEMP' not in env: 353 env['TEMP'] = tempfile.gettempdir() 354 355 # Isolate build's temp directory to a particular location so we can clobber 356 # the whole thing predictably and so we have a record of who's leaking 357 # temporary files. 358 nacl_tmp = os.path.join(env['TEMP'], 'nacl_tmp') 359 if not os.path.exists(nacl_tmp): 360 os.mkdir(nacl_tmp) 361 env['TEMP'] = os.path.join(nacl_tmp, builder) 362 if not os.path.exists(env['TEMP']): 363 os.mkdir(env['TEMP']) 364 365 # Set all temp directory variants to the same thing. 366 env['TMPDIR'] = env['TEMP'] 367 env['TMP'] = env['TEMP'] 368 print('TEMP=%s' % env['TEMP']) 369 sys.stdout.flush() 370 371 print("%s runs: %s\n" % (builder, cmd)) 372 sys.stdout.flush() 373 retcode = subprocess.call(cmd, env=env, shell=True) 374 sys.exit(retcode) 375 376 377if __name__ == '__main__': 378 Main() 379