1#!/usr/bin/env python3 2# run tests on all Samba subprojects and push to a git tree on success 3# Copyright Andrew Tridgell 2010 4# released under GNU GPL v3 or later 5 6from __future__ import print_function 7from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError 8import os 9import tarfile 10import sys 11import time 12import random 13from optparse import OptionParser 14import smtplib 15import email 16from email.mime.text import MIMEText 17from email.mime.base import MIMEBase 18from email.mime.application import MIMEApplication 19from email.mime.multipart import MIMEMultipart 20from distutils.sysconfig import get_python_lib 21import platform 22 23try: 24 from waflib.Build import CACHE_SUFFIX 25except ImportError: 26 sys.path.insert(0, "./third_party/waf") 27 from waflib.Build import CACHE_SUFFIX 28 29 30os.environ["PYTHONUNBUFFERED"] = "1" 31 32# This speeds up testing remarkably. 33os.environ['TDB_NO_FSYNC'] = '1' 34 35 36def find_git_root(): 37 '''get to the top of the git repo''' 38 p = os.getcwd() 39 while p != '/': 40 if os.path.exists(os.path.join(p, ".git")): 41 return p 42 p = os.path.abspath(os.path.join(p, '..')) 43 return None 44 45 46gitroot = find_git_root() 47if gitroot is None: 48 raise Exception("Failed to find git root") 49 50 51def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER')) 52 53parser = OptionParser() 54parser.add_option("--tail", help="show output while running", default=False, action="store_true") 55parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true") 56parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true") 57parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase, 58 default=def_testbase) 59parser.add_option("--passcmd", help="command to run on success", default=None) 60parser.add_option("--verbose", help="show all commands as they are run", 61 default=False, action="store_true") 62parser.add_option("--rebase", help="rebase on the given tree before testing", 63 default=None, type='str') 64parser.add_option("--pushto", help="push to a git url on success", 65 default=None, type='str') 66parser.add_option("--mark", help="add a Tested-By signoff before pushing", 67 default=False, action="store_true") 68parser.add_option("--fix-whitespace", help="fix whitespace on rebase", 69 default=False, action="store_true") 70parser.add_option("--retry", help="automatically retry if master changes", 71 default=False, action="store_true") 72parser.add_option("--email", help="send email to the given address on failure", 73 type='str', default=None) 74parser.add_option("--email-from", help="send email from the given address", 75 type='str', default="autobuild@samba.org") 76parser.add_option("--email-server", help="send email via the given server", 77 type='str', default='localhost') 78parser.add_option("--always-email", help="always send email, even on success", 79 action="store_true") 80parser.add_option("--daemon", help="daemonize after initial setup", 81 action="store_true") 82parser.add_option("--branch", help="the branch to work on (default=master)", 83 default="master", type='str') 84parser.add_option("--log-base", help="location where the logs can be found (default=cwd)", 85 default=gitroot, type='str') 86parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?", 87 default=False, action="store_true") 88parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex", 89 default='') 90parser.add_option("--enable-coverage", dest='enable_coverage', 91 action="store_const", const='--enable-coverage', default='', 92 help="Add --enable-coverage option while configure") 93 94(options, args) = parser.parse_args() 95 96if options.retry: 97 if options.rebase is None: 98 raise Exception('You can only use --retry if you also rebase') 99 100testbase = "%s/b%u" % (options.testbase, os.getpid()) 101test_master = "%s/master" % testbase 102test_prefix = "%s/prefix" % testbase 103test_tmpdir = "%s/tmp" % testbase 104os.environ['TMPDIR'] = test_tmpdir 105 106if options.enable_coverage: 107 LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'" 108else: 109 LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"' 110 111if args: 112 # If we are only running specific test, 113 # do not sleep randomly to wait for it to start 114 def random_sleep(low, high): 115 return 'sleep 1' 116else: 117 def random_sleep(low, high): 118 return 'sleep {}'.format(random.randint(low, high)) 119 120cleanup_list = [] 121 122builddirs = { 123 "ctdb": "ctdb", 124 "ldb": "lib/ldb", 125 "tdb": "lib/tdb", 126 "talloc": "lib/talloc", 127 "replace": "lib/replace", 128 "tevent": "lib/tevent", 129 "pidl": "pidl" 130} 131 132ctdb_configure_params = " --enable-developer ${PREFIX}" 133samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data" 134 135samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH" 136samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig" 137samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'" 138samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}" 139samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE" 140samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!ldb,!pyldb,!pyldb-util,!tevent,!pytevent,!popt" 141samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs 142 143 144def format_option(name, value=None): 145 """Format option as str list.""" 146 if value is None: # boolean option 147 return [name] 148 if not isinstance(value, list): # single value option 149 value = [value] 150 # repeatable option 151 return ['{}={}'.format(name, item) for item in value] 152 153 154def make_test( 155 cmd='make test', 156 FAIL_IMMEDIATELY=1, 157 TESTS='', 158 include_envs=None, 159 exclude_envs=None): 160 161 test_options = [] 162 if include_envs: 163 test_options = format_option('--include-env', include_envs) 164 if exclude_envs: 165 test_options = format_option('--exclude-env', exclude_envs) 166 if test_options: 167 # join envs options to original test options 168 TESTS = (TESTS + ' ' + ' '.join(test_options)).strip() 169 170 _options = [] 171 if FAIL_IMMEDIATELY: 172 _options.append('FAIL_IMMEDIATELY=1') 173 if TESTS: 174 _options.append("TESTS='{}'".format(TESTS)) 175 176 return ' '.join([cmd] + _options) 177 178 179# When updating this list, also update .gitlab-ci.yml to add the job 180# and to make it a dependency of 'page' for the coverage report. 181 182tasks = { 183 "ctdb": [ 184 ("random-sleep", random_sleep(300, 900)), 185 ("configure", "./configure " + ctdb_configure_params), 186 ("make", "make all"), 187 ("install", "make install"), 188 ("test", "make autotest"), 189 ("check-clean-tree", "../script/clean-source-tree.sh"), 190 ("clean", "make clean"), 191 ], 192 193 # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)' 194 "samba": [ 195 ("random-sleep", random_sleep(300, 900)), 196 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 197 ("make", "make -j"), 198 ("test", make_test(exclude_envs=[ 199 "none", 200 "nt4_dc", 201 "nt4_dc_schannel", 202 "nt4_member", 203 "ad_dc", 204 "ad_dc_backup", 205 "ad_dc_ntvfs", 206 "ad_dc_default", 207 "ad_dc_slowtests", 208 "ad_dc_no_nss", 209 "ad_dc_no_ntlm", 210 "fl2003dc", 211 "fl2008dc", 212 "fl2008r2dc", 213 "ad_member", 214 "ad_member_idmap_rid", 215 "ad_member_idmap_ad", 216 "ad_member_rfc2307", 217 "chgdcpass", 218 "vampire_2000_dc", 219 "fl2000dc", 220 "fileserver", 221 "maptoguest", 222 "simpleserver", 223 "backupfromdc", 224 "restoredc", 225 "renamedc", 226 "offlinebackupdc", 227 "labdc", 228 "preforkrestartdc", 229 "proclimitdc", 230 "promoted_dc", 231 "vampire_dc", 232 "rodc", 233 "ad_dc_default", 234 "ad_dc_slowtests", 235 "schema_pair_dc", 236 "schema_dc", 237 ])), 238 ("lcov", LCOV_CMD), 239 ("install", "make install"), 240 ("check-clean-tree", "script/clean-source-tree.sh"), 241 ("clean", "make clean"), 242 ], 243 244 # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)' 245 "samba-mitkrb5": [ 246 ("random-sleep", random_sleep(300, 900)), 247 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params), 248 ("make", "make -j"), 249 ("test", make_test(exclude_envs=[ 250 "none", 251 "nt4_dc", 252 "nt4_dc_schannel", 253 "nt4_member", 254 "ad_dc", 255 "ad_dc_backup", 256 "ad_dc_ntvfs", 257 "ad_dc_default", 258 "ad_dc_slowtests", 259 "ad_dc_no_nss", 260 "ad_dc_no_ntlm", 261 "fl2003dc", 262 "fl2008dc", 263 "fl2008r2dc", 264 "ad_member", 265 "ad_member_idmap_rid", 266 "ad_member_idmap_ad", 267 "ad_member_rfc2307", 268 "chgdcpass", 269 "vampire_2000_dc", 270 "fl2000dc", 271 "fileserver", 272 "maptoguest", 273 "simpleserver", 274 "backupfromdc", 275 "restoredc", 276 "renamedc", 277 "offlinebackupdc", 278 "labdc", 279 "preforkrestartdc", 280 "proclimitdc", 281 "promoted_dc", 282 "vampire_dc", 283 "rodc", 284 "ad_dc_default", 285 "ad_dc_slowtests", 286 "schema_pair_dc", 287 "schema_dc", 288 ])), 289 ("lcov", LCOV_CMD), 290 ("install", "make install"), 291 ("check-clean-tree", "script/clean-source-tree.sh"), 292 ("clean", "make clean"), 293 ], 294 295 "samba-nt4": [ 296 ("random-sleep", random_sleep(300, 900)), 297 ("configure", "./configure.developer --without-ads --with-selftest-prefix=./bin/ab" + samba_configure_params), 298 ("make", "make -j"), 299 ("test", make_test(include_envs=[ 300 "nt4_dc", 301 "nt4_dc_schannel", 302 "nt4_member", 303 ])), 304 ("lcov", LCOV_CMD), 305 ("install", "make install"), 306 ("check-clean-tree", "script/clean-source-tree.sh"), 307 ("clean", "make clean"), 308 ], 309 310 "samba-simpleserver": [ 311 ("random-sleep", random_sleep(300, 900)), 312 ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json --with-selftest-prefix=./bin/ab" + samba_configure_params), 313 ("make", "make -j"), 314 ("test", make_test(include_envs=[ 315 "simpleserver", 316 ])), 317 ("lcov", LCOV_CMD), 318 ("check-clean-tree", "script/clean-source-tree.sh"), 319 ], 320 321 "samba-fileserver": [ 322 ("random-sleep", random_sleep(300, 900)), 323 ("configure", "./configure.developer --without-ad-dc --with-selftest-prefix=./bin/ab" + samba_configure_params), 324 ("make", "make -j"), 325 ("test", make_test(include_envs=[ 326 "fileserver", 327 "maptoguest", 328 ])), 329 ("lcov", LCOV_CMD), 330 ("check-clean-tree", "script/clean-source-tree.sh"), 331 ], 332 333 "samba-ktest-heimdal": [ 334 ("random-sleep", random_sleep(300, 900)), 335 ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5 --with-selftest-prefix=./bin/ab" + samba_configure_params), 336 ("make", "make -j"), 337 ("test", make_test(include_envs=[ 338 "ktest", 339 ])), 340 ("lcov", LCOV_CMD), 341 ("check-clean-tree", "script/clean-source-tree.sh"), 342 ], 343 344 "samba-admem": [ 345 ("random-sleep", random_sleep(300, 900)), 346 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 347 ("make", "make -j"), 348 ("test", make_test(include_envs=[ 349 "ad_member", 350 "ad_member_idmap_rid", 351 "ad_member_idmap_ad", 352 "ad_member_rfc2307", 353 ])), 354 ("lcov", LCOV_CMD), 355 ("check-clean-tree", "script/clean-source-tree.sh"), 356 ], 357 358 "samba-ad-dc-1": [ 359 ("random-sleep", random_sleep(1, 1)), 360 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 361 ("make", "make -j"), 362 ("test", make_test(include_envs=[ 363 "ad_dc", 364 "ad_dc_no_nss", 365 "ad_dc_no_ntlm", 366 ])), 367 ("lcov", LCOV_CMD), 368 ("check-clean-tree", "script/clean-source-tree.sh"), 369 ], 370 371 "samba-ad-dc-2": [ 372 ("random-sleep", random_sleep(1, 1)), 373 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 374 ("make", "make -j"), 375 ("test", make_test(include_envs=[ 376 "vampire_dc", 377 "vampire_2000_dc", 378 "rodc", 379 ])), 380 ("lcov", LCOV_CMD), 381 ("check-clean-tree", "script/clean-source-tree.sh"), 382 ], 383 384 "samba-ad-dc-3": [ 385 ("random-sleep", random_sleep(1, 1)), 386 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 387 ("make", "make -j"), 388 ("test", make_test(include_envs=[ 389 "promoted_dc", 390 "chgdcpass", 391 "preforkrestartdc", 392 "proclimitdc", 393 ])), 394 ("lcov", LCOV_CMD), 395 ("check-clean-tree", "script/clean-source-tree.sh"), 396 ], 397 398 "samba-ad-dc-4": [ 399 ("random-sleep", random_sleep(1, 1)), 400 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 401 ("make", "make -j"), 402 ("test", make_test(include_envs=[ 403 "fl2000dc", 404 "fl2003dc", 405 "fl2008dc", 406 "fl2008r2dc", 407 ])), 408 ("lcov", LCOV_CMD), 409 ("check-clean-tree", "script/clean-source-tree.sh"), 410 ], 411 412 "samba-ad-dc-5": [ 413 ("random-sleep", random_sleep(1, 1)), 414 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 415 ("make", "make -j"), 416 ("test", make_test(include_envs=["ad_dc_default"])), 417 ("lcov", LCOV_CMD), 418 ("check-clean-tree", "script/clean-source-tree.sh"), 419 ], 420 421 "samba-ad-dc-6": [ 422 ("random-sleep", random_sleep(1, 1)), 423 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 424 ("make", "make -j"), 425 ("test", make_test(include_envs=["ad_dc_slowtests"])), 426 ("lcov", LCOV_CMD), 427 ("check-clean-tree", "script/clean-source-tree.sh"), 428 ], 429 430 "samba-schemaupgrade": [ 431 ("random-sleep", random_sleep(1, 1)), 432 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 433 ("make", "make -j"), 434 ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])), 435 ("lcov", LCOV_CMD), 436 ("check-clean-tree", "script/clean-source-tree.sh"), 437 ], 438 439 # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait 440 # This is currently the longest task, so we don't randomly delay it. 441 "samba-ad-dc-ntvfs": [ 442 ("random-sleep", random_sleep(1, 1)), 443 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 444 ("make", "make -j"), 445 ("test", make_test(include_envs=["ad_dc_ntvfs"])), 446 ("lcov", LCOV_CMD), 447 ("check-clean-tree", "script/clean-source-tree.sh"), 448 ], 449 450 # run the backup/restore testenvs separately as they're fairly standalone 451 # (and CI seems to max out at ~8 different DCs running at once) 452 "samba-ad-dc-backup": [ 453 ("random-sleep", random_sleep(300, 900)), 454 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 455 ("make", "make -j"), 456 ("test", make_test(include_envs=[ 457 "backupfromdc", 458 "restoredc", 459 "renamedc", 460 "offlinebackupdc", 461 "labdc", 462 "ad_dc_backup", 463 ])), 464 ("lcov", LCOV_CMD), 465 ("check-clean-tree", "script/clean-source-tree.sh"), 466 ], 467 468 "samba-admem-mit": [ 469 ("random-sleep", random_sleep(300, 900)), 470 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params), 471 ("make", "make -j"), 472 ("test", make_test(include_envs=[ 473 "ad_member", 474 "ad_member_idmap_rid", 475 "ad_member_idmap_ad", 476 "ad_member_rfc2307", 477 ])), 478 ("lcov", LCOV_CMD), 479 ("check-clean-tree", "script/clean-source-tree.sh"), 480 ], 481 482 "samba-ad-dc-1-mitkrb5": [ 483 ("random-sleep", random_sleep(1, 1)), 484 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params), 485 ("make", "make -j"), 486 ("test", make_test(include_envs=[ 487 "ad_dc", 488 "ad_dc_no_nss", 489 "ad_dc_no_ntlm", 490 ])), 491 ("lcov", LCOV_CMD), 492 ("check-clean-tree", "script/clean-source-tree.sh"), 493 ], 494 495 "samba-ad-dc-4-mitkrb5": [ 496 ("random-sleep", random_sleep(1, 1)), 497 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params), 498 ("make", "make -j"), 499 ("test", make_test(include_envs=[ 500 "fl2000dc", 501 "fl2003dc", 502 "fl2008dc", 503 "fl2008r2dc", 504 ])), 505 ("lcov", LCOV_CMD), 506 ("check-clean-tree", "script/clean-source-tree.sh"), 507 ], 508 509 "samba-test-only": [ 510 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params), 511 ("make", "make -j"), 512 ("test", make_test(TESTS="${TESTS}")), 513 ("lcov", LCOV_CMD), 514 ], 515 516 # Test cross-compile infrastructure 517 "samba-xc": [ 518 ("random-sleep", random_sleep(900, 1500)), 519 ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 520 ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \ 521 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params), 522 ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"), 523 ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \ 524 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params), 525 ("compare-results", "script/compare_cc_results.py " 526 "./bin/c4che/default{} " 527 "./bin-xe/c4che/default{} " 528 "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))), 529 ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"), 530 ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \ 531 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params), 532 ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \ 533 " = \"'1234'\"".format(CACHE_SUFFIX)), 534 ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"), 535 ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \ 536 " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \ 537 " ; test $? -ne 0"), 538 ], 539 540 # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments 541 "samba-o3": [ 542 ("random-sleep", random_sleep(300, 900)), 543 ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --with-selftest-prefix=./bin/ab --abi-check-disable" + samba_configure_params), 544 ("make", "make -j"), 545 ("test", make_test(cmd='make quicktest', include_envs=["ad_dc"])), 546 ("lcov", LCOV_CMD), 547 ("install", "make install"), 548 ("check-clean-tree", "script/clean-source-tree.sh"), 549 ("clean", "make clean"), 550 ], 551 552 "samba-ctdb": [ 553 ("random-sleep", random_sleep(900, 1500)), 554 555 # make sure we have tdb around: 556 ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"), 557 ("tdb-make", "cd lib/tdb && make"), 558 ("tdb-install", "cd lib/tdb && make install"), 559 560 # build samba with cluster support (also building ctdb): 561 ("samba-configure", "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} ./configure.developer ${PREFIX} --with-selftest-prefix=./bin/ab --with-cluster-support --bundled-libraries=!tdb"), 562 ("samba-make", "make"), 563 ("samba-check", "./bin/smbd -b | grep CLUSTER_SUPPORT"), 564 ("samba-install", "make install"), 565 ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"), 566 567 # clean up: 568 ("check-clean-tree", "script/clean-source-tree.sh"), 569 ("clean", "make clean"), 570 ("ctdb-clean", "cd ./ctdb && make clean"), 571 ], 572 573 "samba-libs": [ 574 ("random-sleep", random_sleep(300, 900)), 575 ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs), 576 ("talloc-make", "cd lib/talloc && make"), 577 ("talloc-install", "cd lib/talloc && make install"), 578 579 ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs), 580 ("tdb-make", "cd lib/tdb && make"), 581 ("tdb-install", "cd lib/tdb && make install"), 582 583 ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs), 584 ("tevent-make", "cd lib/tevent && make"), 585 ("tevent-install", "cd lib/tevent && make install"), 586 587 ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_libs), 588 ("ldb-make", "cd lib/ldb && make"), 589 ("ldb-install", "cd lib/ldb && make install"), 590 591 ("nondevel-configure", "./configure ${PREFIX}"), 592 ("nondevel-make", "make -j"), 593 ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"), 594 ("nondevel-install", "make install"), 595 ("nondevel-dist", "make dist"), 596 597 # retry with all modules shared 598 ("allshared-distclean", "make distclean"), 599 ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"), 600 ("allshared-make", "make -j"), 601 ], 602 603 "samba-none-env": [ 604 ("random-sleep", random_sleep(1, 1)), 605 ("configure", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params), 606 ("make", "make -j"), 607 ("test", make_test(include_envs=["none"])), 608 ("lcov", LCOV_CMD), 609 ], 610 611 "samba-static": [ 612 ("random-sleep", random_sleep(1, 1)), 613 # build with all modules static 614 ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"), 615 ("allstatic-make", "make -j"), 616 ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")), 617 ("lcov", LCOV_CMD), 618 619 # retry without any required modules 620 ("none-distclean", "make distclean"), 621 ("none-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"), 622 ("none-make", "make -j"), 623 624 # retry with nonshared smbd and smbtorture 625 ("nonshared-distclean", "make distclean"), 626 ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"), 627 ("nonshared-make", "make -j") 628 ], 629 630 "samba-fuzz": [ 631 # build the fuzzers (static) via the oss-fuzz script 632 ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"), 633 ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl"), 634 ("fuzzers-check", "./lib/fuzzing/oss-fuzz/check_build.sh ${PREFIX_DIR}") 635 ], 636 637 # Test Samba without python still builds. When this test fails 638 # due to more use of Python, the expectations is that the newly 639 # failing part of the code should be disabled when 640 # --disable-python is set (rather than major work being done to 641 # support this environment). The target here is for vendors 642 # shipping a minimal smbd. 643 "samba-nopython": [ 644 ("random-sleep", random_sleep(300, 900)), 645 ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"), 646 ("make", "make -j"), 647 ("install", "make install"), 648 ("find-python", "script/find_python.sh ${PREFIX}"), 649 ("test", "make test-nopython"), 650 ("lcov", LCOV_CMD), 651 ("check-clean-tree", "script/clean-source-tree.sh"), 652 ("clean", "make clean"), 653 654 ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), 655 ("talloc-make", "cd lib/talloc && make"), 656 ("talloc-install", "cd lib/talloc && make install"), 657 658 ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), 659 ("tdb-make", "cd lib/tdb && make"), 660 ("tdb-install", "cd lib/tdb && make install"), 661 662 ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), 663 ("tevent-make", "cd lib/tevent && make"), 664 ("tevent-install", "cd lib/tevent && make install"), 665 666 ("ldb-configure", "cd lib/ldb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), 667 ("ldb-make", "cd lib/ldb && make"), 668 ("ldb-install", "cd lib/ldb && make install"), 669 670 # retry against installed library packages 671 ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"), 672 ("libs-make", "make -j"), 673 ("libs-install", "make install"), 674 ("libs-check-clean-tree", "script/clean-source-tree.sh"), 675 ("libs-clean", "make clean"), 676 ], 677 678 # check we can do the same thing using python2 679 "samba-nopython-py2": [ 680 ("random-sleep", random_sleep(300, 900)), 681 ("configure", "PYTHON=python2 ./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"), 682 ("make", "PYTHON=python2 make -j"), 683 ("install", "PYTHON=python2 make install"), 684 ("find-python", "script/find_python.sh ${PREFIX}"), 685 ("test", "make test-nopython"), 686 ("lcov", LCOV_CMD), 687 ("check-clean-tree", "script/clean-source-tree.sh"), 688 ("clean", "PYTHON=python2 make clean"), 689 690 ("talloc-configure", "cd lib/talloc && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), 691 ("talloc-make", "cd lib/talloc && PYTHON=python2 make"), 692 ("talloc-install", "cd lib/talloc && PYTHON=python2 make install"), 693 694 ("tdb-configure", "cd lib/tdb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), 695 ("tdb-make", "cd lib/tdb && PYTHON=python2 make"), 696 ("tdb-install", "cd lib/tdb && PYTHON=python2 make install"), 697 698 ("tevent-configure", "cd lib/tevent && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), 699 ("tevent-make", "cd lib/tevent && PYTHON=python2 make"), 700 ("tevent-install", "cd lib/tevent && PYTHON=python2 make install"), 701 702 ("ldb-configure", "cd lib/ldb && PYTHON=python2 " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"), 703 ("ldb-make", "cd lib/ldb && PYTHON=python2 make"), 704 ("ldb-install", "cd lib/ldb && PYTHON=python2 make install"), 705 706 # retry against installed library packages 707 ("libs-configure", "PYTHON=python2 " + samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc"), 708 ("libs-make", "PYTHON=python2 make -j"), 709 ("libs-install", "PYTHON=python2 make install"), 710 ("libs-check-clean-tree", "script/clean-source-tree.sh"), 711 ("libs-clean", "PYTHON=python2 make clean"), 712 ], 713 714 "ldb": [ 715 ("random-sleep", random_sleep(60, 600)), 716 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"), 717 ("make", "make"), 718 ("install", "make install"), 719 ("test", "make test"), 720 ("lcov", LCOV_CMD), 721 ("clean", "make clean"), 722 ("configure-no-lmdb", "./configure ${ENABLE_COVERAGE} --enable-developer --without-ldb-lmdb -C ${PREFIX}"), 723 ("make-no-lmdb", "make"), 724 ("test-no-lmdb", "make test"), 725 ("lcov-no-lmdb", LCOV_CMD), 726 ("install-no-lmdb", "make install"), 727 ("check-clean-tree", "../../script/clean-source-tree.sh"), 728 ("distcheck", "make distcheck"), 729 ("clean", "make clean"), 730 ], 731 732 "tdb": [ 733 ("random-sleep", random_sleep(60, 600)), 734 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"), 735 ("make", "make"), 736 ("install", "make install"), 737 ("test", "make test"), 738 ("lcov", LCOV_CMD), 739 ("check-clean-tree", "../../script/clean-source-tree.sh"), 740 ("distcheck", "make distcheck"), 741 ("clean", "make clean"), 742 ], 743 744 "talloc": [ 745 ("random-sleep", random_sleep(60, 600)), 746 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"), 747 ("make", "make"), 748 ("install", "make install"), 749 ("test", "make test"), 750 ("lcov", LCOV_CMD), 751 ("check-clean-tree", "../../script/clean-source-tree.sh"), 752 ("distcheck", "make distcheck"), 753 ("clean", "make clean"), 754 ], 755 756 "replace": [ 757 ("random-sleep", random_sleep(60, 600)), 758 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"), 759 ("make", "make"), 760 ("install", "make install"), 761 ("test", "make test"), 762 ("lcov", LCOV_CMD), 763 ("check-clean-tree", "../../script/clean-source-tree.sh"), 764 ("distcheck", "make distcheck"), 765 ("clean", "make clean"), 766 ], 767 768 "tevent": [ 769 ("random-sleep", random_sleep(60, 600)), 770 ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"), 771 ("make", "make"), 772 ("install", "make install"), 773 ("test", "make test"), 774 ("lcov", LCOV_CMD), 775 ("check-clean-tree", "../../script/clean-source-tree.sh"), 776 ("distcheck", "make distcheck"), 777 ("clean", "make clean"), 778 ], 779 780 "pidl": [ 781 ("random-sleep", random_sleep(60, 600)), 782 ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"), 783 ("touch", "touch *.yp"), 784 ("make", "make"), 785 ("test", "make test"), 786 ("install", "make install"), 787 ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"), 788 ("check-clean-tree", "../script/clean-source-tree.sh"), 789 ("clean", "make clean"), 790 ], 791 792 # these are useful for debugging autobuild 793 'pass': [("pass", 'echo passing && /bin/true')], 794 'fail': [("fail", 'echo failing && /bin/false')], 795} 796 797defaulttasks = list(tasks.keys()) 798 799defaulttasks.remove("pass") 800defaulttasks.remove("fail") 801defaulttasks.remove("samba-test-only") 802defaulttasks.remove("samba-fuzz") 803if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1": 804 defaulttasks.remove("samba-o3") 805 806 807def do_print(msg): 808 print("%s" % msg) 809 sys.stdout.flush() 810 sys.stderr.flush() 811 812 813def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True): 814 if show is None: 815 show = options.verbose 816 if show: 817 do_print("Running: '%s' in '%s'" % (cmd, dir)) 818 if output: 819 out = check_output([cmd], shell=True, cwd=dir) 820 return out.decode(encoding='utf-8', errors='backslashreplace') 821 elif checkfail: 822 return check_call(cmd, shell=True, cwd=dir) 823 else: 824 return call(cmd, shell=True, cwd=dir) 825 826def rmdir_force(dirname, re_raise=True): 827 try: 828 run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % ( 829 dirname, dirname, dirname), output=True, show=True) 830 except CalledProcessError as e: 831 do_print("Failed: '%s'" % (str(e))) 832 run_cmd("tree %s" % dirname, output=True, show=True) 833 if re_raise: 834 raise 835 return False 836 return True 837 838class builder(object): 839 '''handle build of one directory''' 840 841 def __init__(self, name, sequence, cp=True): 842 self.name = name 843 self.dir = builddirs.get(name, '.') 844 self.tag = self.name.replace('/', '_') 845 self.sequence = sequence 846 self.next = 0 847 self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag) 848 self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag) 849 if options.verbose: 850 do_print("stdout for %s in %s" % (self.name, self.stdout_path)) 851 do_print("stderr for %s in %s" % (self.name, self.stderr_path)) 852 run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path)) 853 self.stdout = open(self.stdout_path, 'w') 854 self.stderr = open(self.stderr_path, 'w') 855 self.stdin = open("/dev/null", 'r') 856 self.test_source_dir = "%s/%s" % (testbase, self.tag) 857 self.cwd = "%s/%s" % (self.test_source_dir, self.dir) 858 self.prefix = "%s/%s" % (test_prefix, self.tag) 859 rmdir_force(self.test_source_dir) 860 rmdir_force(self.prefix) 861 if cp: 862 run_cmd("cp -R -a -l %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True) 863 else: 864 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.test_source_dir), dir=test_master, show=True) 865 self.start_next() 866 867 def start_next(self): 868 if self.next == len(self.sequence): 869 if not options.nocleanup: 870 rmdir_force(self.test_source_dir) 871 rmdir_force(self.prefix) 872 do_print('%s: Completed OK' % self.name) 873 self.done = True 874 return 875 (self.stage, self.cmd) = self.sequence[self.next] 876 self.cmd = self.cmd.replace("${PYTHON_PREFIX}", get_python_lib(plat_specific=1, standard_lib=0, prefix=self.prefix)) 877 self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix) 878 self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix) 879 self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests) 880 self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir) 881 self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base) 882 self.cmd = self.cmd.replace("${NAME}", self.name) 883 self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage) 884 do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd)) 885 self.proc = Popen(self.cmd, shell=True, 886 close_fds=True, cwd=self.cwd, 887 stdout=self.stdout, stderr=self.stderr, stdin=self.stdin) 888 self.next += 1 889 890 891class buildlist(object): 892 '''handle build of multiple directories''' 893 894 def __init__(self, tasknames, rebase_url, rebase_branch="master"): 895 self.tail_proc = None 896 self.retry = None 897 if not tasknames: 898 if options.restrict_tests: 899 tasknames = ["samba-test-only"] 900 else: 901 tasknames = defaulttasks 902 903 self.tlist = [builder(n, tasks[n], cp=(n != "pidl")) for n in tasknames] 904 905 if options.retry: 906 rebase_remote = "rebaseon" 907 retry_task = [("retry", 908 '''set -e 909 git remote add -t %s %s %s 910 git fetch %s 911 while :; do 912 sleep 60 913 git describe %s/%s > old_remote_branch.desc 914 git fetch %s 915 git describe %s/%s > remote_branch.desc 916 diff old_remote_branch.desc remote_branch.desc 917 done 918 ''' % ( 919 rebase_branch, rebase_remote, rebase_url, 920 rebase_remote, 921 rebase_remote, rebase_branch, 922 rebase_remote, 923 rebase_remote, rebase_branch 924 ))] 925 926 self.retry = builder('retry', retry_task, cp=False) 927 self.need_retry = False 928 929 def kill_kids(self): 930 if self.tail_proc is not None: 931 self.tail_proc.terminate() 932 self.tail_proc.wait() 933 self.tail_proc = None 934 if self.retry is not None: 935 self.retry.proc.terminate() 936 self.retry.proc.wait() 937 self.retry = None 938 for b in self.tlist: 939 if b.proc is not None: 940 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False) 941 b.proc.terminate() 942 b.proc.wait() 943 b.proc = None 944 945 def wait_one(self): 946 while True: 947 none_running = True 948 for b in self.tlist: 949 if b.proc is None: 950 continue 951 none_running = False 952 b.status = b.proc.poll() 953 if b.status is None: 954 continue 955 b.proc = None 956 return b 957 if options.retry: 958 ret = self.retry.proc.poll() 959 if ret is not None: 960 self.need_retry = True 961 self.retry = None 962 return None 963 if none_running: 964 return None 965 time.sleep(0.1) 966 967 def run(self): 968 while True: 969 b = self.wait_one() 970 if options.retry and self.need_retry: 971 self.kill_kids() 972 do_print("retry needed") 973 return (0, None, None, None, "retry") 974 if b is None: 975 break 976 if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0: 977 self.kill_kids() 978 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status)) 979 b.start_next() 980 self.kill_kids() 981 return (0, None, None, None, "All OK") 982 983 def write_system_info(self, filename): 984 with open(filename, 'w') as f: 985 for cmd in ['uname -a', 986 'lsb_release -a', 987 'free', 988 'mount', 989 'cat /proc/cpuinfo', 990 'cc --version', 991 'df -m .', 992 'df -m %s' % testbase]: 993 try: 994 out = run_cmd(cmd, output=True, checkfail=False) 995 except CalledProcessError as e: 996 out = "<failed: %s>" % str(e) 997 print('### %s' % cmd, file=f) 998 print(out, file=f) 999 print(file=f) 1000 1001 def tarlogs(self, fname): 1002 with tarfile.open(fname, "w:gz") as tar: 1003 for b in self.tlist: 1004 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag) 1005 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag) 1006 if os.path.exists("autobuild.log"): 1007 tar.add("autobuild.log") 1008 filename = 'system-info.txt' 1009 self.write_system_info(filename) 1010 tar.add(filename) 1011 1012 def remove_logs(self): 1013 for b in self.tlist: 1014 os.unlink(b.stdout_path) 1015 os.unlink(b.stderr_path) 1016 1017 def start_tail(self): 1018 cmd = ["tail", "-f"] 1019 for b in self.tlist: 1020 cmd.append(b.stdout_path) 1021 cmd.append(b.stderr_path) 1022 self.tail_proc = Popen(cmd, close_fds=True) 1023 1024 1025def cleanup(do_raise=False): 1026 if options.nocleanup: 1027 return 1028 run_cmd("stat %s || true" % test_tmpdir, show=True) 1029 run_cmd("stat %s" % testbase, show=True) 1030 do_print("Cleaning up %r" % cleanup_list) 1031 for d in cleanup_list: 1032 ok = rmdir_force(d, re_raise=False) 1033 if ok: 1034 continue 1035 if os.path.isdir(d): 1036 do_print("Killing, waiting and retry") 1037 run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False) 1038 else: 1039 do_print("Waiting and retry") 1040 time.sleep(1) 1041 rmdir_force(d, re_raise=do_raise) 1042 1043 1044def daemonize(logfile): 1045 pid = os.fork() 1046 if pid == 0: # Parent 1047 os.setsid() 1048 pid = os.fork() 1049 if pid != 0: # Actual daemon 1050 os._exit(0) 1051 else: # Grandparent 1052 os._exit(0) 1053 1054 import resource # Resource usage information. 1055 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] 1056 if maxfd == resource.RLIM_INFINITY: 1057 maxfd = 1024 # Rough guess at maximum number of open file descriptors. 1058 for fd in range(0, maxfd): 1059 try: 1060 os.close(fd) 1061 except OSError: 1062 pass 1063 os.open(logfile, os.O_RDWR | os.O_CREAT) 1064 os.dup2(0, 1) 1065 os.dup2(0, 2) 1066 1067 1068def write_pidfile(fname): 1069 '''write a pid file, cleanup on exit''' 1070 with open(fname, mode='w') as f: 1071 f.write("%u\n" % os.getpid()) 1072 1073 1074def rebase_tree(rebase_url, rebase_branch="master"): 1075 rebase_remote = "rebaseon" 1076 do_print("Rebasing on %s" % rebase_url) 1077 run_cmd("git describe HEAD", show=True, dir=test_master) 1078 run_cmd("git remote add -t %s %s %s" % 1079 (rebase_branch, rebase_remote, rebase_url), 1080 show=True, dir=test_master) 1081 run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master) 1082 if options.fix_whitespace: 1083 run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" % 1084 (rebase_remote, rebase_branch), 1085 show=True, dir=test_master) 1086 else: 1087 run_cmd("git rebase --force-rebase %s/%s" % 1088 (rebase_remote, rebase_branch), 1089 show=True, dir=test_master) 1090 diff = run_cmd("git --no-pager diff HEAD %s/%s" % 1091 (rebase_remote, rebase_branch), 1092 dir=test_master, output=True) 1093 if diff == '': 1094 do_print("No differences between HEAD and %s/%s - exiting" % 1095 (rebase_remote, rebase_branch)) 1096 sys.exit(0) 1097 run_cmd("git describe %s/%s" % 1098 (rebase_remote, rebase_branch), 1099 show=True, dir=test_master) 1100 run_cmd("git describe HEAD", show=True, dir=test_master) 1101 run_cmd("git --no-pager diff --stat HEAD %s/%s" % 1102 (rebase_remote, rebase_branch), 1103 show=True, dir=test_master) 1104 1105 1106def push_to(push_url, push_branch="master"): 1107 push_remote = "pushto" 1108 do_print("Pushing to %s" % push_url) 1109 if options.mark: 1110 run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master) 1111 run_cmd("git commit --amend -c HEAD", dir=test_master) 1112 # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master 1113 # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master) 1114 run_cmd("git remote add -t %s %s %s" % 1115 (push_branch, push_remote, push_url), 1116 show=True, dir=test_master) 1117 run_cmd("git push %s +HEAD:%s" % 1118 (push_remote, push_branch), 1119 show=True, dir=test_master) 1120 1121 1122def send_email(subject, text, log_tar): 1123 if options.email is None: 1124 do_print("not sending email because the recipient is not set") 1125 do_print("the text content would have been:\n\nSubject: %s\n\n%s" % 1126 (subject, text)) 1127 return 1128 outer = MIMEMultipart() 1129 outer['Subject'] = subject 1130 outer['To'] = options.email 1131 outer['From'] = options.email_from 1132 outer['Date'] = email.utils.formatdate(localtime=True) 1133 outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n' 1134 outer.attach(MIMEText(text, 'plain')) 1135 if options.attach_logs: 1136 with open(log_tar, 'rb') as fp: 1137 msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64) 1138 # Set the filename parameter 1139 msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar)) 1140 outer.attach(msg) 1141 content = outer.as_string() 1142 s = smtplib.SMTP(options.email_server) 1143 email_user = os.getenv('SMTP_USERNAME') 1144 email_password = os.getenv('SMTP_PASSWORD') 1145 if email_user is not None: 1146 s.starttls() 1147 s.login(email_user, email_password) 1148 1149 s.sendmail(options.email_from, [options.email], content) 1150 s.set_debuglevel(1) 1151 s.quit() 1152 1153 1154def email_failure(status, failed_task, failed_stage, failed_tag, errstr, 1155 elapsed_time, log_base=None, add_log_tail=True): 1156 '''send an email to options.email about the failure''' 1157 elapsed_minutes = elapsed_time / 60.0 1158 if log_base is None: 1159 log_base = gitroot 1160 text = ''' 1161Dear Developer, 1162 1163Your autobuild on %s failed after %.1f minutes 1164when trying to test %s with the following error: 1165 1166 %s 1167 1168the autobuild has been abandoned. Please fix the error and resubmit. 1169 1170A summary of the autobuild process is here: 1171 1172 %s/autobuild.log 1173''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base) 1174 1175 if options.restrict_tests: 1176 text += """ 1177The build was restricted to tests matching %s\n""" % options.restrict_tests 1178 1179 if failed_task != 'rebase': 1180 text += ''' 1181You can see logs of the failed task here: 1182 1183 %s/%s.stdout 1184 %s/%s.stderr 1185 1186or you can get full logs of all tasks in this job here: 1187 1188 %s/logs.tar.gz 1189 1190The top commit for the tree that was built was: 1191 1192%s 1193 1194''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg) 1195 1196 if add_log_tail: 1197 f = open("%s/%s.stdout" % (gitroot, failed_tag), 'r') 1198 lines = f.readlines() 1199 log_tail = "".join(lines[-50:]) 1200 num_lines = len(lines) 1201 if num_lines < 50: 1202 # Also include stderr (compile failures) if < 50 lines of stdout 1203 f = open("%s/%s.stderr" % (gitroot, failed_tag), 'r') 1204 log_tail += "".join(f.readlines()[-(50 - num_lines):]) 1205 1206 text += ''' 1207The last 50 lines of log messages: 1208 1209%s 1210 ''' % log_tail 1211 f.close() 1212 1213 logs = os.path.join(gitroot, 'logs.tar.gz') 1214 send_email('autobuild[%s] failure on %s for task %s during %s' 1215 % (options.branch, platform.node(), failed_task, failed_stage), 1216 text, logs) 1217 1218 1219def email_success(elapsed_time, log_base=None): 1220 '''send an email to options.email about a successful build''' 1221 if log_base is None: 1222 log_base = gitroot 1223 text = ''' 1224Dear Developer, 1225 1226Your autobuild on %s has succeeded after %.1f minutes. 1227 1228''' % (platform.node(), elapsed_time / 60.) 1229 1230 if options.restrict_tests: 1231 text += """ 1232The build was restricted to tests matching %s\n""" % options.restrict_tests 1233 1234 if options.keeplogs: 1235 text += ''' 1236 1237you can get full logs of all tasks in this job here: 1238 1239 %s/logs.tar.gz 1240 1241''' % log_base 1242 1243 text += ''' 1244The top commit for the tree that was built was: 1245 1246%s 1247''' % top_commit_msg 1248 1249 logs = os.path.join(gitroot, 'logs.tar.gz') 1250 send_email('autobuild[%s] success on %s' % (options.branch, platform.node()), 1251 text, logs) 1252 1253 1254# get the top commit message, for emails 1255top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True) 1256 1257try: 1258 os.makedirs(testbase) 1259except Exception as reason: 1260 raise Exception("Unable to create %s : %s" % (testbase, reason)) 1261cleanup_list.append(testbase) 1262 1263if options.daemon: 1264 logfile = os.path.join(testbase, "log") 1265 do_print("Forking into the background, writing progress to %s" % logfile) 1266 daemonize(logfile) 1267 1268write_pidfile(gitroot + "/autobuild.pid") 1269 1270start_time = time.time() 1271 1272while True: 1273 try: 1274 run_cmd("rm -rf %s" % test_tmpdir, show=True) 1275 os.makedirs(test_tmpdir) 1276 # The waf uninstall code removes empty directories all the way 1277 # up the tree. Creating a file in test_tmpdir stops it from 1278 # being removed. 1279 run_cmd("touch %s" % os.path.join(test_tmpdir, 1280 ".directory-is-not-empty"), show=True) 1281 run_cmd("stat %s" % test_tmpdir, show=True) 1282 run_cmd("stat %s" % testbase, show=True) 1283 run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot) 1284 except Exception: 1285 cleanup() 1286 raise 1287 1288 try: 1289 if options.rebase is not None: 1290 rebase_tree(options.rebase, rebase_branch=options.branch) 1291 except Exception: 1292 cleanup_list.append(gitroot + "/autobuild.pid") 1293 cleanup() 1294 elapsed_time = time.time() - start_time 1295 email_failure(-1, 'rebase', 'rebase', 'rebase', 1296 'rebase on %s failed' % options.branch, 1297 elapsed_time, log_base=options.log_base) 1298 sys.exit(1) 1299 1300 try: 1301 blist = buildlist(args, options.rebase, rebase_branch=options.branch) 1302 if options.tail: 1303 blist.start_tail() 1304 (status, failed_task, failed_stage, failed_tag, errstr) = blist.run() 1305 if status != 0 or errstr != "retry": 1306 break 1307 cleanup(do_raise=True) 1308 except Exception: 1309 cleanup() 1310 raise 1311 1312cleanup_list.append(gitroot + "/autobuild.pid") 1313 1314do_print(errstr) 1315 1316blist.kill_kids() 1317if options.tail: 1318 do_print("waiting for tail to flush") 1319 time.sleep(1) 1320 1321elapsed_time = time.time() - start_time 1322if status == 0: 1323 if options.passcmd is not None: 1324 do_print("Running passcmd: %s" % options.passcmd) 1325 run_cmd(options.passcmd, dir=test_master) 1326 if options.pushto is not None: 1327 push_to(options.pushto, push_branch=options.branch) 1328 if options.keeplogs or options.attach_logs: 1329 blist.tarlogs("logs.tar.gz") 1330 do_print("Logs in logs.tar.gz") 1331 if options.always_email: 1332 email_success(elapsed_time, log_base=options.log_base) 1333 blist.remove_logs() 1334 cleanup() 1335 do_print(errstr) 1336 sys.exit(0) 1337 1338# something failed, gather a tar of the logs 1339blist.tarlogs("logs.tar.gz") 1340 1341if options.email is not None: 1342 email_failure(status, failed_task, failed_stage, failed_tag, errstr, 1343 elapsed_time, log_base=options.log_base) 1344else: 1345 elapsed_minutes = elapsed_time / 60.0 1346 print(''' 1347 1348#################################################################### 1349 1350AUTOBUILD FAILURE 1351 1352Your autobuild[%s] on %s failed after %.1f minutes 1353when trying to test %s with the following error: 1354 1355 %s 1356 1357the autobuild has been abandoned. Please fix the error and resubmit. 1358 1359#################################################################### 1360 1361''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr)) 1362 1363cleanup() 1364do_print(errstr) 1365do_print("Logs in logs.tar.gz") 1366sys.exit(status) 1367