1#!/usr/bin/env python3 2# 3# Test case executor 4# Copyright (c) 2013-2019, Jouni Malinen <j@w1.fi> 5# 6# This software may be distributed under the terms of the BSD license. 7# See README for more details. 8 9import os 10import re 11import sys 12import time 13from datetime import datetime 14import argparse 15import subprocess 16import termios 17 18import logging 19logger = logging.getLogger() 20 21try: 22 import sqlite3 23 sqlite3_imported = True 24except ImportError: 25 sqlite3_imported = False 26 27scriptsdir = os.path.dirname(os.path.realpath(sys.modules[__name__].__file__)) 28sys.path.append(os.path.join(scriptsdir, '..', '..', 'wpaspy')) 29 30from wpasupplicant import WpaSupplicant 31from hostapd import HostapdGlobal 32from check_kernel import check_kernel 33from wlantest import Wlantest 34from utils import HwsimSkip 35 36def set_term_echo(fd, enabled): 37 [iflag, oflag, cflag, lflag, ispeed, ospeed, cc] = termios.tcgetattr(fd) 38 if enabled: 39 lflag |= termios.ECHO 40 else: 41 lflag &= ~termios.ECHO 42 termios.tcsetattr(fd, termios.TCSANOW, 43 [iflag, oflag, cflag, lflag, ispeed, ospeed, cc]) 44 45def reset_devs(dev, apdev): 46 ok = True 47 for d in dev: 48 try: 49 d.reset() 50 except Exception as e: 51 logger.info("Failed to reset device " + d.ifname) 52 print(str(e)) 53 ok = False 54 55 wpas = None 56 try: 57 wpas = WpaSupplicant(global_iface='/tmp/wpas-wlan5', monitor=False) 58 ifaces = wpas.global_request("INTERFACES").splitlines() 59 for iface in ifaces: 60 if iface.startswith("wlan"): 61 wpas.interface_remove(iface) 62 except Exception as e: 63 pass 64 if wpas: 65 wpas.close_ctrl() 66 del wpas 67 68 try: 69 hapd = HostapdGlobal() 70 hapd.flush() 71 hapd.remove('wlan3-6') 72 hapd.remove('wlan3-5') 73 hapd.remove('wlan3-4') 74 hapd.remove('wlan3-3') 75 hapd.remove('wlan3-2') 76 for ap in apdev: 77 hapd.remove(ap['ifname']) 78 hapd.remove('as-erp') 79 except Exception as e: 80 logger.info("Failed to remove hostapd interface") 81 print(str(e)) 82 ok = False 83 return ok 84 85def add_log_file(conn, test, run, type, path): 86 if not os.path.exists(path): 87 return 88 contents = None 89 with open(path, 'rb') as f: 90 contents = f.read() 91 if contents is None: 92 return 93 sql = "INSERT INTO logs(test,run,type,contents) VALUES(?, ?, ?, ?)" 94 params = (test, run, type, sqlite3.Binary(contents)) 95 try: 96 conn.execute(sql, params) 97 conn.commit() 98 except Exception as e: 99 print("sqlite: " + str(e)) 100 print("sql: %r" % (params, )) 101 102def report(conn, prefill, build, commit, run, test, result, duration, logdir, 103 sql_commit=True): 104 if conn: 105 if not build: 106 build = '' 107 if not commit: 108 commit = '' 109 if prefill: 110 conn.execute('DELETE FROM results WHERE test=? AND run=? AND result=?', (test, run, 'NOTRUN')) 111 sql = "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES(?, ?, ?, ?, ?, ?, ?)" 112 params = (test, result, run, time.time(), duration, build, commit) 113 try: 114 conn.execute(sql, params) 115 if sql_commit: 116 conn.commit() 117 except Exception as e: 118 print("sqlite: " + str(e)) 119 print("sql: %r" % (params, )) 120 121 if result == "FAIL": 122 for log in ["log", "log0", "log1", "log2", "log3", "log5", 123 "hostapd", "dmesg", "hwsim0", "hwsim0.pcapng"]: 124 add_log_file(conn, test, run, log, 125 logdir + "/" + test + "." + log) 126 127class DataCollector(object): 128 def __init__(self, logdir, testname, args): 129 self._logdir = logdir 130 self._testname = testname 131 self._tracing = args.tracing 132 self._dmesg = args.dmesg 133 self._dbus = args.dbus 134 def __enter__(self): 135 if self._tracing: 136 output = os.path.abspath(os.path.join(self._logdir, '%s.dat' % (self._testname, ))) 137 self._trace_cmd = subprocess.Popen(['trace-cmd', 'record', '-o', output, '-e', 'mac80211', '-e', 'cfg80211', '-e', 'printk', 'sh', '-c', 'echo STARTED ; read l'], 138 stdin=subprocess.PIPE, 139 stdout=subprocess.PIPE, 140 stderr=open('/dev/null', 'w'), 141 cwd=self._logdir) 142 l = self._trace_cmd.stdout.read(7) 143 while self._trace_cmd.poll() is None and b'STARTED' not in l: 144 l += self._trace_cmd.stdout.read(1) 145 res = self._trace_cmd.returncode 146 if res: 147 print("Failed calling trace-cmd: returned exit status %d" % res) 148 sys.exit(1) 149 if self._dbus: 150 output = os.path.abspath(os.path.join(self._logdir, '%s.dbus' % (self._testname, ))) 151 self._dbus_cmd = subprocess.Popen(['dbus-monitor', '--system'], 152 stdout=open(output, 'w'), 153 stderr=open('/dev/null', 'w'), 154 cwd=self._logdir) 155 res = self._dbus_cmd.returncode 156 if res: 157 print("Failed calling dbus-monitor: returned exit status %d" % res) 158 sys.exit(1) 159 def __exit__(self, type, value, traceback): 160 if self._tracing: 161 self._trace_cmd.stdin.write(b'DONE\n') 162 self._trace_cmd.stdin.flush() 163 self._trace_cmd.wait() 164 if self._dmesg: 165 output = os.path.join(self._logdir, '%s.dmesg' % (self._testname, )) 166 num = 0 167 while os.path.exists(output): 168 output = os.path.join(self._logdir, '%s.dmesg-%d' % (self._testname, num)) 169 num += 1 170 subprocess.call(['dmesg', '-c'], stdout=open(output, 'w')) 171 172def rename_log(logdir, basename, testname, dev): 173 try: 174 import getpass 175 srcname = os.path.join(logdir, basename) 176 dstname = os.path.join(logdir, testname + '.' + basename) 177 num = 0 178 while os.path.exists(dstname): 179 dstname = os.path.join(logdir, 180 testname + '.' + basename + '-' + str(num)) 181 num = num + 1 182 os.rename(srcname, dstname) 183 if dev: 184 dev.relog() 185 subprocess.call(['chown', '-f', getpass.getuser(), srcname]) 186 except Exception as e: 187 logger.info("Failed to rename log files") 188 logger.info(e) 189 190def is_long_duration_test(t): 191 return hasattr(t, "long_duration_test") and t.long_duration_test 192 193def get_test_description(t): 194 if t.__doc__ is None: 195 desc = "MISSING DESCRIPTION" 196 else: 197 desc = t.__doc__ 198 if is_long_duration_test(t): 199 desc += " [long]" 200 return desc 201 202def main(): 203 tests = [] 204 test_modules = [] 205 files = os.listdir(scriptsdir) 206 for t in files: 207 m = re.match(r'(test_.*)\.py$', t) 208 if m: 209 logger.debug("Import test cases from " + t) 210 mod = __import__(m.group(1)) 211 test_modules.append(mod.__name__.replace('test_', '', 1)) 212 for key, val in mod.__dict__.items(): 213 if key.startswith("test_"): 214 tests.append(val) 215 test_names = list(set([t.__name__.replace('test_', '', 1) for t in tests])) 216 217 run = None 218 219 parser = argparse.ArgumentParser(description='hwsim test runner') 220 parser.add_argument('--logdir', metavar='<directory>', 221 help='log output directory for all other options, ' + 222 'must be given if other log options are used') 223 group = parser.add_mutually_exclusive_group() 224 group.add_argument('-d', const=logging.DEBUG, action='store_const', 225 dest='loglevel', default=logging.INFO, 226 help="verbose debug output") 227 group.add_argument('-q', const=logging.WARNING, action='store_const', 228 dest='loglevel', help="be quiet") 229 230 parser.add_argument('-S', metavar='<sqlite3 db>', dest='database', 231 help='database to write results to') 232 parser.add_argument('--prefill-tests', action='store_true', dest='prefill', 233 help='prefill test database with NOTRUN before all tests') 234 parser.add_argument('--commit', metavar='<commit id>', 235 help='commit ID, only for database') 236 parser.add_argument('-b', metavar='<build>', dest='build', help='build ID') 237 parser.add_argument('-L', action='store_true', dest='update_tests_db', 238 help='List tests (and update descriptions in DB)') 239 parser.add_argument('-T', action='store_true', dest='tracing', 240 help='collect tracing per test case (in log directory)') 241 parser.add_argument('-D', action='store_true', dest='dmesg', 242 help='collect dmesg per test case (in log directory)') 243 parser.add_argument('--dbus', action='store_true', dest='dbus', 244 help='collect dbus per test case (in log directory)') 245 parser.add_argument('--shuffle-tests', action='store_true', 246 dest='shuffle_tests', 247 help='Shuffle test cases to randomize order') 248 parser.add_argument('--split', help='split tests for parallel execution (<server number>/<total servers>)') 249 parser.add_argument('--no-reset', action='store_true', dest='no_reset', 250 help='Do not reset devices at the end of the test') 251 parser.add_argument('--long', action='store_true', 252 help='Include test cases that take long time') 253 parser.add_argument('-f', dest='testmodules', metavar='<test module>', 254 help='execute only tests from these test modules', 255 type=str, choices=[[]] + test_modules, nargs='+') 256 parser.add_argument('-l', metavar='<modules file>', dest='mfile', 257 help='test modules file name') 258 parser.add_argument('-i', action='store_true', dest='stdin_ctrl', 259 help='stdin-controlled test case execution') 260 parser.add_argument('tests', metavar='<test>', nargs='*', type=str, 261 help='tests to run (only valid without -f)') 262 263 args = parser.parse_args() 264 265 if (args.tests and args.testmodules) or (args.tests and args.mfile) or (args.testmodules and args.mfile): 266 print('Invalid arguments - only one of (test, test modules, modules file) can be given.') 267 sys.exit(2) 268 269 if args.tests: 270 fail = False 271 for t in args.tests: 272 if t.endswith('*'): 273 prefix = t.rstrip('*') 274 found = False 275 for tn in test_names: 276 if tn.startswith(prefix): 277 found = True 278 break 279 if not found: 280 print('Invalid arguments - test "%s" wildcard did not match' % t) 281 fail = True 282 elif t not in test_names: 283 print('Invalid arguments - test "%s" not known' % t) 284 fail = True 285 if fail: 286 sys.exit(2) 287 288 if args.database: 289 if not sqlite3_imported: 290 print("No sqlite3 module found") 291 sys.exit(2) 292 conn = sqlite3.connect(args.database) 293 conn.execute('CREATE TABLE IF NOT EXISTS results (test,result,run,time,duration,build,commitid)') 294 conn.execute('CREATE TABLE IF NOT EXISTS tests (test,description)') 295 conn.execute('CREATE TABLE IF NOT EXISTS logs (test,run,type,contents)') 296 else: 297 conn = None 298 299 if conn: 300 run = int(time.time()) 301 302 # read the modules from the modules file 303 if args.mfile: 304 args.testmodules = [] 305 with open(args.mfile) as f: 306 for line in f.readlines(): 307 line = line.strip() 308 if not line or line.startswith('#'): 309 continue 310 args.testmodules.append(line) 311 312 tests_to_run = [] 313 if args.tests: 314 for selected in args.tests: 315 for t in tests: 316 name = t.__name__.replace('test_', '', 1) 317 if selected.endswith('*'): 318 prefix = selected.rstrip('*') 319 if name.startswith(prefix): 320 tests_to_run.append(t) 321 elif name == selected: 322 tests_to_run.append(t) 323 else: 324 for t in tests: 325 name = t.__name__.replace('test_', '', 1) 326 if args.testmodules: 327 if t.__module__.replace('test_', '', 1) not in args.testmodules: 328 continue 329 tests_to_run.append(t) 330 331 if args.update_tests_db: 332 for t in tests_to_run: 333 name = t.__name__.replace('test_', '', 1) 334 print(name + " - " + get_test_description(t)) 335 if conn: 336 sql = 'INSERT OR REPLACE INTO tests(test,description) VALUES (?, ?)' 337 params = (name, get_test_description(t)) 338 try: 339 conn.execute(sql, params) 340 except Exception as e: 341 print("sqlite: " + str(e)) 342 print("sql: %r" % (params,)) 343 if conn: 344 conn.commit() 345 conn.close() 346 sys.exit(0) 347 348 if not args.logdir: 349 if os.path.exists('logs/current'): 350 args.logdir = 'logs/current' 351 else: 352 args.logdir = 'logs' 353 354 # Write debug level log to a file and configurable verbosity to stdout 355 logger.setLevel(logging.DEBUG) 356 357 stdout_handler = logging.StreamHandler() 358 stdout_handler.setLevel(args.loglevel) 359 logger.addHandler(stdout_handler) 360 361 file_name = os.path.join(args.logdir, 'run-tests.log') 362 log_handler = logging.FileHandler(file_name, encoding='utf-8') 363 log_handler.setLevel(logging.DEBUG) 364 fmt = "%(asctime)s %(levelname)s %(message)s" 365 log_formatter = logging.Formatter(fmt) 366 log_handler.setFormatter(log_formatter) 367 logger.addHandler(log_handler) 368 369 dev0 = WpaSupplicant('wlan0', '/tmp/wpas-wlan0') 370 dev1 = WpaSupplicant('wlan1', '/tmp/wpas-wlan1') 371 dev2 = WpaSupplicant('wlan2', '/tmp/wpas-wlan2') 372 dev = [dev0, dev1, dev2] 373 apdev = [] 374 apdev.append({"ifname": 'wlan3', "bssid": "02:00:00:00:03:00"}) 375 apdev.append({"ifname": 'wlan4', "bssid": "02:00:00:00:04:00"}) 376 377 for d in dev: 378 if not d.ping(): 379 logger.info(d.ifname + ": No response from wpa_supplicant") 380 return 381 logger.info("DEV: " + d.ifname + ": " + d.p2p_dev_addr()) 382 for ap in apdev: 383 logger.info("APDEV: " + ap['ifname']) 384 385 passed = [] 386 skipped = [] 387 failed = [] 388 389 # make sure nothing is left over from previous runs 390 # (if there were any other manual runs or we crashed) 391 if not reset_devs(dev, apdev): 392 if conn: 393 conn.close() 394 conn = None 395 sys.exit(1) 396 397 if args.dmesg: 398 subprocess.call(['dmesg', '-c'], stdout=open('/dev/null', 'w')) 399 400 if conn and args.prefill: 401 for t in tests_to_run: 402 name = t.__name__.replace('test_', '', 1) 403 report(conn, False, args.build, args.commit, run, name, 'NOTRUN', 0, 404 args.logdir, sql_commit=False) 405 conn.commit() 406 407 if args.split: 408 vals = args.split.split('/') 409 split_server = int(vals[0]) 410 split_total = int(vals[1]) 411 logger.info("Parallel execution - %d/%d" % (split_server, split_total)) 412 split_server -= 1 413 tests_to_run.sort(key=lambda t: t.__name__) 414 tests_to_run = [x for i, x in enumerate(tests_to_run) if i % split_total == split_server] 415 416 if args.shuffle_tests: 417 from random import shuffle 418 shuffle(tests_to_run) 419 420 count = 0 421 if args.stdin_ctrl: 422 print("READY") 423 sys.stdout.flush() 424 num_tests = 0 425 else: 426 num_tests = len(tests_to_run) 427 if args.stdin_ctrl: 428 set_term_echo(sys.stdin.fileno(), False) 429 430 check_country_00 = True 431 for d in dev: 432 if d.get_driver_status_field("country") != "00": 433 check_country_00 = False 434 435 while True: 436 if args.stdin_ctrl: 437 test = sys.stdin.readline() 438 if not test: 439 break 440 test = test.splitlines()[0] 441 if test == '': 442 break 443 t = None 444 for tt in tests: 445 name = tt.__name__.replace('test_', '', 1) 446 if name == test: 447 t = tt 448 break 449 if not t: 450 print("NOT-FOUND") 451 sys.stdout.flush() 452 continue 453 else: 454 if len(tests_to_run) == 0: 455 break 456 t = tests_to_run.pop(0) 457 458 if dev[0].get_driver_status_field("country") == "98": 459 # Work around cfg80211 regulatory issues in clearing intersected 460 # country code 98. Need to make station disconnect without any 461 # other wiphy being active in the system. 462 logger.info("country=98 workaround - try to clear state") 463 id = dev[1].add_network() 464 dev[1].set_network(id, "mode", "2") 465 dev[1].set_network_quoted(id, "ssid", "country98") 466 dev[1].set_network(id, "key_mgmt", "NONE") 467 dev[1].set_network(id, "frequency", "2412") 468 dev[1].set_network(id, "scan_freq", "2412") 469 dev[1].select_network(id) 470 ev = dev[1].wait_event(["CTRL-EVENT-CONNECTED"]) 471 if ev: 472 dev[0].connect("country98", key_mgmt="NONE", scan_freq="2412") 473 dev[1].request("DISCONNECT") 474 dev[0].wait_disconnected() 475 dev[0].disconnect_and_stop_scan() 476 dev[0].reset() 477 dev[1].reset() 478 dev[0].dump_monitor() 479 dev[1].dump_monitor() 480 481 name = t.__name__.replace('test_', '', 1) 482 open('/dev/kmsg', 'w').write('running hwsim test case %s\n' % name) 483 if log_handler: 484 log_handler.stream.close() 485 logger.removeHandler(log_handler) 486 file_name = os.path.join(args.logdir, name + '.log') 487 log_handler = logging.FileHandler(file_name, encoding='utf-8') 488 log_handler.setLevel(logging.DEBUG) 489 log_handler.setFormatter(log_formatter) 490 logger.addHandler(log_handler) 491 492 reset_ok = True 493 with DataCollector(args.logdir, name, args): 494 count = count + 1 495 msg = "START {} {}/{}".format(name, count, num_tests) 496 logger.info(msg) 497 if args.loglevel == logging.WARNING: 498 print(msg) 499 sys.stdout.flush() 500 if t.__doc__: 501 logger.info("Test: " + t.__doc__) 502 start = datetime.now() 503 open('/dev/kmsg', 'w').write('TEST-START %s @%.6f\n' % (name, time.time())) 504 for d in dev: 505 try: 506 d.dump_monitor() 507 if not d.ping(): 508 raise Exception("PING failed for {}".format(d.ifname)) 509 if not d.global_ping(): 510 raise Exception("Global PING failed for {}".format(d.ifname)) 511 d.request("NOTE TEST-START " + name) 512 except Exception as e: 513 logger.info("Failed to issue TEST-START before " + name + " for " + d.ifname) 514 logger.info(e) 515 print("FAIL " + name + " - could not start test") 516 if conn: 517 conn.close() 518 conn = None 519 if args.stdin_ctrl: 520 set_term_echo(sys.stdin.fileno(), True) 521 sys.exit(1) 522 skip_reason = None 523 try: 524 if is_long_duration_test(t) and not args.long: 525 raise HwsimSkip("Skip test case with long duration due to --long not specified") 526 if t.__code__.co_argcount > 2: 527 params = {} 528 params['logdir'] = args.logdir 529 params['name'] = name 530 params['prefix'] = os.path.join(args.logdir, name) 531 t(dev, apdev, params) 532 elif t.__code__.co_argcount > 1: 533 t(dev, apdev) 534 else: 535 t(dev) 536 result = "PASS" 537 if check_country_00: 538 for d in dev: 539 country = d.get_driver_status_field("country") 540 if country is None: 541 logger.info(d.ifname + ": Could not fetch country code after the test case run") 542 elif country != "00": 543 d.dump_monitor() 544 logger.info(d.ifname + ": Country code not reset back to 00: is " + country) 545 print(d.ifname + ": Country code not reset back to 00: is " + country) 546 result = "FAIL" 547 548 # Try to wait for cfg80211 regulatory state to 549 # clear. 550 d.cmd_execute(['iw', 'reg', 'set', '00']) 551 for i in range(5): 552 time.sleep(1) 553 country = d.get_driver_status_field("country") 554 if country == "00": 555 break 556 if country == "00": 557 print(d.ifname + ": Country code cleared back to 00") 558 logger.info(d.ifname + ": Country code cleared back to 00") 559 else: 560 print("Country code remains set - expect following test cases to fail") 561 logger.info("Country code remains set - expect following test cases to fail") 562 break 563 except HwsimSkip as e: 564 logger.info("Skip test case: %s" % e) 565 skip_reason = e 566 result = "SKIP" 567 except NameError as e: 568 import traceback 569 logger.info(e) 570 traceback.print_exc() 571 result = "FAIL" 572 except Exception as e: 573 import traceback 574 logger.info(e) 575 traceback.print_exc() 576 if args.loglevel == logging.WARNING: 577 print("Exception: " + str(e)) 578 result = "FAIL" 579 open('/dev/kmsg', 'w').write('TEST-STOP %s @%.6f\n' % (name, time.time())) 580 for d in dev: 581 try: 582 d.dump_monitor() 583 d.request("NOTE TEST-STOP " + name) 584 except Exception as e: 585 logger.info("Failed to issue TEST-STOP after {} for {}".format(name, d.ifname)) 586 logger.info(e) 587 result = "FAIL" 588 if args.no_reset: 589 print("Leaving devices in current state") 590 else: 591 reset_ok = reset_devs(dev, apdev) 592 wpas = None 593 try: 594 wpas = WpaSupplicant(global_iface="/tmp/wpas-wlan5", 595 monitor=False) 596 rename_log(args.logdir, 'log5', name, wpas) 597 if not args.no_reset: 598 wpas.remove_ifname() 599 except Exception as e: 600 pass 601 if wpas: 602 wpas.close_ctrl() 603 del wpas 604 605 for i in range(0, 3): 606 rename_log(args.logdir, 'log' + str(i), name, dev[i]) 607 try: 608 hapd = HostapdGlobal() 609 except Exception as e: 610 print("Failed to connect to hostapd interface") 611 print(str(e)) 612 reset_ok = False 613 result = "FAIL" 614 hapd = None 615 rename_log(args.logdir, 'hostapd', name, hapd) 616 if hapd: 617 del hapd 618 hapd = None 619 620 # Use None here since this instance of Wlantest() will never be 621 # used for remote host hwsim tests on real hardware. 622 Wlantest.setup(None) 623 wt = Wlantest() 624 rename_log(args.logdir, 'hwsim0.pcapng', name, wt) 625 rename_log(args.logdir, 'hwsim0', name, wt) 626 if os.path.exists(os.path.join(args.logdir, 'fst-wpa_supplicant')): 627 rename_log(args.logdir, 'fst-wpa_supplicant', name, None) 628 if os.path.exists(os.path.join(args.logdir, 'fst-hostapd')): 629 rename_log(args.logdir, 'fst-hostapd', name, None) 630 if os.path.exists(os.path.join(args.logdir, 'wmediumd.log')): 631 rename_log(args.logdir, 'wmediumd.log', name, None) 632 633 end = datetime.now() 634 diff = end - start 635 636 if result == 'PASS' and args.dmesg: 637 if not check_kernel(os.path.join(args.logdir, name + '.dmesg')): 638 logger.info("Kernel issue found in dmesg - mark test failed") 639 result = 'FAIL' 640 641 if result == 'PASS': 642 passed.append(name) 643 elif result == 'SKIP': 644 skipped.append(name) 645 else: 646 failed.append(name) 647 648 report(conn, args.prefill, args.build, args.commit, run, name, result, 649 diff.total_seconds(), args.logdir) 650 result = "{} {} {} {}".format(result, name, diff.total_seconds(), end) 651 logger.info(result) 652 if args.loglevel == logging.WARNING: 653 print(result) 654 if skip_reason: 655 print("REASON", skip_reason) 656 sys.stdout.flush() 657 658 if not reset_ok: 659 print("Terminating early due to device reset failure") 660 break 661 if args.stdin_ctrl: 662 set_term_echo(sys.stdin.fileno(), True) 663 664 if log_handler: 665 log_handler.stream.close() 666 logger.removeHandler(log_handler) 667 file_name = os.path.join(args.logdir, 'run-tests.log') 668 log_handler = logging.FileHandler(file_name, encoding='utf-8') 669 log_handler.setLevel(logging.DEBUG) 670 log_handler.setFormatter(log_formatter) 671 logger.addHandler(log_handler) 672 673 if conn: 674 conn.close() 675 676 if len(failed): 677 logger.info("passed {} test case(s)".format(len(passed))) 678 logger.info("skipped {} test case(s)".format(len(skipped))) 679 logger.info("failed tests: " + ' '.join(failed)) 680 if args.loglevel == logging.WARNING: 681 print("failed tests: " + ' '.join(failed)) 682 sys.exit(1) 683 logger.info("passed all {} test case(s)".format(len(passed))) 684 if len(skipped): 685 logger.info("skipped {} test case(s)".format(len(skipped))) 686 if args.loglevel == logging.WARNING: 687 print("passed all {} test case(s)".format(len(passed))) 688 if len(skipped): 689 print("skipped {} test case(s)".format(len(skipped))) 690 691if __name__ == "__main__": 692 main() 693