1# Things that would be nice: 2# - less hard-coding of paths here 3 4import argparse 5import copy 6import errno 7import json 8import os.path 9import socket 10import subprocess 11import sys 12import time 13from typing import Dict, List, Tuple 14 15PORT = 8642 16 17CLIENT_CONFIG = { 18 "options": {"failByDrop": False}, 19 "outdir": "./reports/servers", 20 "servers": [ 21 { 22 "agent": "wsproto", 23 "url": f"ws://localhost:{PORT}", 24 "options": {"version": 18}, 25 } 26 ], 27 "cases": ["*"], 28 "exclude-cases": ["13.3.*", "13.5.*", "13.7.*"], 29 "exclude-agent-cases": {}, 30} 31 32SERVER_CONFIG = { 33 "url": f"ws://localhost:{PORT}", 34 "options": {"failByDrop": False}, 35 "outdir": "./reports/clients", 36 "webport": 8080, 37 "cases": ["*"], 38 "exclude-cases": ["13.3.*", "13.5.*", "13.7.*"], 39 "exclude-agent-cases": {}, 40} 41 42CASES = { 43 "all": ["*"], 44 "fast": [ 45 # The core functionality tests 46 *[f"{i}.*" for i in range(1, 12)], 47 # Compression tests -- in each section, the tests get progressively 48 # slower until they're taking 10s of seconds apiece. And it's 49 # mostly stress tests, without much extra coverage to show for 50 # it. (Weird trick: autobahntestsuite treats these as regexps 51 # except that . is quoted and * becomes .*) 52 "12.*.[1234]$", 53 "13.*.[1234]$", 54 # At one point these were catching a unique bug that none of the 55 # above were -- they're relatively quick and involve 56 # fragmentation. 57 "12.1.11", 58 "12.1.12", 59 "13.1.11", 60 "13.1.12", 61 ], 62} 63 64 65def say(*args: object) -> None: 66 print("run-autobahn-tests.py:", *args) 67 68 69def setup_venv() -> None: 70 if not os.path.exists("autobahntestsuite-venv"): 71 say("Creating Python 2.7 environment and installing autobahntestsuite") 72 subprocess.check_call( 73 ["virtualenv", "-p", "python2.7", "autobahntestsuite-venv"] 74 ) 75 subprocess.check_call( 76 ["autobahntestsuite-venv/bin/pip", "install", "autobahntestsuite>=0.8.0"] 77 ) 78 79 80def wait_for_listener(port: int) -> None: 81 while True: 82 sock = socket.socket() 83 try: 84 sock.connect(("localhost", port)) 85 except OSError as exc: 86 if exc.errno == errno.ECONNREFUSED: 87 time.sleep(0.01) 88 else: 89 raise 90 else: 91 return 92 finally: 93 sock.close() 94 95 96def coverage(command: List[str], coverage_settings: Dict[str, str]) -> List[str]: 97 if not coverage_settings["enabled"]: 98 return [sys.executable] + command 99 100 return [ 101 sys.executable, 102 "-m", 103 "coverage", 104 "run", 105 "--include", 106 coverage_settings["wsproto-path"], 107 ] + command 108 109 110def summarize(report_path: str) -> Tuple[int, int]: 111 with open(os.path.join(report_path, "index.json")) as f: 112 result_summary = json.load(f)["wsproto"] 113 failed = 0 114 total = 0 115 PASS = {"OK", "INFORMATIONAL"} 116 for test_name, results in sorted(result_summary.items()): 117 total += 1 118 if results["behavior"] not in PASS or results["behaviorClose"] not in PASS: 119 say("FAIL:", test_name, results) 120 say("Details:") 121 with open(os.path.join(report_path, results["reportfile"])) as f: 122 print(f.read()) 123 failed += 1 124 125 speed_ordered = sorted(result_summary.items(), key=lambda kv: -kv[1]["duration"]) 126 say("Slowest tests:") 127 for test_name, results in speed_ordered[:5]: 128 say(" {}: {} seconds".format(test_name, results["duration"] / 1000)) 129 130 return failed, total 131 132 133def run_client_tests( 134 cases: List[str], coverage_settings: Dict[str, str] 135) -> Tuple[int, int]: 136 say("Starting autobahntestsuite server") 137 server_config = copy.deepcopy(SERVER_CONFIG) 138 server_config["cases"] = cases 139 with open("auto-tests-server-config.json", "w") as f: 140 json.dump(server_config, f) 141 server = subprocess.Popen( 142 [ 143 "autobahntestsuite-venv/bin/wstest", 144 "-m", 145 "fuzzingserver", 146 "-s", 147 "auto-tests-server-config.json", 148 ] 149 ) 150 say("Waiting for server to start") 151 wait_for_listener(PORT) 152 try: 153 say("Running wsproto test client") 154 subprocess.check_call(coverage(["./test_client.py"], coverage_settings)) 155 # the client doesn't exit until the server closes the connection on the 156 # /updateReports call, and the server doesn't close the connection until 157 # after it writes the reports, so there's no race condition here. 158 finally: 159 say("Stopping server...") 160 server.terminate() 161 server.wait() 162 163 return summarize("reports/clients") 164 165 166def run_server_tests( 167 cases: List[str], coverage_settings: Dict[str, str] 168) -> Tuple[int, int]: 169 say("Starting wsproto test server") 170 server = subprocess.Popen(coverage(["./test_server.py"], coverage_settings)) 171 try: 172 say("Waiting for server to start") 173 wait_for_listener(PORT) 174 175 client_config = copy.deepcopy(CLIENT_CONFIG) 176 client_config["cases"] = cases 177 with open("auto-tests-client-config.json", "w") as f: 178 json.dump(client_config, f) 179 say("Starting autobahntestsuite client") 180 subprocess.check_call( 181 [ 182 "autobahntestsuite-venv/bin/wstest", 183 "-m", 184 "fuzzingclient", 185 "-s", 186 "auto-tests-client-config.json", 187 ] 188 ) 189 finally: 190 say("Stopping server...") 191 # Connection on this port triggers a shutdown 192 sock = socket.socket() 193 sock.connect(("localhost", PORT + 1)) 194 sock.close() 195 server.wait() 196 197 return summarize("reports/servers") 198 199 200def main() -> None: 201 if not os.path.exists("test_client.py"): 202 say("Run me from the compliance/ directory") 203 sys.exit(2) 204 coverage_settings = {"coveragerc": "../.coveragerc"} 205 try: 206 import wsproto # pylint: disable=import-outside-toplevel 207 except ImportError: 208 say("wsproto must be on python path -- set PYTHONPATH or install it") 209 sys.exit(2) 210 else: 211 coverage_settings["wsproto-path"] = os.path.dirname(wsproto.__file__) 212 213 parser = argparse.ArgumentParser() 214 215 parser.add_argument("MODE", help="'client' or 'server'") 216 # can do e.g. 217 # --cases='["1.*"]' 218 parser.add_argument( 219 "--cases", help="'fast' or 'all' or a JSON list", default="fast" 220 ) 221 parser.add_argument("--cov", help="enable coverage", action="store_true") 222 223 args = parser.parse_args() 224 225 coverage_settings["enabled"] = args.cov 226 cases = args.cases 227 # pylint: disable=consider-using-get 228 if cases in CASES: 229 cases = CASES[cases] 230 else: 231 cases = json.loads(cases) 232 233 setup_venv() 234 235 if args.MODE == "client": 236 failed, total = run_client_tests(cases, coverage_settings) 237 elif args.MODE == "server": 238 failed, total = run_server_tests(cases, coverage_settings) 239 else: 240 say("Unrecognized mode, try 'client' or 'server'") 241 sys.exit(2) 242 243 say(f"in {args.MODE.upper()} mode: failed {failed} out of {total} total") 244 245 if failed: 246 say("Test failed") 247 sys.exit(1) 248 else: 249 say("SUCCESS!") 250 251 252if __name__ == "__main__": 253 main() 254