1# Things that would be nice:
2# - less hard-coding of paths here
3
4import argparse
5import copy
6import errno
7import json
8import os.path
9import socket
10import subprocess
11import sys
12import time
13from typing import Dict, List, Tuple
14
15PORT = 8642
16
17CLIENT_CONFIG = {
18    "options": {"failByDrop": False},
19    "outdir": "./reports/servers",
20    "servers": [
21        {
22            "agent": "wsproto",
23            "url": f"ws://localhost:{PORT}",
24            "options": {"version": 18},
25        }
26    ],
27    "cases": ["*"],
28    "exclude-cases": ["13.3.*", "13.5.*", "13.7.*"],
29    "exclude-agent-cases": {},
30}
31
32SERVER_CONFIG = {
33    "url": f"ws://localhost:{PORT}",
34    "options": {"failByDrop": False},
35    "outdir": "./reports/clients",
36    "webport": 8080,
37    "cases": ["*"],
38    "exclude-cases": ["13.3.*", "13.5.*", "13.7.*"],
39    "exclude-agent-cases": {},
40}
41
42CASES = {
43    "all": ["*"],
44    "fast": [
45        # The core functionality tests
46        *[f"{i}.*" for i in range(1, 12)],
47        # Compression tests -- in each section, the tests get progressively
48        # slower until they're taking 10s of seconds apiece. And it's
49        # mostly stress tests, without much extra coverage to show for
50        # it. (Weird trick: autobahntestsuite treats these as regexps
51        # except that . is quoted and * becomes .*)
52        "12.*.[1234]$",
53        "13.*.[1234]$",
54        # At one point these were catching a unique bug that none of the
55        # above were -- they're relatively quick and involve
56        # fragmentation.
57        "12.1.11",
58        "12.1.12",
59        "13.1.11",
60        "13.1.12",
61    ],
62}
63
64
65def say(*args: object) -> None:
66    print("run-autobahn-tests.py:", *args)
67
68
69def setup_venv() -> None:
70    if not os.path.exists("autobahntestsuite-venv"):
71        say("Creating Python 2.7 environment and installing autobahntestsuite")
72        subprocess.check_call(
73            ["virtualenv", "-p", "python2.7", "autobahntestsuite-venv"]
74        )
75        subprocess.check_call(
76            ["autobahntestsuite-venv/bin/pip", "install", "autobahntestsuite>=0.8.0"]
77        )
78
79
80def wait_for_listener(port: int) -> None:
81    while True:
82        sock = socket.socket()
83        try:
84            sock.connect(("localhost", port))
85        except OSError as exc:
86            if exc.errno == errno.ECONNREFUSED:
87                time.sleep(0.01)
88            else:
89                raise
90        else:
91            return
92        finally:
93            sock.close()
94
95
96def coverage(command: List[str], coverage_settings: Dict[str, str]) -> List[str]:
97    if not coverage_settings["enabled"]:
98        return [sys.executable] + command
99
100    return [
101        sys.executable,
102        "-m",
103        "coverage",
104        "run",
105        "--include",
106        coverage_settings["wsproto-path"],
107    ] + command
108
109
110def summarize(report_path: str) -> Tuple[int, int]:
111    with open(os.path.join(report_path, "index.json")) as f:
112        result_summary = json.load(f)["wsproto"]
113    failed = 0
114    total = 0
115    PASS = {"OK", "INFORMATIONAL"}
116    for test_name, results in sorted(result_summary.items()):
117        total += 1
118        if results["behavior"] not in PASS or results["behaviorClose"] not in PASS:
119            say("FAIL:", test_name, results)
120            say("Details:")
121            with open(os.path.join(report_path, results["reportfile"])) as f:
122                print(f.read())
123            failed += 1
124
125    speed_ordered = sorted(result_summary.items(), key=lambda kv: -kv[1]["duration"])
126    say("Slowest tests:")
127    for test_name, results in speed_ordered[:5]:
128        say("    {}: {} seconds".format(test_name, results["duration"] / 1000))
129
130    return failed, total
131
132
133def run_client_tests(
134    cases: List[str], coverage_settings: Dict[str, str]
135) -> Tuple[int, int]:
136    say("Starting autobahntestsuite server")
137    server_config = copy.deepcopy(SERVER_CONFIG)
138    server_config["cases"] = cases
139    with open("auto-tests-server-config.json", "w") as f:
140        json.dump(server_config, f)
141    server = subprocess.Popen(
142        [
143            "autobahntestsuite-venv/bin/wstest",
144            "-m",
145            "fuzzingserver",
146            "-s",
147            "auto-tests-server-config.json",
148        ]
149    )
150    say("Waiting for server to start")
151    wait_for_listener(PORT)
152    try:
153        say("Running wsproto test client")
154        subprocess.check_call(coverage(["./test_client.py"], coverage_settings))
155        # the client doesn't exit until the server closes the connection on the
156        # /updateReports call, and the server doesn't close the connection until
157        # after it writes the reports, so there's no race condition here.
158    finally:
159        say("Stopping server...")
160        server.terminate()
161        server.wait()
162
163    return summarize("reports/clients")
164
165
166def run_server_tests(
167    cases: List[str], coverage_settings: Dict[str, str]
168) -> Tuple[int, int]:
169    say("Starting wsproto test server")
170    server = subprocess.Popen(coverage(["./test_server.py"], coverage_settings))
171    try:
172        say("Waiting for server to start")
173        wait_for_listener(PORT)
174
175        client_config = copy.deepcopy(CLIENT_CONFIG)
176        client_config["cases"] = cases
177        with open("auto-tests-client-config.json", "w") as f:
178            json.dump(client_config, f)
179        say("Starting autobahntestsuite client")
180        subprocess.check_call(
181            [
182                "autobahntestsuite-venv/bin/wstest",
183                "-m",
184                "fuzzingclient",
185                "-s",
186                "auto-tests-client-config.json",
187            ]
188        )
189    finally:
190        say("Stopping server...")
191        # Connection on this port triggers a shutdown
192        sock = socket.socket()
193        sock.connect(("localhost", PORT + 1))
194        sock.close()
195        server.wait()
196
197    return summarize("reports/servers")
198
199
200def main() -> None:
201    if not os.path.exists("test_client.py"):
202        say("Run me from the compliance/ directory")
203        sys.exit(2)
204    coverage_settings = {"coveragerc": "../.coveragerc"}
205    try:
206        import wsproto  # pylint: disable=import-outside-toplevel
207    except ImportError:
208        say("wsproto must be on python path -- set PYTHONPATH or install it")
209        sys.exit(2)
210    else:
211        coverage_settings["wsproto-path"] = os.path.dirname(wsproto.__file__)
212
213    parser = argparse.ArgumentParser()
214
215    parser.add_argument("MODE", help="'client' or 'server'")
216    # can do e.g.
217    #   --cases='["1.*"]'
218    parser.add_argument(
219        "--cases", help="'fast' or 'all' or a JSON list", default="fast"
220    )
221    parser.add_argument("--cov", help="enable coverage", action="store_true")
222
223    args = parser.parse_args()
224
225    coverage_settings["enabled"] = args.cov
226    cases = args.cases
227    # pylint: disable=consider-using-get
228    if cases in CASES:
229        cases = CASES[cases]
230    else:
231        cases = json.loads(cases)
232
233    setup_venv()
234
235    if args.MODE == "client":
236        failed, total = run_client_tests(cases, coverage_settings)
237    elif args.MODE == "server":
238        failed, total = run_server_tests(cases, coverage_settings)
239    else:
240        say("Unrecognized mode, try 'client' or 'server'")
241        sys.exit(2)
242
243    say(f"in {args.MODE.upper()} mode: failed {failed} out of {total} total")
244
245    if failed:
246        say("Test failed")
247        sys.exit(1)
248    else:
249        say("SUCCESS!")
250
251
252if __name__ == "__main__":
253    main()
254