1#!/usr/bin/env python3
2
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this
5# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7import runner
8import pytest
9
10
11def test_log_parser():
12    mock_url = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
13    mock_log = b'''
14[PERF] perf block start
15[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/index.html
16[PERF],navigationStart,1460358376
17[PERF],unloadEventStart,undefined
18[PERF],unloadEventEnd,undefined
19[PERF],redirectStart,undefined
20[PERF],redirectEnd,undefined
21[PERF],fetchStart,undefined
22[PERF],domainLookupStart,undefined
23[PERF],domainLookupEnd,undefined
24[PERF],connectStart,undefined
25[PERF],connectEnd,undefined
26[PERF],secureConnectionStart,undefined
27[PERF],requestStart,undefined
28[PERF],responseStart,undefined
29[PERF],responseEnd,undefined
30[PERF],domLoading,1460358376000
31[PERF],domInteractive,1460358388000
32[PERF],domContentLoadedEventStart,1460358388000
33[PERF],domContentLoadedEventEnd,1460358388000
34[PERF],domComplete,1460358389000
35[PERF],loadEventStart,undefined
36[PERF],loadEventEnd,undefined
37[PERF] perf block end
38Shutting down the Constellation after generating an output file or exit flag specified
39'''
40
41    expected = [{
42        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
43        "navigationStart": 1460358376,
44        "unloadEventStart": None,
45        "unloadEventEnd": None,
46        "redirectStart": None,
47        "redirectEnd": None,
48        "fetchStart": None,
49        "domainLookupStart": None,
50        "domainLookupEnd": None,
51        "connectStart": None,
52        "connectEnd": None,
53        "secureConnectionStart": None,
54        "requestStart": None,
55        "responseStart": None,
56        "responseEnd": None,
57        "domLoading": 1460358376000,
58        "domInteractive": 1460358388000,
59        "domContentLoadedEventStart": 1460358388000,
60        "domContentLoadedEventEnd": 1460358388000,
61        "domComplete": 1460358389000,
62        "loadEventStart": None,
63        "loadEventEnd": None
64    }]
65    result = runner.parse_log(mock_log, mock_url)
66    assert(expected == list(result))
67
68
69def test_log_parser_complex():
70    mock_log = b'''
71[PERF] perf block start
72[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/content.html
73[PERF],navigationStart,1460358300
74[PERF],unloadEventStart,undefined
75[PERF],unloadEventEnd,undefined
76[PERF],redirectStart,undefined
77[PERF],redirectEnd,undefined
78[PERF],fetchStart,undefined
79[PERF],domainLookupStart,undefined
80[PERF],domainLookupEnd,undefined
81[PERF],connectStart,undefined
82[PERF],connectEnd,undefined
83[PERF],secureConnectionStart,undefined
84[PERF],requestStart,undefined
85[PERF],responseStart,undefined
86[PERF],responseEnd,undefined
87[PERF],domLoading,1460358376000
88[PERF],domInteractive,1460358388000
89[PERF],domContentLoadedEventStart,1460358388000
90[PERF],domContentLoadedEventEnd,1460358388000
91[PERF],domComplete,1460358389000
92[PERF],loadEventStart,undefined
93[PERF],loadEventEnd,undefined
94[PERF] perf block end
95Some other js error logs here
96
97[PERF] perf block start
98[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/index.html
99[PERF],navigationStart,1460358376
100[PERF],unloadEventStart,undefined
101[PERF],unloadEventEnd,undefined
102[PERF],redirectStart,undefined
103[PERF],redirectEnd,undefined
104[PERF],fetchStart,undefined
105[PERF],domainLookupStart,undefined
106[PERF],domainLookupEnd,undefined
107[PERF],connectStart,undefined
108[PERF],connectEnd,undefined
109[PERF],secureConnectionStart,undefined
110[PERF],requestStart,undefined
111[PERF],responseStart,undefined
112[PERF],responseEnd,undefined
113[PERF],domLoading,1460358376000
114[PERF],domInteractive,1460358388000
115[PERF],domContentLoadedEventStart,1460358388000
116[PERF],domContentLoadedEventEnd,1460358388000
117[PERF],domComplete,1460358389000
118[PERF],loadEventStart,undefined
119[PERF],loadEventEnd,undefined
120[PERF] perf block end
121Shutting down the Constellation after generating an output file or exit flag specified
122'''
123    mock_url = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
124    expected = [{
125        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
126        "navigationStart": 1460358376,
127        "unloadEventStart": None,
128        "unloadEventEnd": None,
129        "redirectStart": None,
130        "redirectEnd": None,
131        "fetchStart": None,
132        "domainLookupStart": None,
133        "domainLookupEnd": None,
134        "connectStart": None,
135        "connectEnd": None,
136        "secureConnectionStart": None,
137        "requestStart": None,
138        "responseStart": None,
139        "responseEnd": None,
140        "domLoading": 1460358376000,
141        "domInteractive": 1460358388000,
142        "domContentLoadedEventStart": 1460358388000,
143        "domContentLoadedEventEnd": 1460358388000,
144        "domComplete": 1460358389000,
145        "loadEventStart": None,
146        "loadEventEnd": None
147    }]
148    result = runner.parse_log(mock_log, mock_url)
149    assert(expected == list(result))
150
151
152def test_log_parser_empty():
153    mock_log = b'''
154[PERF] perf block start
155[PERF]BROKEN!!!!!!!!!1
156[PERF]BROKEN!!!!!!!!!1
157[PERF]BROKEN!!!!!!!!!1
158[PERF]BROKEN!!!!!!!!!1
159[PERF]BROKEN!!!!!!!!!1
160[PERF] perf block end
161'''
162    mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
163
164    expected = [{
165        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
166        "title": "",
167        "navigationStart": 0,
168        "unloadEventStart": -1,
169        "unloadEventEnd": -1,
170        "redirectStart": -1,
171        "redirectEnd": -1,
172        "fetchStart": -1,
173        "domainLookupStart": -1,
174        "domainLookupEnd": -1,
175        "connectStart": -1,
176        "connectEnd": -1,
177        "secureConnectionStart": -1,
178        "requestStart": -1,
179        "responseStart": -1,
180        "responseEnd": -1,
181        "domLoading": -1,
182        "domInteractive": -1,
183        "domContentLoadedEventStart": -1,
184        "domContentLoadedEventEnd": -1,
185        "domComplete": -1,
186        "loadEventStart": -1,
187        "loadEventEnd": -1
188    }]
189    result = runner.parse_log(mock_log, mock_testcase)
190    assert(expected == list(result))
191
192
193def test_log_parser_error():
194    mock_log = b'Nothing here! Test failed!'
195    mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
196
197    expected = [{
198        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
199        "title": "",
200        "navigationStart": 0,
201        "unloadEventStart": -1,
202        "unloadEventEnd": -1,
203        "redirectStart": -1,
204        "redirectEnd": -1,
205        "fetchStart": -1,
206        "domainLookupStart": -1,
207        "domainLookupEnd": -1,
208        "connectStart": -1,
209        "connectEnd": -1,
210        "secureConnectionStart": -1,
211        "requestStart": -1,
212        "responseStart": -1,
213        "responseEnd": -1,
214        "domLoading": -1,
215        "domInteractive": -1,
216        "domContentLoadedEventStart": -1,
217        "domContentLoadedEventEnd": -1,
218        "domComplete": -1,
219        "loadEventStart": -1,
220        "loadEventEnd": -1
221    }]
222    result = runner.parse_log(mock_log, mock_testcase)
223    assert(expected == list(result))
224
225
226def test_log_parser_bad_testcase_name():
227    mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
228    # Notice the testcase is about:blank, servo crashed
229    mock_log = b'''
230[PERF] perf block start
231[PERF],testcase,about:blank
232[PERF],navigationStart,1460358376
233[PERF],unloadEventStart,undefined
234[PERF],unloadEventEnd,undefined
235[PERF],redirectStart,undefined
236[PERF],redirectEnd,undefined
237[PERF],fetchStart,undefined
238[PERF],domainLookupStart,undefined
239[PERF],domainLookupEnd,undefined
240[PERF],connectStart,undefined
241[PERF],connectEnd,undefined
242[PERF],secureConnectionStart,undefined
243[PERF],requestStart,undefined
244[PERF],responseStart,undefined
245[PERF],responseEnd,undefined
246[PERF],domLoading,1460358376000
247[PERF],domInteractive,1460358388000
248[PERF],domContentLoadedEventStart,1460358388000
249[PERF],domContentLoadedEventEnd,1460358388000
250[PERF],domComplete,1460358389000
251[PERF],loadEventStart,undefined
252[PERF],loadEventEnd,undefined
253[PERF] perf block end
254Shutting down the Constellation after generating an output file or exit flag specified
255'''
256
257    expected = [{
258        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
259        "title": "",
260        "navigationStart": 0,
261        "unloadEventStart": -1,
262        "unloadEventEnd": -1,
263        "redirectStart": -1,
264        "redirectEnd": -1,
265        "fetchStart": -1,
266        "domainLookupStart": -1,
267        "domainLookupEnd": -1,
268        "connectStart": -1,
269        "connectEnd": -1,
270        "secureConnectionStart": -1,
271        "requestStart": -1,
272        "responseStart": -1,
273        "responseEnd": -1,
274        "domLoading": -1,
275        "domInteractive": -1,
276        "domContentLoadedEventStart": -1,
277        "domContentLoadedEventEnd": -1,
278        "domComplete": -1,
279        "loadEventStart": -1,
280        "loadEventEnd": -1
281    }]
282    result = runner.parse_log(mock_log, mock_testcase)
283    assert(expected == list(result))
284
285
286def test_manifest_loader():
287
288    text = '''
289http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html
290http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html
291
292http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html
293# Disabled! http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html
294'''
295    expected = [
296        ("http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html", False),
297        ("http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html", False),
298        ("http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html", False)
299    ]
300    assert(expected == list(runner.parse_manifest(text)))
301
302
303def test_manifest_loader_async():
304
305    text = '''
306http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html
307async http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html
308'''
309    expected = [
310        ("http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html", False),
311        ("http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html", True),
312    ]
313    assert(expected == list(runner.parse_manifest(text)))
314
315
316def test_filter_result_by_manifest():
317    input_json = [{
318        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/content.html",
319        "domComplete": 1460358389000,
320    }, {
321        "testcase": "non-existing-html",
322        "domComplete": 1460358389000,
323    }, {
324        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
325        "domComplete": 1460358389000,
326    }]
327
328    expected = [{
329        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
330        "domComplete": 1460358389000,
331    }]
332
333    manifest = [
334        ("http://localhost:8000/page_load_test/56.com/www.56.com/index.html", False)
335    ]
336
337    assert(expected == runner.filter_result_by_manifest(input_json, manifest))
338
339
340def test_filter_result_by_manifest_error():
341    input_json = [{
342        "testcase": "1.html",
343        "domComplete": 1460358389000,
344    }]
345
346    manifest = [
347        ("1.html", False),
348        ("2.html", False)
349    ]
350
351    with pytest.raises(Exception) as execinfo:
352        runner.filter_result_by_manifest(input_json, manifest)
353    assert "Missing test result" in str(execinfo.value)
354
355
356def test_take_result_median_odd():
357    input_json = [{
358        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
359        "domComplete": 1460358389001,
360        "domLoading": 1460358380002
361    }, {
362        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
363        "domComplete": 1460358389002,
364        "domLoading": 1460358380001
365    }, {
366        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
367        "domComplete": 1460358389003,
368        "domLoading": 1460358380003
369    }]
370
371    expected = [{
372        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
373        "domComplete": 1460358389002,
374        "domLoading": 1460358380002
375    }]
376
377    assert(expected == runner.take_result_median(input_json, len(input_json)))
378
379
380def test_take_result_median_even():
381    input_json = [{
382        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
383        "domComplete": 1460358389001,
384        "domLoading": 1460358380002
385    }, {
386        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
387        "domComplete": 1460358389002,
388        "domLoading": 1460358380001
389    }]
390
391    expected = [{
392        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
393        "domComplete": 1460358389001.5,
394        "domLoading": 1460358380001.5
395    }]
396
397    assert(expected == runner.take_result_median(input_json, len(input_json)))
398
399
400def test_take_result_median_error():
401    input_json = [{
402        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
403        "domComplete": None,
404        "domLoading": 1460358380002
405    }, {
406        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
407        "domComplete": 1460358389002,
408        "domLoading": 1460358380001
409    }]
410
411    expected = [{
412        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
413        "domComplete": 1460358389002,
414        "domLoading": 1460358380001.5
415    }]
416
417    assert(expected == runner.take_result_median(input_json, len(input_json)))
418
419
420def test_log_result():
421    results = [{
422        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
423        "domComplete": -1
424    }, {
425        "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
426        "domComplete": -1
427    }, {
428        "testcase": "http://localhost:8000/page_load_test/104.com/www.104.com/index.html",
429        "domComplete": 123456789
430    }]
431
432    expected = """
433========================================
434Total 3 tests; 1 succeeded, 2 failed.
435
436Failure summary:
437 - http://localhost:8000/page_load_test/56.com/www.56.com/index.html
438========================================
439"""
440    assert(expected == runner.format_result_summary(results))
441