1#!/usr/bin/env python3 2 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this 5# file, You can obtain one at http://mozilla.org/MPL/2.0/. 6 7import runner 8import pytest 9 10 11def test_log_parser(): 12 mock_url = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html" 13 mock_log = b''' 14[PERF] perf block start 15[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/index.html 16[PERF],navigationStart,1460358376 17[PERF],unloadEventStart,undefined 18[PERF],unloadEventEnd,undefined 19[PERF],redirectStart,undefined 20[PERF],redirectEnd,undefined 21[PERF],fetchStart,undefined 22[PERF],domainLookupStart,undefined 23[PERF],domainLookupEnd,undefined 24[PERF],connectStart,undefined 25[PERF],connectEnd,undefined 26[PERF],secureConnectionStart,undefined 27[PERF],requestStart,undefined 28[PERF],responseStart,undefined 29[PERF],responseEnd,undefined 30[PERF],domLoading,1460358376000 31[PERF],domInteractive,1460358388000 32[PERF],domContentLoadedEventStart,1460358388000 33[PERF],domContentLoadedEventEnd,1460358388000 34[PERF],domComplete,1460358389000 35[PERF],loadEventStart,undefined 36[PERF],loadEventEnd,undefined 37[PERF] perf block end 38Shutting down the Constellation after generating an output file or exit flag specified 39''' 40 41 expected = [{ 42 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 43 "navigationStart": 1460358376, 44 "unloadEventStart": None, 45 "unloadEventEnd": None, 46 "redirectStart": None, 47 "redirectEnd": None, 48 "fetchStart": None, 49 "domainLookupStart": None, 50 "domainLookupEnd": None, 51 "connectStart": None, 52 "connectEnd": None, 53 "secureConnectionStart": None, 54 "requestStart": None, 55 "responseStart": None, 56 "responseEnd": None, 57 "domLoading": 1460358376000, 58 "domInteractive": 1460358388000, 59 "domContentLoadedEventStart": 1460358388000, 60 "domContentLoadedEventEnd": 1460358388000, 61 "domComplete": 1460358389000, 62 "loadEventStart": None, 63 "loadEventEnd": None 64 }] 65 result = runner.parse_log(mock_log, mock_url) 66 assert(expected == list(result)) 67 68 69def test_log_parser_complex(): 70 mock_log = b''' 71[PERF] perf block start 72[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/content.html 73[PERF],navigationStart,1460358300 74[PERF],unloadEventStart,undefined 75[PERF],unloadEventEnd,undefined 76[PERF],redirectStart,undefined 77[PERF],redirectEnd,undefined 78[PERF],fetchStart,undefined 79[PERF],domainLookupStart,undefined 80[PERF],domainLookupEnd,undefined 81[PERF],connectStart,undefined 82[PERF],connectEnd,undefined 83[PERF],secureConnectionStart,undefined 84[PERF],requestStart,undefined 85[PERF],responseStart,undefined 86[PERF],responseEnd,undefined 87[PERF],domLoading,1460358376000 88[PERF],domInteractive,1460358388000 89[PERF],domContentLoadedEventStart,1460358388000 90[PERF],domContentLoadedEventEnd,1460358388000 91[PERF],domComplete,1460358389000 92[PERF],loadEventStart,undefined 93[PERF],loadEventEnd,undefined 94[PERF] perf block end 95Some other js error logs here 96 97[PERF] perf block start 98[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/index.html 99[PERF],navigationStart,1460358376 100[PERF],unloadEventStart,undefined 101[PERF],unloadEventEnd,undefined 102[PERF],redirectStart,undefined 103[PERF],redirectEnd,undefined 104[PERF],fetchStart,undefined 105[PERF],domainLookupStart,undefined 106[PERF],domainLookupEnd,undefined 107[PERF],connectStart,undefined 108[PERF],connectEnd,undefined 109[PERF],secureConnectionStart,undefined 110[PERF],requestStart,undefined 111[PERF],responseStart,undefined 112[PERF],responseEnd,undefined 113[PERF],domLoading,1460358376000 114[PERF],domInteractive,1460358388000 115[PERF],domContentLoadedEventStart,1460358388000 116[PERF],domContentLoadedEventEnd,1460358388000 117[PERF],domComplete,1460358389000 118[PERF],loadEventStart,undefined 119[PERF],loadEventEnd,undefined 120[PERF] perf block end 121Shutting down the Constellation after generating an output file or exit flag specified 122''' 123 mock_url = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html" 124 expected = [{ 125 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 126 "navigationStart": 1460358376, 127 "unloadEventStart": None, 128 "unloadEventEnd": None, 129 "redirectStart": None, 130 "redirectEnd": None, 131 "fetchStart": None, 132 "domainLookupStart": None, 133 "domainLookupEnd": None, 134 "connectStart": None, 135 "connectEnd": None, 136 "secureConnectionStart": None, 137 "requestStart": None, 138 "responseStart": None, 139 "responseEnd": None, 140 "domLoading": 1460358376000, 141 "domInteractive": 1460358388000, 142 "domContentLoadedEventStart": 1460358388000, 143 "domContentLoadedEventEnd": 1460358388000, 144 "domComplete": 1460358389000, 145 "loadEventStart": None, 146 "loadEventEnd": None 147 }] 148 result = runner.parse_log(mock_log, mock_url) 149 assert(expected == list(result)) 150 151 152def test_log_parser_empty(): 153 mock_log = b''' 154[PERF] perf block start 155[PERF]BROKEN!!!!!!!!!1 156[PERF]BROKEN!!!!!!!!!1 157[PERF]BROKEN!!!!!!!!!1 158[PERF]BROKEN!!!!!!!!!1 159[PERF]BROKEN!!!!!!!!!1 160[PERF] perf block end 161''' 162 mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html" 163 164 expected = [{ 165 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 166 "title": "", 167 "navigationStart": 0, 168 "unloadEventStart": -1, 169 "unloadEventEnd": -1, 170 "redirectStart": -1, 171 "redirectEnd": -1, 172 "fetchStart": -1, 173 "domainLookupStart": -1, 174 "domainLookupEnd": -1, 175 "connectStart": -1, 176 "connectEnd": -1, 177 "secureConnectionStart": -1, 178 "requestStart": -1, 179 "responseStart": -1, 180 "responseEnd": -1, 181 "domLoading": -1, 182 "domInteractive": -1, 183 "domContentLoadedEventStart": -1, 184 "domContentLoadedEventEnd": -1, 185 "domComplete": -1, 186 "loadEventStart": -1, 187 "loadEventEnd": -1 188 }] 189 result = runner.parse_log(mock_log, mock_testcase) 190 assert(expected == list(result)) 191 192 193def test_log_parser_error(): 194 mock_log = b'Nothing here! Test failed!' 195 mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html" 196 197 expected = [{ 198 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 199 "title": "", 200 "navigationStart": 0, 201 "unloadEventStart": -1, 202 "unloadEventEnd": -1, 203 "redirectStart": -1, 204 "redirectEnd": -1, 205 "fetchStart": -1, 206 "domainLookupStart": -1, 207 "domainLookupEnd": -1, 208 "connectStart": -1, 209 "connectEnd": -1, 210 "secureConnectionStart": -1, 211 "requestStart": -1, 212 "responseStart": -1, 213 "responseEnd": -1, 214 "domLoading": -1, 215 "domInteractive": -1, 216 "domContentLoadedEventStart": -1, 217 "domContentLoadedEventEnd": -1, 218 "domComplete": -1, 219 "loadEventStart": -1, 220 "loadEventEnd": -1 221 }] 222 result = runner.parse_log(mock_log, mock_testcase) 223 assert(expected == list(result)) 224 225 226def test_log_parser_bad_testcase_name(): 227 mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html" 228 # Notice the testcase is about:blank, servo crashed 229 mock_log = b''' 230[PERF] perf block start 231[PERF],testcase,about:blank 232[PERF],navigationStart,1460358376 233[PERF],unloadEventStart,undefined 234[PERF],unloadEventEnd,undefined 235[PERF],redirectStart,undefined 236[PERF],redirectEnd,undefined 237[PERF],fetchStart,undefined 238[PERF],domainLookupStart,undefined 239[PERF],domainLookupEnd,undefined 240[PERF],connectStart,undefined 241[PERF],connectEnd,undefined 242[PERF],secureConnectionStart,undefined 243[PERF],requestStart,undefined 244[PERF],responseStart,undefined 245[PERF],responseEnd,undefined 246[PERF],domLoading,1460358376000 247[PERF],domInteractive,1460358388000 248[PERF],domContentLoadedEventStart,1460358388000 249[PERF],domContentLoadedEventEnd,1460358388000 250[PERF],domComplete,1460358389000 251[PERF],loadEventStart,undefined 252[PERF],loadEventEnd,undefined 253[PERF] perf block end 254Shutting down the Constellation after generating an output file or exit flag specified 255''' 256 257 expected = [{ 258 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 259 "title": "", 260 "navigationStart": 0, 261 "unloadEventStart": -1, 262 "unloadEventEnd": -1, 263 "redirectStart": -1, 264 "redirectEnd": -1, 265 "fetchStart": -1, 266 "domainLookupStart": -1, 267 "domainLookupEnd": -1, 268 "connectStart": -1, 269 "connectEnd": -1, 270 "secureConnectionStart": -1, 271 "requestStart": -1, 272 "responseStart": -1, 273 "responseEnd": -1, 274 "domLoading": -1, 275 "domInteractive": -1, 276 "domContentLoadedEventStart": -1, 277 "domContentLoadedEventEnd": -1, 278 "domComplete": -1, 279 "loadEventStart": -1, 280 "loadEventEnd": -1 281 }] 282 result = runner.parse_log(mock_log, mock_testcase) 283 assert(expected == list(result)) 284 285 286def test_manifest_loader(): 287 288 text = ''' 289http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html 290http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html 291 292http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html 293# Disabled! http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html 294''' 295 expected = [ 296 ("http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html", False), 297 ("http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html", False), 298 ("http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html", False) 299 ] 300 assert(expected == list(runner.parse_manifest(text))) 301 302 303def test_manifest_loader_async(): 304 305 text = ''' 306http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html 307async http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html 308''' 309 expected = [ 310 ("http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html", False), 311 ("http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html", True), 312 ] 313 assert(expected == list(runner.parse_manifest(text))) 314 315 316def test_filter_result_by_manifest(): 317 input_json = [{ 318 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/content.html", 319 "domComplete": 1460358389000, 320 }, { 321 "testcase": "non-existing-html", 322 "domComplete": 1460358389000, 323 }, { 324 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 325 "domComplete": 1460358389000, 326 }] 327 328 expected = [{ 329 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 330 "domComplete": 1460358389000, 331 }] 332 333 manifest = [ 334 ("http://localhost:8000/page_load_test/56.com/www.56.com/index.html", False) 335 ] 336 337 assert(expected == runner.filter_result_by_manifest(input_json, manifest)) 338 339 340def test_filter_result_by_manifest_error(): 341 input_json = [{ 342 "testcase": "1.html", 343 "domComplete": 1460358389000, 344 }] 345 346 manifest = [ 347 ("1.html", False), 348 ("2.html", False) 349 ] 350 351 with pytest.raises(Exception) as execinfo: 352 runner.filter_result_by_manifest(input_json, manifest) 353 assert "Missing test result" in str(execinfo.value) 354 355 356def test_take_result_median_odd(): 357 input_json = [{ 358 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 359 "domComplete": 1460358389001, 360 "domLoading": 1460358380002 361 }, { 362 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 363 "domComplete": 1460358389002, 364 "domLoading": 1460358380001 365 }, { 366 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 367 "domComplete": 1460358389003, 368 "domLoading": 1460358380003 369 }] 370 371 expected = [{ 372 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 373 "domComplete": 1460358389002, 374 "domLoading": 1460358380002 375 }] 376 377 assert(expected == runner.take_result_median(input_json, len(input_json))) 378 379 380def test_take_result_median_even(): 381 input_json = [{ 382 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 383 "domComplete": 1460358389001, 384 "domLoading": 1460358380002 385 }, { 386 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 387 "domComplete": 1460358389002, 388 "domLoading": 1460358380001 389 }] 390 391 expected = [{ 392 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 393 "domComplete": 1460358389001.5, 394 "domLoading": 1460358380001.5 395 }] 396 397 assert(expected == runner.take_result_median(input_json, len(input_json))) 398 399 400def test_take_result_median_error(): 401 input_json = [{ 402 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 403 "domComplete": None, 404 "domLoading": 1460358380002 405 }, { 406 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 407 "domComplete": 1460358389002, 408 "domLoading": 1460358380001 409 }] 410 411 expected = [{ 412 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 413 "domComplete": 1460358389002, 414 "domLoading": 1460358380001.5 415 }] 416 417 assert(expected == runner.take_result_median(input_json, len(input_json))) 418 419 420def test_log_result(): 421 results = [{ 422 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 423 "domComplete": -1 424 }, { 425 "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", 426 "domComplete": -1 427 }, { 428 "testcase": "http://localhost:8000/page_load_test/104.com/www.104.com/index.html", 429 "domComplete": 123456789 430 }] 431 432 expected = """ 433======================================== 434Total 3 tests; 1 succeeded, 2 failed. 435 436Failure summary: 437 - http://localhost:8000/page_load_test/56.com/www.56.com/index.html 438======================================== 439""" 440 assert(expected == runner.format_result_summary(results)) 441