1#!/usr/bin/env python 2# Copyright 2018, Google Inc. 3# All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: 8# 9# * Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# * Redistributions in binary form must reproduce the above 12# copyright notice, this list of conditions and the following disclaimer 13# in the documentation and/or other materials provided with the 14# distribution. 15# * Neither the name of Google Inc. nor the names of its 16# contributors may be used to endorse or promote products derived from 17# this software without specific prior written permission. 18# 19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31"""Unit test for the gtest_json_output module.""" 32 33import datetime 34import errno 35import json 36import os 37import re 38import sys 39 40import gtest_json_test_utils 41import gtest_test_utils 42 43GTEST_FILTER_FLAG = '--gtest_filter' 44GTEST_LIST_TESTS_FLAG = '--gtest_list_tests' 45GTEST_OUTPUT_FLAG = '--gtest_output' 46GTEST_DEFAULT_OUTPUT_FILE = 'test_detail.json' 47GTEST_PROGRAM_NAME = 'gtest_xml_output_unittest_' 48 49# The flag indicating stacktraces are not supported 50NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support' 51 52SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv 53 54if SUPPORTS_STACK_TRACES: 55 STACK_TRACE_TEMPLATE = '\nStack trace:\n*' 56else: 57 STACK_TRACE_TEMPLATE = '' 58 59EXPECTED_NON_EMPTY = { 60 u'tests': 61 24, 62 u'failures': 63 4, 64 u'disabled': 65 2, 66 u'errors': 67 0, 68 u'timestamp': 69 u'*', 70 u'time': 71 u'*', 72 u'ad_hoc_property': 73 u'42', 74 u'name': 75 u'AllTests', 76 u'testsuites': [ 77 { 78 u'name': 79 u'SuccessfulTest', 80 u'tests': 81 1, 82 u'failures': 83 0, 84 u'disabled': 85 0, 86 u'errors': 87 0, 88 u'time': 89 u'*', 90 u'testsuite': [{ 91 u'name': u'Succeeds', 92 u'status': u'RUN', 93 u'result': u'COMPLETED', 94 u'time': u'*', 95 u'classname': u'SuccessfulTest' 96 }] 97 }, 98 { 99 u'name': 100 u'FailedTest', 101 u'tests': 102 1, 103 u'failures': 104 1, 105 u'disabled': 106 0, 107 u'errors': 108 0, 109 u'time': 110 u'*', 111 u'testsuite': [{ 112 u'name': 113 u'Fails', 114 u'status': 115 u'RUN', 116 u'result': 117 u'COMPLETED', 118 u'time': 119 u'*', 120 u'classname': 121 u'FailedTest', 122 u'failures': [{ 123 u'failure': u'gtest_xml_output_unittest_.cc:*\n' 124 u'Expected equality of these values:\n' 125 u' 1\n 2' + STACK_TRACE_TEMPLATE, 126 u'type': u'' 127 }] 128 }] 129 }, 130 { 131 u'name': 132 u'DisabledTest', 133 u'tests': 134 1, 135 u'failures': 136 0, 137 u'disabled': 138 1, 139 u'errors': 140 0, 141 u'time': 142 u'*', 143 u'testsuite': [{ 144 u'name': u'DISABLED_test_not_run', 145 u'status': u'NOTRUN', 146 u'result': u'SUPPRESSED', 147 u'time': u'*', 148 u'classname': u'DisabledTest' 149 }] 150 }, 151 { 152 u'name': 153 u'SkippedTest', 154 u'tests': 155 1, 156 u'failures': 157 0, 158 u'disabled': 159 0, 160 u'errors': 161 0, 162 u'time': 163 u'*', 164 u'testsuite': [{ 165 u'name': u'Skipped', 166 u'status': u'RUN', 167 u'result': u'SKIPPED', 168 u'time': u'*', 169 u'classname': u'SkippedTest' 170 }] 171 }, 172 { 173 u'name': 174 u'MixedResultTest', 175 u'tests': 176 3, 177 u'failures': 178 1, 179 u'disabled': 180 1, 181 u'errors': 182 0, 183 u'time': 184 u'*', 185 u'testsuite': [ 186 { 187 u'name': u'Succeeds', 188 u'status': u'RUN', 189 u'result': u'COMPLETED', 190 u'time': u'*', 191 u'classname': u'MixedResultTest' 192 }, 193 { 194 u'name': 195 u'Fails', 196 u'status': 197 u'RUN', 198 u'result': 199 u'COMPLETED', 200 u'time': 201 u'*', 202 u'classname': 203 u'MixedResultTest', 204 u'failures': 205 [{ 206 u'failure': u'gtest_xml_output_unittest_.cc:*\n' 207 u'Expected equality of these values:\n' 208 u' 1\n 2' + STACK_TRACE_TEMPLATE, 209 u'type': u'' 210 }, 211 { 212 u'failure': u'gtest_xml_output_unittest_.cc:*\n' 213 u'Expected equality of these values:\n' 214 u' 2\n 3' + STACK_TRACE_TEMPLATE, 215 u'type': u'' 216 }] 217 }, 218 { 219 u'name': u'DISABLED_test', 220 u'status': u'NOTRUN', 221 u'result': u'SUPPRESSED', 222 u'time': u'*', 223 u'classname': u'MixedResultTest' 224 } 225 ] 226 }, 227 { 228 u'name': 229 u'XmlQuotingTest', 230 u'tests': 231 1, 232 u'failures': 233 1, 234 u'disabled': 235 0, 236 u'errors': 237 0, 238 u'time': 239 u'*', 240 u'testsuite': [{ 241 u'name': 242 u'OutputsCData', 243 u'status': 244 u'RUN', 245 u'result': 246 u'COMPLETED', 247 u'time': 248 u'*', 249 u'classname': 250 u'XmlQuotingTest', 251 u'failures': [{ 252 u'failure': u'gtest_xml_output_unittest_.cc:*\n' 253 u'Failed\nXML output: <?xml encoding="utf-8">' 254 u'<top><![CDATA[cdata text]]></top>' + 255 STACK_TRACE_TEMPLATE, 256 u'type': u'' 257 }] 258 }] 259 }, 260 { 261 u'name': 262 u'InvalidCharactersTest', 263 u'tests': 264 1, 265 u'failures': 266 1, 267 u'disabled': 268 0, 269 u'errors': 270 0, 271 u'time': 272 u'*', 273 u'testsuite': [{ 274 u'name': 275 u'InvalidCharactersInMessage', 276 u'status': 277 u'RUN', 278 u'result': 279 u'COMPLETED', 280 u'time': 281 u'*', 282 u'classname': 283 u'InvalidCharactersTest', 284 u'failures': [{ 285 u'failure': u'gtest_xml_output_unittest_.cc:*\n' 286 u'Failed\nInvalid characters in brackets' 287 u' [\x01\x02]' + STACK_TRACE_TEMPLATE, 288 u'type': u'' 289 }] 290 }] 291 }, 292 { 293 u'name': 294 u'PropertyRecordingTest', 295 u'tests': 296 4, 297 u'failures': 298 0, 299 u'disabled': 300 0, 301 u'errors': 302 0, 303 u'time': 304 u'*', 305 u'SetUpTestSuite': 306 u'yes', 307 u'TearDownTestSuite': 308 u'aye', 309 u'testsuite': [{ 310 u'name': u'OneProperty', 311 u'status': u'RUN', 312 u'result': u'COMPLETED', 313 u'time': u'*', 314 u'classname': u'PropertyRecordingTest', 315 u'key_1': u'1' 316 }, 317 { 318 u'name': u'IntValuedProperty', 319 u'status': u'RUN', 320 u'result': u'COMPLETED', 321 u'time': u'*', 322 u'classname': u'PropertyRecordingTest', 323 u'key_int': u'1' 324 }, 325 { 326 u'name': u'ThreeProperties', 327 u'status': u'RUN', 328 u'result': u'COMPLETED', 329 u'time': u'*', 330 u'classname': u'PropertyRecordingTest', 331 u'key_1': u'1', 332 u'key_2': u'2', 333 u'key_3': u'3' 334 }, 335 { 336 u'name': u'TwoValuesForOneKeyUsesLastValue', 337 u'status': u'RUN', 338 u'result': u'COMPLETED', 339 u'time': u'*', 340 u'classname': u'PropertyRecordingTest', 341 u'key_1': u'2' 342 }] 343 }, 344 { 345 u'name': 346 u'NoFixtureTest', 347 u'tests': 348 3, 349 u'failures': 350 0, 351 u'disabled': 352 0, 353 u'errors': 354 0, 355 u'time': 356 u'*', 357 u'testsuite': [ 358 { 359 u'name': u'RecordProperty', 360 u'status': u'RUN', 361 u'result': u'COMPLETED', 362 u'time': u'*', 363 u'classname': u'NoFixtureTest', 364 u'key': u'1' 365 }, 366 { 367 u'name': u'ExternalUtilityThatCallsRecordIntValuedProperty', 368 u'status': u'RUN', 369 u'result': u'COMPLETED', 370 u'time': u'*', 371 u'classname': u'NoFixtureTest', 372 u'key_for_utility_int': u'1' 373 }, 374 { 375 u'name': 376 u'ExternalUtilityThatCallsRecordStringValuedProperty', 377 u'status': 378 u'RUN', 379 u'result': 380 u'COMPLETED', 381 u'time': 382 u'*', 383 u'classname': 384 u'NoFixtureTest', 385 u'key_for_utility_string': 386 u'1' 387 } 388 ] 389 }, 390 { 391 u'name': 392 u'TypedTest/0', 393 u'tests': 394 1, 395 u'failures': 396 0, 397 u'disabled': 398 0, 399 u'errors': 400 0, 401 u'time': 402 u'*', 403 u'testsuite': [{ 404 u'name': u'HasTypeParamAttribute', 405 u'type_param': u'int', 406 u'status': u'RUN', 407 u'result': u'COMPLETED', 408 u'time': u'*', 409 u'classname': u'TypedTest/0' 410 }] 411 }, 412 { 413 u'name': 414 u'TypedTest/1', 415 u'tests': 416 1, 417 u'failures': 418 0, 419 u'disabled': 420 0, 421 u'errors': 422 0, 423 u'time': 424 u'*', 425 u'testsuite': [{ 426 u'name': u'HasTypeParamAttribute', 427 u'type_param': u'long', 428 u'status': u'RUN', 429 u'result': u'COMPLETED', 430 u'time': u'*', 431 u'classname': u'TypedTest/1' 432 }] 433 }, 434 { 435 u'name': 436 u'Single/TypeParameterizedTestSuite/0', 437 u'tests': 438 1, 439 u'failures': 440 0, 441 u'disabled': 442 0, 443 u'errors': 444 0, 445 u'time': 446 u'*', 447 u'testsuite': [{ 448 u'name': u'HasTypeParamAttribute', 449 u'type_param': u'int', 450 u'status': u'RUN', 451 u'result': u'COMPLETED', 452 u'time': u'*', 453 u'classname': u'Single/TypeParameterizedTestSuite/0' 454 }] 455 }, 456 { 457 u'name': 458 u'Single/TypeParameterizedTestSuite/1', 459 u'tests': 460 1, 461 u'failures': 462 0, 463 u'disabled': 464 0, 465 u'errors': 466 0, 467 u'time': 468 u'*', 469 u'testsuite': [{ 470 u'name': u'HasTypeParamAttribute', 471 u'type_param': u'long', 472 u'status': u'RUN', 473 u'result': u'COMPLETED', 474 u'time': u'*', 475 u'classname': u'Single/TypeParameterizedTestSuite/1' 476 }] 477 }, 478 { 479 u'name': 480 u'Single/ValueParamTest', 481 u'tests': 482 4, 483 u'failures': 484 0, 485 u'disabled': 486 0, 487 u'errors': 488 0, 489 u'time': 490 u'*', 491 u'testsuite': [ 492 { 493 u'name': u'HasValueParamAttribute/0', 494 u'value_param': u'33', 495 u'status': u'RUN', 496 u'result': u'COMPLETED', 497 u'time': u'*', 498 u'classname': u'Single/ValueParamTest' 499 }, 500 { 501 u'name': u'HasValueParamAttribute/1', 502 u'value_param': u'42', 503 u'status': u'RUN', 504 u'result': u'COMPLETED', 505 u'time': u'*', 506 u'classname': u'Single/ValueParamTest' 507 }, 508 { 509 u'name': u'AnotherTestThatHasValueParamAttribute/0', 510 u'value_param': u'33', 511 u'status': u'RUN', 512 u'result': u'COMPLETED', 513 u'time': u'*', 514 u'classname': u'Single/ValueParamTest' 515 }, 516 { 517 u'name': u'AnotherTestThatHasValueParamAttribute/1', 518 u'value_param': u'42', 519 u'status': u'RUN', 520 u'result': u'COMPLETED', 521 u'time': u'*', 522 u'classname': u'Single/ValueParamTest' 523 } 524 ] 525 } 526 ] 527} 528 529EXPECTED_FILTERED = { 530 u'tests': 531 1, 532 u'failures': 533 0, 534 u'disabled': 535 0, 536 u'errors': 537 0, 538 u'time': 539 u'*', 540 u'timestamp': 541 u'*', 542 u'name': 543 u'AllTests', 544 u'ad_hoc_property': 545 u'42', 546 u'testsuites': [{ 547 u'name': 548 u'SuccessfulTest', 549 u'tests': 550 1, 551 u'failures': 552 0, 553 u'disabled': 554 0, 555 u'errors': 556 0, 557 u'time': 558 u'*', 559 u'testsuite': [{ 560 u'name': u'Succeeds', 561 u'status': u'RUN', 562 u'result': u'COMPLETED', 563 u'time': u'*', 564 u'classname': u'SuccessfulTest', 565 }] 566 }], 567} 568 569EXPECTED_EMPTY = { 570 u'tests': 0, 571 u'failures': 0, 572 u'disabled': 0, 573 u'errors': 0, 574 u'time': u'*', 575 u'timestamp': u'*', 576 u'name': u'AllTests', 577 u'testsuites': [], 578} 579 580GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME) 581 582SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess( 583 [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output 584 585 586class GTestJsonOutputUnitTest(gtest_test_utils.TestCase): 587 """Unit test for Google Test's JSON output functionality. 588 """ 589 590 # This test currently breaks on platforms that do not support typed and 591 # type-parameterized tests, so we don't run it under them. 592 if SUPPORTS_TYPED_TESTS: 593 594 def testNonEmptyJsonOutput(self): 595 """Verifies JSON output for a Google Test binary with non-empty output. 596 597 Runs a test program that generates a non-empty JSON output, and 598 tests that the JSON output is expected. 599 """ 600 self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY, 1) 601 602 def testEmptyJsonOutput(self): 603 """Verifies JSON output for a Google Test binary without actual tests. 604 605 Runs a test program that generates an empty JSON output, and 606 tests that the JSON output is expected. 607 """ 608 609 self._TestJsonOutput('gtest_no_test_unittest', EXPECTED_EMPTY, 0) 610 611 def testTimestampValue(self): 612 """Checks whether the timestamp attribute in the JSON output is valid. 613 614 Runs a test program that generates an empty JSON output, and checks if 615 the timestamp attribute in the testsuites tag is valid. 616 """ 617 actual = self._GetJsonOutput('gtest_no_test_unittest', [], 0) 618 date_time_str = actual['timestamp'] 619 # datetime.strptime() is only available in Python 2.5+ so we have to 620 # parse the expected datetime manually. 621 match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str) 622 self.assertTrue( 623 re.match, 624 'JSON datettime string %s has incorrect format' % date_time_str) 625 date_time_from_json = datetime.datetime( 626 year=int(match.group(1)), month=int(match.group(2)), 627 day=int(match.group(3)), hour=int(match.group(4)), 628 minute=int(match.group(5)), second=int(match.group(6))) 629 630 time_delta = abs(datetime.datetime.now() - date_time_from_json) 631 # timestamp value should be near the current local time 632 self.assertTrue(time_delta < datetime.timedelta(seconds=600), 633 'time_delta is %s' % time_delta) 634 635 def testDefaultOutputFile(self): 636 """Verifies the default output file name. 637 638 Confirms that Google Test produces an JSON output file with the expected 639 default name if no name is explicitly specified. 640 """ 641 output_file = os.path.join(gtest_test_utils.GetTempDir(), 642 GTEST_DEFAULT_OUTPUT_FILE) 643 gtest_prog_path = gtest_test_utils.GetTestExecutablePath( 644 'gtest_no_test_unittest') 645 try: 646 os.remove(output_file) 647 except OSError: 648 e = sys.exc_info()[1] 649 if e.errno != errno.ENOENT: 650 raise 651 652 p = gtest_test_utils.Subprocess( 653 [gtest_prog_path, '%s=json' % GTEST_OUTPUT_FLAG], 654 working_dir=gtest_test_utils.GetTempDir()) 655 self.assert_(p.exited) 656 self.assertEquals(0, p.exit_code) 657 self.assert_(os.path.isfile(output_file)) 658 659 def testSuppressedJsonOutput(self): 660 """Verifies that no JSON output is generated. 661 662 Tests that no JSON file is generated if the default JSON listener is 663 shut down before RUN_ALL_TESTS is invoked. 664 """ 665 666 json_path = os.path.join(gtest_test_utils.GetTempDir(), 667 GTEST_PROGRAM_NAME + 'out.json') 668 if os.path.isfile(json_path): 669 os.remove(json_path) 670 671 command = [GTEST_PROGRAM_PATH, 672 '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path), 673 '--shut_down_xml'] 674 p = gtest_test_utils.Subprocess(command) 675 if p.terminated_by_signal: 676 # p.signal is available only if p.terminated_by_signal is True. 677 self.assertFalse( 678 p.terminated_by_signal, 679 '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal)) 680 else: 681 self.assert_(p.exited) 682 self.assertEquals(1, p.exit_code, 683 "'%s' exited with code %s, which doesn't match " 684 'the expected exit code %s.' 685 % (command, p.exit_code, 1)) 686 687 self.assert_(not os.path.isfile(json_path)) 688 689 def testFilteredTestJsonOutput(self): 690 """Verifies JSON output when a filter is applied. 691 692 Runs a test program that executes only some tests and verifies that 693 non-selected tests do not show up in the JSON output. 694 """ 695 696 self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED, 0, 697 extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG]) 698 699 def _GetJsonOutput(self, gtest_prog_name, extra_args, expected_exit_code): 700 """Returns the JSON output generated by running the program gtest_prog_name. 701 702 Furthermore, the program's exit code must be expected_exit_code. 703 704 Args: 705 gtest_prog_name: Google Test binary name. 706 extra_args: extra arguments to binary invocation. 707 expected_exit_code: program's exit code. 708 """ 709 json_path = os.path.join(gtest_test_utils.GetTempDir(), 710 gtest_prog_name + 'out.json') 711 gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name) 712 713 command = ( 714 [gtest_prog_path, '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path)] + 715 extra_args 716 ) 717 p = gtest_test_utils.Subprocess(command) 718 if p.terminated_by_signal: 719 self.assert_(False, 720 '%s was killed by signal %d' % (gtest_prog_name, p.signal)) 721 else: 722 self.assert_(p.exited) 723 self.assertEquals(expected_exit_code, p.exit_code, 724 "'%s' exited with code %s, which doesn't match " 725 'the expected exit code %s.' 726 % (command, p.exit_code, expected_exit_code)) 727 with open(json_path) as f: 728 actual = json.load(f) 729 return actual 730 731 def _TestJsonOutput(self, gtest_prog_name, expected, 732 expected_exit_code, extra_args=None): 733 """Checks the JSON output generated by the Google Test binary. 734 735 Asserts that the JSON document generated by running the program 736 gtest_prog_name matches expected_json, a string containing another 737 JSON document. Furthermore, the program's exit code must be 738 expected_exit_code. 739 740 Args: 741 gtest_prog_name: Google Test binary name. 742 expected: expected output. 743 expected_exit_code: program's exit code. 744 extra_args: extra arguments to binary invocation. 745 """ 746 747 actual = self._GetJsonOutput(gtest_prog_name, extra_args or [], 748 expected_exit_code) 749 self.assertEqual(expected, gtest_json_test_utils.normalize(actual)) 750 751 752if __name__ == '__main__': 753 if NO_STACKTRACE_SUPPORT_FLAG in sys.argv: 754 # unittest.main() can't handle unknown flags 755 sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG) 756 757 os.environ['GTEST_STACK_TRACE_DEPTH'] = '1' 758 gtest_test_utils.Main() 759