1#!/usr/bin/env python 2# Copyright 2018, Google Inc. 3# All rights reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: 8# 9# * Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# * Redistributions in binary form must reproduce the above 12# copyright notice, this list of conditions and the following disclaimer 13# in the documentation and/or other materials provided with the 14# distribution. 15# * Neither the name of Google Inc. nor the names of its 16# contributors may be used to endorse or promote products derived from 17# this software without specific prior written permission. 18# 19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31"""Unit test for the gtest_json_output module.""" 32 33import datetime 34import errno 35import json 36import os 37import re 38import sys 39 40import gtest_json_test_utils 41import gtest_test_utils 42 43GTEST_FILTER_FLAG = '--gtest_filter' 44GTEST_LIST_TESTS_FLAG = '--gtest_list_tests' 45GTEST_OUTPUT_FLAG = '--gtest_output' 46GTEST_DEFAULT_OUTPUT_FILE = 'test_detail.json' 47GTEST_PROGRAM_NAME = 'gtest_xml_output_unittest_' 48 49# The flag indicating stacktraces are not supported 50NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support' 51 52SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv 53 54if SUPPORTS_STACK_TRACES: 55 STACK_TRACE_TEMPLATE = '\nStack trace:\n*' 56else: 57 STACK_TRACE_TEMPLATE = '' 58 59EXPECTED_NON_EMPTY = { 60 u'tests': 23, 61 u'failures': 4, 62 u'disabled': 2, 63 u'errors': 0, 64 u'timestamp': u'*', 65 u'time': u'*', 66 u'ad_hoc_property': u'42', 67 u'name': u'AllTests', 68 u'testsuites': [ 69 { 70 u'name': u'SuccessfulTest', 71 u'tests': 1, 72 u'failures': 0, 73 u'disabled': 0, 74 u'errors': 0, 75 u'time': u'*', 76 u'testsuite': [ 77 { 78 u'name': u'Succeeds', 79 u'status': u'RUN', 80 u'time': u'*', 81 u'classname': u'SuccessfulTest' 82 } 83 ] 84 }, 85 { 86 u'name': u'FailedTest', 87 u'tests': 1, 88 u'failures': 1, 89 u'disabled': 0, 90 u'errors': 0, 91 u'time': u'*', 92 u'testsuite': [ 93 { 94 u'name': u'Fails', 95 u'status': u'RUN', 96 u'time': u'*', 97 u'classname': u'FailedTest', 98 u'failures': [ 99 { 100 u'failure': 101 u'gtest_xml_output_unittest_.cc:*\n' 102 u'Expected equality of these values:\n' 103 u' 1\n 2' + STACK_TRACE_TEMPLATE, 104 u'type': u'' 105 } 106 ] 107 } 108 ] 109 }, 110 { 111 u'name': u'DisabledTest', 112 u'tests': 1, 113 u'failures': 0, 114 u'disabled': 1, 115 u'errors': 0, 116 u'time': u'*', 117 u'testsuite': [ 118 { 119 u'name': u'DISABLED_test_not_run', 120 u'status': u'NOTRUN', 121 u'time': u'*', 122 u'classname': u'DisabledTest' 123 } 124 ] 125 }, 126 { 127 u'name': u'MixedResultTest', 128 u'tests': 3, 129 u'failures': 1, 130 u'disabled': 1, 131 u'errors': 0, 132 u'time': u'*', 133 u'testsuite': [ 134 { 135 u'name': u'Succeeds', 136 u'status': u'RUN', 137 u'time': u'*', 138 u'classname': u'MixedResultTest' 139 }, 140 { 141 u'name': u'Fails', 142 u'status': u'RUN', 143 u'time': u'*', 144 u'classname': u'MixedResultTest', 145 u'failures': [ 146 { 147 u'failure': 148 u'gtest_xml_output_unittest_.cc:*\n' 149 u'Expected equality of these values:\n' 150 u' 1\n 2' + STACK_TRACE_TEMPLATE, 151 u'type': u'' 152 }, 153 { 154 u'failure': 155 u'gtest_xml_output_unittest_.cc:*\n' 156 u'Expected equality of these values:\n' 157 u' 2\n 3' + STACK_TRACE_TEMPLATE, 158 u'type': u'' 159 } 160 ] 161 }, 162 { 163 u'name': u'DISABLED_test', 164 u'status': u'NOTRUN', 165 u'time': u'*', 166 u'classname': u'MixedResultTest' 167 } 168 ] 169 }, 170 { 171 u'name': u'XmlQuotingTest', 172 u'tests': 1, 173 u'failures': 1, 174 u'disabled': 0, 175 u'errors': 0, 176 u'time': u'*', 177 u'testsuite': [ 178 { 179 u'name': u'OutputsCData', 180 u'status': u'RUN', 181 u'time': u'*', 182 u'classname': u'XmlQuotingTest', 183 u'failures': [ 184 { 185 u'failure': 186 u'gtest_xml_output_unittest_.cc:*\n' 187 u'Failed\nXML output: <?xml encoding="utf-8">' 188 u'<top><![CDATA[cdata text]]></top>' + 189 STACK_TRACE_TEMPLATE, 190 u'type': u'' 191 } 192 ] 193 } 194 ] 195 }, 196 { 197 u'name': u'InvalidCharactersTest', 198 u'tests': 1, 199 u'failures': 1, 200 u'disabled': 0, 201 u'errors': 0, 202 u'time': u'*', 203 u'testsuite': [ 204 { 205 u'name': u'InvalidCharactersInMessage', 206 u'status': u'RUN', 207 u'time': u'*', 208 u'classname': u'InvalidCharactersTest', 209 u'failures': [ 210 { 211 u'failure': 212 u'gtest_xml_output_unittest_.cc:*\n' 213 u'Failed\nInvalid characters in brackets' 214 u' [\x01\x02]' + STACK_TRACE_TEMPLATE, 215 u'type': u'' 216 } 217 ] 218 } 219 ] 220 }, 221 { 222 u'name': u'PropertyRecordingTest', 223 u'tests': 4, 224 u'failures': 0, 225 u'disabled': 0, 226 u'errors': 0, 227 u'time': u'*', 228 u'SetUpTestCase': u'yes', 229 u'TearDownTestCase': u'aye', 230 u'testsuite': [ 231 { 232 u'name': u'OneProperty', 233 u'status': u'RUN', 234 u'time': u'*', 235 u'classname': u'PropertyRecordingTest', 236 u'key_1': u'1' 237 }, 238 { 239 u'name': u'IntValuedProperty', 240 u'status': u'RUN', 241 u'time': u'*', 242 u'classname': u'PropertyRecordingTest', 243 u'key_int': u'1' 244 }, 245 { 246 u'name': u'ThreeProperties', 247 u'status': u'RUN', 248 u'time': u'*', 249 u'classname': u'PropertyRecordingTest', 250 u'key_1': u'1', 251 u'key_2': u'2', 252 u'key_3': u'3' 253 }, 254 { 255 u'name': u'TwoValuesForOneKeyUsesLastValue', 256 u'status': u'RUN', 257 u'time': u'*', 258 u'classname': u'PropertyRecordingTest', 259 u'key_1': u'2' 260 } 261 ] 262 }, 263 { 264 u'name': u'NoFixtureTest', 265 u'tests': 3, 266 u'failures': 0, 267 u'disabled': 0, 268 u'errors': 0, 269 u'time': u'*', 270 u'testsuite': [ 271 { 272 u'name': u'RecordProperty', 273 u'status': u'RUN', 274 u'time': u'*', 275 u'classname': u'NoFixtureTest', 276 u'key': u'1' 277 }, 278 { 279 u'name': u'ExternalUtilityThatCallsRecordIntValuedProperty', 280 u'status': u'RUN', 281 u'time': u'*', 282 u'classname': u'NoFixtureTest', 283 u'key_for_utility_int': u'1' 284 }, 285 { 286 u'name': 287 u'ExternalUtilityThatCallsRecordStringValuedProperty', 288 u'status': u'RUN', 289 u'time': u'*', 290 u'classname': u'NoFixtureTest', 291 u'key_for_utility_string': u'1' 292 } 293 ] 294 }, 295 { 296 u'name': u'TypedTest/0', 297 u'tests': 1, 298 u'failures': 0, 299 u'disabled': 0, 300 u'errors': 0, 301 u'time': u'*', 302 u'testsuite': [ 303 { 304 u'name': u'HasTypeParamAttribute', 305 u'type_param': u'int', 306 u'status': u'RUN', 307 u'time': u'*', 308 u'classname': u'TypedTest/0' 309 } 310 ] 311 }, 312 { 313 u'name': u'TypedTest/1', 314 u'tests': 1, 315 u'failures': 0, 316 u'disabled': 0, 317 u'errors': 0, 318 u'time': u'*', 319 u'testsuite': [ 320 { 321 u'name': u'HasTypeParamAttribute', 322 u'type_param': u'long', 323 u'status': u'RUN', 324 u'time': u'*', 325 u'classname': u'TypedTest/1' 326 } 327 ] 328 }, 329 { 330 u'name': u'Single/TypeParameterizedTestCase/0', 331 u'tests': 1, 332 u'failures': 0, 333 u'disabled': 0, 334 u'errors': 0, 335 u'time': u'*', 336 u'testsuite': [ 337 { 338 u'name': u'HasTypeParamAttribute', 339 u'type_param': u'int', 340 u'status': u'RUN', 341 u'time': u'*', 342 u'classname': u'Single/TypeParameterizedTestCase/0' 343 } 344 ] 345 }, 346 { 347 u'name': u'Single/TypeParameterizedTestCase/1', 348 u'tests': 1, 349 u'failures': 0, 350 u'disabled': 0, 351 u'errors': 0, 352 u'time': u'*', 353 u'testsuite': [ 354 { 355 u'name': u'HasTypeParamAttribute', 356 u'type_param': u'long', 357 u'status': u'RUN', 358 u'time': u'*', 359 u'classname': u'Single/TypeParameterizedTestCase/1' 360 } 361 ] 362 }, 363 { 364 u'name': u'Single/ValueParamTest', 365 u'tests': 4, 366 u'failures': 0, 367 u'disabled': 0, 368 u'errors': 0, 369 u'time': u'*', 370 u'testsuite': [ 371 { 372 u'name': u'HasValueParamAttribute/0', 373 u'value_param': u'33', 374 u'status': u'RUN', 375 u'time': u'*', 376 u'classname': u'Single/ValueParamTest' 377 }, 378 { 379 u'name': u'HasValueParamAttribute/1', 380 u'value_param': u'42', 381 u'status': u'RUN', 382 u'time': u'*', 383 u'classname': u'Single/ValueParamTest' 384 }, 385 { 386 u'name': u'AnotherTestThatHasValueParamAttribute/0', 387 u'value_param': u'33', 388 u'status': u'RUN', 389 u'time': u'*', 390 u'classname': u'Single/ValueParamTest' 391 }, 392 { 393 u'name': u'AnotherTestThatHasValueParamAttribute/1', 394 u'value_param': u'42', 395 u'status': u'RUN', 396 u'time': u'*', 397 u'classname': u'Single/ValueParamTest' 398 } 399 ] 400 } 401 ] 402} 403 404EXPECTED_FILTERED = { 405 u'tests': 1, 406 u'failures': 0, 407 u'disabled': 0, 408 u'errors': 0, 409 u'time': u'*', 410 u'timestamp': u'*', 411 u'name': u'AllTests', 412 u'ad_hoc_property': u'42', 413 u'testsuites': [{ 414 u'name': u'SuccessfulTest', 415 u'tests': 1, 416 u'failures': 0, 417 u'disabled': 0, 418 u'errors': 0, 419 u'time': u'*', 420 u'testsuite': [{ 421 u'name': u'Succeeds', 422 u'status': u'RUN', 423 u'time': u'*', 424 u'classname': u'SuccessfulTest', 425 }] 426 }], 427} 428 429EXPECTED_EMPTY = { 430 u'tests': 0, 431 u'failures': 0, 432 u'disabled': 0, 433 u'errors': 0, 434 u'time': u'*', 435 u'timestamp': u'*', 436 u'name': u'AllTests', 437 u'testsuites': [], 438} 439 440GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME) 441 442SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess( 443 [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output 444 445 446class GTestJsonOutputUnitTest(gtest_test_utils.TestCase): 447 """Unit test for Google Test's JSON output functionality. 448 """ 449 450 # This test currently breaks on platforms that do not support typed and 451 # type-parameterized tests, so we don't run it under them. 452 if SUPPORTS_TYPED_TESTS: 453 454 def testNonEmptyJsonOutput(self): 455 """Verifies JSON output for a Google Test binary with non-empty output. 456 457 Runs a test program that generates a non-empty JSON output, and 458 tests that the JSON output is expected. 459 """ 460 self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY, 1) 461 462 def testEmptyJsonOutput(self): 463 """Verifies JSON output for a Google Test binary without actual tests. 464 465 Runs a test program that generates an empty JSON output, and 466 tests that the JSON output is expected. 467 """ 468 469 self._TestJsonOutput('gtest_no_test_unittest', EXPECTED_EMPTY, 0) 470 471 def testTimestampValue(self): 472 """Checks whether the timestamp attribute in the JSON output is valid. 473 474 Runs a test program that generates an empty JSON output, and checks if 475 the timestamp attribute in the testsuites tag is valid. 476 """ 477 actual = self._GetJsonOutput('gtest_no_test_unittest', [], 0) 478 date_time_str = actual['timestamp'] 479 # datetime.strptime() is only available in Python 2.5+ so we have to 480 # parse the expected datetime manually. 481 match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str) 482 self.assertTrue( 483 re.match, 484 'JSON datettime string %s has incorrect format' % date_time_str) 485 date_time_from_json = datetime.datetime( 486 year=int(match.group(1)), month=int(match.group(2)), 487 day=int(match.group(3)), hour=int(match.group(4)), 488 minute=int(match.group(5)), second=int(match.group(6))) 489 490 time_delta = abs(datetime.datetime.now() - date_time_from_json) 491 # timestamp value should be near the current local time 492 self.assertTrue(time_delta < datetime.timedelta(seconds=600), 493 'time_delta is %s' % time_delta) 494 495 def testDefaultOutputFile(self): 496 """Verifies the default output file name. 497 498 Confirms that Google Test produces an JSON output file with the expected 499 default name if no name is explicitly specified. 500 """ 501 output_file = os.path.join(gtest_test_utils.GetTempDir(), 502 GTEST_DEFAULT_OUTPUT_FILE) 503 gtest_prog_path = gtest_test_utils.GetTestExecutablePath( 504 'gtest_no_test_unittest') 505 try: 506 os.remove(output_file) 507 except OSError: 508 e = sys.exc_info()[1] 509 if e.errno != errno.ENOENT: 510 raise 511 512 p = gtest_test_utils.Subprocess( 513 [gtest_prog_path, '%s=json' % GTEST_OUTPUT_FLAG], 514 working_dir=gtest_test_utils.GetTempDir()) 515 self.assert_(p.exited) 516 self.assertEquals(0, p.exit_code) 517 self.assert_(os.path.isfile(output_file)) 518 519 def testSuppressedJsonOutput(self): 520 """Verifies that no JSON output is generated. 521 522 Tests that no JSON file is generated if the default JSON listener is 523 shut down before RUN_ALL_TESTS is invoked. 524 """ 525 526 json_path = os.path.join(gtest_test_utils.GetTempDir(), 527 GTEST_PROGRAM_NAME + 'out.json') 528 if os.path.isfile(json_path): 529 os.remove(json_path) 530 531 command = [GTEST_PROGRAM_PATH, 532 '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path), 533 '--shut_down_xml'] 534 p = gtest_test_utils.Subprocess(command) 535 if p.terminated_by_signal: 536 # p.signal is available only if p.terminated_by_signal is True. 537 self.assertFalse( 538 p.terminated_by_signal, 539 '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal)) 540 else: 541 self.assert_(p.exited) 542 self.assertEquals(1, p.exit_code, 543 "'%s' exited with code %s, which doesn't match " 544 'the expected exit code %s.' 545 % (command, p.exit_code, 1)) 546 547 self.assert_(not os.path.isfile(json_path)) 548 549 def testFilteredTestJsonOutput(self): 550 """Verifies JSON output when a filter is applied. 551 552 Runs a test program that executes only some tests and verifies that 553 non-selected tests do not show up in the JSON output. 554 """ 555 556 self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED, 0, 557 extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG]) 558 559 def _GetJsonOutput(self, gtest_prog_name, extra_args, expected_exit_code): 560 """Returns the JSON output generated by running the program gtest_prog_name. 561 562 Furthermore, the program's exit code must be expected_exit_code. 563 564 Args: 565 gtest_prog_name: Google Test binary name. 566 extra_args: extra arguments to binary invocation. 567 expected_exit_code: program's exit code. 568 """ 569 json_path = os.path.join(gtest_test_utils.GetTempDir(), 570 gtest_prog_name + 'out.json') 571 gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name) 572 573 command = ( 574 [gtest_prog_path, '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path)] + 575 extra_args 576 ) 577 p = gtest_test_utils.Subprocess(command) 578 if p.terminated_by_signal: 579 self.assert_(False, 580 '%s was killed by signal %d' % (gtest_prog_name, p.signal)) 581 else: 582 self.assert_(p.exited) 583 self.assertEquals(expected_exit_code, p.exit_code, 584 "'%s' exited with code %s, which doesn't match " 585 'the expected exit code %s.' 586 % (command, p.exit_code, expected_exit_code)) 587 with open(json_path) as f: 588 actual = json.load(f) 589 return actual 590 591 def _TestJsonOutput(self, gtest_prog_name, expected, 592 expected_exit_code, extra_args=None): 593 """Checks the JSON output generated by the Google Test binary. 594 595 Asserts that the JSON document generated by running the program 596 gtest_prog_name matches expected_json, a string containing another 597 JSON document. Furthermore, the program's exit code must be 598 expected_exit_code. 599 600 Args: 601 gtest_prog_name: Google Test binary name. 602 expected: expected output. 603 expected_exit_code: program's exit code. 604 extra_args: extra arguments to binary invocation. 605 """ 606 607 actual = self._GetJsonOutput(gtest_prog_name, extra_args or [], 608 expected_exit_code) 609 self.assertEqual(expected, gtest_json_test_utils.normalize(actual)) 610 611 612if __name__ == '__main__': 613 if NO_STACKTRACE_SUPPORT_FLAG in sys.argv: 614 # unittest.main() can't handle unknown flags 615 sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG) 616 617 os.environ['GTEST_STACK_TRACE_DEPTH'] = '1' 618 gtest_test_utils.Main() 619