1#!/usr/bin/env python
2# Copyright 2018, Google Inc.
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9#     * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11#     * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15#     * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31"""Unit test for the gtest_json_output module."""
32
33import datetime
34import errno
35import json
36import os
37import re
38import sys
39
40import gtest_json_test_utils
41import gtest_test_utils
42
43GTEST_FILTER_FLAG = '--gtest_filter'
44GTEST_LIST_TESTS_FLAG = '--gtest_list_tests'
45GTEST_OUTPUT_FLAG = '--gtest_output'
46GTEST_DEFAULT_OUTPUT_FILE = 'test_detail.json'
47GTEST_PROGRAM_NAME = 'gtest_xml_output_unittest_'
48
49# The flag indicating stacktraces are not supported
50NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support'
51
52SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv
53
54if SUPPORTS_STACK_TRACES:
55  STACK_TRACE_TEMPLATE = '\nStack trace:\n*'
56else:
57  STACK_TRACE_TEMPLATE = ''
58
59EXPECTED_NON_EMPTY = {
60    u'tests':
61        24,
62    u'failures':
63        4,
64    u'disabled':
65        2,
66    u'errors':
67        0,
68    u'timestamp':
69        u'*',
70    u'time':
71        u'*',
72    u'ad_hoc_property':
73        u'42',
74    u'name':
75        u'AllTests',
76    u'testsuites': [
77        {
78            u'name':
79                u'SuccessfulTest',
80            u'tests':
81                1,
82            u'failures':
83                0,
84            u'disabled':
85                0,
86            u'errors':
87                0,
88            u'time':
89                u'*',
90            u'testsuite': [{
91                u'name': u'Succeeds',
92                u'status': u'RUN',
93                u'result': u'COMPLETED',
94                u'time': u'*',
95                u'classname': u'SuccessfulTest'
96            }]
97        },
98        {
99            u'name':
100                u'FailedTest',
101            u'tests':
102                1,
103            u'failures':
104                1,
105            u'disabled':
106                0,
107            u'errors':
108                0,
109            u'time':
110                u'*',
111            u'testsuite': [{
112                u'name':
113                    u'Fails',
114                u'status':
115                    u'RUN',
116                u'result':
117                    u'COMPLETED',
118                u'time':
119                    u'*',
120                u'classname':
121                    u'FailedTest',
122                u'failures': [{
123                    u'failure': u'gtest_xml_output_unittest_.cc:*\n'
124                                u'Expected equality of these values:\n'
125                                u'  1\n  2' + STACK_TRACE_TEMPLATE,
126                    u'type': u''
127                }]
128            }]
129        },
130        {
131            u'name':
132                u'DisabledTest',
133            u'tests':
134                1,
135            u'failures':
136                0,
137            u'disabled':
138                1,
139            u'errors':
140                0,
141            u'time':
142                u'*',
143            u'testsuite': [{
144                u'name': u'DISABLED_test_not_run',
145                u'status': u'NOTRUN',
146                u'result': u'SUPPRESSED',
147                u'time': u'*',
148                u'classname': u'DisabledTest'
149            }]
150        },
151        {
152            u'name':
153                u'SkippedTest',
154            u'tests':
155                1,
156            u'failures':
157                0,
158            u'disabled':
159                0,
160            u'errors':
161                0,
162            u'time':
163                u'*',
164            u'testsuite': [{
165                u'name': u'Skipped',
166                u'status': u'RUN',
167                u'result': u'SKIPPED',
168                u'time': u'*',
169                u'classname': u'SkippedTest'
170            }]
171        },
172        {
173            u'name':
174                u'MixedResultTest',
175            u'tests':
176                3,
177            u'failures':
178                1,
179            u'disabled':
180                1,
181            u'errors':
182                0,
183            u'time':
184                u'*',
185            u'testsuite': [
186                {
187                    u'name': u'Succeeds',
188                    u'status': u'RUN',
189                    u'result': u'COMPLETED',
190                    u'time': u'*',
191                    u'classname': u'MixedResultTest'
192                },
193                {
194                    u'name':
195                        u'Fails',
196                    u'status':
197                        u'RUN',
198                    u'result':
199                        u'COMPLETED',
200                    u'time':
201                        u'*',
202                    u'classname':
203                        u'MixedResultTest',
204                    u'failures':
205                        [{
206                            u'failure': u'gtest_xml_output_unittest_.cc:*\n'
207                                        u'Expected equality of these values:\n'
208                                        u'  1\n  2' + STACK_TRACE_TEMPLATE,
209                            u'type': u''
210                        },
211                         {
212                             u'failure': u'gtest_xml_output_unittest_.cc:*\n'
213                                         u'Expected equality of these values:\n'
214                                         u'  2\n  3' + STACK_TRACE_TEMPLATE,
215                             u'type': u''
216                         }]
217                },
218                {
219                    u'name': u'DISABLED_test',
220                    u'status': u'NOTRUN',
221                    u'result': u'SUPPRESSED',
222                    u'time': u'*',
223                    u'classname': u'MixedResultTest'
224                }
225            ]
226        },
227        {
228            u'name':
229                u'XmlQuotingTest',
230            u'tests':
231                1,
232            u'failures':
233                1,
234            u'disabled':
235                0,
236            u'errors':
237                0,
238            u'time':
239                u'*',
240            u'testsuite': [{
241                u'name':
242                    u'OutputsCData',
243                u'status':
244                    u'RUN',
245                u'result':
246                    u'COMPLETED',
247                u'time':
248                    u'*',
249                u'classname':
250                    u'XmlQuotingTest',
251                u'failures': [{
252                    u'failure': u'gtest_xml_output_unittest_.cc:*\n'
253                                u'Failed\nXML output: <?xml encoding="utf-8">'
254                                u'<top><![CDATA[cdata text]]></top>' +
255                                STACK_TRACE_TEMPLATE,
256                    u'type': u''
257                }]
258            }]
259        },
260        {
261            u'name':
262                u'InvalidCharactersTest',
263            u'tests':
264                1,
265            u'failures':
266                1,
267            u'disabled':
268                0,
269            u'errors':
270                0,
271            u'time':
272                u'*',
273            u'testsuite': [{
274                u'name':
275                    u'InvalidCharactersInMessage',
276                u'status':
277                    u'RUN',
278                u'result':
279                    u'COMPLETED',
280                u'time':
281                    u'*',
282                u'classname':
283                    u'InvalidCharactersTest',
284                u'failures': [{
285                    u'failure': u'gtest_xml_output_unittest_.cc:*\n'
286                                u'Failed\nInvalid characters in brackets'
287                                u' [\x01\x02]' + STACK_TRACE_TEMPLATE,
288                    u'type': u''
289                }]
290            }]
291        },
292        {
293            u'name':
294                u'PropertyRecordingTest',
295            u'tests':
296                4,
297            u'failures':
298                0,
299            u'disabled':
300                0,
301            u'errors':
302                0,
303            u'time':
304                u'*',
305            u'SetUpTestSuite':
306                u'yes',
307            u'TearDownTestSuite':
308                u'aye',
309            u'testsuite': [{
310                u'name': u'OneProperty',
311                u'status': u'RUN',
312                u'result': u'COMPLETED',
313                u'time': u'*',
314                u'classname': u'PropertyRecordingTest',
315                u'key_1': u'1'
316            },
317                           {
318                               u'name': u'IntValuedProperty',
319                               u'status': u'RUN',
320                               u'result': u'COMPLETED',
321                               u'time': u'*',
322                               u'classname': u'PropertyRecordingTest',
323                               u'key_int': u'1'
324                           },
325                           {
326                               u'name': u'ThreeProperties',
327                               u'status': u'RUN',
328                               u'result': u'COMPLETED',
329                               u'time': u'*',
330                               u'classname': u'PropertyRecordingTest',
331                               u'key_1': u'1',
332                               u'key_2': u'2',
333                               u'key_3': u'3'
334                           },
335                           {
336                               u'name': u'TwoValuesForOneKeyUsesLastValue',
337                               u'status': u'RUN',
338                               u'result': u'COMPLETED',
339                               u'time': u'*',
340                               u'classname': u'PropertyRecordingTest',
341                               u'key_1': u'2'
342                           }]
343        },
344        {
345            u'name':
346                u'NoFixtureTest',
347            u'tests':
348                3,
349            u'failures':
350                0,
351            u'disabled':
352                0,
353            u'errors':
354                0,
355            u'time':
356                u'*',
357            u'testsuite': [
358                {
359                    u'name': u'RecordProperty',
360                    u'status': u'RUN',
361                    u'result': u'COMPLETED',
362                    u'time': u'*',
363                    u'classname': u'NoFixtureTest',
364                    u'key': u'1'
365                },
366                {
367                    u'name': u'ExternalUtilityThatCallsRecordIntValuedProperty',
368                    u'status': u'RUN',
369                    u'result': u'COMPLETED',
370                    u'time': u'*',
371                    u'classname': u'NoFixtureTest',
372                    u'key_for_utility_int': u'1'
373                },
374                {
375                    u'name':
376                        u'ExternalUtilityThatCallsRecordStringValuedProperty',
377                    u'status':
378                        u'RUN',
379                    u'result':
380                        u'COMPLETED',
381                    u'time':
382                        u'*',
383                    u'classname':
384                        u'NoFixtureTest',
385                    u'key_for_utility_string':
386                        u'1'
387                }
388            ]
389        },
390        {
391            u'name':
392                u'TypedTest/0',
393            u'tests':
394                1,
395            u'failures':
396                0,
397            u'disabled':
398                0,
399            u'errors':
400                0,
401            u'time':
402                u'*',
403            u'testsuite': [{
404                u'name': u'HasTypeParamAttribute',
405                u'type_param': u'int',
406                u'status': u'RUN',
407                u'result': u'COMPLETED',
408                u'time': u'*',
409                u'classname': u'TypedTest/0'
410            }]
411        },
412        {
413            u'name':
414                u'TypedTest/1',
415            u'tests':
416                1,
417            u'failures':
418                0,
419            u'disabled':
420                0,
421            u'errors':
422                0,
423            u'time':
424                u'*',
425            u'testsuite': [{
426                u'name': u'HasTypeParamAttribute',
427                u'type_param': u'long',
428                u'status': u'RUN',
429                u'result': u'COMPLETED',
430                u'time': u'*',
431                u'classname': u'TypedTest/1'
432            }]
433        },
434        {
435            u'name':
436                u'Single/TypeParameterizedTestSuite/0',
437            u'tests':
438                1,
439            u'failures':
440                0,
441            u'disabled':
442                0,
443            u'errors':
444                0,
445            u'time':
446                u'*',
447            u'testsuite': [{
448                u'name': u'HasTypeParamAttribute',
449                u'type_param': u'int',
450                u'status': u'RUN',
451                u'result': u'COMPLETED',
452                u'time': u'*',
453                u'classname': u'Single/TypeParameterizedTestSuite/0'
454            }]
455        },
456        {
457            u'name':
458                u'Single/TypeParameterizedTestSuite/1',
459            u'tests':
460                1,
461            u'failures':
462                0,
463            u'disabled':
464                0,
465            u'errors':
466                0,
467            u'time':
468                u'*',
469            u'testsuite': [{
470                u'name': u'HasTypeParamAttribute',
471                u'type_param': u'long',
472                u'status': u'RUN',
473                u'result': u'COMPLETED',
474                u'time': u'*',
475                u'classname': u'Single/TypeParameterizedTestSuite/1'
476            }]
477        },
478        {
479            u'name':
480                u'Single/ValueParamTest',
481            u'tests':
482                4,
483            u'failures':
484                0,
485            u'disabled':
486                0,
487            u'errors':
488                0,
489            u'time':
490                u'*',
491            u'testsuite': [
492                {
493                    u'name': u'HasValueParamAttribute/0',
494                    u'value_param': u'33',
495                    u'status': u'RUN',
496                    u'result': u'COMPLETED',
497                    u'time': u'*',
498                    u'classname': u'Single/ValueParamTest'
499                },
500                {
501                    u'name': u'HasValueParamAttribute/1',
502                    u'value_param': u'42',
503                    u'status': u'RUN',
504                    u'result': u'COMPLETED',
505                    u'time': u'*',
506                    u'classname': u'Single/ValueParamTest'
507                },
508                {
509                    u'name': u'AnotherTestThatHasValueParamAttribute/0',
510                    u'value_param': u'33',
511                    u'status': u'RUN',
512                    u'result': u'COMPLETED',
513                    u'time': u'*',
514                    u'classname': u'Single/ValueParamTest'
515                },
516                {
517                    u'name': u'AnotherTestThatHasValueParamAttribute/1',
518                    u'value_param': u'42',
519                    u'status': u'RUN',
520                    u'result': u'COMPLETED',
521                    u'time': u'*',
522                    u'classname': u'Single/ValueParamTest'
523                }
524            ]
525        }
526    ]
527}
528
529EXPECTED_FILTERED = {
530    u'tests':
531        1,
532    u'failures':
533        0,
534    u'disabled':
535        0,
536    u'errors':
537        0,
538    u'time':
539        u'*',
540    u'timestamp':
541        u'*',
542    u'name':
543        u'AllTests',
544    u'ad_hoc_property':
545        u'42',
546    u'testsuites': [{
547        u'name':
548            u'SuccessfulTest',
549        u'tests':
550            1,
551        u'failures':
552            0,
553        u'disabled':
554            0,
555        u'errors':
556            0,
557        u'time':
558            u'*',
559        u'testsuite': [{
560            u'name': u'Succeeds',
561            u'status': u'RUN',
562            u'result': u'COMPLETED',
563            u'time': u'*',
564            u'classname': u'SuccessfulTest',
565        }]
566    }],
567}
568
569EXPECTED_EMPTY = {
570    u'tests': 0,
571    u'failures': 0,
572    u'disabled': 0,
573    u'errors': 0,
574    u'time': u'*',
575    u'timestamp': u'*',
576    u'name': u'AllTests',
577    u'testsuites': [],
578}
579
580GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME)
581
582SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess(
583    [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output
584
585
586class GTestJsonOutputUnitTest(gtest_test_utils.TestCase):
587  """Unit test for Google Test's JSON output functionality.
588  """
589
590  # This test currently breaks on platforms that do not support typed and
591  # type-parameterized tests, so we don't run it under them.
592  if SUPPORTS_TYPED_TESTS:
593
594    def testNonEmptyJsonOutput(self):
595      """Verifies JSON output for a Google Test binary with non-empty output.
596
597      Runs a test program that generates a non-empty JSON output, and
598      tests that the JSON output is expected.
599      """
600      self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY, 1)
601
602  def testEmptyJsonOutput(self):
603    """Verifies JSON output for a Google Test binary without actual tests.
604
605    Runs a test program that generates an empty JSON output, and
606    tests that the JSON output is expected.
607    """
608
609    self._TestJsonOutput('gtest_no_test_unittest', EXPECTED_EMPTY, 0)
610
611  def testTimestampValue(self):
612    """Checks whether the timestamp attribute in the JSON output is valid.
613
614    Runs a test program that generates an empty JSON output, and checks if
615    the timestamp attribute in the testsuites tag is valid.
616    """
617    actual = self._GetJsonOutput('gtest_no_test_unittest', [], 0)
618    date_time_str = actual['timestamp']
619    # datetime.strptime() is only available in Python 2.5+ so we have to
620    # parse the expected datetime manually.
621    match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str)
622    self.assertTrue(
623        re.match,
624        'JSON datettime string %s has incorrect format' % date_time_str)
625    date_time_from_json = datetime.datetime(
626        year=int(match.group(1)), month=int(match.group(2)),
627        day=int(match.group(3)), hour=int(match.group(4)),
628        minute=int(match.group(5)), second=int(match.group(6)))
629
630    time_delta = abs(datetime.datetime.now() - date_time_from_json)
631    # timestamp value should be near the current local time
632    self.assertTrue(time_delta < datetime.timedelta(seconds=600),
633                    'time_delta is %s' % time_delta)
634
635  def testDefaultOutputFile(self):
636    """Verifies the default output file name.
637
638    Confirms that Google Test produces an JSON output file with the expected
639    default name if no name is explicitly specified.
640    """
641    output_file = os.path.join(gtest_test_utils.GetTempDir(),
642                               GTEST_DEFAULT_OUTPUT_FILE)
643    gtest_prog_path = gtest_test_utils.GetTestExecutablePath(
644        'gtest_no_test_unittest')
645    try:
646      os.remove(output_file)
647    except OSError:
648      e = sys.exc_info()[1]
649      if e.errno != errno.ENOENT:
650        raise
651
652    p = gtest_test_utils.Subprocess(
653        [gtest_prog_path, '%s=json' % GTEST_OUTPUT_FLAG],
654        working_dir=gtest_test_utils.GetTempDir())
655    self.assert_(p.exited)
656    self.assertEquals(0, p.exit_code)
657    self.assert_(os.path.isfile(output_file))
658
659  def testSuppressedJsonOutput(self):
660    """Verifies that no JSON output is generated.
661
662    Tests that no JSON file is generated if the default JSON listener is
663    shut down before RUN_ALL_TESTS is invoked.
664    """
665
666    json_path = os.path.join(gtest_test_utils.GetTempDir(),
667                             GTEST_PROGRAM_NAME + 'out.json')
668    if os.path.isfile(json_path):
669      os.remove(json_path)
670
671    command = [GTEST_PROGRAM_PATH,
672               '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path),
673               '--shut_down_xml']
674    p = gtest_test_utils.Subprocess(command)
675    if p.terminated_by_signal:
676      # p.signal is available only if p.terminated_by_signal is True.
677      self.assertFalse(
678          p.terminated_by_signal,
679          '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal))
680    else:
681      self.assert_(p.exited)
682      self.assertEquals(1, p.exit_code,
683                        "'%s' exited with code %s, which doesn't match "
684                        'the expected exit code %s.'
685                        % (command, p.exit_code, 1))
686
687    self.assert_(not os.path.isfile(json_path))
688
689  def testFilteredTestJsonOutput(self):
690    """Verifies JSON output when a filter is applied.
691
692    Runs a test program that executes only some tests and verifies that
693    non-selected tests do not show up in the JSON output.
694    """
695
696    self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED, 0,
697                         extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG])
698
699  def _GetJsonOutput(self, gtest_prog_name, extra_args, expected_exit_code):
700    """Returns the JSON output generated by running the program gtest_prog_name.
701
702    Furthermore, the program's exit code must be expected_exit_code.
703
704    Args:
705      gtest_prog_name: Google Test binary name.
706      extra_args: extra arguments to binary invocation.
707      expected_exit_code: program's exit code.
708    """
709    json_path = os.path.join(gtest_test_utils.GetTempDir(),
710                             gtest_prog_name + 'out.json')
711    gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name)
712
713    command = (
714        [gtest_prog_path, '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path)] +
715        extra_args
716    )
717    p = gtest_test_utils.Subprocess(command)
718    if p.terminated_by_signal:
719      self.assert_(False,
720                   '%s was killed by signal %d' % (gtest_prog_name, p.signal))
721    else:
722      self.assert_(p.exited)
723      self.assertEquals(expected_exit_code, p.exit_code,
724                        "'%s' exited with code %s, which doesn't match "
725                        'the expected exit code %s.'
726                        % (command, p.exit_code, expected_exit_code))
727    with open(json_path) as f:
728      actual = json.load(f)
729    return actual
730
731  def _TestJsonOutput(self, gtest_prog_name, expected,
732                      expected_exit_code, extra_args=None):
733    """Checks the JSON output generated by the Google Test binary.
734
735    Asserts that the JSON document generated by running the program
736    gtest_prog_name matches expected_json, a string containing another
737    JSON document.  Furthermore, the program's exit code must be
738    expected_exit_code.
739
740    Args:
741      gtest_prog_name: Google Test binary name.
742      expected: expected output.
743      expected_exit_code: program's exit code.
744      extra_args: extra arguments to binary invocation.
745    """
746
747    actual = self._GetJsonOutput(gtest_prog_name, extra_args or [],
748                                 expected_exit_code)
749    self.assertEqual(expected, gtest_json_test_utils.normalize(actual))
750
751
752if __name__ == '__main__':
753  if NO_STACKTRACE_SUPPORT_FLAG in sys.argv:
754    # unittest.main() can't handle unknown flags
755    sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)
756
757  os.environ['GTEST_STACK_TRACE_DEPTH'] = '1'
758  gtest_test_utils.Main()
759