1#!/usr/bin/env python
2#
3# Copyright 2006, Google Inc.
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are
8# met:
9#
10#     * Redistributions of source code must retain the above copyright
11# notice, this list of conditions and the following disclaimer.
12#     * Redistributions in binary form must reproduce the above
13# copyright notice, this list of conditions and the following disclaimer
14# in the documentation and/or other materials provided with the
15# distribution.
16#     * Neither the name of Google Inc. nor the names of its
17# contributors may be used to endorse or promote products derived from
18# this software without specific prior written permission.
19#
20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
32"""Unit test for the gtest_xml_output module"""
33
34import datetime
35import errno
36import os
37import re
38import sys
39from xml.dom import minidom, Node
40
41import gtest_test_utils
42import gtest_xml_test_utils
43
44GTEST_FILTER_FLAG = '--gtest_filter'
45GTEST_LIST_TESTS_FLAG = '--gtest_list_tests'
46GTEST_OUTPUT_FLAG = '--gtest_output'
47GTEST_DEFAULT_OUTPUT_FILE = 'test_detail.xml'
48GTEST_PROGRAM_NAME = 'gtest_xml_output_unittest_'
49
50# The flag indicating stacktraces are not supported
51NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support'
52
53# The environment variables for test sharding.
54TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
55SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
56SHARD_STATUS_FILE_ENV_VAR = 'GTEST_SHARD_STATUS_FILE'
57
58SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv
59
60if SUPPORTS_STACK_TRACES:
61  STACK_TRACE_TEMPLATE = '\nStack trace:\n*'
62else:
63  STACK_TRACE_TEMPLATE = ''
64  # unittest.main() can't handle unknown flags
65  sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)
66
67EXPECTED_NON_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?>
68<testsuites tests="23" failures="4" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
69  <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" errors="0" time="*">
70    <testcase name="Succeeds" status="run" time="*" classname="SuccessfulTest"/>
71  </testsuite>
72  <testsuite name="FailedTest" tests="1" failures="1" disabled="0" errors="0" time="*">
73    <testcase name="Fails" status="run" time="*" classname="FailedTest">
74      <failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A;  1&#x0A;  2" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
75Expected equality of these values:
76  1
77  2%(stack)s]]></failure>
78    </testcase>
79  </testsuite>
80  <testsuite name="MixedResultTest" tests="3" failures="1" disabled="1" errors="0" time="*">
81    <testcase name="Succeeds" status="run" time="*" classname="MixedResultTest"/>
82    <testcase name="Fails" status="run" time="*" classname="MixedResultTest">
83      <failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A;  1&#x0A;  2" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
84Expected equality of these values:
85  1
86  2%(stack)s]]></failure>
87      <failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A;  2&#x0A;  3" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
88Expected equality of these values:
89  2
90  3%(stack)s]]></failure>
91    </testcase>
92    <testcase name="DISABLED_test" status="notrun" time="*" classname="MixedResultTest"/>
93  </testsuite>
94  <testsuite name="XmlQuotingTest" tests="1" failures="1" disabled="0" errors="0" time="*">
95    <testcase name="OutputsCData" status="run" time="*" classname="XmlQuotingTest">
96      <failure message="gtest_xml_output_unittest_.cc:*&#x0A;Failed&#x0A;XML output: &lt;?xml encoding=&quot;utf-8&quot;&gt;&lt;top&gt;&lt;![CDATA[cdata text]]&gt;&lt;/top&gt;" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
97Failed
98XML output: <?xml encoding="utf-8"><top><![CDATA[cdata text]]>]]&gt;<![CDATA[</top>%(stack)s]]></failure>
99    </testcase>
100  </testsuite>
101  <testsuite name="InvalidCharactersTest" tests="1" failures="1" disabled="0" errors="0" time="*">
102    <testcase name="InvalidCharactersInMessage" status="run" time="*" classname="InvalidCharactersTest">
103      <failure message="gtest_xml_output_unittest_.cc:*&#x0A;Failed&#x0A;Invalid characters in brackets []" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
104Failed
105Invalid characters in brackets []%(stack)s]]></failure>
106    </testcase>
107  </testsuite>
108  <testsuite name="DisabledTest" tests="1" failures="0" disabled="1" errors="0" time="*">
109    <testcase name="DISABLED_test_not_run" status="notrun" time="*" classname="DisabledTest"/>
110  </testsuite>
111  <testsuite name="PropertyRecordingTest" tests="4" failures="0" disabled="0" errors="0" time="*" SetUpTestCase="yes" TearDownTestCase="aye">
112    <testcase name="OneProperty" status="run" time="*" classname="PropertyRecordingTest">
113      <properties>
114        <property name="key_1" value="1"/>
115      </properties>
116    </testcase>
117    <testcase name="IntValuedProperty" status="run" time="*" classname="PropertyRecordingTest">
118      <properties>
119        <property name="key_int" value="1"/>
120      </properties>
121    </testcase>
122    <testcase name="ThreeProperties" status="run" time="*" classname="PropertyRecordingTest">
123      <properties>
124        <property name="key_1" value="1"/>
125        <property name="key_2" value="2"/>
126        <property name="key_3" value="3"/>
127      </properties>
128    </testcase>
129    <testcase name="TwoValuesForOneKeyUsesLastValue" status="run" time="*" classname="PropertyRecordingTest">
130      <properties>
131        <property name="key_1" value="2"/>
132      </properties>
133    </testcase>
134  </testsuite>
135  <testsuite name="NoFixtureTest" tests="3" failures="0" disabled="0" errors="0" time="*">
136     <testcase name="RecordProperty" status="run" time="*" classname="NoFixtureTest">
137       <properties>
138         <property name="key" value="1"/>
139       </properties>
140     </testcase>
141     <testcase name="ExternalUtilityThatCallsRecordIntValuedProperty" status="run" time="*" classname="NoFixtureTest">
142       <properties>
143         <property name="key_for_utility_int" value="1"/>
144       </properties>
145     </testcase>
146     <testcase name="ExternalUtilityThatCallsRecordStringValuedProperty" status="run" time="*" classname="NoFixtureTest">
147       <properties>
148         <property name="key_for_utility_string" value="1"/>
149       </properties>
150     </testcase>
151  </testsuite>
152  <testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" errors="0" time="*">
153    <testcase name="HasValueParamAttribute/0" value_param="33" status="run" time="*" classname="Single/ValueParamTest" />
154    <testcase name="HasValueParamAttribute/1" value_param="42" status="run" time="*" classname="Single/ValueParamTest" />
155    <testcase name="AnotherTestThatHasValueParamAttribute/0" value_param="33" status="run" time="*" classname="Single/ValueParamTest" />
156    <testcase name="AnotherTestThatHasValueParamAttribute/1" value_param="42" status="run" time="*" classname="Single/ValueParamTest" />
157  </testsuite>
158  <testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" errors="0" time="*">
159    <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="TypedTest/0" />
160  </testsuite>
161  <testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" errors="0" time="*">
162    <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="TypedTest/1" />
163  </testsuite>
164  <testsuite name="Single/TypeParameterizedTestCase/0" tests="1" failures="0" disabled="0" errors="0" time="*">
165    <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="Single/TypeParameterizedTestCase/0" />
166  </testsuite>
167  <testsuite name="Single/TypeParameterizedTestCase/1" tests="1" failures="0" disabled="0" errors="0" time="*">
168    <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="Single/TypeParameterizedTestCase/1" />
169  </testsuite>
170</testsuites>""" % {'stack': STACK_TRACE_TEMPLATE}
171
172EXPECTED_FILTERED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
173<testsuites tests="1" failures="0" disabled="0" errors="0" time="*"
174            timestamp="*" name="AllTests" ad_hoc_property="42">
175  <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0"
176             errors="0" time="*">
177    <testcase name="Succeeds" status="run" time="*" classname="SuccessfulTest"/>
178  </testsuite>
179</testsuites>"""
180
181EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
182<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
183  <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" errors="0" time="*">
184    <testcase name="Succeeds" status="run" time="*" classname="SuccessfulTest"/>
185  </testsuite>
186  <testsuite name="NoFixtureTest" tests="1" failures="0" disabled="0" errors="0" time="*">
187     <testcase name="RecordProperty" status="run" time="*" classname="NoFixtureTest">
188       <properties>
189         <property name="key" value="1"/>
190       </properties>
191     </testcase>
192  </testsuite>
193  <testsuite name="Single/ValueParamTest" tests="1" failures="0" disabled="0" errors="0" time="*">
194    <testcase name="AnotherTestThatHasValueParamAttribute/1" value_param="42" status="run" time="*" classname="Single/ValueParamTest" />
195  </testsuite>
196</testsuites>"""
197
198EXPECTED_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?>
199<testsuites tests="0" failures="0" disabled="0" errors="0" time="*"
200            timestamp="*" name="AllTests">
201</testsuites>"""
202
203GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME)
204
205SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess(
206    [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output
207
208
209class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase):
210  """
211  Unit test for Google Test's XML output functionality.
212  """
213
214  # This test currently breaks on platforms that do not support typed and
215  # type-parameterized tests, so we don't run it under them.
216  if SUPPORTS_TYPED_TESTS:
217    def testNonEmptyXmlOutput(self):
218      """
219      Runs a test program that generates a non-empty XML output, and
220      tests that the XML output is expected.
221      """
222      self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY_XML, 1)
223
224  def testEmptyXmlOutput(self):
225    """Verifies XML output for a Google Test binary without actual tests.
226
227    Runs a test program that generates an empty XML output, and
228    tests that the XML output is expected.
229    """
230
231    self._TestXmlOutput('gtest_no_test_unittest', EXPECTED_EMPTY_XML, 0)
232
233  def testTimestampValue(self):
234    """Checks whether the timestamp attribute in the XML output is valid.
235
236    Runs a test program that generates an empty XML output, and checks if
237    the timestamp attribute in the testsuites tag is valid.
238    """
239    actual = self._GetXmlOutput('gtest_no_test_unittest', [], {}, 0)
240    date_time_str = actual.documentElement.getAttributeNode('timestamp').value
241    # datetime.strptime() is only available in Python 2.5+ so we have to
242    # parse the expected datetime manually.
243    match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str)
244    self.assertTrue(
245        re.match,
246        'XML datettime string %s has incorrect format' % date_time_str)
247    date_time_from_xml = datetime.datetime(
248        year=int(match.group(1)), month=int(match.group(2)),
249        day=int(match.group(3)), hour=int(match.group(4)),
250        minute=int(match.group(5)), second=int(match.group(6)))
251
252    time_delta = abs(datetime.datetime.now() - date_time_from_xml)
253    # timestamp value should be near the current local time
254    self.assertTrue(time_delta < datetime.timedelta(seconds=600),
255                    'time_delta is %s' % time_delta)
256    actual.unlink()
257
258  def testDefaultOutputFile(self):
259    """
260    Confirms that Google Test produces an XML output file with the expected
261    default name if no name is explicitly specified.
262    """
263    output_file = os.path.join(gtest_test_utils.GetTempDir(),
264                               GTEST_DEFAULT_OUTPUT_FILE)
265    gtest_prog_path = gtest_test_utils.GetTestExecutablePath(
266        'gtest_no_test_unittest')
267    try:
268      os.remove(output_file)
269    except OSError, e:
270      if e.errno != errno.ENOENT:
271        raise
272
273    p = gtest_test_utils.Subprocess(
274        [gtest_prog_path, '%s=xml' % GTEST_OUTPUT_FLAG],
275        working_dir=gtest_test_utils.GetTempDir())
276    self.assert_(p.exited)
277    self.assertEquals(0, p.exit_code)
278    self.assert_(os.path.isfile(output_file))
279
280  def testSuppressedXmlOutput(self):
281    """
282    Tests that no XML file is generated if the default XML listener is
283    shut down before RUN_ALL_TESTS is invoked.
284    """
285
286    xml_path = os.path.join(gtest_test_utils.GetTempDir(),
287                            GTEST_PROGRAM_NAME + 'out.xml')
288    if os.path.isfile(xml_path):
289      os.remove(xml_path)
290
291    command = [GTEST_PROGRAM_PATH,
292               '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path),
293               '--shut_down_xml']
294    p = gtest_test_utils.Subprocess(command)
295    if p.terminated_by_signal:
296      # p.signal is available only if p.terminated_by_signal is True.
297      self.assertFalse(
298          p.terminated_by_signal,
299          '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal))
300    else:
301      self.assert_(p.exited)
302      self.assertEquals(1, p.exit_code,
303                        "'%s' exited with code %s, which doesn't match "
304                        'the expected exit code %s.'
305                        % (command, p.exit_code, 1))
306
307    self.assert_(not os.path.isfile(xml_path))
308
309  def testFilteredTestXmlOutput(self):
310    """Verifies XML output when a filter is applied.
311
312    Runs a test program that executes only some tests and verifies that
313    non-selected tests do not show up in the XML output.
314    """
315
316    self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED_TEST_XML, 0,
317                        extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG])
318
319  def testShardedTestXmlOutput(self):
320    """Verifies XML output when run using multiple shards.
321
322    Runs a test program that executes only one shard and verifies that tests
323    from other shards do not show up in the XML output.
324    """
325
326    self._TestXmlOutput(
327        GTEST_PROGRAM_NAME,
328        EXPECTED_SHARDED_TEST_XML,
329        0,
330        extra_env={SHARD_INDEX_ENV_VAR: '0',
331                   TOTAL_SHARDS_ENV_VAR: '10'})
332
333  def _GetXmlOutput(self, gtest_prog_name, extra_args, extra_env,
334                    expected_exit_code):
335    """
336    Returns the xml output generated by running the program gtest_prog_name.
337    Furthermore, the program's exit code must be expected_exit_code.
338    """
339    xml_path = os.path.join(gtest_test_utils.GetTempDir(),
340                            gtest_prog_name + 'out.xml')
341    gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name)
342
343    command = ([gtest_prog_path, '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path)] +
344               extra_args)
345    environ_copy = os.environ.copy()
346    if extra_env:
347      environ_copy.update(extra_env)
348    p = gtest_test_utils.Subprocess(command, env=environ_copy)
349
350    if p.terminated_by_signal:
351      self.assert_(False,
352                   '%s was killed by signal %d' % (gtest_prog_name, p.signal))
353    else:
354      self.assert_(p.exited)
355      self.assertEquals(expected_exit_code, p.exit_code,
356                        "'%s' exited with code %s, which doesn't match "
357                        'the expected exit code %s.'
358                        % (command, p.exit_code, expected_exit_code))
359    actual = minidom.parse(xml_path)
360    return actual
361
362  def _TestXmlOutput(self, gtest_prog_name, expected_xml,
363                     expected_exit_code, extra_args=None, extra_env=None):
364    """
365    Asserts that the XML document generated by running the program
366    gtest_prog_name matches expected_xml, a string containing another
367    XML document.  Furthermore, the program's exit code must be
368    expected_exit_code.
369    """
370
371    actual = self._GetXmlOutput(gtest_prog_name, extra_args or [],
372                                extra_env or {}, expected_exit_code)
373    expected = minidom.parseString(expected_xml)
374    self.NormalizeXml(actual.documentElement)
375    self.AssertEquivalentNodes(expected.documentElement,
376                               actual.documentElement)
377    expected.unlink()
378    actual.unlink()
379
380
381if __name__ == '__main__':
382  os.environ['GTEST_STACK_TRACE_DEPTH'] = '1'
383  gtest_test_utils.Main()
384