1#!/usr/bin/env python 2# 3# Copyright 2006, Google Inc. 4# All rights reserved. 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions are 8# met: 9# 10# * Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# * Redistributions in binary form must reproduce the above 13# copyright notice, this list of conditions and the following disclaimer 14# in the documentation and/or other materials provided with the 15# distribution. 16# * Neither the name of Google Inc. nor the names of its 17# contributors may be used to endorse or promote products derived from 18# this software without specific prior written permission. 19# 20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32"""Unit test for the gtest_xml_output module""" 33 34import datetime 35import errno 36import os 37import re 38import sys 39from xml.dom import minidom, Node 40 41import gtest_test_utils 42import gtest_xml_test_utils 43 44GTEST_FILTER_FLAG = '--gtest_filter' 45GTEST_LIST_TESTS_FLAG = '--gtest_list_tests' 46GTEST_OUTPUT_FLAG = '--gtest_output' 47GTEST_DEFAULT_OUTPUT_FILE = 'test_detail.xml' 48GTEST_PROGRAM_NAME = 'gtest_xml_output_unittest_' 49 50# The flag indicating stacktraces are not supported 51NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support' 52 53# The environment variables for test sharding. 54TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 55SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 56SHARD_STATUS_FILE_ENV_VAR = 'GTEST_SHARD_STATUS_FILE' 57 58SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv 59 60if SUPPORTS_STACK_TRACES: 61 STACK_TRACE_TEMPLATE = '\nStack trace:\n*' 62else: 63 STACK_TRACE_TEMPLATE = '' 64 # unittest.main() can't handle unknown flags 65 sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG) 66 67EXPECTED_NON_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?> 68<testsuites tests="23" failures="4" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42"> 69 <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" errors="0" time="*"> 70 <testcase name="Succeeds" status="run" time="*" classname="SuccessfulTest"/> 71 </testsuite> 72 <testsuite name="FailedTest" tests="1" failures="1" disabled="0" errors="0" time="*"> 73 <testcase name="Fails" status="run" time="*" classname="FailedTest"> 74 <failure message="gtest_xml_output_unittest_.cc:*
Expected equality of these values:
 1
 2" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 75Expected equality of these values: 76 1 77 2%(stack)s]]></failure> 78 </testcase> 79 </testsuite> 80 <testsuite name="MixedResultTest" tests="3" failures="1" disabled="1" errors="0" time="*"> 81 <testcase name="Succeeds" status="run" time="*" classname="MixedResultTest"/> 82 <testcase name="Fails" status="run" time="*" classname="MixedResultTest"> 83 <failure message="gtest_xml_output_unittest_.cc:*
Expected equality of these values:
 1
 2" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 84Expected equality of these values: 85 1 86 2%(stack)s]]></failure> 87 <failure message="gtest_xml_output_unittest_.cc:*
Expected equality of these values:
 2
 3" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 88Expected equality of these values: 89 2 90 3%(stack)s]]></failure> 91 </testcase> 92 <testcase name="DISABLED_test" status="notrun" time="*" classname="MixedResultTest"/> 93 </testsuite> 94 <testsuite name="XmlQuotingTest" tests="1" failures="1" disabled="0" errors="0" time="*"> 95 <testcase name="OutputsCData" status="run" time="*" classname="XmlQuotingTest"> 96 <failure message="gtest_xml_output_unittest_.cc:*
Failed
XML output: <?xml encoding="utf-8"><top><![CDATA[cdata text]]></top>" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 97Failed 98XML output: <?xml encoding="utf-8"><top><![CDATA[cdata text]]>]]><![CDATA[</top>%(stack)s]]></failure> 99 </testcase> 100 </testsuite> 101 <testsuite name="InvalidCharactersTest" tests="1" failures="1" disabled="0" errors="0" time="*"> 102 <testcase name="InvalidCharactersInMessage" status="run" time="*" classname="InvalidCharactersTest"> 103 <failure message="gtest_xml_output_unittest_.cc:*
Failed
Invalid characters in brackets []" type=""><![CDATA[gtest_xml_output_unittest_.cc:* 104Failed 105Invalid characters in brackets []%(stack)s]]></failure> 106 </testcase> 107 </testsuite> 108 <testsuite name="DisabledTest" tests="1" failures="0" disabled="1" errors="0" time="*"> 109 <testcase name="DISABLED_test_not_run" status="notrun" time="*" classname="DisabledTest"/> 110 </testsuite> 111 <testsuite name="PropertyRecordingTest" tests="4" failures="0" disabled="0" errors="0" time="*" SetUpTestCase="yes" TearDownTestCase="aye"> 112 <testcase name="OneProperty" status="run" time="*" classname="PropertyRecordingTest"> 113 <properties> 114 <property name="key_1" value="1"/> 115 </properties> 116 </testcase> 117 <testcase name="IntValuedProperty" status="run" time="*" classname="PropertyRecordingTest"> 118 <properties> 119 <property name="key_int" value="1"/> 120 </properties> 121 </testcase> 122 <testcase name="ThreeProperties" status="run" time="*" classname="PropertyRecordingTest"> 123 <properties> 124 <property name="key_1" value="1"/> 125 <property name="key_2" value="2"/> 126 <property name="key_3" value="3"/> 127 </properties> 128 </testcase> 129 <testcase name="TwoValuesForOneKeyUsesLastValue" status="run" time="*" classname="PropertyRecordingTest"> 130 <properties> 131 <property name="key_1" value="2"/> 132 </properties> 133 </testcase> 134 </testsuite> 135 <testsuite name="NoFixtureTest" tests="3" failures="0" disabled="0" errors="0" time="*"> 136 <testcase name="RecordProperty" status="run" time="*" classname="NoFixtureTest"> 137 <properties> 138 <property name="key" value="1"/> 139 </properties> 140 </testcase> 141 <testcase name="ExternalUtilityThatCallsRecordIntValuedProperty" status="run" time="*" classname="NoFixtureTest"> 142 <properties> 143 <property name="key_for_utility_int" value="1"/> 144 </properties> 145 </testcase> 146 <testcase name="ExternalUtilityThatCallsRecordStringValuedProperty" status="run" time="*" classname="NoFixtureTest"> 147 <properties> 148 <property name="key_for_utility_string" value="1"/> 149 </properties> 150 </testcase> 151 </testsuite> 152 <testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" errors="0" time="*"> 153 <testcase name="HasValueParamAttribute/0" value_param="33" status="run" time="*" classname="Single/ValueParamTest" /> 154 <testcase name="HasValueParamAttribute/1" value_param="42" status="run" time="*" classname="Single/ValueParamTest" /> 155 <testcase name="AnotherTestThatHasValueParamAttribute/0" value_param="33" status="run" time="*" classname="Single/ValueParamTest" /> 156 <testcase name="AnotherTestThatHasValueParamAttribute/1" value_param="42" status="run" time="*" classname="Single/ValueParamTest" /> 157 </testsuite> 158 <testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" errors="0" time="*"> 159 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="TypedTest/0" /> 160 </testsuite> 161 <testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" errors="0" time="*"> 162 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="TypedTest/1" /> 163 </testsuite> 164 <testsuite name="Single/TypeParameterizedTestCase/0" tests="1" failures="0" disabled="0" errors="0" time="*"> 165 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="Single/TypeParameterizedTestCase/0" /> 166 </testsuite> 167 <testsuite name="Single/TypeParameterizedTestCase/1" tests="1" failures="0" disabled="0" errors="0" time="*"> 168 <testcase name="HasTypeParamAttribute" type_param="*" status="run" time="*" classname="Single/TypeParameterizedTestCase/1" /> 169 </testsuite> 170</testsuites>""" % {'stack': STACK_TRACE_TEMPLATE} 171 172EXPECTED_FILTERED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?> 173<testsuites tests="1" failures="0" disabled="0" errors="0" time="*" 174 timestamp="*" name="AllTests" ad_hoc_property="42"> 175 <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" 176 errors="0" time="*"> 177 <testcase name="Succeeds" status="run" time="*" classname="SuccessfulTest"/> 178 </testsuite> 179</testsuites>""" 180 181EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?> 182<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42"> 183 <testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" errors="0" time="*"> 184 <testcase name="Succeeds" status="run" time="*" classname="SuccessfulTest"/> 185 </testsuite> 186 <testsuite name="NoFixtureTest" tests="1" failures="0" disabled="0" errors="0" time="*"> 187 <testcase name="RecordProperty" status="run" time="*" classname="NoFixtureTest"> 188 <properties> 189 <property name="key" value="1"/> 190 </properties> 191 </testcase> 192 </testsuite> 193 <testsuite name="Single/ValueParamTest" tests="1" failures="0" disabled="0" errors="0" time="*"> 194 <testcase name="AnotherTestThatHasValueParamAttribute/1" value_param="42" status="run" time="*" classname="Single/ValueParamTest" /> 195 </testsuite> 196</testsuites>""" 197 198EXPECTED_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?> 199<testsuites tests="0" failures="0" disabled="0" errors="0" time="*" 200 timestamp="*" name="AllTests"> 201</testsuites>""" 202 203GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME) 204 205SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess( 206 [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output 207 208 209class GTestXMLOutputUnitTest(gtest_xml_test_utils.GTestXMLTestCase): 210 """ 211 Unit test for Google Test's XML output functionality. 212 """ 213 214 # This test currently breaks on platforms that do not support typed and 215 # type-parameterized tests, so we don't run it under them. 216 if SUPPORTS_TYPED_TESTS: 217 def testNonEmptyXmlOutput(self): 218 """ 219 Runs a test program that generates a non-empty XML output, and 220 tests that the XML output is expected. 221 """ 222 self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY_XML, 1) 223 224 def testEmptyXmlOutput(self): 225 """Verifies XML output for a Google Test binary without actual tests. 226 227 Runs a test program that generates an empty XML output, and 228 tests that the XML output is expected. 229 """ 230 231 self._TestXmlOutput('gtest_no_test_unittest', EXPECTED_EMPTY_XML, 0) 232 233 def testTimestampValue(self): 234 """Checks whether the timestamp attribute in the XML output is valid. 235 236 Runs a test program that generates an empty XML output, and checks if 237 the timestamp attribute in the testsuites tag is valid. 238 """ 239 actual = self._GetXmlOutput('gtest_no_test_unittest', [], {}, 0) 240 date_time_str = actual.documentElement.getAttributeNode('timestamp').value 241 # datetime.strptime() is only available in Python 2.5+ so we have to 242 # parse the expected datetime manually. 243 match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str) 244 self.assertTrue( 245 re.match, 246 'XML datettime string %s has incorrect format' % date_time_str) 247 date_time_from_xml = datetime.datetime( 248 year=int(match.group(1)), month=int(match.group(2)), 249 day=int(match.group(3)), hour=int(match.group(4)), 250 minute=int(match.group(5)), second=int(match.group(6))) 251 252 time_delta = abs(datetime.datetime.now() - date_time_from_xml) 253 # timestamp value should be near the current local time 254 self.assertTrue(time_delta < datetime.timedelta(seconds=600), 255 'time_delta is %s' % time_delta) 256 actual.unlink() 257 258 def testDefaultOutputFile(self): 259 """ 260 Confirms that Google Test produces an XML output file with the expected 261 default name if no name is explicitly specified. 262 """ 263 output_file = os.path.join(gtest_test_utils.GetTempDir(), 264 GTEST_DEFAULT_OUTPUT_FILE) 265 gtest_prog_path = gtest_test_utils.GetTestExecutablePath( 266 'gtest_no_test_unittest') 267 try: 268 os.remove(output_file) 269 except OSError, e: 270 if e.errno != errno.ENOENT: 271 raise 272 273 p = gtest_test_utils.Subprocess( 274 [gtest_prog_path, '%s=xml' % GTEST_OUTPUT_FLAG], 275 working_dir=gtest_test_utils.GetTempDir()) 276 self.assert_(p.exited) 277 self.assertEquals(0, p.exit_code) 278 self.assert_(os.path.isfile(output_file)) 279 280 def testSuppressedXmlOutput(self): 281 """ 282 Tests that no XML file is generated if the default XML listener is 283 shut down before RUN_ALL_TESTS is invoked. 284 """ 285 286 xml_path = os.path.join(gtest_test_utils.GetTempDir(), 287 GTEST_PROGRAM_NAME + 'out.xml') 288 if os.path.isfile(xml_path): 289 os.remove(xml_path) 290 291 command = [GTEST_PROGRAM_PATH, 292 '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path), 293 '--shut_down_xml'] 294 p = gtest_test_utils.Subprocess(command) 295 if p.terminated_by_signal: 296 # p.signal is available only if p.terminated_by_signal is True. 297 self.assertFalse( 298 p.terminated_by_signal, 299 '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal)) 300 else: 301 self.assert_(p.exited) 302 self.assertEquals(1, p.exit_code, 303 "'%s' exited with code %s, which doesn't match " 304 'the expected exit code %s.' 305 % (command, p.exit_code, 1)) 306 307 self.assert_(not os.path.isfile(xml_path)) 308 309 def testFilteredTestXmlOutput(self): 310 """Verifies XML output when a filter is applied. 311 312 Runs a test program that executes only some tests and verifies that 313 non-selected tests do not show up in the XML output. 314 """ 315 316 self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED_TEST_XML, 0, 317 extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG]) 318 319 def testShardedTestXmlOutput(self): 320 """Verifies XML output when run using multiple shards. 321 322 Runs a test program that executes only one shard and verifies that tests 323 from other shards do not show up in the XML output. 324 """ 325 326 self._TestXmlOutput( 327 GTEST_PROGRAM_NAME, 328 EXPECTED_SHARDED_TEST_XML, 329 0, 330 extra_env={SHARD_INDEX_ENV_VAR: '0', 331 TOTAL_SHARDS_ENV_VAR: '10'}) 332 333 def _GetXmlOutput(self, gtest_prog_name, extra_args, extra_env, 334 expected_exit_code): 335 """ 336 Returns the xml output generated by running the program gtest_prog_name. 337 Furthermore, the program's exit code must be expected_exit_code. 338 """ 339 xml_path = os.path.join(gtest_test_utils.GetTempDir(), 340 gtest_prog_name + 'out.xml') 341 gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name) 342 343 command = ([gtest_prog_path, '%s=xml:%s' % (GTEST_OUTPUT_FLAG, xml_path)] + 344 extra_args) 345 environ_copy = os.environ.copy() 346 if extra_env: 347 environ_copy.update(extra_env) 348 p = gtest_test_utils.Subprocess(command, env=environ_copy) 349 350 if p.terminated_by_signal: 351 self.assert_(False, 352 '%s was killed by signal %d' % (gtest_prog_name, p.signal)) 353 else: 354 self.assert_(p.exited) 355 self.assertEquals(expected_exit_code, p.exit_code, 356 "'%s' exited with code %s, which doesn't match " 357 'the expected exit code %s.' 358 % (command, p.exit_code, expected_exit_code)) 359 actual = minidom.parse(xml_path) 360 return actual 361 362 def _TestXmlOutput(self, gtest_prog_name, expected_xml, 363 expected_exit_code, extra_args=None, extra_env=None): 364 """ 365 Asserts that the XML document generated by running the program 366 gtest_prog_name matches expected_xml, a string containing another 367 XML document. Furthermore, the program's exit code must be 368 expected_exit_code. 369 """ 370 371 actual = self._GetXmlOutput(gtest_prog_name, extra_args or [], 372 extra_env or {}, expected_exit_code) 373 expected = minidom.parseString(expected_xml) 374 self.NormalizeXml(actual.documentElement) 375 self.AssertEquivalentNodes(expected.documentElement, 376 actual.documentElement) 377 expected.unlink() 378 actual.unlink() 379 380 381if __name__ == '__main__': 382 os.environ['GTEST_STACK_TRACE_DEPTH'] = '1' 383 gtest_test_utils.Main() 384