1 /*
2  *  Created by Phil on 26/11/2010.
3  *  Copyright 2010 Two Blue Cubes Ltd. All rights reserved.
4  *
5  *  Distributed under the Boost Software License, Version 1.0. (See accompanying
6  *  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7  */
8 
9 #include "catch_reporter_bases.hpp"
10 
11 #include "catch_reporter_junit.h"
12 
13 #include "../internal/catch_tostring.h"
14 #include "../internal/catch_reporter_registrars.hpp"
15 #include "../internal/catch_text.h"
16 
17 #include <cassert>
18 #include <sstream>
19 #include <ctime>
20 #include <algorithm>
21 
22 namespace Catch {
23 
24     namespace {
getCurrentTimestamp()25         std::string getCurrentTimestamp() {
26             // Beware, this is not reentrant because of backward compatibility issues
27             // Also, UTC only, again because of backward compatibility (%z is C++11)
28             time_t rawtime;
29             std::time(&rawtime);
30             auto const timeStampSize = sizeof("2017-01-16T17:06:45Z");
31 
32 #ifdef _MSC_VER
33             std::tm timeInfo = {};
34             gmtime_s(&timeInfo, &rawtime);
35 #else
36             std::tm* timeInfo;
37             timeInfo = std::gmtime(&rawtime);
38 #endif
39 
40             char timeStamp[timeStampSize];
41             const char * const fmt = "%Y-%m-%dT%H:%M:%SZ";
42 
43 #ifdef _MSC_VER
44             std::strftime(timeStamp, timeStampSize, fmt, &timeInfo);
45 #else
46             std::strftime(timeStamp, timeStampSize, fmt, timeInfo);
47 #endif
48             return std::string(timeStamp);
49         }
50 
fileNameTag(const std::vector<std::string> & tags)51         std::string fileNameTag(const std::vector<std::string> &tags) {
52             auto it = std::find_if(begin(tags),
53                                    end(tags),
54                                    [] (std::string const& tag) {return tag.front() == '#'; });
55             if (it != tags.end())
56                 return it->substr(1);
57             return std::string();
58         }
59     } // anonymous namespace
60 
JunitReporter(ReporterConfig const & _config)61     JunitReporter::JunitReporter( ReporterConfig const& _config )
62         :   CumulativeReporterBase( _config ),
63             xml( _config.stream() )
64         {
65             m_reporterPrefs.shouldRedirectStdOut = true;
66             m_reporterPrefs.shouldReportAllAssertions = true;
67         }
68 
~JunitReporter()69     JunitReporter::~JunitReporter() {}
70 
getDescription()71     std::string JunitReporter::getDescription() {
72         return "Reports test results in an XML format that looks like Ant's junitreport target";
73     }
74 
noMatchingTestCases(std::string const &)75     void JunitReporter::noMatchingTestCases( std::string const& /*spec*/ ) {}
76 
testRunStarting(TestRunInfo const & runInfo)77     void JunitReporter::testRunStarting( TestRunInfo const& runInfo )  {
78         CumulativeReporterBase::testRunStarting( runInfo );
79         xml.startElement( "testsuites" );
80     }
81 
testGroupStarting(GroupInfo const & groupInfo)82     void JunitReporter::testGroupStarting( GroupInfo const& groupInfo ) {
83         suiteTimer.start();
84         stdOutForSuite.clear();
85         stdErrForSuite.clear();
86         unexpectedExceptions = 0;
87         CumulativeReporterBase::testGroupStarting( groupInfo );
88     }
89 
testCaseStarting(TestCaseInfo const & testCaseInfo)90     void JunitReporter::testCaseStarting( TestCaseInfo const& testCaseInfo ) {
91         m_okToFail = testCaseInfo.okToFail();
92     }
93 
assertionEnded(AssertionStats const & assertionStats)94     bool JunitReporter::assertionEnded( AssertionStats const& assertionStats ) {
95         if( assertionStats.assertionResult.getResultType() == ResultWas::ThrewException && !m_okToFail )
96             unexpectedExceptions++;
97         return CumulativeReporterBase::assertionEnded( assertionStats );
98     }
99 
testCaseEnded(TestCaseStats const & testCaseStats)100     void JunitReporter::testCaseEnded( TestCaseStats const& testCaseStats ) {
101         stdOutForSuite += testCaseStats.stdOut;
102         stdErrForSuite += testCaseStats.stdErr;
103         CumulativeReporterBase::testCaseEnded( testCaseStats );
104     }
105 
testGroupEnded(TestGroupStats const & testGroupStats)106     void JunitReporter::testGroupEnded( TestGroupStats const& testGroupStats ) {
107         double suiteTime = suiteTimer.getElapsedSeconds();
108         CumulativeReporterBase::testGroupEnded( testGroupStats );
109         writeGroup( *m_testGroups.back(), suiteTime );
110     }
111 
testRunEndedCumulative()112     void JunitReporter::testRunEndedCumulative() {
113         xml.endElement();
114     }
115 
writeGroup(TestGroupNode const & groupNode,double suiteTime)116     void JunitReporter::writeGroup( TestGroupNode const& groupNode, double suiteTime ) {
117         XmlWriter::ScopedElement e = xml.scopedElement( "testsuite" );
118 
119         TestGroupStats const& stats = groupNode.value;
120         xml.writeAttribute( "name", stats.groupInfo.name );
121         xml.writeAttribute( "errors", unexpectedExceptions );
122         xml.writeAttribute( "failures", stats.totals.assertions.failed-unexpectedExceptions );
123         xml.writeAttribute( "tests", stats.totals.assertions.total() );
124         xml.writeAttribute( "hostname", "tbd" ); // !TBD
125         if( m_config->showDurations() == ShowDurations::Never )
126             xml.writeAttribute( "time", "" );
127         else
128             xml.writeAttribute( "time", suiteTime );
129         xml.writeAttribute( "timestamp", getCurrentTimestamp() );
130 
131         // Write properties if there are any
132         if (m_config->hasTestFilters() || m_config->rngSeed() != 0) {
133             auto properties = xml.scopedElement("properties");
134             if (m_config->hasTestFilters()) {
135                 xml.scopedElement("property")
136                     .writeAttribute("name", "filters")
137                     .writeAttribute("value", serializeFilters(m_config->getTestsOrTags()));
138             }
139             if (m_config->rngSeed() != 0) {
140                 xml.scopedElement("property")
141                     .writeAttribute("name", "random-seed")
142                     .writeAttribute("value", m_config->rngSeed());
143             }
144         }
145 
146         // Write test cases
147         for( auto const& child : groupNode.children )
148             writeTestCase( *child );
149 
150         xml.scopedElement( "system-out" ).writeText( trim( stdOutForSuite ), XmlFormatting::Newline );
151         xml.scopedElement( "system-err" ).writeText( trim( stdErrForSuite ), XmlFormatting::Newline );
152     }
153 
writeTestCase(TestCaseNode const & testCaseNode)154     void JunitReporter::writeTestCase( TestCaseNode const& testCaseNode ) {
155         TestCaseStats const& stats = testCaseNode.value;
156 
157         // All test cases have exactly one section - which represents the
158         // test case itself. That section may have 0-n nested sections
159         assert( testCaseNode.children.size() == 1 );
160         SectionNode const& rootSection = *testCaseNode.children.front();
161 
162         std::string className = stats.testInfo.className;
163 
164         if( className.empty() ) {
165             className = fileNameTag(stats.testInfo.tags);
166             if ( className.empty() )
167                 className = "global";
168         }
169 
170         if ( !m_config->name().empty() )
171             className = m_config->name() + "." + className;
172 
173         writeSection( className, "", rootSection );
174     }
175 
writeSection(std::string const & className,std::string const & rootName,SectionNode const & sectionNode)176     void JunitReporter::writeSection(  std::string const& className,
177                         std::string const& rootName,
178                         SectionNode const& sectionNode ) {
179         std::string name = trim( sectionNode.stats.sectionInfo.name );
180         if( !rootName.empty() )
181             name = rootName + '/' + name;
182 
183         if( !sectionNode.assertions.empty() ||
184             !sectionNode.stdOut.empty() ||
185             !sectionNode.stdErr.empty() ) {
186             XmlWriter::ScopedElement e = xml.scopedElement( "testcase" );
187             if( className.empty() ) {
188                 xml.writeAttribute( "classname", name );
189                 xml.writeAttribute( "name", "root" );
190             }
191             else {
192                 xml.writeAttribute( "classname", className );
193                 xml.writeAttribute( "name", name );
194             }
195             xml.writeAttribute( "time", ::Catch::Detail::stringify( sectionNode.stats.durationInSeconds ) );
196             // This is not ideal, but it should be enough to mimic gtest's
197             // junit output.
198             // Ideally the JUnit reporter would also handle `skipTest`
199             // events and write those out appropriately.
200             xml.writeAttribute( "status", "run" );
201 
202             writeAssertions( sectionNode );
203 
204             if( !sectionNode.stdOut.empty() )
205                 xml.scopedElement( "system-out" ).writeText( trim( sectionNode.stdOut ), XmlFormatting::Newline );
206             if( !sectionNode.stdErr.empty() )
207                 xml.scopedElement( "system-err" ).writeText( trim( sectionNode.stdErr ), XmlFormatting::Newline );
208         }
209         for( auto const& childNode : sectionNode.childSections )
210             if( className.empty() )
211                 writeSection( name, "", *childNode );
212             else
213                 writeSection( className, name, *childNode );
214     }
215 
writeAssertions(SectionNode const & sectionNode)216     void JunitReporter::writeAssertions( SectionNode const& sectionNode ) {
217         for( auto const& assertion : sectionNode.assertions )
218             writeAssertion( assertion );
219     }
220 
writeAssertion(AssertionStats const & stats)221     void JunitReporter::writeAssertion( AssertionStats const& stats ) {
222         AssertionResult const& result = stats.assertionResult;
223         if( !result.isOk() ) {
224             std::string elementName;
225             switch( result.getResultType() ) {
226                 case ResultWas::ThrewException:
227                 case ResultWas::FatalErrorCondition:
228                     elementName = "error";
229                     break;
230                 case ResultWas::ExplicitFailure:
231                 case ResultWas::ExpressionFailed:
232                 case ResultWas::DidntThrowException:
233                     elementName = "failure";
234                     break;
235 
236                 // We should never see these here:
237                 case ResultWas::Info:
238                 case ResultWas::Warning:
239                 case ResultWas::Ok:
240                 case ResultWas::Unknown:
241                 case ResultWas::FailureBit:
242                 case ResultWas::Exception:
243                     elementName = "internalError";
244                     break;
245             }
246 
247             XmlWriter::ScopedElement e = xml.scopedElement( elementName );
248 
249             xml.writeAttribute( "message", result.getExpression() );
250             xml.writeAttribute( "type", result.getTestMacroName() );
251 
252             ReusableStringStream rss;
253             if (stats.totals.assertions.total() > 0) {
254                 rss << "FAILED" << ":\n";
255                 if (result.hasExpression()) {
256                     rss << "  ";
257                     rss << result.getExpressionInMacro();
258                     rss << '\n';
259                 }
260                 if (result.hasExpandedExpression()) {
261                     rss << "with expansion:\n";
262                     rss << Column(result.getExpandedExpression()).indent(2) << '\n';
263                 }
264             } else {
265                 rss << '\n';
266             }
267 
268             if( !result.getMessage().empty() )
269                 rss << result.getMessage() << '\n';
270             for( auto const& msg : stats.infoMessages )
271                 if( msg.type == ResultWas::Info )
272                     rss << msg.message << '\n';
273 
274             rss << "at " << result.getSourceInfo();
275             xml.writeText( rss.str(), XmlFormatting::Newline );
276         }
277     }
278 
279     CATCH_REGISTER_REPORTER( "junit", JunitReporter )
280 
281 } // end namespace Catch
282