1 /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
2 /*
3 * Copyright (c) 2009 University of Washington
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation;
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */
18
19 #include "test.h"
20 #include "assert.h"
21 #include "abort.h"
22 #include "singleton.h"
23 #include "system-path.h"
24 #include "log.h"
25 #include "des-metrics.h"
26 #include <cmath>
27 #include <cstring>
28 #include <vector>
29 #include <list>
30 #include <map>
31
32
33 /**
34 * \file
35 * \ingroup testing
36 * \brief ns3::TestCase, ns3::TestSuite, ns3::TestRunner implementations,
37 */
38
39 namespace ns3 {
40
41 NS_LOG_COMPONENT_DEFINE ("Test");
42
43 bool
TestDoubleIsEqual(const double x1,const double x2,const double epsilon)44 TestDoubleIsEqual (const double x1, const double x2, const double epsilon)
45 {
46 NS_LOG_FUNCTION (x1 << x2 << epsilon);
47 int exponent;
48 double delta, difference;
49
50 //
51 // Find exponent of largest absolute value
52 //
53 {
54 double max = (std::fabs (x1) > std::fabs (x2)) ? x1 : x2;
55 (void)std::frexp (max, &exponent);
56 }
57
58 //
59 // Form a neighborhood of size 2 * delta
60 //
61 delta = std::ldexp (epsilon, exponent);
62 difference = x1 - x2;
63
64 if (difference > delta || difference < -delta)
65 {
66 return false;
67 }
68 return true;
69 }
70
71 /**
72 * \ingroup testingimpl
73 * Container for details of a test failure.
74 */
75 struct TestCaseFailure
76 {
77 /**
78 * Constructor.
79 *
80 * \param [in] _cond The name of the condition being tested.
81 * \param [in] _actual The actual value returned by the test.
82 * \param [in] _limit The expected value.
83 * \param [in] _message The associated message.
84 * \param [in] _file The source file.
85 * \param [in] _line The source line.
86 */
87 TestCaseFailure (std::string _cond, std::string _actual,
88 std::string _limit, std::string _message,
89 std::string _file, int32_t _line);
90 std::string cond; /**< The name of the condition being tested. */
91 std::string actual; /**< The actual value returned by the test. */
92 std::string limit; /**< The expected value. */
93 std::string message; /**< The associated message. */
94 std::string file; /**< The source file. */
95 int32_t line; /**< The source line. */
96 };
97 /**
98 * Output streamer for TestCaseFailure.
99 *
100 * \param [in,out] os The output stream.
101 * \param [in] failure The TestCaseFailure to print.
102 * \returns The stream.
103 */
operator <<(std::ostream & os,const TestCaseFailure & failure)104 std::ostream & operator << (std::ostream & os, const TestCaseFailure & failure)
105 {
106 os << " test=\"" << failure.cond
107 << "\" actual=\"" << failure.actual
108 << "\" limit=\"" << failure.limit
109 << "\" in=\"" << failure.file
110 << ":" << failure.line
111 << "\" " << failure.message;
112
113 return os;
114 }
115
116 /**
117 * \ingroup testingimpl
118 * Container for results from a TestCase.
119 */
120 struct TestCase::Result
121 {
122 /** Constructor. */
123 Result ();
124
125 /** Test running time. */
126 SystemWallClockMs clock;
127 /** TestCaseFailure records for each child. */
128 std::vector<TestCaseFailure> failure;
129 /** \c true if any child TestCases failed. */
130 bool childrenFailed;
131 };
132
133 /**
134 * \ingroup testingimpl
135 * Container for all tests.
136 * \todo Move TestRunnerImpl to separate file.
137 */
138 class TestRunnerImpl : public Singleton<TestRunnerImpl>
139 {
140 public:
141 /** Constructor. */
142 TestRunnerImpl ();
143
144 /**
145 * Add a new top-level TestSuite.
146 * \param [in] testSuite The new TestSuite.
147 */
148 void AddTestSuite (TestSuite *testSuite);
149 /** \copydoc TestCase::MustAssertOnFailure() */
150 bool MustAssertOnFailure (void) const;
151 /** \copydoc TestCase::MustContinueOnFailure() */
152 bool MustContinueOnFailure (void) const;
153 /**
154 * Check if this run should update the reference data.
155 * \return \c true if we should update the reference data.
156 */
157 bool MustUpdateData (void) const;
158 /**
159 * Get the path to the root of the source tree.
160 *
161 * The root directory is defined by the presence of two files:
162 * "VERSION" and "LICENSE".
163 *
164 * \returns The path to the root.
165 */
166 std::string GetTopLevelSourceDir (void) const;
167 /**
168 * Get the path to temporary directory.
169 * \return The temporary directory path.
170 */
171 std::string GetTempDir (void) const;
172 /** \copydoc TestRunner::Run() */
173 int Run (int argc, char *argv[]);
174
175 private:
176
177 /**
178 * Check if this is the root of the source tree.
179 * \param [in] path The path to test.
180 * \returns \c true if \pname{path} is the root.
181 */
182 bool IsTopLevelSourceDir (std::string path) const;
183 /**
184 * Clean up characters not allowed in XML.
185 *
186 * XML files have restrictions on certain characters that may be present in
187 * data. We need to replace these characters with their alternate
188 * representation on the way into the XML file.
189 *
190 * Specifically, we make these replacements:
191 * Raw Source | Replacement
192 * :--------: | :---------:
193 * '<' | "<"
194 * '>' | ">"
195 * '&' | "&"
196 * '"' | "&39;"
197 * '\' | """
198 *
199 * \param [in] xml The raw string.
200 * \returns The sanitized string.
201 */
202 std::string ReplaceXmlSpecialCharacters (std::string xml) const;
203 /**
204 * Print the test report.
205 *
206 * \param [in] test The TestCase to print.
207 * \param [in,out] os The output stream.
208 * \param [in] xml Generate XML output if \c true.
209 * \param [in] level Indentation level.
210 */
211 void PrintReport (TestCase *test, std::ostream *os, bool xml, int level);
212 /**
213 * Print the list of all requested test suites.
214 *
215 * \param [in] begin Iterator to the first TestCase to print.
216 * \param [in] end Iterator to the end of the list.
217 * \param [in] printTestType Preprend the test type label if \c true.
218 */
219 void PrintTestNameList (std::list<TestCase *>::const_iterator begin,
220 std::list<TestCase *>::const_iterator end,
221 bool printTestType) const;
222 /** Print the list of test types. */
223 void PrintTestTypeList (void) const;
224 /**
225 * Print the help text.
226 * \param [in] programName The name of the invoking program.
227 */
228 void PrintHelp (const char *programName) const;
229 /**
230 * Generate the list of tests matching the constraints.
231 *
232 * Test name and type constraints are or'ed. The duration constraint
233 * is and'ed.
234 *
235 * \param [in] testName Include a specific test by name.
236 * \param [in] testType Include all tests of give type.
237 * \param [in] maximumTestDuration Restrict to tests shorter than this.
238 * \returns The list of tests matching the filter constraints.
239 */
240 std::list<TestCase *> FilterTests (std::string testName,
241 enum TestSuite::Type testType,
242 enum TestCase::TestDuration maximumTestDuration);
243
244
245 /** Container type for the test. */
246 typedef std::vector<TestSuite *> TestSuiteVector;
247
248 TestSuiteVector m_suites; //!< The list of tests.
249 std::string m_tempDir; //!< The temporary directory.
250 bool m_verbose; //!< Produce verbose output.
251 bool m_assertOnFailure; //!< \c true if we should assert on failure.
252 bool m_continueOnFailure; //!< \c true if we should continue on failure.
253 bool m_updateData; //!< \c true if we should update reference data.
254 };
255
256
257
TestCaseFailure(std::string _cond,std::string _actual,std::string _limit,std::string _message,std::string _file,int32_t _line)258 TestCaseFailure::TestCaseFailure (std::string _cond, std::string _actual,
259 std::string _limit, std::string _message,
260 std::string _file, int32_t _line)
261 : cond (_cond), actual (_actual), limit (_limit),
262 message (_message), file (_file), line (_line)
263 {
264 NS_LOG_FUNCTION (this << _cond << _actual << _limit << _message << _file << _line);
265 }
Result()266 TestCase::Result::Result ()
267 : childrenFailed (false)
268 {
269 NS_LOG_FUNCTION (this);
270 }
271
272
273
TestCase(std::string name)274 TestCase::TestCase (std::string name)
275 : m_parent (0),
276 m_dataDir (""),
277 m_runner (0),
278 m_result (0),
279 m_name (name),
280 m_duration (TestCase::QUICK)
281 {
282 NS_LOG_FUNCTION (this << name);
283 }
284
~TestCase()285 TestCase::~TestCase ()
286 {
287 NS_LOG_FUNCTION (this);
288 NS_ASSERT (m_runner == 0);
289 m_parent = 0;
290 delete m_result;
291 for (std::vector<TestCase *>::const_iterator i = m_children.begin (); i != m_children.end (); ++i)
292 {
293 delete *i;
294 }
295 m_children.clear ();
296 }
297
298 void
AddTestCase(TestCase * testCase,enum TestCase::TestDuration duration)299 TestCase::AddTestCase (TestCase *testCase, enum TestCase::TestDuration duration)
300 {
301 NS_LOG_FUNCTION (&testCase << duration);
302
303 // Test names are used to create temporary directories,
304 // so we test for illegal characters.
305 //
306 // Windows: <>:"/\|?*
307 // http://msdn.microsoft.com/en-us/library/aa365247(v=vs.85).aspx
308 // Mac: : (deprecated, was path separator in Mac OS Classic, pre X)
309 // Unix: / (and .. may give trouble?)
310 //
311 // The Windows list is too restrictive: we like to label
312 // tests with "val = v1 * v2" or "v1 < 3" or "case: foo --> bar"
313 // So we allow ':<>*"
314
315 std::string badchars = "\"/\\|?";
316 // Badchar Class Regex Count of failing test names
317 // All ":<>\"/\\|?*" 611
318 // Allow ':' "<>\"/\\|?*" 128
319 // Allow ':<>' "\"/\\|?*" 12
320 // Allow ':<>*' "\"/\\|?" 0
321
322 std::string::size_type badch = testCase->m_name.find_first_of (badchars);
323 if (badch != std::string::npos)
324 {
325 /*
326 To count the bad test names, use NS_LOG_UNCOND instead
327 of NS_FATAL_ERROR, and the command
328 $ ./waf --run "test-runner --list" 2>&1 | grep "^Invalid" | wc
329 */
330 NS_LOG_UNCOND ("Invalid test name: cannot contain any of '"
331 << badchars << "': " << testCase->m_name);
332 }
333
334 testCase->m_duration = duration;
335 testCase->m_parent = this;
336 m_children.push_back (testCase);
337 }
338
339 bool
IsFailed(void) const340 TestCase::IsFailed (void) const
341 {
342 NS_LOG_FUNCTION (this);
343 return m_result->childrenFailed || !m_result->failure.empty ();
344 }
345
346 void
Run(TestRunnerImpl * runner)347 TestCase::Run (TestRunnerImpl *runner)
348 {
349 NS_LOG_FUNCTION (this << runner);
350 m_result = new Result ();
351 m_runner = runner;
352 DoSetup ();
353 m_result->clock.Start ();
354 for (std::vector<TestCase *>::const_iterator i = m_children.begin (); i != m_children.end (); ++i)
355 {
356 TestCase *test = *i;
357 test->Run (runner);
358 if (IsFailed ())
359 {
360 goto out;
361 }
362 }
363 DoRun ();
364 out:
365 m_result->clock.End ();
366 DoTeardown ();
367 m_runner = 0;
368 }
369 std::string
GetName(void) const370 TestCase::GetName (void) const
371 {
372 NS_LOG_FUNCTION (this);
373 return m_name;
374 }
375 TestCase *
GetParent() const376 TestCase::GetParent () const
377 {
378 return m_parent;
379 }
380
381 void
ReportTestFailure(std::string cond,std::string actual,std::string limit,std::string message,std::string file,int32_t line)382 TestCase::ReportTestFailure (std::string cond, std::string actual,
383 std::string limit, std::string message,
384 std::string file, int32_t line)
385 {
386 NS_LOG_FUNCTION (this << cond << actual << limit << message << file << line);
387 m_result->failure.push_back (TestCaseFailure (cond, actual, limit,
388 message, file, line));
389 // set childrenFailed flag on parents.
390 TestCase *current = m_parent;
391 while (current != 0)
392 {
393 current->m_result->childrenFailed = true;
394 current = current->m_parent;
395 }
396
397 }
398 bool
MustAssertOnFailure(void) const399 TestCase::MustAssertOnFailure (void) const
400 {
401 NS_LOG_FUNCTION (this);
402 return m_runner->MustAssertOnFailure ();
403 }
404 bool
MustContinueOnFailure(void) const405 TestCase::MustContinueOnFailure (void) const
406 {
407 NS_LOG_FUNCTION (this);
408 return m_runner->MustContinueOnFailure ();
409 }
410
411 std::string
CreateDataDirFilename(std::string filename)412 TestCase::CreateDataDirFilename (std::string filename)
413 {
414 NS_LOG_FUNCTION (this << filename);
415 const TestCase *current = this;
416 while (current != 0 && current->m_dataDir == "")
417 {
418 current = current->m_parent;
419 }
420 if (current == 0)
421 {
422 NS_FATAL_ERROR ("No one called SetDataDir prior to calling this function");
423 }
424
425 std::string a = SystemPath::Append (m_runner->GetTopLevelSourceDir (), current->m_dataDir);
426 std::string b = SystemPath::Append (a, filename);
427 return b;
428 }
429 std::string
CreateTempDirFilename(std::string filename)430 TestCase::CreateTempDirFilename (std::string filename)
431 {
432 NS_LOG_FUNCTION (this << filename);
433 if (m_runner->MustUpdateData ())
434 {
435 return CreateDataDirFilename (filename);
436 }
437 else
438 {
439 std::list<std::string> names;
440 const TestCase *current = this;
441 while (current != 0)
442 {
443 names.push_front (current->m_name);
444 current = current->m_parent;
445 }
446 std::string tempDir = SystemPath::Append (m_runner->GetTempDir (), SystemPath::Join (names.begin (), names.end ()));
447 SystemPath::MakeDirectories (tempDir);
448 return SystemPath::Append (tempDir, filename);
449 }
450 }
451 bool
IsStatusFailure(void) const452 TestCase::IsStatusFailure (void) const
453 {
454 NS_LOG_FUNCTION (this);
455 return !IsStatusSuccess ();
456 }
457 bool
IsStatusSuccess(void) const458 TestCase::IsStatusSuccess (void) const
459 {
460 NS_LOG_FUNCTION (this);
461 return m_result->failure.empty ();
462 }
463
464 void
SetDataDir(std::string directory)465 TestCase::SetDataDir (std::string directory)
466 {
467 NS_LOG_FUNCTION (this << directory);
468 m_dataDir = directory;
469 }
470
471 void
DoSetup(void)472 TestCase::DoSetup (void)
473 {
474 NS_LOG_FUNCTION (this);
475 }
476 void
DoTeardown(void)477 TestCase::DoTeardown (void)
478 {
479 NS_LOG_FUNCTION (this);
480 }
481
482
TestSuite(std::string name,TestSuite::Type type)483 TestSuite::TestSuite (std::string name, TestSuite::Type type)
484 : TestCase (name),
485 m_type (type)
486 {
487 NS_LOG_FUNCTION (this << name << type);
488 TestRunnerImpl::Get ()->AddTestSuite (this);
489 }
490
491 TestSuite::Type
GetTestType(void)492 TestSuite::GetTestType (void)
493 {
494 NS_LOG_FUNCTION (this);
495 return m_type;
496 }
497
498 void
DoRun(void)499 TestSuite::DoRun (void)
500 {
501 NS_LOG_FUNCTION (this);
502 }
503
TestRunnerImpl()504 TestRunnerImpl::TestRunnerImpl ()
505 : m_tempDir (""),
506 m_assertOnFailure (false),
507 m_continueOnFailure (true),
508 m_updateData (false)
509 {
510 NS_LOG_FUNCTION (this);
511 }
512
513 void
AddTestSuite(TestSuite * testSuite)514 TestRunnerImpl::AddTestSuite (TestSuite *testSuite)
515 {
516 NS_LOG_FUNCTION (this << testSuite);
517 m_suites.push_back (testSuite);
518 }
519
520
521 bool
MustAssertOnFailure(void) const522 TestRunnerImpl::MustAssertOnFailure (void) const
523 {
524 NS_LOG_FUNCTION (this);
525 return m_assertOnFailure;
526 }
527 bool
MustContinueOnFailure(void) const528 TestRunnerImpl::MustContinueOnFailure (void) const
529 {
530 NS_LOG_FUNCTION (this);
531 return m_continueOnFailure;
532 }
533
534 bool
MustUpdateData(void) const535 TestRunnerImpl::MustUpdateData (void) const
536 {
537 NS_LOG_FUNCTION (this);
538 return m_updateData;
539 }
540 std::string
GetTempDir(void) const541 TestRunnerImpl::GetTempDir (void) const
542 {
543 NS_LOG_FUNCTION (this);
544 return m_tempDir;
545 }
546 bool
IsTopLevelSourceDir(std::string path) const547 TestRunnerImpl::IsTopLevelSourceDir (std::string path) const
548 {
549 NS_LOG_FUNCTION (this << path);
550 bool haveVersion = false;
551 bool haveLicense = false;
552
553 //
554 // If there's a file named VERSION and a file named LICENSE in this
555 // directory, we assume it's our top level source directory.
556 //
557
558 std::list<std::string> files = SystemPath::ReadFiles (path);
559 for (std::list<std::string>::const_iterator i = files.begin (); i != files.end (); ++i)
560 {
561 if (*i == "VERSION")
562 {
563 haveVersion = true;
564 }
565 else if (*i == "LICENSE")
566 {
567 haveLicense = true;
568 }
569 }
570
571 return haveVersion && haveLicense;
572 }
573
574 std::string
GetTopLevelSourceDir(void) const575 TestRunnerImpl::GetTopLevelSourceDir (void) const
576 {
577 NS_LOG_FUNCTION (this);
578 std::string self = SystemPath::FindSelfDirectory ();
579 std::list<std::string> elements = SystemPath::Split (self);
580 while (!elements.empty ())
581 {
582 std::string path = SystemPath::Join (elements.begin (), elements.end ());
583 if (IsTopLevelSourceDir (path))
584 {
585 return path;
586 }
587 elements.pop_back ();
588 }
589 NS_FATAL_ERROR ("Could not find source directory from self=" << self);
590 }
591
592 //
593 // XML files have restrictions on certain characters that may be present in
594 // data. We need to replace these characters with their alternate
595 // representation on the way into the XML file.
596 //
597 std::string
ReplaceXmlSpecialCharacters(std::string xml) const598 TestRunnerImpl::ReplaceXmlSpecialCharacters (std::string xml) const
599 {
600 NS_LOG_FUNCTION (this << xml);
601 typedef std::map <char, std::string> specials_map;
602 specials_map specials;
603 specials['<'] = "<";
604 specials['>'] = ">";
605 specials['&'] = "&";
606 specials['"'] = "'";
607 specials['\''] = """;
608
609 std::string result;
610 std::size_t length = xml.length ();
611
612 for (size_t i = 0; i < length; ++i)
613 {
614 char character = xml[i];
615
616 specials_map::const_iterator it = specials.find (character);
617
618 if (it == specials.end ())
619 {
620 result.push_back (character);
621 }
622 else
623 {
624 result += it->second;
625 }
626 }
627 return result;
628 }
629
630 /** Helper to indent output a specified number of steps. */
631 struct Indent
632 {
633 /**
634 * Constructor.
635 * \param [in] level The number of steps. A step is " ".
636 */
637 Indent (int level);
638 /** The number of steps. */
639 int level;
640 };
Indent(int _level)641 Indent::Indent (int _level)
642 : level (_level)
643 {
644 NS_LOG_FUNCTION (this << _level);
645 }
646 /**
647 * Output streamer for Indent.
648 * \param [in,out] os The output stream.
649 * \param [in] val The Indent object.
650 * \returns The stream.
651 */
operator <<(std::ostream & os,const Indent & val)652 std::ostream &operator << (std::ostream &os, const Indent &val)
653 {
654 for (int i = 0; i < val.level; i++)
655 {
656 os << " ";
657 }
658 return os;
659 }
660
661 void
PrintReport(TestCase * test,std::ostream * os,bool xml,int level)662 TestRunnerImpl::PrintReport (TestCase *test, std::ostream *os, bool xml, int level)
663 {
664 NS_LOG_FUNCTION (this << test << os << xml << level);
665 if (test->m_result == 0)
666 {
667 // Do not print reports for tests that were not run.
668 return;
669 }
670 // Report times in seconds, from ms timer
671 const double MS_PER_SEC = 1000.;
672 double real = test->m_result->clock.GetElapsedReal () / MS_PER_SEC;
673 double user = test->m_result->clock.GetElapsedUser () / MS_PER_SEC;
674 double system = test->m_result->clock.GetElapsedSystem () / MS_PER_SEC;
675
676 std::streamsize oldPrecision = (*os).precision (3);
677 *os << std::fixed;
678
679 std::string statusString = test->IsFailed () ? "FAIL" : "PASS";
680 if (xml)
681 {
682 *os << Indent (level) << "<Test>" << std::endl;
683 *os << Indent (level + 1) << "<Name>" << ReplaceXmlSpecialCharacters (test->m_name)
684 << "</Name>" << std::endl;
685 *os << Indent (level + 1) << "<Result>" << statusString << "</Result>" << std::endl;
686 *os << Indent (level + 1) << "<Time real=\"" << real << "\" user=\"" << user
687 << "\" system=\"" << system << "\"/>" << std::endl;
688 for (uint32_t i = 0; i < test->m_result->failure.size (); i++)
689 {
690 TestCaseFailure failure = test->m_result->failure[i];
691 *os << Indent (level + 2) << "<FailureDetails>" << std::endl
692 << Indent (level + 3) << "<Condition>"
693 << ReplaceXmlSpecialCharacters (failure.cond) << "</Condition>" << std::endl
694 << Indent (level + 3) << "<Actual>"
695 << ReplaceXmlSpecialCharacters (failure.actual) << "</Actual>" << std::endl
696 << Indent (level + 3) << "<Limit>"
697 << ReplaceXmlSpecialCharacters (failure.limit) << "</Limit>" << std::endl
698 << Indent (level + 3) << "<Message>"
699 << ReplaceXmlSpecialCharacters (failure.message) << "</Message>" << std::endl
700 << Indent (level + 3) << "<File>"
701 << ReplaceXmlSpecialCharacters (failure.file) << "</File>" << std::endl
702 << Indent (level + 3) << "<Line>" << failure.line << "</Line>" << std::endl
703 << Indent (level + 2) << "</FailureDetails>" << std::endl;
704 }
705 for (uint32_t i = 0; i < test->m_children.size (); i++)
706 {
707 TestCase *child = test->m_children[i];
708 PrintReport (child, os, xml, level + 1);
709 }
710 *os << Indent (level) << "</Test>" << std::endl;
711 }
712 else
713 {
714 *os << Indent (level) << statusString << " " << test->GetName ()
715 << " " << real << " s" << std::endl;
716 if (m_verbose)
717 {
718 for (uint32_t i = 0; i < test->m_result->failure.size (); i++)
719 {
720 *os << Indent (level) << test->m_result->failure[i] << std::endl;
721 }
722 for (uint32_t i = 0; i < test->m_children.size (); i++)
723 {
724 TestCase *child = test->m_children[i];
725 PrintReport (child, os, xml, level + 1);
726 }
727 }
728 }
729
730 (*os).unsetf (std::ios_base::floatfield);
731 (*os).precision (oldPrecision);
732 }
733
734 void
PrintHelp(const char * program_name) const735 TestRunnerImpl::PrintHelp (const char *program_name) const
736 {
737 NS_LOG_FUNCTION (this << program_name);
738 std::cout << "Usage: " << program_name << " [OPTIONS]" << std::endl
739 << std::endl
740 << "Options: " << std::endl
741 << " --help : print these options" << std::endl
742 << " --print-test-name-list : print the list of names of tests available" << std::endl
743 << " --list : an alias for --print-test-name-list" << std::endl
744 << " --print-test-types : print the type of tests along with their names" << std::endl
745 << " --print-test-type-list : print the list of types of tests available" << std::endl
746 << " --print-temp-dir : print name of temporary directory before running " << std::endl
747 << " the tests" << std::endl
748 << " --test-type=TYPE : process only tests of type TYPE" << std::endl
749 << " --test-name=NAME : process only test whose name matches NAME" << std::endl
750 << " --suite=NAME : an alias (here for compatibility reasons only) " << std::endl
751 << " for --test-name=NAME" << std::endl
752 << " --assert-on-failure : when a test fails, crash immediately (useful" << std::endl
753 << " when running under a debugger" << std::endl
754 << " --stop-on-failure : when a test fails, stop immediately" << std::endl
755 << " --fullness=FULLNESS : choose the duration of tests to run: QUICK, " << std::endl
756 << " EXTENSIVE, or TAKES_FOREVER, where EXTENSIVE " << std::endl
757 << " includes QUICK and TAKES_FOREVER includes " << std::endl
758 << " QUICK and EXTENSIVE (only QUICK tests are " << std::endl
759 << " run by default)" << std::endl
760 << " --verbose : print details of test execution" << std::endl
761 << " --xml : format test run output as xml" << std::endl
762 << " --tempdir=DIR : set temp dir for tests to store output files" << std::endl
763 << " --datadir=DIR : set data dir for tests to read reference files" << std::endl
764 << " --out=FILE : send test result to FILE instead of standard "
765 << "output" << std::endl
766 << " --append=FILE : append test result to FILE instead of standard "
767 << "output" << std::endl
768 ;
769 }
770
771 void
PrintTestNameList(std::list<TestCase * >::const_iterator begin,std::list<TestCase * >::const_iterator end,bool printTestType) const772 TestRunnerImpl::PrintTestNameList (std::list<TestCase *>::const_iterator begin,
773 std::list<TestCase *>::const_iterator end,
774 bool printTestType) const
775 {
776 NS_LOG_FUNCTION (this << &begin << &end << printTestType);
777 std::map<TestSuite::Type, std::string> label;
778
779 label[TestSuite::ALL] = "all ";
780 label[TestSuite::UNIT] = "unit ";
781 label[TestSuite::SYSTEM] = "system ";
782 label[TestSuite::EXAMPLE] = "example ";
783 label[TestSuite::PERFORMANCE] = "performance ";
784
785 for (std::list<TestCase *>::const_iterator i = begin; i != end; ++i)
786 {
787 TestSuite * test = dynamic_cast<TestSuite *> (*i);
788 NS_ASSERT (test != 0);
789 if (printTestType)
790 {
791 std::cout << label[test->GetTestType ()];
792 }
793 std::cout << test->GetName () << std::endl;
794 }
795 }
796
797 void
PrintTestTypeList(void) const798 TestRunnerImpl::PrintTestTypeList (void) const
799 {
800 NS_LOG_FUNCTION (this);
801 std::cout << " core: Run all TestSuite-based tests (exclude examples)" << std::endl;
802 std::cout << " example: Examples (to see if example programs run successfully)" << std::endl;
803 std::cout << " performance: Performance Tests (check to see if the system is as fast as expected)" << std::endl;
804 std::cout << " system: System Tests (spans modules to check integration of modules)" << std::endl;
805 std::cout << " unit: Unit Tests (within modules to check basic functionality)" << std::endl;
806 }
807
808
809 std::list<TestCase *>
FilterTests(std::string testName,enum TestSuite::Type testType,enum TestCase::TestDuration maximumTestDuration)810 TestRunnerImpl::FilterTests (std::string testName,
811 enum TestSuite::Type testType,
812 enum TestCase::TestDuration maximumTestDuration)
813 {
814 NS_LOG_FUNCTION (this << testName << testType);
815 std::list<TestCase *> tests;
816 for (uint32_t i = 0; i < m_suites.size (); ++i)
817 {
818 TestSuite *test = m_suites[i];
819 if (testType != TestSuite::ALL && test->GetTestType () != testType)
820 {
821 // skip test
822 continue;
823 }
824 if (testName != "" && test->GetName () != testName)
825 {
826 // skip test
827 continue;
828 }
829
830 // Remove any test cases that should be skipped.
831 std::vector<TestCase *>::iterator j;
832 for (j = test->m_children.begin (); j != test->m_children.end ();)
833 {
834 TestCase *testCase = *j;
835
836 // If this test case takes longer than the maximum test
837 // duration that should be run, then don't run it.
838 if (testCase->m_duration > maximumTestDuration)
839 {
840 // Free this test case's memory.
841 delete *j;
842
843 // Remove this test case from the test suite.
844 j = test->m_children.erase (j);
845 }
846 else
847 {
848 // Only advance through the vector elements if this test
849 // case wasn't deleted.
850 ++j;
851 }
852 }
853
854 // Add this test suite.
855 tests.push_back (test);
856 }
857 return tests;
858 }
859
860
861 int
Run(int argc,char * argv[])862 TestRunnerImpl::Run (int argc, char *argv[])
863 {
864 NS_LOG_FUNCTION (this << argc << argv);
865 std::string testName = "";
866 std::string testTypeString = "";
867 std::string out = "";
868 std::string fullness = "";
869 bool xml = false;
870 bool append = false;
871 bool printTempDir = false;
872 bool printTestTypeList = false;
873 bool printTestNameList = false;
874 bool printTestTypeAndName = false;
875 enum TestCase::TestDuration maximumTestDuration = TestCase::QUICK;
876 char *progname = argv[0];
877
878 char ** argi = argv;
879 ++argi;
880
881 while (*argi != 0)
882 {
883 char *arg = *argi;
884
885 if (strcmp (arg, "--assert-on-failure") == 0)
886 {
887 m_assertOnFailure = true;
888 }
889 else if (strcmp (arg, "--stop-on-failure") == 0)
890 {
891 m_continueOnFailure = false;
892 }
893 else if (strcmp (arg, "--verbose") == 0)
894 {
895 m_verbose = true;
896 }
897 else if (strcmp (arg, "--print-temp-dir") == 0)
898 {
899 printTempDir = true;
900 }
901 else if (strcmp (arg, "--update-data") == 0)
902 {
903 m_updateData = true;
904 }
905 else if (strcmp (arg, "--help") == 0)
906 {
907 PrintHelp (progname);
908 return 0;
909 }
910 else if (strcmp (arg, "--print-test-name-list") == 0
911 || strcmp (arg, "--list") == 0)
912 {
913 printTestNameList = true;
914 }
915 else if (strcmp (arg, "--print-test-types") == 0)
916 {
917 printTestTypeAndName = true;
918 }
919 else if (strcmp (arg, "--print-test-type-list") == 0)
920 {
921 printTestTypeList = true;
922 }
923 else if (strcmp (arg, "--append") == 0)
924 {
925 append = true;
926 }
927 else if (strcmp (arg, "--xml") == 0)
928 {
929 xml = true;
930 }
931 else if (strncmp (arg, "--test-type=", strlen ("--test-type=")) == 0)
932 {
933 testTypeString = arg + strlen ("--test-type=");
934 }
935 else if (strncmp (arg, "--test-name=", strlen ("--test-name=")) == 0)
936 {
937 testName = arg + strlen ("--test-name=");
938 }
939 else if (strncmp (arg, "--suite=", strlen ("--suite=")) == 0)
940 {
941 testName = arg + strlen ("--suite=");
942 }
943 else if (strncmp (arg, "--tempdir=", strlen ("--tempdir=")) == 0)
944 {
945 m_tempDir = arg + strlen ("--tempdir=");
946 }
947 else if (strncmp (arg, "--out=", strlen ("--out=")) == 0)
948 {
949 out = arg + strlen ("--out=");
950 }
951 else if (strncmp (arg, "--fullness=", strlen ("--fullness=")) == 0)
952 {
953 fullness = arg + strlen ("--fullness=");
954
955 // Set the maximum test length allowed.
956 if (fullness == "QUICK")
957 {
958 maximumTestDuration = TestCase::QUICK;
959 }
960 else if (fullness == "EXTENSIVE")
961 {
962 maximumTestDuration = TestCase::EXTENSIVE;
963 }
964 else if (fullness == "TAKES_FOREVER")
965 {
966 maximumTestDuration = TestCase::TAKES_FOREVER;
967 }
968 else
969 {
970 // Wrong fullness option
971 PrintHelp (progname);
972 return 3;
973 }
974 }
975 else
976 {
977 // un-recognized command-line argument
978 PrintHelp (progname);
979 return 0;
980 }
981 argi++;
982 }
983 enum TestSuite::Type testType;
984 if (testTypeString == "")
985 {
986 testType = TestSuite::ALL;
987 }
988 else if (testTypeString == "core")
989 {
990 testType = TestSuite::ALL;
991 }
992 else if (testTypeString == "example")
993 {
994 testType = TestSuite::EXAMPLE;
995 }
996 else if (testTypeString == "unit")
997 {
998 testType = TestSuite::UNIT;
999 }
1000 else if (testTypeString == "system")
1001 {
1002 testType = TestSuite::SYSTEM;
1003 }
1004 else if (testTypeString == "performance")
1005 {
1006 testType = TestSuite::PERFORMANCE;
1007 }
1008 else
1009 {
1010 std::cout << "Invalid test type specified: " << testTypeString << std::endl;
1011 PrintTestTypeList ();
1012 return 1;
1013 }
1014
1015 std::list<TestCase *> tests = FilterTests (testName, testType, maximumTestDuration);
1016
1017 if (m_tempDir == "")
1018 {
1019 m_tempDir = SystemPath::MakeTemporaryDirectoryName ();
1020 }
1021 if (printTempDir)
1022 {
1023 std::cout << m_tempDir << std::endl;
1024 }
1025 if (printTestNameList)
1026 {
1027 PrintTestNameList (tests.begin (), tests.end (), printTestTypeAndName);
1028 return 0;
1029 }
1030 if (printTestTypeList)
1031 {
1032 PrintTestTypeList ();
1033 return 0;
1034 }
1035
1036
1037 std::ostream *os;
1038 if (out != "")
1039 {
1040 std::ofstream *ofs;
1041 ofs = new std::ofstream ();
1042 std::ios_base::openmode mode = std::ios_base::out;
1043 if (append)
1044 {
1045 mode |= std::ios_base::app;
1046 }
1047 else
1048 {
1049 mode |= std::ios_base::trunc;
1050 }
1051 ofs->open (out.c_str (), mode);
1052 os = ofs;
1053 }
1054 else
1055 {
1056 os = &std::cout;
1057 }
1058
1059 // let's run our tests now.
1060 bool failed = false;
1061 if (tests.size () == 0)
1062 {
1063 std::cerr << "Error: no tests match the requested string" << std::endl;
1064 return 1;
1065 }
1066 for (std::list<TestCase *>::const_iterator i = tests.begin (); i != tests.end (); ++i)
1067 {
1068 TestCase *test = *i;
1069
1070 #ifdef ENABLE_DES_METRICS
1071 {
1072 /*
1073 Reorganize argv
1074 Since DES Metrics uses argv[0] for the trace file name,
1075 grab the test name and put it in argv[0],
1076 with test-runner as argv[1]
1077 then the rest of the original arguments.
1078 */
1079 std::string testname = test->GetName ();
1080 std::string runner = "[" + SystemPath::Split (argv[0]).back () + "]";
1081
1082 std::vector<std::string> desargs;
1083 desargs.push_back (testname);
1084 desargs.push_back (runner);
1085 for (int i = 1; i < argc; ++i)
1086 {
1087 desargs.push_back (argv[i]);
1088 }
1089
1090 DesMetrics::Get ()->Initialize (desargs, m_tempDir);
1091 }
1092 #endif
1093
1094 test->Run (this);
1095 PrintReport (test, os, xml, 0);
1096 if (test->IsFailed ())
1097 {
1098 failed = true;
1099 if (!m_continueOnFailure)
1100 {
1101 return 1;
1102 }
1103 }
1104 }
1105
1106 if (out != "")
1107 {
1108 delete os;
1109 }
1110
1111 return failed ? 1 : 0;
1112 }
1113
1114 int
Run(int argc,char * argv[])1115 TestRunner::Run (int argc, char *argv[])
1116 {
1117 NS_LOG_FUNCTION (argc << argv);
1118 return TestRunnerImpl::Get ()->Run (argc, argv);
1119 }
1120
1121 } // namespace ns3
1122