1#  _________________________________________________________________________
2#
3#  UTILIB: A utility library for developing portable C++ codes.
4#  Copyright (c) 2008 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  For more information, see the README file in the top UTILIB directory.
9#  _________________________________________________________________________
10#
11# Typical format to describe tests that are available to run, tests that
12# are being requested to be run, and tests that have been run.
13#
14# <level0  type="directory" select="yes" name="tests/testdir" desc="testdir" >
15#   <results>tests/testdir/testResults.txt</results>
16#   <level1 type="testsuite" select="yes" name="Tfeature.h" desc="feature suite">
17#     <level2 type="test" select="yes" name="testname" desc="testname test">
18#        <status>Fail</status>
19#        <output></output>
20#        <error></error>
21#     </level2>
22#   </level1>
23# </level0>
24#
25# Number of levels is fixed for a test type class, but can be
26# different for different test types.  Names are for the test class,
27# descriptions are for the user interface.  "Select" means the user
28# can choose that type for execution (all tests in a directory, all
29# tests defined in a suite, etc.)  When received back from the
30# user, "select" is "yes" if that directory, test, etc has been selected,
31# or "no" otherwise.  When tests have been run, the status of
32# a test is either "pass", "fail" or "notRun".  If detail is
33# requested, that is in the "detail".  This could be test output,
34# valgrind output, coverage info, html, etc.
35#
36# If only one of "name" and "desc" are given, then that is to
37# be used both internally and by the user interface.
38#
39# TODO:
40# Given directory name list, maybe we should search recursively
41# for test directories.
42#
43# TODO:
44# A signal handler can reply with the tests results so far while
45# tests are being run.  The UI can display progress
46
47
48class testRunner:
49
50  dirs=["."]
51  testDefs=""
52
53  def setDirectories(self, dl):
54    """ Provide the list of directories that tests may be found in. """
55
56    self.dirs=dl
57
58  def parseTestDefinitions(self):
59    """ Find all tests in the list of directories.
60
61        Save enough information about them, in a hierarchical
62        XML format, so that we can run the tests, or find the
63        test results.
64    """
65
66  def getTestDefinitionDepth(self):
67    """ How many levels are in the XML hierarchical test description? """
68
69  def getTestDefinitions(self):
70    """ Return the current list of test definitions. """
71
72  def runAllTests(self):
73    """ Run all tests found when parsing the test definitions.
74
75        Return the test descriptions with brief results.
76    """
77
78  def runTests(self, selectedTests):
79    """ Run the selected tests in the provided list.
80
81        Return the test descriptions with brief results.
82    """
83
84  def getDetailedResults(self, selectedTests):
85    """ If more detailed test results are possible, get these.
86
87        Some test types may have lengthy output, like valgrind results.
88    """
89
90  def cleanTestResults(self):
91    """ Remove files created by the last run of tests. """
92
93  def writeDetailToDartFormatXML(self, selectedTests):
94    """ For the tests in the list, write Dart format XML with results.
95
96        Return the name of a directory with the resulting XML files.
97        Caller is responsible for deleting this directory after sending
98        files to a Dart server.
99    """
100