1 //
2 // Copyright 2019 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // TestSuite:
7 //   Basic implementation of a test harness in ANGLE.
8 
9 #ifndef ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
10 #define ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
11 
12 #include <map>
13 #include <memory>
14 #include <mutex>
15 #include <queue>
16 #include <string>
17 #include <thread>
18 
19 #include "HistogramWriter.h"
20 #include "util/test_utils.h"
21 
22 namespace angle
23 {
24 struct TestIdentifier
25 {
26     TestIdentifier();
27     TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn);
28     TestIdentifier(const TestIdentifier &other);
29     ~TestIdentifier();
30 
31     TestIdentifier &operator=(const TestIdentifier &other);
32 
33     static bool ParseFromString(const std::string &str, TestIdentifier *idOut);
34 
validTestIdentifier35     bool valid() const { return !testName.empty(); }
36     void sprintfName(char *outBuffer) const;
37 
38     std::string testSuiteName;
39     std::string testName;
40 };
41 
42 inline bool operator<(const TestIdentifier &a, const TestIdentifier &b)
43 {
44     return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName);
45 }
46 
47 inline bool operator==(const TestIdentifier &a, const TestIdentifier &b)
48 {
49     return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName);
50 }
51 
52 inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id)
53 {
54     return os << id.testSuiteName << "." << id.testName;
55 }
56 
57 enum class TestResultType
58 {
59     Crash,
60     Fail,
61     NoResult,
62     Pass,
63     Timeout,
64     Unknown,
65 };
66 
67 const char *TestResultTypeToString(TestResultType type);
68 
69 struct TestResult
70 {
71     TestResultType type       = TestResultType::NoResult;
72     double elapsedTimeSeconds = 0.0;
73     uint32_t flakyFailures    = 0;
74 };
75 
76 inline bool operator==(const TestResult &a, const TestResult &b)
77 {
78     return a.type == b.type;
79 }
80 
81 inline std::ostream &operator<<(std::ostream &os, const TestResult &result)
82 {
83     return os << TestResultTypeToString(result.type);
84 }
85 
86 struct TestResults
87 {
88     TestResults();
89     ~TestResults();
90 
91     std::map<TestIdentifier, TestResult> results;
92     std::mutex currentTestMutex;
93     TestIdentifier currentTest;
94     Timer currentTestTimer;
95     bool allDone = false;
96 };
97 
98 struct FileLine
99 {
100     const char *file;
101     int line;
102 };
103 
104 struct ProcessInfo : angle::NonCopyable
105 {
106     ProcessInfo();
107     ~ProcessInfo();
108     ProcessInfo(ProcessInfo &&other);
109     ProcessInfo &operator=(ProcessInfo &&rhs);
110 
111     ProcessHandle process;
112     std::vector<TestIdentifier> testsInBatch;
113     std::string resultsFileName;
114     std::string filterFileName;
115     std::string commandLine;
116     std::string filterString;
117 };
118 
119 using TestQueue = std::queue<std::vector<TestIdentifier>>;
120 
121 class TestSuite
122 {
123   public:
124     TestSuite(int *argc, char **argv);
125     ~TestSuite();
126 
127     int run();
128     void onCrashOrTimeout(TestResultType crashOrTimeout);
129     void addHistogramSample(const std::string &measurement,
130                             const std::string &story,
131                             double value,
132                             const std::string &units);
133 
GetInstance()134     static TestSuite *GetInstance() { return mInstance; }
135 
136   private:
137     bool parseSingleArg(const char *argument);
138     bool launchChildTestProcess(uint32_t batchId, const std::vector<TestIdentifier> &testsInBatch);
139     bool finishProcess(ProcessInfo *processInfo);
140     int printFailuresAndReturnCount() const;
141     void startWatchdog();
142 
143     static TestSuite *mInstance;
144 
145     std::string mTestExecutableName;
146     std::string mTestSuiteName;
147     TestQueue mTestQueue;
148     std::string mFilterString;
149     std::string mFilterFile;
150     std::string mResultsDirectory;
151     std::string mResultsFile;
152     std::string mHistogramJsonFile;
153     int mShardCount;
154     int mShardIndex;
155     angle::CrashCallback mCrashCallback;
156     TestResults mTestResults;
157     bool mBotMode;
158     bool mDebugTestGroups;
159     bool mGTestListTests;
160     bool mListTests;
161     bool mPrintTestStdout;
162     bool mDisableCrashHandler;
163     int mBatchSize;
164     int mCurrentResultCount;
165     int mTotalResultCount;
166     int mMaxProcesses;
167     int mTestTimeout;
168     int mBatchTimeout;
169     int mBatchId;
170     int mFlakyRetries;
171     std::vector<std::string> mChildProcessArgs;
172     std::map<TestIdentifier, FileLine> mTestFileLines;
173     std::vector<ProcessInfo> mCurrentProcesses;
174     std::thread mWatchdogThread;
175     HistogramWriter mHistogramWriter;
176 };
177 
178 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut);
179 }  // namespace angle
180 
181 #endif  // ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
182