1 // 2 // Copyright 2019 The ANGLE Project Authors. All rights reserved. 3 // Use of this source code is governed by a BSD-style license that can be 4 // found in the LICENSE file. 5 // 6 // TestSuite: 7 // Basic implementation of a test harness in ANGLE. 8 9 #ifndef ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_ 10 #define ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_ 11 12 #include <map> 13 #include <memory> 14 #include <mutex> 15 #include <queue> 16 #include <string> 17 #include <thread> 18 19 #include "HistogramWriter.h" 20 #include "util/test_utils.h" 21 22 namespace angle 23 { 24 struct TestIdentifier 25 { 26 TestIdentifier(); 27 TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn); 28 TestIdentifier(const TestIdentifier &other); 29 ~TestIdentifier(); 30 31 TestIdentifier &operator=(const TestIdentifier &other); 32 33 static bool ParseFromString(const std::string &str, TestIdentifier *idOut); 34 validTestIdentifier35 bool valid() const { return !testName.empty(); } 36 void sprintfName(char *outBuffer) const; 37 38 std::string testSuiteName; 39 std::string testName; 40 }; 41 42 inline bool operator<(const TestIdentifier &a, const TestIdentifier &b) 43 { 44 return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName); 45 } 46 47 inline bool operator==(const TestIdentifier &a, const TestIdentifier &b) 48 { 49 return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName); 50 } 51 52 inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id) 53 { 54 return os << id.testSuiteName << "." << id.testName; 55 } 56 57 enum class TestResultType 58 { 59 Crash, 60 Fail, 61 NoResult, 62 Pass, 63 Timeout, 64 Unknown, 65 }; 66 67 const char *TestResultTypeToString(TestResultType type); 68 69 struct TestResult 70 { 71 TestResultType type = TestResultType::NoResult; 72 double elapsedTimeSeconds = 0.0; 73 uint32_t flakyFailures = 0; 74 }; 75 76 inline bool operator==(const TestResult &a, const TestResult &b) 77 { 78 return a.type == b.type; 79 } 80 81 inline std::ostream &operator<<(std::ostream &os, const TestResult &result) 82 { 83 return os << TestResultTypeToString(result.type); 84 } 85 86 struct TestResults 87 { 88 TestResults(); 89 ~TestResults(); 90 91 std::map<TestIdentifier, TestResult> results; 92 std::mutex currentTestMutex; 93 TestIdentifier currentTest; 94 Timer currentTestTimer; 95 bool allDone = false; 96 }; 97 98 struct FileLine 99 { 100 const char *file; 101 int line; 102 }; 103 104 struct ProcessInfo : angle::NonCopyable 105 { 106 ProcessInfo(); 107 ~ProcessInfo(); 108 ProcessInfo(ProcessInfo &&other); 109 ProcessInfo &operator=(ProcessInfo &&rhs); 110 111 ProcessHandle process; 112 std::vector<TestIdentifier> testsInBatch; 113 std::string resultsFileName; 114 std::string filterFileName; 115 std::string commandLine; 116 std::string filterString; 117 }; 118 119 using TestQueue = std::queue<std::vector<TestIdentifier>>; 120 121 class TestSuite 122 { 123 public: 124 TestSuite(int *argc, char **argv); 125 ~TestSuite(); 126 127 int run(); 128 void onCrashOrTimeout(TestResultType crashOrTimeout); 129 void addHistogramSample(const std::string &measurement, 130 const std::string &story, 131 double value, 132 const std::string &units); 133 GetInstance()134 static TestSuite *GetInstance() { return mInstance; } 135 136 private: 137 bool parseSingleArg(const char *argument); 138 bool launchChildTestProcess(uint32_t batchId, const std::vector<TestIdentifier> &testsInBatch); 139 bool finishProcess(ProcessInfo *processInfo); 140 int printFailuresAndReturnCount() const; 141 void startWatchdog(); 142 143 static TestSuite *mInstance; 144 145 std::string mTestExecutableName; 146 std::string mTestSuiteName; 147 TestQueue mTestQueue; 148 std::string mFilterString; 149 std::string mFilterFile; 150 std::string mResultsDirectory; 151 std::string mResultsFile; 152 std::string mHistogramJsonFile; 153 int mShardCount; 154 int mShardIndex; 155 angle::CrashCallback mCrashCallback; 156 TestResults mTestResults; 157 bool mBotMode; 158 bool mDebugTestGroups; 159 bool mGTestListTests; 160 bool mListTests; 161 bool mPrintTestStdout; 162 bool mDisableCrashHandler; 163 int mBatchSize; 164 int mCurrentResultCount; 165 int mTotalResultCount; 166 int mMaxProcesses; 167 int mTestTimeout; 168 int mBatchTimeout; 169 int mBatchId; 170 int mFlakyRetries; 171 std::vector<std::string> mChildProcessArgs; 172 std::map<TestIdentifier, FileLine> mTestFileLines; 173 std::vector<ProcessInfo> mCurrentProcesses; 174 std::thread mWatchdogThread; 175 HistogramWriter mHistogramWriter; 176 }; 177 178 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut); 179 } // namespace angle 180 181 #endif // ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_ 182