1 //
2 // Copyright 2015 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // angle_deqp_gtest:
7 // dEQP and GoogleTest integration logic. Calls through to the random
8 // order executor.
9
10 #include <fstream>
11 #include <stdint.h>
12
13 #include <gtest/gtest.h>
14
15 #include "angle_deqp_libtester.h"
16 #include "common/angleutils.h"
17 #include "common/debug.h"
18 #include "common/Optional.h"
19 #include "common/platform.h"
20 #include "common/string_utils.h"
21 #include "gpu_test_expectations_parser.h"
22 #include "system_utils.h"
23
24 namespace
25 {
26
27 #if defined(ANGLE_PLATFORM_ANDROID)
28 const char *g_CaseListRelativePath =
29 "/../../sdcard/chromium_tests_root/third_party/deqp/src/android/cts/master/";
30 #else
31 const char *g_CaseListRelativePath = "/../../third_party/deqp/src/android/cts/master/";
32 #endif
33
34 const char *g_TestExpectationsSearchPaths[] = {
35 "/../../src/tests/deqp_support/", "/../../third_party/angle/src/tests/deqp_support/",
36 "/deqp_support/", "/../../sdcard/chromium_tests_root/third_party/angle/src/tests/deqp_support/",
37 };
38
39 const char *g_CaseListFiles[] = {
40 "gles2-master.txt", "gles3-master.txt", "gles31-master.txt", "egl-master.txt",
41 };
42
43 const char *g_TestExpectationsFiles[] = {
44 "deqp_gles2_test_expectations.txt", "deqp_gles3_test_expectations.txt",
45 "deqp_gles31_test_expectations.txt", "deqp_egl_test_expectations.txt",
46 };
47
48 using APIInfo = std::pair<const char *, gpu::GPUTestConfig::API>;
49
50 const APIInfo g_eglDisplayAPIs[] = {
51 {"angle-d3d9", gpu::GPUTestConfig::kAPID3D9},
52 {"angle-d3d11", gpu::GPUTestConfig::kAPID3D11},
53 {"angle-gl", gpu::GPUTestConfig::kAPIGLDesktop},
54 {"angle-gles", gpu::GPUTestConfig::kAPIGLES},
55 };
56
57 const APIInfo *g_initAPI = nullptr;
58
59 // Returns the default API for a platform.
GetDefaultAPIName()60 const char *GetDefaultAPIName()
61 {
62 #if defined(ANGLE_PLATFORM_WINDOWS)
63 return "angle-d3d11";
64 #elif defined(ANGLE_PLATFORM_APPLE) || defined(ANGLE_PLATFORM_LINUX)
65 return "angle-gl";
66 #elif defined(ANGLE_PLATFORM_ANDROID)
67 return "angle-gles";
68 #else
69 #error Unknown platform.
70 #endif
71 }
72
FindAPIInfo(const std::string & arg)73 const APIInfo *FindAPIInfo(const std::string &arg)
74 {
75 for (auto &displayAPI : g_eglDisplayAPIs)
76 {
77 if (arg == displayAPI.first)
78 {
79 return &displayAPI;
80 }
81 }
82 return nullptr;
83 }
84
GetDefaultAPIInfo()85 const APIInfo *GetDefaultAPIInfo()
86 {
87 const APIInfo *defaultInfo = FindAPIInfo(GetDefaultAPIName());
88 ASSERT(defaultInfo);
89 return defaultInfo;
90 }
91
92 // During the CaseList initialization we cannot use the GTEST FAIL macro to quit the program because
93 // the initialization is called outside of tests the first time.
Die()94 void Die()
95 {
96 exit(EXIT_FAILURE);
97 }
98
FindTestExpectationsPath(const std::string & exeDir,size_t testModuleIndex)99 Optional<std::string> FindTestExpectationsPath(const std::string &exeDir,
100 size_t testModuleIndex)
101 {
102 for (const char *testPath : g_TestExpectationsSearchPaths)
103 {
104 std::stringstream testExpectationsPathStr;
105 testExpectationsPathStr << exeDir << testPath << g_TestExpectationsFiles[testModuleIndex];
106
107 std::string path = testExpectationsPathStr.str();
108 std::ifstream inFile(path.c_str());
109 if (!inFile.fail())
110 {
111 inFile.close();
112 return Optional<std::string>(path);
113 }
114 }
115
116 return Optional<std::string>::Invalid();
117 }
118
119 class dEQPCaseList
120 {
121 public:
122 dEQPCaseList(size_t testModuleIndex);
123
124 struct CaseInfo
125 {
CaseInfo__anona1b147650111::dEQPCaseList::CaseInfo126 CaseInfo(const std::string &dEQPName,
127 const std::string &gTestName,
128 int expectation)
129 : mDEQPName(dEQPName),
130 mGTestName(gTestName),
131 mExpectation(expectation)
132 {
133 }
134
135 std::string mDEQPName;
136 std::string mGTestName;
137 int mExpectation;
138 };
139
140 void initialize();
141
getCaseInfo(size_t caseIndex) const142 const CaseInfo &getCaseInfo(size_t caseIndex) const
143 {
144 ASSERT(mInitialized);
145 ASSERT(caseIndex < mCaseInfoList.size());
146 return mCaseInfoList[caseIndex];
147 }
148
numCases() const149 size_t numCases() const
150 {
151 ASSERT(mInitialized);
152 return mCaseInfoList.size();
153 }
154
155 private:
156 std::vector<CaseInfo> mCaseInfoList;
157 gpu::GPUTestExpectationsParser mTestExpectationsParser;
158 gpu::GPUTestBotConfig mTestConfig;
159 size_t mTestModuleIndex;
160 bool mInitialized;
161 };
162
dEQPCaseList(size_t testModuleIndex)163 dEQPCaseList::dEQPCaseList(size_t testModuleIndex)
164 : mTestModuleIndex(testModuleIndex),
165 mInitialized(false)
166 {
167 }
168
initialize()169 void dEQPCaseList::initialize()
170 {
171 if (mInitialized)
172 {
173 return;
174 }
175
176 mInitialized = true;
177
178 std::string exeDir = angle::GetExecutableDirectory();
179
180 std::stringstream caseListPathStr;
181 caseListPathStr << exeDir << g_CaseListRelativePath << g_CaseListFiles[mTestModuleIndex];
182 std::string caseListPath = caseListPathStr.str();
183
184 Optional<std::string> testExpectationsPath = FindTestExpectationsPath(exeDir, mTestModuleIndex);
185 if (!testExpectationsPath.valid())
186 {
187 std::cerr << "Failed to find test expectations file." << std::endl;
188 Die();
189 }
190
191 if (!mTestExpectationsParser.LoadTestExpectationsFromFile(testExpectationsPath.value()))
192 {
193 std::stringstream errorMsgStream;
194 for (const auto &message : mTestExpectationsParser.GetErrorMessages())
195 {
196 errorMsgStream << std::endl << " " << message;
197 }
198
199 std::cerr << "Failed to load test expectations." << errorMsgStream.str() << std::endl;
200 Die();
201 }
202
203 if (!mTestConfig.LoadCurrentConfig(nullptr))
204 {
205 std::cerr << "Failed to load test configuration." << std::endl;
206 Die();
207 }
208
209 // Set the API from the command line, or using the default platform API.
210 if (g_initAPI)
211 {
212 mTestConfig.set_api(g_initAPI->second);
213 }
214 else
215 {
216 mTestConfig.set_api(GetDefaultAPIInfo()->second);
217 }
218
219 std::ifstream caseListStream(caseListPath);
220 if (caseListStream.fail())
221 {
222 std::cerr << "Failed to load the case list." << std::endl;
223 Die();
224 }
225
226 while (!caseListStream.eof())
227 {
228 std::string inString;
229 std::getline(caseListStream, inString);
230
231 std::string dEQPName = angle::TrimString(inString, angle::kWhitespaceASCII);
232 if (dEQPName.empty())
233 continue;
234 std::string gTestName = dEQPName.substr(dEQPName.find('.') + 1);
235 if (gTestName.empty())
236 continue;
237 std::replace(gTestName.begin(), gTestName.end(), '.', '_');
238
239 // Occurs in some luminance tests
240 gTestName.erase(std::remove(gTestName.begin(), gTestName.end(), '-'), gTestName.end());
241
242 int expectation = mTestExpectationsParser.GetTestExpectation(dEQPName, mTestConfig);
243 if (expectation != gpu::GPUTestExpectationsParser::kGpuTestSkip)
244 {
245 mCaseInfoList.push_back(CaseInfo(dEQPName, gTestName, expectation));
246 }
247 }
248 }
249
250 template <size_t TestModuleIndex>
251 class dEQPTest : public testing::TestWithParam<size_t>
252 {
253 public:
GetTestingRange()254 static testing::internal::ParamGenerator<size_t> GetTestingRange()
255 {
256 return testing::Range<size_t>(0, GetCaseList().numCases());
257 }
258
GetCaseGTestName(size_t caseIndex)259 static std::string GetCaseGTestName(size_t caseIndex)
260 {
261 const auto &caseInfo = GetCaseList().getCaseInfo(caseIndex);
262 return caseInfo.mGTestName;
263 }
264
GetCaseList()265 static const dEQPCaseList &GetCaseList()
266 {
267 static dEQPCaseList sCaseList(TestModuleIndex);
268 sCaseList.initialize();
269 return sCaseList;
270 }
271
272 static void SetUpTestCase();
273 static void TearDownTestCase();
274
275 protected:
runTest()276 void runTest()
277 {
278 const auto &caseInfo = GetCaseList().getCaseInfo(GetParam());
279 std::cout << caseInfo.mDEQPName << std::endl;
280
281 bool result = deqp_libtester_run(caseInfo.mDEQPName.c_str());
282
283 if (caseInfo.mExpectation == gpu::GPUTestExpectationsParser::kGpuTestPass)
284 {
285 EXPECT_TRUE(result);
286 sPasses += (result ? 1u : 0u);
287 sFails += (!result ? 1u : 0u);
288 }
289 else if (result)
290 {
291 std::cout << "Test expected to fail but passed!" << std::endl;
292 sUnexpectedPasses++;
293 }
294 else
295 {
296 sFails++;
297 }
298 }
299
300 static unsigned int sPasses;
301 static unsigned int sFails;
302 static unsigned int sUnexpectedPasses;
303 };
304
305 template <size_t TestModuleIndex>
306 unsigned int dEQPTest<TestModuleIndex>::sPasses = 0;
307 template <size_t TestModuleIndex>
308 unsigned int dEQPTest<TestModuleIndex>::sFails = 0;
309 template <size_t TestModuleIndex>
310 unsigned int dEQPTest<TestModuleIndex>::sUnexpectedPasses = 0;
311
312 // static
313 template <size_t TestModuleIndex>
SetUpTestCase()314 void dEQPTest<TestModuleIndex>::SetUpTestCase()
315 {
316 sPasses = 0;
317 sFails = 0;
318 sUnexpectedPasses = 0;
319
320 int argc = 0;
321 std::vector<const char *> argv;
322
323 // Reserve one argument for the binary name.
324 argc++;
325 argv.push_back("");
326
327 // Add init api.
328 argc++;
329 argv.push_back(g_initAPI ? g_initAPI->first : GetDefaultAPIName());
330
331 // Init the platform.
332 if (!deqp_libtester_init_platform(argc, argv.data()))
333 {
334 std::cout << "Aborting test due to dEQP initialization error." << std::endl;
335 exit(1);
336 }
337 }
338
339 // static
340 template <size_t TestModuleIndex>
TearDownTestCase()341 void dEQPTest<TestModuleIndex>::TearDownTestCase()
342 {
343 unsigned int total = sPasses + sFails;
344 float passFrac = static_cast<float>(sPasses) / static_cast<float>(total) * 100.0f;
345 float failFrac = static_cast<float>(sFails) / static_cast<float>(total) * 100.0f;
346 std::cout << "Passed: " << sPasses << "/" << total << " tests. (" << passFrac << "%)"
347 << std::endl;
348 if (sFails > 0)
349 {
350 std::cout << "Failed: " << sFails << "/" << total << " tests. (" << failFrac << "%)"
351 << std::endl;
352 }
353 if (sUnexpectedPasses > 0)
354 {
355 std::cout << sUnexpectedPasses << " tests unexpectedly passed." << std::endl;
356 }
357
358 deqp_libtester_shutdown_platform();
359 }
360
361 #define ANGLE_INSTANTIATE_DEQP_TEST_CASE(DEQP_TEST, N) \
362 class DEQP_TEST : public dEQPTest<N> \
363 { \
364 }; \
365 TEST_P(DEQP_TEST, Default) { runTest(); } \
366 \
367 INSTANTIATE_TEST_CASE_P(, DEQP_TEST, DEQP_TEST::GetTestingRange(), \
368 [](const testing::TestParamInfo<size_t> &info) { \
369 return DEQP_TEST::GetCaseGTestName(info.param); \
370 })
371
372 #ifdef ANGLE_DEQP_GLES2_TESTS
373 ANGLE_INSTANTIATE_DEQP_TEST_CASE(dEQP_GLES2, 0);
374 #endif
375
376 #ifdef ANGLE_DEQP_GLES3_TESTS
377 ANGLE_INSTANTIATE_DEQP_TEST_CASE(dEQP_GLES3, 1);
378 #endif
379
380 #ifdef ANGLE_DEQP_GLES31_TESTS
381 ANGLE_INSTANTIATE_DEQP_TEST_CASE(dEQP_GLES31, 2);
382 #endif
383
384 #ifdef ANGLE_DEQP_EGL_TESTS
385 ANGLE_INSTANTIATE_DEQP_TEST_CASE(dEQP_EGL, 3);
386 #endif
387
388 const char *g_deqpEGLString = "--deqp-egl-display-type=";
389 const char *g_angleEGLString = "--use-angle=";
390
HandleDisplayType(const char * displayTypeString)391 void HandleDisplayType(const char *displayTypeString)
392 {
393 std::stringstream argStream;
394
395 if (g_initAPI)
396 {
397 std::cout << "Cannot specify two EGL displays!" << std::endl;
398 exit(1);
399 }
400
401 if (strncmp(displayTypeString, "angle-", strlen("angle-")) != 0)
402 {
403 argStream << "angle-";
404 }
405
406 argStream << displayTypeString;
407 std::string arg = argStream.str();
408
409 g_initAPI = FindAPIInfo(arg);
410
411 if (!g_initAPI)
412 {
413 std::cout << "Unknown ANGLE back-end API: " << displayTypeString << std::endl;
414 exit(1);
415 }
416 }
417
DeleteArg(int * argc,int argIndex,char ** argv)418 void DeleteArg(int *argc, int argIndex, char **argv)
419 {
420 (*argc)--;
421 for (int moveIndex = argIndex; moveIndex < *argc; ++moveIndex)
422 {
423 argv[moveIndex] = argv[moveIndex + 1];
424 }
425 }
426
427 } // anonymous namespace
428
429 // Called from main() to process command-line arguments.
430 namespace angle
431 {
InitTestHarness(int * argc,char ** argv)432 void InitTestHarness(int *argc, char **argv)
433 {
434 int argIndex = 0;
435 while (argIndex < *argc)
436 {
437 if (strncmp(argv[argIndex], g_deqpEGLString, strlen(g_deqpEGLString)) == 0)
438 {
439 HandleDisplayType(argv[argIndex] + strlen(g_deqpEGLString));
440 DeleteArg(argc, argIndex, argv);
441 }
442 else if (strncmp(argv[argIndex], g_angleEGLString, strlen(g_angleEGLString)) == 0)
443 {
444 HandleDisplayType(argv[argIndex] + strlen(g_angleEGLString));
445 DeleteArg(argc, argIndex, argv);
446 }
447 else
448 {
449 argIndex++;
450 }
451 }
452 }
453 } // namespace angle
454