1 //
2 // Copyright 2015 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // angle_deqp_gtest:
7 // dEQP and GoogleTest integration logic. Calls through to the random
8 // order executor.
9
10 #include <stdint.h>
11 #include <array>
12 #include <fstream>
13
14 #include <gtest/gtest.h>
15
16 #include "angle_deqp_libtester.h"
17 #include "common/Optional.h"
18 #include "common/angleutils.h"
19 #include "common/base/anglebase/no_destructor.h"
20 #include "common/debug.h"
21 #include "common/platform.h"
22 #include "common/string_utils.h"
23 #include "common/system_utils.h"
24 #include "platform/PlatformMethods.h"
25 #include "tests/test_utils/runner/TestSuite.h"
26 #include "util/OSWindow.h"
27 #include "util/test_utils.h"
28
29 namespace angle
30 {
31 namespace
32 {
33 #if !defined(NDEBUG)
34 constexpr bool kIsDebug = true;
35 #else
36 constexpr bool kIsDebug = false;
37 #endif // !defined(NDEBUG)
38
39 bool gGlobalError = false;
40 bool gExpectError = false;
41 bool gVerbose = false;
42
43 // Set this to true temporarily to enable image logging in release. Useful for diagnosing
44 // errors.
45 bool gLogImages = kIsDebug;
46
47 constexpr char kInfoTag[] = "*RESULT";
48
HandlePlatformError(PlatformMethods * platform,const char * errorMessage)49 void HandlePlatformError(PlatformMethods *platform, const char *errorMessage)
50 {
51 if (!gExpectError)
52 {
53 FAIL() << errorMessage;
54 }
55 gGlobalError = true;
56 }
57
58 // Relative to the ANGLE root folder.
59 constexpr char kCTSRootPath[] = "third_party/VK-GL-CTS/src/";
60 constexpr char kSupportPath[] = "src/tests/deqp_support/";
61
62 #define GLES_CTS_DIR(PATH) "external/openglcts/data/gl_cts/data/mustpass/gles/" PATH
63 #define GL_CTS_DIR(PATH) "external/openglcts/data/gl_cts/data/mustpass/gl/" PATH
64 #define EGL_CTS_DIR(PATH) "external/openglcts/data/gl_cts/data/mustpass/egl/" PATH
65
66 const char *gCaseListFiles[] = {
67 EGL_CTS_DIR("aosp_mustpass/main/egl-main.txt"),
68 GLES_CTS_DIR("aosp_mustpass/main/gles2-main.txt"),
69 GLES_CTS_DIR("aosp_mustpass/main/gles3-main.txt"),
70 GLES_CTS_DIR("aosp_mustpass/main/gles31-main.txt"),
71 GLES_CTS_DIR("khronos_mustpass/main/gles2-khr-main.txt"),
72 GLES_CTS_DIR("khronos_mustpass/main/gles3-khr-main.txt"),
73 GLES_CTS_DIR("khronos_mustpass/main/gles31-khr-main.txt"),
74 GLES_CTS_DIR("khronos_mustpass/main/gles32-khr-main.txt"),
75 GLES_CTS_DIR("khronos_mustpass_noctx/main/gles2-khr-noctx-main.txt"),
76 GLES_CTS_DIR("khronos_mustpass_noctx/main/gles32-khr-noctx-main.txt"),
77 GLES_CTS_DIR("khronos_mustpass_single/main/gles32-khr-single.txt"),
78 GLES_CTS_DIR("aosp_mustpass/main/gles3-rotate-landscape.txt"),
79 GLES_CTS_DIR("aosp_mustpass/main/gles3-rotate-reverse-portrait.txt"),
80 GLES_CTS_DIR("aosp_mustpass/main/gles3-rotate-reverse-landscape.txt"),
81 GLES_CTS_DIR("aosp_mustpass/main/gles31-rotate-landscape.txt"),
82 GLES_CTS_DIR("aosp_mustpass/main/gles31-rotate-reverse-portrait.txt"),
83 GLES_CTS_DIR("aosp_mustpass/main/gles31-rotate-reverse-landscape.txt"),
84 GLES_CTS_DIR("aosp_mustpass/main/gles3-multisample.txt"),
85 GLES_CTS_DIR("aosp_mustpass/main/gles3-565-no-depth-no-stencil.txt"),
86 GLES_CTS_DIR("aosp_mustpass/main/gles31-multisample.txt"),
87 GLES_CTS_DIR("aosp_mustpass/main/gles31-565-no-depth-no-stencil.txt"),
88 };
89
90 const std::vector<const char *> gTestSuiteConfigParameters[] = {
91 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // egl
92 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles2
93 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles3
94 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles31
95 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles2_khr
96 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles3_khr
97 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles31_khr
98 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles32_khr
99 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles2_khr_noctx
100 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles32_khr_noctx
101 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles32_khr_single
102 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles3_rotate90
103 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles3_rotate180
104 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles3_rotate270
105 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles31_rotate90
106 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles31_rotate180
107 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles31_rotate270
108 {"--deqp-gl-config-name=rgba8888d24s8ms4"}, // gles3_multisample
109 {"--deqp-gl-config-name=rgb565d0s0ms0"}, // gles3_rgb565_no_depth_no_stencil
110 {"--deqp-gl-config-name=rgba8888d24s8ms4"}, // gles31_multisample
111 {"--deqp-gl-config-name=rgb565d0s0ms0"}, // gles31_rgb565_no_depth_no_stencil
112 };
113
114 #undef GLES_CTS_DIR
115 #undef GL_CTS_DIR
116
117 const char *gTestExpectationsFiles[] = {
118 "deqp_egl_test_expectations.txt",
119 "deqp_gles2_test_expectations.txt",
120 "deqp_gles3_test_expectations.txt",
121 "deqp_gles31_test_expectations.txt",
122 "deqp_khr_gles2_test_expectations.txt",
123 "deqp_khr_gles3_test_expectations.txt",
124 "deqp_khr_gles31_test_expectations.txt",
125 "deqp_khr_gles32_test_expectations.txt",
126 "deqp_khr_noctx_gles2_test_expectations.txt",
127 "deqp_khr_noctx_gles32_test_expectations.txt",
128 "deqp_khr_single_gles32_test_expectations.txt",
129 "deqp_gles3_rotate_test_expectations.txt",
130 "deqp_gles3_rotate_test_expectations.txt",
131 "deqp_gles3_rotate_test_expectations.txt",
132 "deqp_gles31_rotate_test_expectations.txt",
133 "deqp_gles31_rotate_test_expectations.txt",
134 "deqp_gles31_rotate_test_expectations.txt",
135 "deqp_gles3_multisample_test_expectations.txt",
136 "deqp_gles3_565_no_depth_no_stencil_test_expectations.txt",
137 "deqp_gles31_multisample_test_expectations.txt",
138 "deqp_gles31_565_no_depth_no_stencil_test_expectations.txt",
139 };
140
141 using APIInfo = std::pair<const char *, GPUTestConfig::API>;
142
143 constexpr APIInfo kEGLDisplayAPIs[] = {
144 {"angle-d3d9", GPUTestConfig::kAPID3D9},
145 {"angle-d3d11", GPUTestConfig::kAPID3D11},
146 {"angle-d3d11-ref", GPUTestConfig::kAPID3D11},
147 {"angle-gl", GPUTestConfig::kAPIGLDesktop},
148 {"angle-gles", GPUTestConfig::kAPIGLES},
149 {"angle-metal", GPUTestConfig::kAPIMetal},
150 {"angle-null", GPUTestConfig::kAPIUnknown},
151 {"angle-swiftshader", GPUTestConfig::kAPISwiftShader},
152 {"angle-vulkan", GPUTestConfig::kAPIVulkan},
153 {"angle-webgpu", GPUTestConfig::kAPIWgpu},
154 {"win32", GPUTestConfig::kAPIUnknown},
155 {"x11", GPUTestConfig::kAPIUnknown},
156 };
157
158 constexpr char kdEQPEGLString[] = "--deqp-egl-display-type=";
159 constexpr char kANGLEEGLString[] = "--use-angle=";
160 constexpr char kANGLEPreRotation[] = "--emulated-pre-rotation=";
161 constexpr char kdEQPCaseString[] = "--deqp-case=";
162 constexpr char kVerboseString[] = "--verbose";
163 constexpr char kRenderDocString[] = "--renderdoc";
164 constexpr char kNoRenderDocString[] = "--no-renderdoc";
165 constexpr char kdEQPFlagsPrefix[] = "--deqp-";
166 constexpr char kGTestFilter[] = "--gtest_filter=";
167 constexpr char kdEQPSurfaceWidth[] = "--deqp-surface-width=";
168 constexpr char kdEQPSurfaceHeight[] = "--deqp-surface-height=";
169 constexpr char kdEQPBaseSeed[] = "--deqp-base-seed";
170 constexpr const char gdEQPLogImagesString[] = "--deqp-log-images=";
171
172 // Use the config name defined in gTestSuiteConfigParameters by default
173 // If gEGLConfigNameFromCmdLine is overwritten by --deqp-gl-config-name passed from command
174 // line arguments, for example:
175 // out/Debug/angle_deqp_egl_tests --verbose --deqp-gl-config-name=rgba8888d24s8
176 // use gEGLConfigNameFromCmdLine (rgba8888d24s8) instead.
177 // Invalid --deqp-gl-config-name value passed from command line arguments will be caught by
178 // glu::parseConfigBitsFromName() defined in gluRenderConfig.cpp, and it will cause tests
179 // to fail
180 constexpr const char gdEQPEGLConfigNameString[] = "--deqp-gl-config-name=";
181 const char *gEGLConfigNameFromCmdLine = "";
182
183 angle::base::NoDestructor<std::vector<char>> gFilterStringBuffer;
184
185 // For angle_deqp_gles3*_rotateN_tests, default gOptions.preRotation to N.
186 #if defined(ANGLE_DEQP_GLES3_ROTATE90_TESTS) || defined(ANGLE_DEQP_GLES31_ROTATE90_TESTS)
187 constexpr uint32_t kDefaultPreRotation = 90;
188 #elif defined(ANGLE_DEQP_GLES3_ROTATE180_TESTS) || defined(ANGLE_DEQP_GLES31_ROTATE180_TESTS)
189 constexpr uint32_t kDefaultPreRotation = 180;
190 #elif defined(ANGLE_DEQP_GLES3_ROTATE270_TESTS) || defined(ANGLE_DEQP_GLES31_ROTATE270_TESTS)
191 constexpr uint32_t kDefaultPreRotation = 270;
192 #else
193 constexpr uint32_t kDefaultPreRotation = 0;
194 #endif
195
196 #if defined(ANGLE_TEST_ENABLE_RENDERDOC_CAPTURE)
197 constexpr bool kEnableRenderDocCapture = true;
198 #else
199 constexpr bool kEnableRenderDocCapture = false;
200 #endif
201
202 const APIInfo *gInitAPI = nullptr;
203 dEQPOptions gOptions = {
204 kDefaultPreRotation, // preRotation
205 kEnableRenderDocCapture, // enableRenderDocCapture
206 };
207
208 std::vector<const char *> gdEQPForwardFlags;
209
210 // Returns the default API for a platform.
GetDefaultAPIName()211 const char *GetDefaultAPIName()
212 {
213 #if defined(ANGLE_PLATFORM_ANDROID) || defined(ANGLE_PLATFORM_LINUX) || \
214 defined(ANGLE_PLATFORM_WINDOWS)
215 return "angle-vulkan";
216 #elif defined(ANGLE_PLATFORM_APPLE)
217 return "angle-gl";
218 #else
219 # error Unknown platform.
220 #endif
221 }
222
FindAPIInfo(const std::string & arg)223 const APIInfo *FindAPIInfo(const std::string &arg)
224 {
225 for (auto &displayAPI : kEGLDisplayAPIs)
226 {
227 if (arg == displayAPI.first)
228 {
229 return &displayAPI;
230 }
231 }
232 return nullptr;
233 }
234
GetDefaultAPIInfo()235 const APIInfo *GetDefaultAPIInfo()
236 {
237 const APIInfo *defaultInfo = FindAPIInfo(GetDefaultAPIName());
238 ASSERT(defaultInfo);
239 return defaultInfo;
240 }
241
GetTestStatLine(const std::string & key,const std::string & value)242 std::string GetTestStatLine(const std::string &key, const std::string &value)
243 {
244 return std::string(kInfoTag) + ": " + key + ": " + value + "\n";
245 }
246
247 // During the CaseList initialization we cannot use the GTEST FAIL macro to quit the program
248 // because the initialization is called outside of tests the first time.
Die()249 void Die()
250 {
251 exit(EXIT_FAILURE);
252 }
253
FindFileFromPath(const char * dirPath,const char * filePath)254 Optional<std::string> FindFileFromPath(const char *dirPath, const char *filePath)
255 {
256 std::stringstream strstr;
257 strstr << dirPath << filePath;
258 std::string path = strstr.str();
259
260 constexpr size_t kMaxFoundPathLen = 1000;
261 char foundPath[kMaxFoundPathLen];
262 if (angle::FindTestDataPath(path.c_str(), foundPath, kMaxFoundPathLen))
263 {
264 return std::string(foundPath);
265 }
266
267 return Optional<std::string>::Invalid();
268 }
269
FindCaseListPath(size_t testModuleIndex)270 Optional<std::string> FindCaseListPath(size_t testModuleIndex)
271 {
272 return FindFileFromPath(kCTSRootPath, gCaseListFiles[testModuleIndex]);
273 }
274
FindTestExpectationsPath(size_t testModuleIndex)275 Optional<std::string> FindTestExpectationsPath(size_t testModuleIndex)
276 {
277 return FindFileFromPath(kSupportPath, gTestExpectationsFiles[testModuleIndex]);
278 }
279
GetTestModuleIndex()280 size_t GetTestModuleIndex()
281 {
282 #ifdef ANGLE_DEQP_EGL_TESTS
283 return 0;
284 #endif
285
286 #ifdef ANGLE_DEQP_GLES2_TESTS
287 return 1;
288 #endif
289
290 #ifdef ANGLE_DEQP_GLES3_TESTS
291 return 2;
292 #endif
293
294 #ifdef ANGLE_DEQP_GLES31_TESTS
295 return 3;
296 #endif
297
298 #ifdef ANGLE_DEQP_KHR_GLES2_TESTS
299 return 4;
300 #endif
301
302 #ifdef ANGLE_DEQP_KHR_GLES3_TESTS
303 return 5;
304 #endif
305
306 #ifdef ANGLE_DEQP_KHR_GLES31_TESTS
307 return 6;
308 #endif
309
310 #ifdef ANGLE_DEQP_KHR_GLES32_TESTS
311 return 7;
312 #endif
313
314 #ifdef ANGLE_DEQP_KHR_NOCTX_GLES2_TESTS
315 return 8;
316 #endif
317
318 #ifdef ANGLE_DEQP_KHR_NOCTX_GLES32_TESTS
319 return 9;
320 #endif
321
322 #ifdef ANGLE_DEQP_KHR_SINGLE_GLES32_TESTS
323 return 10;
324 #endif
325
326 #ifdef ANGLE_DEQP_GLES3_ROTATE90_TESTS
327 return 11;
328 #endif
329
330 #ifdef ANGLE_DEQP_GLES3_ROTATE180_TESTS
331 return 12;
332 #endif
333
334 #ifdef ANGLE_DEQP_GLES3_ROTATE270_TESTS
335 return 13;
336 #endif
337
338 #ifdef ANGLE_DEQP_GLES31_ROTATE90_TESTS
339 return 14;
340 #endif
341
342 #ifdef ANGLE_DEQP_GLES31_ROTATE180_TESTS
343 return 15;
344 #endif
345
346 #ifdef ANGLE_DEQP_GLES31_ROTATE270_TESTS
347 return 16;
348 #endif
349
350 #ifdef ANGLE_DEQP_GLES3_MULTISAMPLE_TESTS
351 return 17;
352 #endif
353
354 #ifdef ANGLE_DEQP_GLES3_565_NO_DEPTH_NO_STENCIL_TESTS
355 return 18;
356 #endif
357
358 #ifdef ANGLE_DEQP_GLES31_MULTISAMPLE_TESTS
359 return 19;
360 #endif
361
362 #ifdef ANGLE_DEQP_GLES31_565_NO_DEPTH_NO_STENCIL_TESTS
363 return 20;
364 #endif
365 }
366
367 class dEQPCaseList
368 {
369 public:
370 dEQPCaseList(size_t testModuleIndex);
371
372 struct CaseInfo
373 {
CaseInfoangle::__anon255d7a720111::dEQPCaseList::CaseInfo374 CaseInfo(const std::string &testNameIn, int expectationIn)
375 : testName(testNameIn), expectation(expectationIn)
376 {}
377
378 std::string testName;
379 int expectation;
380 };
381
382 void initialize();
383
getCaseInfo(size_t caseIndex) const384 const CaseInfo &getCaseInfo(size_t caseIndex) const
385 {
386 ASSERT(mInitialized);
387 ASSERT(caseIndex < mCaseInfoList.size());
388 return mCaseInfoList[caseIndex];
389 }
390
numCases() const391 size_t numCases() const
392 {
393 ASSERT(mInitialized);
394 return mCaseInfoList.size();
395 }
396
397 private:
398 std::vector<CaseInfo> mCaseInfoList;
399 size_t mTestModuleIndex;
400 bool mInitialized = false;
401 };
402
dEQPCaseList(size_t testModuleIndex)403 dEQPCaseList::dEQPCaseList(size_t testModuleIndex) : mTestModuleIndex(testModuleIndex) {}
404
initialize()405 void dEQPCaseList::initialize()
406 {
407 if (mInitialized)
408 {
409 return;
410 }
411
412 mInitialized = true;
413
414 Optional<std::string> caseListPath = FindCaseListPath(mTestModuleIndex);
415 if (!caseListPath.valid())
416 {
417 std::cerr << "Failed to find case list file." << std::endl;
418 Die();
419 }
420
421 Optional<std::string> testExpectationsPath = FindTestExpectationsPath(mTestModuleIndex);
422 if (!testExpectationsPath.valid())
423 {
424 std::cerr << "Failed to find test expectations file." << std::endl;
425 Die();
426 }
427
428 GPUTestConfig::API api = GetDefaultAPIInfo()->second;
429 // Set the API from the command line, or using the default platform API.
430 if (gInitAPI)
431 {
432 api = gInitAPI->second;
433 }
434
435 GPUTestConfig testConfig = GPUTestConfig(api, gOptions.preRotation);
436
437 #if !defined(ANGLE_PLATFORM_ANDROID)
438 // Note: These prints mess up parsing of test list when running on Android.
439 std::cout << "Using test config with:" << std::endl;
440 for (uint32_t condition : testConfig.getConditions())
441 {
442 const char *name = GetConditionName(condition);
443 if (name != nullptr)
444 {
445 std::cout << " " << name << std::endl;
446 }
447 }
448 #endif
449
450 TestSuite *testSuite = TestSuite::GetInstance();
451
452 if (!testSuite->loadTestExpectationsFromFileWithConfig(testConfig,
453 testExpectationsPath.value()))
454 {
455 Die();
456 }
457
458 std::ifstream caseListStream(caseListPath.value());
459 if (caseListStream.fail())
460 {
461 std::cerr << "Failed to load the case list." << std::endl;
462 Die();
463 }
464
465 while (!caseListStream.eof())
466 {
467 std::string inString;
468 std::getline(caseListStream, inString);
469
470 std::string testName = TrimString(inString, kWhitespaceASCII);
471 if (testName.empty())
472 continue;
473 int expectation = testSuite->getTestExpectation(testName);
474 mCaseInfoList.push_back(CaseInfo(testName, expectation));
475 }
476 }
477
IsPassingResult(dEQPTestResult result)478 bool IsPassingResult(dEQPTestResult result)
479 {
480 // Check the global error flag for unexpected platform errors.
481 if (gGlobalError)
482 {
483 gGlobalError = false;
484 return false;
485 }
486
487 switch (result)
488 {
489 case dEQPTestResult::Fail:
490 case dEQPTestResult::Exception:
491 return false;
492
493 default:
494 return true;
495 }
496 }
497
GetTestList(size_t testModuleIndex)498 const dEQPCaseList &GetTestList(size_t testModuleIndex)
499 {
500 static dEQPCaseList sCaseList(testModuleIndex);
501 sCaseList.initialize();
502 return sCaseList;
503 }
504
GetTestCaseName(size_t testModuleIndex,size_t caseIndex)505 const std::string GetTestCaseName(size_t testModuleIndex, size_t caseIndex)
506 {
507 const auto &caseInfo = GetTestList(testModuleIndex).getCaseInfo(caseIndex);
508 return caseInfo.testName;
509 }
510
511 class dEQPTestSuiteStats
512 {
513 public:
dEQPTestSuiteStats()514 dEQPTestSuiteStats() {}
515
516 private:
517 void setUpTestStats();
518 void printTestStats();
519 void countTestResult(dEQPTestResult result);
520
521 uint32_t mTestCount;
522 uint32_t mPassedTestCount;
523 uint32_t mFailedTestCount;
524 uint32_t mTestExceptionCount;
525 uint32_t mNotSupportedTestCount;
526 uint32_t mSkippedTestCount;
527
528 std::vector<std::string> mUnexpectedFailed;
529 std::vector<std::string> mUnexpectedPasses;
530
531 friend class dEQPTest;
532 };
533
setUpTestStats()534 void dEQPTestSuiteStats::setUpTestStats()
535 {
536 mPassedTestCount = 0;
537 mFailedTestCount = 0;
538 mNotSupportedTestCount = 0;
539 mTestExceptionCount = 0;
540 mTestCount = 0;
541 mSkippedTestCount = 0;
542 mUnexpectedPasses.clear();
543 mUnexpectedFailed.clear();
544 }
545
printTestStats()546 void dEQPTestSuiteStats::printTestStats()
547 {
548 uint32_t crashedCount =
549 mTestCount - (mPassedTestCount + mFailedTestCount + mNotSupportedTestCount +
550 mTestExceptionCount + mSkippedTestCount);
551
552 std::cout << GetTestStatLine("Total", std::to_string(mTestCount));
553 std::cout << GetTestStatLine("Passed", std::to_string(mPassedTestCount));
554 std::cout << GetTestStatLine("Failed", std::to_string(mFailedTestCount));
555 std::cout << GetTestStatLine("Skipped", std::to_string(mSkippedTestCount));
556 std::cout << GetTestStatLine("Not Supported", std::to_string(mNotSupportedTestCount));
557 std::cout << GetTestStatLine("Exception", std::to_string(mTestExceptionCount));
558 std::cout << GetTestStatLine("Crashed", std::to_string(crashedCount));
559
560 if (!mUnexpectedPasses.empty())
561 {
562 std::cout << GetTestStatLine("Unexpected Passed Count",
563 std::to_string(mUnexpectedPasses.size()));
564 for (const std::string &testName : mUnexpectedPasses)
565 {
566 std::cout << GetTestStatLine("Unexpected Passed Tests", testName);
567 }
568 }
569
570 if (!mUnexpectedFailed.empty())
571 {
572 std::cout << GetTestStatLine("Unexpected Failed Count",
573 std::to_string(mUnexpectedFailed.size()));
574 for (const std::string &testName : mUnexpectedFailed)
575 {
576 std::cout << GetTestStatLine("Unexpected Failed Tests", testName);
577 }
578 }
579 }
580
countTestResult(dEQPTestResult result)581 void dEQPTestSuiteStats::countTestResult(dEQPTestResult result)
582 {
583 switch (result)
584 {
585 case dEQPTestResult::Pass:
586 mPassedTestCount++;
587 break;
588 case dEQPTestResult::Fail:
589 mFailedTestCount++;
590 break;
591 case dEQPTestResult::NotSupported:
592 mNotSupportedTestCount++;
593 break;
594 case dEQPTestResult::Exception:
595 mTestExceptionCount++;
596 break;
597 default:
598 std::cerr << "Unexpected test result code: " << static_cast<int>(result) << "\n";
599 break;
600 }
601 }
602
603 class dEQPTest : public testing::Test
604 {
605 public:
dEQPTest(size_t testModuleIndex,size_t caseIndex)606 dEQPTest(size_t testModuleIndex, size_t caseIndex)
607 : mTestModuleIndex(testModuleIndex), mTestCaseIndex(caseIndex)
608 {}
609
610 static void SetUpTestSuite();
611 static void TearDownTestSuite();
612
613 protected:
614 void TestBody() override;
615
616 private:
617 size_t mTestModuleIndex = 0;
618 size_t mTestCaseIndex = 0;
619
620 static dEQPTestSuiteStats sTestSuiteData;
621 };
622
623 dEQPTestSuiteStats dEQPTest::sTestSuiteData = dEQPTestSuiteStats();
624
625 // static function called once before running all of dEQPTest under the same test suite
SetUpTestSuite()626 void dEQPTest::SetUpTestSuite()
627 {
628 sTestSuiteData.setUpTestStats();
629
630 std::vector<const char *> argv;
631
632 // Reserve one argument for the binary name.
633 argv.push_back("");
634
635 // Add init api.
636 const char *targetApi = gInitAPI ? gInitAPI->first : GetDefaultAPIName();
637 std::string apiArgString = std::string(kdEQPEGLString) + targetApi;
638 argv.push_back(apiArgString.c_str());
639
640 std::string configNameFromCmdLineString =
641 std::string(gdEQPEGLConfigNameString) + gEGLConfigNameFromCmdLine;
642
643 // Add test config parameters
644 for (const char *configParam : gTestSuiteConfigParameters[GetTestModuleIndex()])
645 {
646 // Check if we pass --deqp-gl-config-name from the command line, if yes, use the one from
647 // command line
648 if (std::strlen(gEGLConfigNameFromCmdLine) > 0 &&
649 std::strncmp(configParam, gdEQPEGLConfigNameString, strlen(gdEQPEGLConfigNameString)) ==
650 0)
651 {
652 configParam = configNameFromCmdLineString.c_str();
653 }
654 argv.push_back(configParam);
655 }
656
657 // Hide SwiftShader window to prevent a race with Xvfb causing hangs on test bots
658 if (gInitAPI && gInitAPI->second == GPUTestConfig::kAPISwiftShader)
659 {
660 argv.push_back("--deqp-visibility=hidden");
661 }
662
663 TestSuite *testSuite = TestSuite::GetInstance();
664
665 std::stringstream logNameStream;
666 logNameStream << "TestResults";
667 if (testSuite->getBatchId() != -1)
668 {
669 logNameStream << "-Batch" << std::setfill('0') << std::setw(3) << testSuite->getBatchId();
670 }
671 logNameStream << ".qpa";
672
673 std::stringstream logArgStream;
674 logArgStream << "--deqp-log-filename="
675 << testSuite->reserveTestArtifactPath(logNameStream.str());
676
677 std::string logNameString = logArgStream.str();
678 argv.push_back(logNameString.c_str());
679
680 if (!gLogImages)
681 {
682 argv.push_back("--deqp-log-images=disable");
683 }
684
685 // Flushing during multi-process execution punishes HDDs. http://anglebug.com/42263718
686 if (testSuite->getBatchId() != -1)
687 {
688 argv.push_back("--deqp-log-flush=disable");
689 }
690
691 // Add any additional flags specified from command line to be forwarded to dEQP.
692 argv.insert(argv.end(), gdEQPForwardFlags.begin(), gdEQPForwardFlags.end());
693
694 // Init the platform.
695 if (!deqp_libtester_init_platform(static_cast<int>(argv.size()), argv.data(),
696 reinterpret_cast<void *>(&HandlePlatformError), gOptions))
697 {
698 std::cout << "Aborting test due to dEQP initialization error." << std::endl;
699 exit(1);
700 }
701 }
702
703 // static function called once after running all of dEQPTest under the same test suite
TearDownTestSuite()704 void dEQPTest::TearDownTestSuite()
705 {
706 sTestSuiteData.printTestStats();
707 deqp_libtester_shutdown_platform();
708 }
709
710 // TestBody() is called once for each dEQPTest
TestBody()711 void dEQPTest::TestBody()
712 {
713 if (sTestSuiteData.mTestExceptionCount > 1)
714 {
715 std::cout << "Too many exceptions, skipping all remaining tests." << std::endl;
716 return;
717 }
718
719 const auto &caseInfo = GetTestList(mTestModuleIndex).getCaseInfo(mTestCaseIndex);
720
721 // Tests that crash exit the harness before collecting the result. To tally the number of
722 // crashed tests we track how many tests we "tried" to run.
723 sTestSuiteData.mTestCount++;
724
725 if (caseInfo.expectation == GPUTestExpectationsParser::kGpuTestSkip)
726 {
727 sTestSuiteData.mSkippedTestCount++;
728 std::cout << "Test skipped.\n";
729 return;
730 }
731
732 TestSuite *testSuite = TestSuite::GetInstance();
733 testSuite->maybeUpdateTestTimeout(caseInfo.expectation);
734
735 gExpectError = (caseInfo.expectation != GPUTestExpectationsParser::kGpuTestPass);
736 dEQPTestResult result = deqp_libtester_run(caseInfo.testName.c_str());
737
738 bool testSucceeded = IsPassingResult(result);
739
740 if (!testSucceeded && caseInfo.expectation == GPUTestExpectationsParser::kGpuTestFlaky)
741 {
742 result = deqp_libtester_run(caseInfo.testName.c_str());
743 testSucceeded = IsPassingResult(result);
744 }
745
746 sTestSuiteData.countTestResult(result);
747
748 if (caseInfo.expectation == GPUTestExpectationsParser::kGpuTestPass ||
749 caseInfo.expectation == GPUTestExpectationsParser::kGpuTestFlaky)
750 {
751 EXPECT_TRUE(testSucceeded);
752
753 if (!testSucceeded)
754 {
755 sTestSuiteData.mUnexpectedFailed.push_back(caseInfo.testName);
756 }
757 }
758 else if (testSucceeded)
759 {
760 std::cout << "Test expected to fail but passed!" << std::endl;
761 sTestSuiteData.mUnexpectedPasses.push_back(caseInfo.testName);
762 }
763 }
764
HandleDisplayType(const char * displayTypeString)765 void HandleDisplayType(const char *displayTypeString)
766 {
767 std::stringstream argStream;
768
769 if (gInitAPI)
770 {
771 std::cout << "Cannot specify two EGL displays!" << std::endl;
772 exit(1);
773 }
774
775 argStream << displayTypeString;
776 std::string arg = argStream.str();
777 gInitAPI = FindAPIInfo(arg);
778
779 if (!gInitAPI && strncmp(displayTypeString, "angle-", strlen("angle-")) != 0)
780 {
781 std::stringstream argStream2;
782 argStream2 << "angle-" << displayTypeString;
783 std::string arg2 = argStream2.str();
784 gInitAPI = FindAPIInfo(arg2);
785
786 if (!gInitAPI)
787 {
788 std::cout << "Unknown API: " << displayTypeString << std::endl;
789 exit(1);
790 }
791 }
792 }
793
HandlePreRotation(const char * preRotationString)794 void HandlePreRotation(const char *preRotationString)
795 {
796 std::istringstream argStream(preRotationString);
797
798 uint32_t preRotation = 0;
799 argStream >> preRotation;
800
801 if (!argStream ||
802 (preRotation != 0 && preRotation != 90 && preRotation != 180 && preRotation != 270))
803 {
804 std::cout << "Invalid PreRotation '" << preRotationString
805 << "'; must be either 0, 90, 180 or 270" << std::endl;
806 exit(1);
807 }
808
809 gOptions.preRotation = preRotation;
810 }
811
HandleEGLConfigName(const char * configNameString)812 void HandleEGLConfigName(const char *configNameString)
813 {
814 gEGLConfigNameFromCmdLine = configNameString;
815 }
816
817 // The --deqp-case flag takes a case expression that is parsed into a --gtest_filter. It
818 // converts the "dEQP" style names (functional.thing.*) into "GoogleTest" style names
819 // (functional_thing_*). Currently it does not handle multiple tests and multiple filters in
820 // different arguments.
HandleFilterArg(const char * filterString,int * argc,int argIndex,char ** argv)821 void HandleFilterArg(const char *filterString, int *argc, int argIndex, char **argv)
822 {
823 std::string googleTestFilter = ReplaceDashesWithQuestionMark(filterString);
824
825 gFilterStringBuffer->resize(googleTestFilter.size() + 3 + strlen(kGTestFilter), 0);
826 std::fill(gFilterStringBuffer->begin(), gFilterStringBuffer->end(), 0);
827
828 int bytesWritten = snprintf(gFilterStringBuffer->data(), gFilterStringBuffer->size() - 1,
829 "%s*%s", kGTestFilter, googleTestFilter.c_str());
830 if (bytesWritten <= 0 || static_cast<size_t>(bytesWritten) >= gFilterStringBuffer->size() - 1)
831 {
832 std::cout << "Error parsing filter string: " << filterString;
833 exit(1);
834 }
835
836 argv[argIndex] = gFilterStringBuffer->data();
837 }
838
HandleLogImages(const char * logImagesString)839 void HandleLogImages(const char *logImagesString)
840 {
841 if (strcmp(logImagesString, "enable") == 0)
842 {
843 gLogImages = true;
844 }
845 else if (strcmp(logImagesString, "disable") == 0)
846 {
847 gLogImages = false;
848 }
849 else
850 {
851 std::cout << "Error parsing log images setting. Use enable/disable.";
852 exit(1);
853 }
854 }
855
RegisterGLCTSTests()856 void RegisterGLCTSTests()
857 {
858 size_t testModuleIndex = GetTestModuleIndex();
859
860 const dEQPCaseList &caseList = GetTestList(testModuleIndex);
861
862 for (size_t caseIndex = 0; caseIndex < caseList.numCases(); ++caseIndex)
863 {
864 auto factory = [testModuleIndex, caseIndex]() {
865 return new dEQPTest(testModuleIndex, caseIndex);
866 };
867
868 const std::string testCaseName = GetTestCaseName(testModuleIndex, caseIndex);
869 size_t pos = testCaseName.find('.');
870 ASSERT(pos != std::string::npos);
871 // testCaseName comes from one of the mustpass files in gCaseListFiles.
872 // All of the testCaseName in the same mustpass file starts with the same testSuiteName
873 // prefix. Which mustpass file to load the set of testCaseName depends on testModuleIndex
874 // compiled into the deqp test application binary. For now, only one testModuleIndex is
875 // compiled in a deqp test application binary, meaning all of the tests invoked by same deqp
876 // test application binary are under the same test suite.
877 std::string testSuiteName = testCaseName.substr(0, pos);
878 std::string testName = testCaseName.substr(pos + 1);
879 testing::RegisterTest(testSuiteName.c_str(), testName.c_str(), nullptr, nullptr, __FILE__,
880 __LINE__, factory);
881 }
882 }
883 } // anonymous namespace
884
885 // Called from main() to process command-line arguments.
RunGLCTSTests(int * argc,char ** argv)886 int RunGLCTSTests(int *argc, char **argv)
887 {
888 int argIndex = 0;
889 while (argIndex < *argc)
890 {
891 if (strncmp(argv[argIndex], kdEQPEGLString, strlen(kdEQPEGLString)) == 0)
892 {
893 HandleDisplayType(argv[argIndex] + strlen(kdEQPEGLString));
894 }
895 else if (strncmp(argv[argIndex], kANGLEEGLString, strlen(kANGLEEGLString)) == 0)
896 {
897 HandleDisplayType(argv[argIndex] + strlen(kANGLEEGLString));
898 }
899 else if (strncmp(argv[argIndex], kANGLEPreRotation, strlen(kANGLEPreRotation)) == 0)
900 {
901 HandlePreRotation(argv[argIndex] + strlen(kANGLEPreRotation));
902 }
903 else if (strncmp(argv[argIndex], gdEQPEGLConfigNameString,
904 strlen(gdEQPEGLConfigNameString)) == 0)
905 {
906 HandleEGLConfigName(argv[argIndex] + strlen(gdEQPEGLConfigNameString));
907 }
908 else if (strncmp(argv[argIndex], kdEQPCaseString, strlen(kdEQPCaseString)) == 0)
909 {
910 HandleFilterArg(argv[argIndex] + strlen(kdEQPCaseString), argc, argIndex, argv);
911 }
912 else if (strncmp(argv[argIndex], kGTestFilter, strlen(kGTestFilter)) == 0)
913 {
914 HandleFilterArg(argv[argIndex] + strlen(kGTestFilter), argc, argIndex, argv);
915 }
916 else if (strncmp(argv[argIndex], kVerboseString, strlen(kVerboseString)) == 0 ||
917 strcmp(argv[argIndex], "-v") == 0)
918 {
919 gVerbose = true;
920 }
921 else if (strncmp(argv[argIndex], gdEQPLogImagesString, strlen(gdEQPLogImagesString)) == 0)
922 {
923 HandleLogImages(argv[argIndex] + strlen(gdEQPLogImagesString));
924 }
925 else if (strncmp(argv[argIndex], kRenderDocString, strlen(kRenderDocString)) == 0)
926 {
927 gOptions.enableRenderDocCapture = true;
928 }
929 else if (strncmp(argv[argIndex], kNoRenderDocString, strlen(kNoRenderDocString)) == 0)
930 {
931 gOptions.enableRenderDocCapture = false;
932 }
933 else if (strncmp(argv[argIndex], kdEQPFlagsPrefix, strlen(kdEQPFlagsPrefix)) == 0)
934 {
935 gdEQPForwardFlags.push_back(argv[argIndex]);
936 }
937 else if (strncmp(argv[argIndex], kdEQPSurfaceWidth, strlen(kdEQPSurfaceWidth)) == 0)
938 {
939 gdEQPForwardFlags.push_back(argv[argIndex]);
940 }
941 else if (strncmp(argv[argIndex], kdEQPSurfaceHeight, strlen(kdEQPSurfaceHeight)) == 0)
942 {
943 gdEQPForwardFlags.push_back(argv[argIndex]);
944 }
945 else if (strncmp(argv[argIndex], kdEQPBaseSeed, strlen(kdEQPBaseSeed)) == 0)
946 {
947 gdEQPForwardFlags.push_back(argv[argIndex]);
948 }
949 argIndex++;
950 }
951
952 GPUTestConfig::API api = GetDefaultAPIInfo()->second;
953 if (gInitAPI)
954 {
955 api = gInitAPI->second;
956 }
957 if (gOptions.preRotation != 0 && api != GPUTestConfig::kAPIVulkan &&
958 api != GPUTestConfig::kAPISwiftShader)
959 {
960 std::cout << "PreRotation is only supported on Vulkan" << std::endl;
961 exit(1);
962 }
963
964 angle::TestSuite testSuite(argc, argv, RegisterGLCTSTests);
965 return testSuite.run();
966 }
967 } // namespace angle
968