1 /*-------------------------------------------------------------------------
2 * drawElements Quality Program Tester Core
3 * ----------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Render target info.
22 *//*--------------------------------------------------------------------*/
23
24 #include "tcuApp.hpp"
25 #include "tcuPlatform.hpp"
26 #include "tcuTestContext.hpp"
27 #include "tcuTestSessionExecutor.hpp"
28 #include "tcuTestHierarchyUtil.hpp"
29 #include "tcuCommandLine.hpp"
30 #include "tcuTestLog.hpp"
31
32 #include "qpInfo.h"
33 #include "qpDebugOut.h"
34
35 #include "deMath.h"
36
37 #include <iostream>
38
39 namespace tcu
40 {
41
42 using std::string;
43
44 /*--------------------------------------------------------------------*//*!
45 * Writes all packages found stdout without any
46 * separations. Recommended to be used with a single package
47 * only. It's possible to use test selectors for limiting the export
48 * to one package in a multipackage binary.
49 *//*--------------------------------------------------------------------*/
writeCaselistsToStdout(TestPackageRoot & root,TestContext & testCtx)50 static void writeCaselistsToStdout(TestPackageRoot &root, TestContext &testCtx)
51 {
52 DefaultHierarchyInflater inflater(testCtx);
53 de::MovePtr<const CaseListFilter> caseListFilter(
54 testCtx.getCommandLine().createCaseListFilter(testCtx.getArchive()));
55 TestHierarchyIterator iter(root, inflater, *caseListFilter);
56
57 while (iter.getState() != TestHierarchyIterator::STATE_FINISHED)
58 {
59 iter.next();
60
61 while (iter.getNode()->getNodeType() != NODETYPE_PACKAGE)
62 {
63 if (iter.getState() == TestHierarchyIterator::STATE_ENTER_NODE)
64 std::cout << (isTestNodeTypeExecutable(iter.getNode()->getNodeType()) ? "TEST" : "GROUP") << ": "
65 << iter.getNodePath() << "\n";
66 iter.next();
67 }
68
69 DE_ASSERT(iter.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
70 iter.getNode()->getNodeType() == NODETYPE_PACKAGE);
71 iter.next();
72 }
73 }
74
75 /*--------------------------------------------------------------------*//*!
76 * Verifies that amber capability requirements in the .amber files
77 * match with capabilities defined on the CTS C code.
78 *//*--------------------------------------------------------------------*/
verifyAmberCapabilityCoherency(TestPackageRoot & root,TestContext & testCtx)79 static void verifyAmberCapabilityCoherency(TestPackageRoot &root, TestContext &testCtx)
80 {
81 DefaultHierarchyInflater inflater(testCtx);
82 de::MovePtr<const CaseListFilter> caseListFilter(
83 testCtx.getCommandLine().createCaseListFilter(testCtx.getArchive()));
84 TestHierarchyIterator iter(root, inflater, *caseListFilter);
85 int count = 0;
86 int errorCount = 0;
87
88 bool ok = true;
89
90 while (iter.getState() != TestHierarchyIterator::STATE_FINISHED)
91 {
92 iter.next();
93
94 while (iter.getNode()->getNodeType() != NODETYPE_PACKAGE)
95 {
96 if (iter.getState() == TestHierarchyIterator::STATE_ENTER_NODE &&
97 isTestNodeTypeExecutable(iter.getNode()->getNodeType()))
98 {
99 std::cout << iter.getNodePath() << "\n";
100 testCtx.getLog() << tcu::TestLog::Message << iter.getNodePath() << tcu::TestLog::EndMessage;
101 if (!iter.getNode()->validateRequirements())
102 {
103 ok = false;
104 errorCount++;
105 }
106 count++;
107 }
108 iter.next();
109 }
110
111 DE_ASSERT(iter.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
112 iter.getNode()->getNodeType() == NODETYPE_PACKAGE);
113 iter.next();
114 }
115 std::cout << count << " amber tests, " << errorCount << " errors.\n";
116 if (!ok)
117 TCU_THROW(InternalError, "One or more CTS and Amber test requirements do not match; check log for details");
118 }
119
120 /*--------------------------------------------------------------------*//*!
121 * \brief Construct test application
122 *
123 * If a fatal error occurs during initialization constructor will call
124 * die() with debug information.
125 *
126 * \param platform Reference to platform implementation.
127 *//*--------------------------------------------------------------------*/
App(Platform & platform,Archive & archive,TestLog & log,const CommandLine & cmdLine)128 App::App(Platform &platform, Archive &archive, TestLog &log, const CommandLine &cmdLine)
129 : m_platform(platform)
130 , m_watchDog(DE_NULL)
131 , m_crashHandler(DE_NULL)
132 , m_crashed(false)
133 , m_testCtx(DE_NULL)
134 , m_testRoot(DE_NULL)
135 , m_testExecutor(DE_NULL)
136 {
137 if (!cmdLine.isSubProcess())
138 {
139 print("dEQP Core %s (0x%08x) starting..\n", qpGetReleaseName(), qpGetReleaseId());
140 print(" target implementation = '%s'\n", qpGetTargetName());
141 }
142
143 if (!deSetRoundingMode(DE_ROUNDINGMODE_TO_NEAREST_EVEN))
144 qpPrintf("WARNING: Failed to set floating-point rounding mode!\n");
145
146 try
147 {
148 const RunMode runMode = cmdLine.getRunMode();
149
150 // Initialize watchdog
151 if (cmdLine.isWatchDogEnabled())
152 TCU_CHECK_INTERNAL(m_watchDog = qpWatchDog_create(onWatchdogTimeout, this, WATCHDOG_TOTAL_TIME_LIMIT_SECS,
153 WATCHDOG_INTERVAL_TIME_LIMIT_SECS));
154
155 // Initialize crash handler.
156 if (cmdLine.isCrashHandlingEnabled())
157 TCU_CHECK_INTERNAL(m_crashHandler = qpCrashHandler_create(onCrash, this));
158
159 // Create test context
160 m_testCtx = new TestContext(m_platform, archive, log, cmdLine, m_watchDog);
161
162 // Create root from registry
163 m_testRoot = new TestPackageRoot(*m_testCtx, TestPackageRegistry::getSingleton());
164
165 // \note No executor is created if runmode is not EXECUTE
166 if (runMode == RUNMODE_EXECUTE)
167 m_testExecutor = new TestSessionExecutor(*m_testRoot, *m_testCtx);
168 else if (runMode == RUNMODE_DUMP_STDOUT_CASELIST)
169 writeCaselistsToStdout(*m_testRoot, *m_testCtx);
170 else if (runMode == RUNMODE_DUMP_XML_CASELIST)
171 writeXmlCaselistsToFiles(*m_testRoot, *m_testCtx, cmdLine);
172 else if (runMode == RUNMODE_DUMP_TEXT_CASELIST)
173 writeTxtCaselistsToFiles(*m_testRoot, *m_testCtx, cmdLine);
174 else if (runMode == RUNMODE_VERIFY_AMBER_COHERENCY)
175 verifyAmberCapabilityCoherency(*m_testRoot, *m_testCtx);
176 else
177 DE_ASSERT(false);
178 }
179 catch (const std::exception &e)
180 {
181 cleanup();
182 die("Failed to initialize dEQP: %s", e.what());
183 }
184 }
185
~App(void)186 App::~App(void)
187 {
188 cleanup();
189 }
190
cleanup(void)191 void App::cleanup(void)
192 {
193 delete m_testExecutor;
194 delete m_testRoot;
195 delete m_testCtx;
196
197 if (m_crashHandler)
198 qpCrashHandler_destroy(m_crashHandler);
199
200 if (m_watchDog)
201 qpWatchDog_destroy(m_watchDog);
202 }
203
204 /*--------------------------------------------------------------------*//*!
205 * \brief Step forward test execution
206 * \return true if application should call iterate() again and false
207 * if test execution session is complete.
208 *//*--------------------------------------------------------------------*/
iterate(void)209 bool App::iterate(void)
210 {
211 if (!m_testExecutor)
212 {
213 DE_ASSERT(m_testCtx->getCommandLine().getRunMode() != RUNMODE_EXECUTE);
214 return false;
215 }
216
217 // Poll platform events
218 const bool platformOk = m_platform.processEvents();
219
220 // Iterate a step.
221 bool testExecOk = false;
222 if (platformOk)
223 {
224 try
225 {
226 testExecOk = m_testExecutor->iterate();
227 }
228 catch (const std::exception &e)
229 {
230 die("%s", e.what());
231 }
232 }
233
234 if ((!platformOk || !testExecOk))
235 {
236 if (!m_testCtx->getCommandLine().isSubProcess())
237 {
238 if (!platformOk)
239 print("\nABORTED!\n");
240 else
241 print("\nDONE!\n");
242 }
243
244 const RunMode runMode = m_testCtx->getCommandLine().getRunMode();
245 if (runMode == RUNMODE_EXECUTE)
246 {
247 const TestRunStatus &result = m_testExecutor->getStatus();
248 if (!m_testCtx->getCommandLine().isSubProcess())
249 {
250 // Report statistics.
251 print("\nTest run totals:\n");
252 print(" Passed: %d/%d (%.1f%%)\n", result.numPassed, result.numExecuted,
253 (result.numExecuted > 0 ? (100.0f * (float)result.numPassed / (float)result.numExecuted) : 0.0f));
254 print(" Failed: %d/%d (%.1f%%)\n", result.numFailed, result.numExecuted,
255 (result.numExecuted > 0 ? (100.0f * (float)result.numFailed / (float)result.numExecuted) : 0.0f));
256 print(" Not supported: %d/%d (%.1f%%)\n", result.numNotSupported, result.numExecuted,
257 (result.numExecuted > 0 ? (100.0f * (float)result.numNotSupported / (float)result.numExecuted) :
258 0.0f));
259 print(
260 " Warnings: %d/%d (%.1f%%)\n", result.numWarnings, result.numExecuted,
261 (result.numExecuted > 0 ? (100.0f * (float)result.numWarnings / (float)result.numExecuted) : 0.0f));
262 print(" Waived: %d/%d (%.1f%%)\n", result.numWaived, result.numExecuted,
263 (result.numExecuted > 0 ? (100.0f * (float)result.numWaived / (float)result.numExecuted) : 0.0f));
264 if (!result.isComplete)
265 print("Test run was ABORTED!\n");
266 }
267 else
268 {
269 // subprocess sends test statisticts through qpa file, so that main process may read it
270 // and add to global statistics ( search for #SubProcessStatus to see how it's done )
271 std::ostringstream str;
272 str << "\n#SubProcessStatus " << result.numExecuted << " " << result.numPassed << " "
273 << result.numFailed << " " << result.numNotSupported << " " << result.numWarnings << " "
274 << result.numWaived << "\n";
275 m_testCtx->getLog().writeRaw(str.str().c_str());
276 }
277 }
278 }
279
280 return platformOk && testExecOk;
281 }
282
getResult(void) const283 const TestRunStatus &App::getResult(void) const
284 {
285 return m_testExecutor->getStatus();
286 }
287
onWatchdogTimeout(qpWatchDog * watchDog,void * userPtr,qpTimeoutReason reason)288 void App::onWatchdogTimeout(qpWatchDog *watchDog, void *userPtr, qpTimeoutReason reason)
289 {
290 DE_UNREF(watchDog);
291 static_cast<App *>(userPtr)->onWatchdogTimeout(reason);
292 }
293
onCrash(qpCrashHandler * crashHandler,void * userPtr)294 void App::onCrash(qpCrashHandler *crashHandler, void *userPtr)
295 {
296 DE_UNREF(crashHandler);
297 static_cast<App *>(userPtr)->onCrash();
298 }
299
onWatchdogTimeout(qpTimeoutReason reason)300 void App::onWatchdogTimeout(qpTimeoutReason reason)
301 {
302 if (!m_crashLock.tryLock() || m_crashed)
303 return; // In crash handler already.
304
305 m_crashed = true;
306
307 m_testCtx->getLog().terminateCase(QP_TEST_RESULT_TIMEOUT);
308 die("Watchdog timer timeout for %s",
309 (reason == QP_TIMEOUT_REASON_INTERVAL_LIMIT ? "touch interval" : "total time"));
310 }
311
writeCrashToLog(void * userPtr,const char * infoString)312 static void writeCrashToLog(void *userPtr, const char *infoString)
313 {
314 // \note THIS IS CALLED BY SIGNAL HANDLER! CALLING MALLOC/FREE IS NOT ALLOWED!
315 TestLog *log = static_cast<TestLog *>(userPtr);
316 log->writeMessage(infoString);
317 }
318
writeCrashToConsole(void * userPtr,const char * infoString)319 static void writeCrashToConsole(void *userPtr, const char *infoString)
320 {
321 // \note THIS IS CALLED BY SIGNAL HANDLER! CALLING MALLOC/FREE IS NOT ALLOWED!
322 DE_UNREF(userPtr);
323 qpPrint(infoString);
324 }
325
onCrash(void)326 void App::onCrash(void)
327 {
328 // \note THIS IS CALLED BY SIGNAL HANDLER! CALLING MALLOC/FREE IS NOT ALLOWED!
329
330 if (!m_crashLock.tryLock() || m_crashed)
331 return; // In crash handler already.
332
333 m_crashed = true;
334
335 bool isInCase = m_testExecutor ? m_testExecutor->isInTestCase() : false;
336
337 if (isInCase)
338 {
339 qpCrashHandler_writeCrashInfo(m_crashHandler, writeCrashToLog, &m_testCtx->getLog());
340 m_testCtx->getLog().terminateCase(QP_TEST_RESULT_CRASH);
341 }
342 else
343 qpCrashHandler_writeCrashInfo(m_crashHandler, writeCrashToConsole, DE_NULL);
344
345 die("Test program crashed");
346 }
347
348 } // namespace tcu
349