1 /*-------------------------------------------------------------------------
2 * drawElements Quality Program Tester Core
3 * ----------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Test executor.
22 *//*--------------------------------------------------------------------*/
23
24 #include "tcuTestSessionExecutor.hpp"
25 #include "qpTestLog.h"
26 #include "tcuCommandLine.hpp"
27 #include "tcuTestLog.hpp"
28
29 #include "deClock.h"
30
31 namespace tcu
32 {
33
34 using std::vector;
35
nodeTypeToTestCaseType(TestNodeType nodeType)36 static qpTestCaseType nodeTypeToTestCaseType(TestNodeType nodeType)
37 {
38 switch (nodeType)
39 {
40 case NODETYPE_SELF_VALIDATE:
41 return QP_TEST_CASE_TYPE_SELF_VALIDATE;
42 case NODETYPE_PERFORMANCE:
43 return QP_TEST_CASE_TYPE_PERFORMANCE;
44 case NODETYPE_CAPABILITY:
45 return QP_TEST_CASE_TYPE_CAPABILITY;
46 case NODETYPE_ACCURACY:
47 return QP_TEST_CASE_TYPE_ACCURACY;
48 default:
49 DE_ASSERT(false);
50 return QP_TEST_CASE_TYPE_LAST;
51 }
52 }
53
TestSessionExecutor(TestPackageRoot & root,TestContext & testCtx)54 TestSessionExecutor::TestSessionExecutor(TestPackageRoot &root, TestContext &testCtx)
55 : m_testCtx(testCtx)
56 , m_inflater(testCtx)
57 , m_caseListFilter(testCtx.getCommandLine().createCaseListFilter(testCtx.getArchive()))
58 , m_iterator(root, m_inflater, *m_caseListFilter)
59 , m_state(STATE_TRAVERSE_HIERARCHY)
60 , m_abortSession(false)
61 , m_isInTestCase(false)
62 , m_testStartTime(0)
63 , m_packageStartTime(0)
64 {
65 }
66
~TestSessionExecutor(void)67 TestSessionExecutor::~TestSessionExecutor(void)
68 {
69 }
70
iterate(void)71 bool TestSessionExecutor::iterate(void)
72 {
73 while (!m_abortSession)
74 {
75 switch (m_state)
76 {
77 case STATE_TRAVERSE_HIERARCHY:
78 {
79 const TestHierarchyIterator::State hierIterState = m_iterator.getState();
80
81 if (hierIterState == TestHierarchyIterator::STATE_ENTER_NODE ||
82 hierIterState == TestHierarchyIterator::STATE_LEAVE_NODE)
83 {
84 TestNode *const curNode = m_iterator.getNode();
85 const TestNodeType nodeType = curNode->getNodeType();
86 const bool isEnter = hierIterState == TestHierarchyIterator::STATE_ENTER_NODE;
87
88 switch (nodeType)
89 {
90 case NODETYPE_PACKAGE:
91 {
92 TestPackage *const testPackage = static_cast<TestPackage *>(curNode);
93 isEnter ? enterTestPackage(testPackage) : leaveTestPackage(testPackage);
94 break;
95 }
96
97 case NODETYPE_GROUP:
98 {
99 isEnter ? enterTestGroup(m_iterator.getNodePath()) : leaveTestGroup(m_iterator.getNodePath());
100 break; // nada
101 }
102
103 case NODETYPE_SELF_VALIDATE:
104 case NODETYPE_PERFORMANCE:
105 case NODETYPE_CAPABILITY:
106 case NODETYPE_ACCURACY:
107 {
108 TestCase *const testCase = static_cast<TestCase *>(curNode);
109
110 if (isEnter)
111 {
112 if (enterTestCase(testCase, m_iterator.getNodePath()))
113 m_state = STATE_EXECUTE_TEST_CASE;
114 // else remain in TRAVERSING_HIERARCHY => node will be exited from in the next iteration
115 }
116 else
117 leaveTestCase(testCase);
118
119 break;
120 }
121
122 default:
123 DE_ASSERT(false);
124 break;
125 }
126
127 m_iterator.next();
128 break;
129 }
130 else
131 {
132 DE_ASSERT(hierIterState == TestHierarchyIterator::STATE_FINISHED);
133 m_status.isComplete = true;
134 return false;
135 }
136 }
137
138 case STATE_EXECUTE_TEST_CASE:
139 {
140 DE_ASSERT(m_iterator.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
141 isTestNodeTypeExecutable(m_iterator.getNode()->getNodeType()));
142
143 TestCase *const testCase = static_cast<TestCase *>(m_iterator.getNode());
144 const TestCase::IterateResult iterResult = iterateTestCase(testCase);
145
146 if (iterResult == TestCase::STOP)
147 m_state = STATE_TRAVERSE_HIERARCHY;
148
149 return true;
150 }
151
152 default:
153 DE_ASSERT(false);
154 break;
155 }
156 }
157
158 return false;
159 }
160
enterTestPackage(TestPackage * testPackage)161 void TestSessionExecutor::enterTestPackage(TestPackage *testPackage)
162 {
163 // Create test case wrapper
164 DE_ASSERT(!m_caseExecutor);
165 m_caseExecutor = de::MovePtr<TestCaseExecutor>(testPackage->createExecutor());
166 testPackage->setCaseListFilter(m_caseListFilter.get());
167 m_packageStartTime = deGetMicroseconds();
168 }
169
leaveTestPackage(TestPackage * testPackage)170 void TestSessionExecutor::leaveTestPackage(TestPackage *testPackage)
171 {
172 DE_UNREF(testPackage);
173 m_caseExecutor->deinitTestPackage(m_testCtx);
174 // If m_caseExecutor uses local status then it may perform some tests in deinitTestPackage(). We have to update TestSessionExecutor::m_status
175 if (m_caseExecutor->usesLocalStatus())
176 m_caseExecutor->updateGlobalStatus(m_status);
177
178 const int64_t duration = deGetMicroseconds() - m_packageStartTime;
179 m_packageStartTime = 0;
180
181 if (!std::string(m_testCtx.getCommandLine().getServerAddress()).empty())
182 m_caseExecutor->reportDurations(m_testCtx, std::string(testPackage->getName()), duration, m_groupsDurationTime);
183
184 m_caseExecutor.clear();
185
186 if (!std::string(m_testCtx.getCommandLine().getServerAddress()).empty())
187 {
188 m_testCtx.getLog().startTestsCasesTime();
189
190 m_testCtx.getLog() << TestLog::Integer(testPackage->getName(), "Total tests case duration in microseconds",
191 "us", QP_KEY_TAG_TIME, duration);
192
193 for (std::map<std::string, uint64_t>::iterator it = m_groupsDurationTime.begin();
194 it != m_groupsDurationTime.end(); ++it)
195 m_testCtx.getLog() << TestLog::Integer(it->first, "The test group case duration in microseconds", "us",
196 QP_KEY_TAG_TIME, it->second);
197
198 m_testCtx.getLog().endTestsCasesTime();
199 }
200 }
201
enterTestGroup(const std::string & casePath)202 void TestSessionExecutor::enterTestGroup(const std::string &casePath)
203 {
204 m_groupsDurationTime[casePath] = deGetMicroseconds();
205 }
206
leaveTestGroup(const std::string & casePath)207 void TestSessionExecutor::leaveTestGroup(const std::string &casePath)
208 {
209 m_groupsDurationTime[casePath] = deGetMicroseconds() - m_groupsDurationTime[casePath];
210 }
211
enterTestCase(TestCase * testCase,const std::string & casePath)212 bool TestSessionExecutor::enterTestCase(TestCase *testCase, const std::string &casePath)
213 {
214 TestLog &log = m_testCtx.getLog();
215 const qpTestCaseType caseType = nodeTypeToTestCaseType(testCase->getNodeType());
216 bool initOk = false;
217
218 print("\nTest case '%s'..\n", casePath.c_str());
219
220 #if (DE_OS == DE_OS_WIN32)
221 fflush(stdout);
222 #endif
223
224 m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
225 m_testCtx.setTerminateAfter(false);
226 log.startCase(casePath.c_str(), caseType);
227
228 m_isInTestCase = true;
229 m_testStartTime = deGetMicroseconds();
230
231 try
232 {
233 m_caseExecutor->init(testCase, casePath);
234 initOk = true;
235 }
236 catch (const std::bad_alloc &)
237 {
238 DE_ASSERT(!initOk);
239 m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory in test case init");
240 m_testCtx.setTerminateAfter(true);
241 }
242 catch (const tcu::TestException &e)
243 {
244 DE_ASSERT(!initOk);
245 DE_ASSERT(e.getTestResult() != QP_TEST_RESULT_LAST);
246 m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
247 m_testCtx.setTerminateAfter(e.isFatal());
248 log << e;
249 }
250 catch (const tcu::Exception &e)
251 {
252 DE_ASSERT(!initOk);
253 m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
254 log << e;
255 }
256
257 DE_ASSERT(initOk || m_testCtx.getTestResult() != QP_TEST_RESULT_LAST);
258
259 return initOk;
260 }
261
leaveTestCase(TestCase * testCase)262 void TestSessionExecutor::leaveTestCase(TestCase *testCase)
263 {
264 TestLog &log = m_testCtx.getLog();
265
266 // De-init case.
267 try
268 {
269 m_caseExecutor->deinit(testCase);
270 }
271 catch (const tcu::Exception &e)
272 {
273 const bool suppressLogging = m_testCtx.getLog().isSupressLogging();
274
275 if (suppressLogging)
276 m_testCtx.getLog().supressLogging(false);
277
278 log << e << TestLog::Message << "Error in test case deinit, test program will terminate."
279 << TestLog::EndMessage;
280 m_testCtx.setTerminateAfter(true);
281
282 m_testCtx.getLog().supressLogging(suppressLogging);
283 }
284
285 {
286 const int64_t duration = deGetMicroseconds() - m_testStartTime;
287 m_testStartTime = 0;
288 m_testCtx.getLog() << TestLog::Integer("TestDuration", "Test case duration in microseconds", "us",
289 QP_KEY_TAG_TIME, duration);
290 }
291
292 {
293 const qpTestResult testResult = m_testCtx.getTestResult();
294 const char *const testResultDesc = m_testCtx.getTestResultDesc();
295 const bool terminateAfter = m_testCtx.getTerminateAfter();
296 DE_ASSERT(testResult != QP_TEST_RESULT_LAST);
297
298 m_isInTestCase = false;
299 m_testCtx.getLog().endCase(testResult, testResultDesc);
300
301 // Update statistics.
302 print(" %s (%s)\n", qpGetTestResultName(testResult), testResultDesc);
303
304 #if (DE_OS == DE_OS_WIN32)
305 fflush(stdout);
306 #endif
307 if (!m_caseExecutor->usesLocalStatus())
308 {
309 m_status.numExecuted += 1;
310 switch (testResult)
311 {
312 case QP_TEST_RESULT_PASS:
313 m_status.numPassed += 1;
314 break;
315 case QP_TEST_RESULT_NOT_SUPPORTED:
316 m_status.numNotSupported += 1;
317 break;
318 case QP_TEST_RESULT_QUALITY_WARNING:
319 m_status.numWarnings += 1;
320 break;
321 case QP_TEST_RESULT_COMPATIBILITY_WARNING:
322 m_status.numWarnings += 1;
323 break;
324 case QP_TEST_RESULT_WAIVER:
325 m_status.numWaived += 1;
326 break;
327 case QP_TEST_RESULT_DEVICE_LOST:
328 m_status.numDeviceLost += 1;
329 m_status.numFailed += 1;
330 break;
331 default:
332 m_status.numFailed += 1;
333 break;
334 }
335 }
336 else
337 {
338 m_caseExecutor->updateGlobalStatus(m_status);
339 }
340
341 // terminateAfter, Resource error or any error in deinit means that execution should end
342 if (terminateAfter || testResult == QP_TEST_RESULT_RESOURCE_ERROR ||
343 (m_status.numFailed > 0 && m_testCtx.getCommandLine().isTerminateOnFailEnabled()) ||
344 (m_status.numDeviceLost > 0 && m_testCtx.getCommandLine().isTerminateOnDeviceLostEnabled()))
345
346 m_abortSession = true;
347 }
348
349 if (m_testCtx.getWatchDog())
350 qpWatchDog_reset(m_testCtx.getWatchDog());
351 }
352
iterateTestCase(TestCase * testCase)353 TestCase::IterateResult TestSessionExecutor::iterateTestCase(TestCase *testCase)
354 {
355 TestLog &log = m_testCtx.getLog();
356 TestCase::IterateResult iterateResult = TestCase::STOP;
357
358 m_testCtx.touchWatchdog();
359
360 try
361 {
362 iterateResult = m_caseExecutor->iterate(testCase);
363 }
364 catch (const std::bad_alloc &)
365 {
366 m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory during test execution");
367 m_testCtx.setTerminateAfter(true);
368 }
369 catch (const tcu::TestException &e)
370 {
371 log << e;
372 m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
373 m_testCtx.setTerminateAfter(e.isFatal());
374 }
375 catch (const tcu::Exception &e)
376 {
377 log << e;
378 m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
379 }
380
381 return iterateResult;
382 }
383
384 } // namespace tcu
385