1 /*------------------------------------------------------------------------
2 * Vulkan Conformance Tests
3 * ------------------------
4 *
5 * Copyright (c) 2019 Google LLC
6 * Copyright (c) 2019 The Khronos Group Inc.
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 *
20 *//*!
21 * \file
22 * \brief Functional tests using amber
23 *//*--------------------------------------------------------------------*/
24
25 #include <amber/amber.h>
26 #include "amber/recipe.h"
27
28 #include <iostream>
29
30 #include "deDefs.hpp"
31 #include "deUniquePtr.hpp"
32 #include "deFilePath.hpp"
33 #include "vktTestCaseUtil.hpp"
34 #include "tcuTestLog.hpp"
35 #include "vktAmberTestCase.hpp"
36 #include "vktAmberHelper.hpp"
37 #include "tcuResource.hpp"
38 #include "tcuTestLog.hpp"
39 #include "vkSpirVProgram.hpp"
40 #include "vkImageUtil.hpp"
41
42 namespace vkt
43 {
44 namespace cts_amber
45 {
46
AmberTestCase(tcu::TestContext & testCtx,const char * name,const char * description,const std::string & readFilename)47 AmberTestCase::AmberTestCase(tcu::TestContext &testCtx, const char *name, const char *description,
48 const std::string &readFilename)
49 : TestCase(testCtx, name)
50 , m_recipe(DE_NULL)
51 , m_readFilename(readFilename)
52 {
53 (void)description;
54 }
55
~AmberTestCase(void)56 AmberTestCase::~AmberTestCase(void)
57 {
58 delete m_recipe;
59 }
60
createInstance(Context & ctx) const61 TestInstance *AmberTestCase::createInstance(Context &ctx) const
62 {
63 return new AmberTestInstance(ctx, m_recipe, nullptr);
64 }
65
createEngineConfig(Context & ctx,vk::VkDevice customDevice)66 static amber::EngineConfig *createEngineConfig(Context &ctx, vk::VkDevice customDevice)
67 {
68 vk::VkDevice dev = customDevice != nullptr ? customDevice : ctx.getDevice();
69 vk::VkQueue queue;
70 vk::DeviceDriver vk(ctx.getPlatformInterface(), ctx.getInstance(), dev, ctx.getUsedApiVersion(),
71 ctx.getTestContext().getCommandLine());
72 vk.getDeviceQueue(dev, ctx.getUniversalQueueFamilyIndex(), 0, &queue);
73
74 amber::EngineConfig *vkConfig =
75 GetVulkanConfig(ctx.getInstance(), ctx.getPhysicalDevice(), dev, &ctx.getDeviceFeatures(),
76 &ctx.getDeviceFeatures2(), ctx.getInstanceExtensions(), ctx.getDeviceExtensions(),
77 ctx.getUniversalQueueFamilyIndex(), queue, ctx.getInstanceProcAddr());
78
79 return vkConfig;
80 }
81
82 // Returns true if the given feature is supported by the device.
83 // Throws an internal error If the feature is not recognized at all.
isFeatureSupported(const vkt::Context & ctx,const std::string & feature)84 static bool isFeatureSupported(const vkt::Context &ctx, const std::string &feature)
85 {
86 if (feature == "Storage16BitFeatures.storageBuffer16BitAccess")
87 return ctx.get16BitStorageFeatures().storageBuffer16BitAccess;
88 if (feature == "Float16Int8Features.shaderFloat16")
89 return ctx.getShaderFloat16Int8Features().shaderFloat16;
90 if (feature == "Float16Int8Features.shaderInt8")
91 return ctx.getShaderFloat16Int8Features().shaderInt8;
92 if (feature == "Features.shaderFloat64")
93 return ctx.getDeviceFeatures().shaderFloat64;
94 if (feature == "Features.shaderInt16")
95 return ctx.getDeviceFeatures().shaderInt16;
96 if (feature == "Features.shaderInt64")
97 return ctx.getDeviceFeatures().shaderInt64;
98 if (feature == "Features.depthClamp")
99 return ctx.getDeviceFeatures().depthClamp;
100 if (feature == "Features.tessellationShader")
101 return ctx.getDeviceFeatures().tessellationShader;
102 if (feature == "Features.shaderTessellationAndGeometryPointSize")
103 return ctx.getDeviceFeatures().shaderTessellationAndGeometryPointSize;
104 if (feature == "Features.geometryShader")
105 return ctx.getDeviceFeatures().geometryShader;
106 if (feature == "Features.fragmentStoresAndAtomics")
107 return ctx.getDeviceFeatures().fragmentStoresAndAtomics;
108 if (feature == "Features.vertexPipelineStoresAndAtomics")
109 return ctx.getDeviceFeatures().vertexPipelineStoresAndAtomics;
110 if (feature == "Features.fillModeNonSolid")
111 return ctx.getDeviceFeatures().fillModeNonSolid;
112 if (feature == "Features.shaderStorageImageMultisample")
113 return ctx.getDeviceFeatures().shaderStorageImageMultisample;
114 if (feature == "Features.sampleRateShading")
115 return ctx.getDeviceFeatures().sampleRateShading;
116 if (feature == "VariablePointerFeatures.variablePointersStorageBuffer")
117 return ctx.getVariablePointersFeatures().variablePointersStorageBuffer;
118 if (feature == "VariablePointerFeatures.variablePointers")
119 return ctx.getVariablePointersFeatures().variablePointers;
120 if (feature == "SubgroupSupportedStages.fragment")
121 return (ctx.getSubgroupProperties().supportedStages & vk::VK_SHADER_STAGE_FRAGMENT_BIT) != 0;
122 if (feature == "SubgroupSupportedOperations.vote")
123 return (ctx.getSubgroupProperties().supportedOperations & vk::VK_SUBGROUP_FEATURE_VOTE_BIT) != 0;
124 if (feature == "SubgroupSupportedOperations.basic")
125 return (ctx.getSubgroupProperties().supportedOperations & vk::VK_SUBGROUP_FEATURE_BASIC_BIT) != 0;
126 if (feature == "SubgroupSupportedOperations.ballot")
127 return (ctx.getSubgroupProperties().supportedOperations & vk::VK_SUBGROUP_FEATURE_BALLOT_BIT) != 0;
128 if (feature == "Storage16BitFeatures.storageBuffer16BitAccess")
129 return ctx.get16BitStorageFeatures().storageBuffer16BitAccess;
130 if (feature == "Storage8BitFeatures.storageBuffer8BitAccess")
131 return ctx.get8BitStorageFeatures().storageBuffer8BitAccess;
132 if (feature == "IndexTypeUint8Features.indexTypeUint8")
133 return ctx.getIndexTypeUint8Features().indexTypeUint8;
134
135 std::string message = std::string("Unexpected feature name: ") + feature;
136 TCU_THROW(InternalError, message.c_str());
137 }
138
delayedInit(void)139 void AmberTestCase::delayedInit(void)
140 {
141 // Make sure the input can be parsed before we use it.
142 if (!parse(m_readFilename))
143 {
144 std::string message = "Failed to parse Amber file: " + m_readFilename;
145 TCU_THROW(InternalError, message.c_str());
146 }
147 }
148
checkSupport(Context & ctx) const149 void AmberTestCase::checkSupport(Context &ctx) const
150 {
151 // Check for instance and device extensions as declared by the test code.
152 if (m_required_extensions.size())
153 {
154 std::set<std::string> device_extensions(ctx.getDeviceExtensions().begin(), ctx.getDeviceExtensions().end());
155 std::set<std::string> instance_extensions(ctx.getInstanceExtensions().begin(),
156 ctx.getInstanceExtensions().end());
157 std::string missing;
158 for (std::set<std::string>::iterator iter = m_required_extensions.begin(); iter != m_required_extensions.end();
159 ++iter)
160 {
161 const std::string extension = *iter;
162 if ((device_extensions.count(extension) == 0) && (instance_extensions.count(extension) == 0))
163 {
164 missing += " " + extension;
165 }
166 }
167 if (missing.size() > 0)
168 {
169 std::string message("Test requires unsupported extensions:");
170 message += missing;
171 TCU_THROW(NotSupportedError, message.c_str());
172 }
173 }
174
175 // Check for required features. Do this after extensions are checked because
176 // some feature checks are only valid when corresponding extensions are enabled.
177 if (m_required_features.size())
178 {
179 std::string missing;
180 for (std::set<std::string>::iterator iter = m_required_features.begin(); iter != m_required_features.end();
181 ++iter)
182 {
183 const std::string feature = *iter;
184 if (!isFeatureSupported(ctx, feature))
185 {
186 missing += " " + feature;
187 }
188 }
189 if (missing.size() > 0)
190 {
191 std::string message("Test requires unsupported features:");
192 message += missing;
193 TCU_THROW(NotSupportedError, message.c_str());
194 }
195 }
196
197 for (auto req : m_imageRequirements)
198 checkImageSupport(ctx.getInstanceInterface(), ctx.getPhysicalDevice(), req);
199
200 for (auto req : m_bufferRequirements)
201 {
202 vk::VkFormatProperties prop;
203 ctx.getInstanceInterface().getPhysicalDeviceFormatProperties(ctx.getPhysicalDevice(), req.m_format, &prop);
204
205 if ((req.m_featureFlags & prop.bufferFeatures) != req.m_featureFlags)
206 {
207 TCU_THROW(NotSupportedError, "Buffer format doesn't support required feature flags");
208 }
209 }
210
211 if (m_checkSupportCallback)
212 (m_checkSupportCallback)(ctx, m_name);
213 }
214
215 class Delegate : public amber::Delegate
216 {
217 public:
218 Delegate(tcu::TestContext &testCtx);
219
220 amber::Result LoadBufferData(const std::string file_name, amber::BufferDataFileType file_type,
221 amber::BufferInfo *buffer) const override;
222
Log(const std::string &)223 void Log(const std::string & /*message*/) override
224 {
225 DE_FATAL("amber::Delegate::Log unimplemented");
226 }
LogGraphicsCalls(void) const227 bool LogGraphicsCalls(void) const override
228 {
229 return m_logGraphicsCalls;
230 }
SetLogGraphicsCalls(bool log_graphics_calls)231 void SetLogGraphicsCalls(bool log_graphics_calls)
232 {
233 m_logGraphicsCalls = log_graphics_calls;
234 }
LogExecuteCalls(void) const235 bool LogExecuteCalls(void) const override
236 {
237 return m_logExecuteCalls;
238 }
SetLogExecuteCalls(bool log_execute_calls)239 void SetLogExecuteCalls(bool log_execute_calls)
240 {
241 m_logExecuteCalls = log_execute_calls;
242 }
LogGraphicsCallsTime(void) const243 bool LogGraphicsCallsTime(void) const override
244 {
245 return m_logGraphicsCallsTime;
246 }
SetLogGraphicsCallsTime(bool log_graphics_calls_time)247 void SetLogGraphicsCallsTime(bool log_graphics_calls_time)
248 {
249 m_logGraphicsCallsTime = log_graphics_calls_time;
250 }
GetTimestampNs(void) const251 uint64_t GetTimestampNs(void) const override
252 {
253 DE_FATAL("amber::Delegate::GetTimestampNs unimplemented");
254 return 0;
255 }
SetScriptPath(std::string path)256 void SetScriptPath(std::string path)
257 {
258 m_path = path;
259 }
260
261 private:
262 tcu::TestContext &m_testCtx;
263 std::string m_path;
264 bool m_logGraphicsCalls;
265 bool m_logGraphicsCallsTime;
266 bool m_logExecuteCalls;
267 };
268
Delegate(tcu::TestContext & testCtx)269 Delegate::Delegate(tcu::TestContext &testCtx)
270 : m_testCtx(testCtx)
271 , m_path("")
272 , m_logGraphicsCalls(false)
273 , m_logGraphicsCallsTime(false)
274 , m_logExecuteCalls(false)
275 {
276 }
277
LoadBufferData(const std::string file_name,amber::BufferDataFileType file_type,amber::BufferInfo * buffer) const278 amber::Result Delegate::LoadBufferData(const std::string file_name, amber::BufferDataFileType file_type,
279 amber::BufferInfo *buffer) const
280 {
281 const tcu::Archive &archive = m_testCtx.getArchive();
282 const de::FilePath filePath = de::FilePath(m_path).join(file_name);
283 de::UniquePtr<tcu::Resource> file(archive.getResource(filePath.getPath()));
284 int numBytes = file->getSize();
285 std::vector<uint8_t> bytes(numBytes);
286
287 if (file_type == amber::BufferDataFileType::kPng)
288 return amber::Result("Amber PNG loading unimplemented");
289
290 file->read(bytes.data(), numBytes);
291
292 if (bytes.empty())
293 return amber::Result("Failed to load buffer data " + file_name);
294
295 for (uint8_t byte : bytes)
296 {
297 amber::Value value;
298 value.SetIntValue(static_cast<uint64_t>(byte));
299 buffer->values.push_back(value);
300 }
301
302 buffer->width = 1;
303 buffer->height = 1;
304
305 return {};
306 }
307
parse(const std::string & readFilename)308 bool AmberTestCase::parse(const std::string &readFilename)
309 {
310 std::string script = ShaderSourceProvider::getSource(m_testCtx.getArchive(), readFilename.c_str());
311 if (script.empty())
312 return false;
313
314 Delegate delegate(m_testCtx);
315 delegate.SetScriptPath(de::FilePath(readFilename).getDirName());
316
317 m_recipe = new amber::Recipe();
318
319 amber::Amber am(&delegate);
320 amber::Result r = am.Parse(script, m_recipe);
321
322 m_recipe->SetFenceTimeout(~0u); // infinity of miliseconds
323
324 if (!r.IsSuccess())
325 {
326 getTestContext().getLog() << tcu::TestLog::Message << "Failed to parse Amber test " << readFilename << ": "
327 << r.Error() << "\n"
328 << tcu::TestLog::EndMessage;
329 // TODO(dneto): Enhance Amber to not require this.
330 m_recipe->SetImpl(DE_NULL);
331 return false;
332 }
333 return true;
334 }
335
initPrograms(vk::SourceCollections & programCollection) const336 void AmberTestCase::initPrograms(vk::SourceCollections &programCollection) const
337 {
338 std::vector<amber::ShaderInfo> shaders = m_recipe->GetShaderInfo();
339 for (size_t i = 0; i < shaders.size(); ++i)
340 {
341 const amber::ShaderInfo &shader = shaders[i];
342
343 vk::SpirvVersion spirvVersion = vk::SPIRV_VERSION_1_0;
344 DE_STATIC_ASSERT(vk::SPIRV_VERSION_LAST == vk::SPIRV_VERSION_1_6 + 1);
345 if (shader.target_env == "spv1.6")
346 spirvVersion = vk::SPIRV_VERSION_1_6;
347 else if (shader.target_env == "spv1.5")
348 spirvVersion = vk::SPIRV_VERSION_1_5;
349 else if (shader.target_env == "spv1.4")
350 spirvVersion = vk::SPIRV_VERSION_1_4;
351 else if (shader.target_env == "spv1.3")
352 spirvVersion = vk::SPIRV_VERSION_1_3;
353 else if (shader.target_env == "spv1.2")
354 spirvVersion = vk::SPIRV_VERSION_1_2;
355 else if (shader.target_env == "spv1.1")
356 spirvVersion = vk::SPIRV_VERSION_1_1;
357
358 /* Hex encoded shaders do not need to be pre-compiled */
359 if (shader.format == amber::kShaderFormatSpirvHex)
360 continue;
361
362 if (shader.format == amber::kShaderFormatSpirvAsm)
363 {
364 programCollection.spirvAsmSources.add(shader.shader_name) << shader.shader_source << m_asm_options;
365 }
366 else if (shader.format == amber::kShaderFormatGlsl)
367 {
368 bool allowSpirv14 = (spirvVersion == vk::SPIRV_VERSION_1_4);
369
370 switch (shader.type)
371 {
372 case amber::kShaderTypeCompute:
373 programCollection.glslSources.add(shader.shader_name)
374 << glu::ComputeSource(shader.shader_source)
375 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
376 break;
377 case amber::kShaderTypeGeometry:
378 programCollection.glslSources.add(shader.shader_name)
379 << glu::GeometrySource(shader.shader_source)
380 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
381 break;
382 case amber::kShaderTypeFragment:
383 programCollection.glslSources.add(shader.shader_name)
384 << glu::FragmentSource(shader.shader_source)
385 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
386 break;
387 case amber::kShaderTypeVertex:
388 programCollection.glslSources.add(shader.shader_name)
389 << glu::VertexSource(shader.shader_source)
390 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
391 break;
392 case amber::kShaderTypeTessellationControl:
393 programCollection.glslSources.add(shader.shader_name)
394 << glu::TessellationControlSource(shader.shader_source)
395 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
396 break;
397 case amber::kShaderTypeTessellationEvaluation:
398 programCollection.glslSources.add(shader.shader_name)
399 << glu::TessellationEvaluationSource(shader.shader_source)
400 << vk::ShaderBuildOptions(programCollection.usedVulkanVersion, spirvVersion, 0u, allowSpirv14);
401 break;
402 case amber::kShaderTypeMulti:
403 DE_ASSERT(false && "Multi shaders not supported");
404 break;
405 }
406 }
407 else
408 {
409 DE_ASSERT(false && "Shader format not supported");
410 }
411 }
412 }
413
iterate(void)414 tcu::TestStatus AmberTestInstance::iterate(void)
415 {
416 amber::Amber am(DE_NULL);
417 amber::Options amber_options;
418 amber::ShaderMap shaderMap;
419 amber::Result r;
420
421 amber_options.engine = amber::kEngineTypeVulkan;
422 amber_options.config = createEngineConfig(m_context, m_customDevice);
423 amber_options.execution_type = amber::ExecutionType::kExecute;
424
425 // Amber should not execute any graphic related shaders when using --deqp-compute-only=enable flag
426 if (m_context.getTestContext().getCommandLine().isComputeOnly())
427 {
428 std::vector<amber::ShaderInfo> shaders_info = m_recipe->GetShaderInfo();
429
430 for (amber::ShaderInfo info : shaders_info)
431 {
432 if (info.type != amber::ShaderType::kShaderTypeCompute)
433 {
434 TCU_THROW(NotSupportedError, "Non compute shaders are not allow when using --deqp-compute-only=enable");
435 }
436 }
437 }
438
439 // Check for extensions as declared by the Amber script itself. Throw an internal
440 // error if that's more demanding.
441 r = am.AreAllRequirementsSupported(m_recipe, &amber_options);
442 if (!r.IsSuccess())
443 {
444 // dEQP does not to rely on external code to determine whether
445 // a test is supported. So throw an internal error here instead
446 // of a NotSupportedError. If an Amber test is not supported, then
447 // you must override this method and throw a NotSupported exception
448 // before reach here.
449 TCU_THROW(InternalError, r.Error().c_str());
450 }
451
452 std::vector<amber::ShaderInfo> shaders = m_recipe->GetShaderInfo();
453 for (size_t i = 0; i < shaders.size(); ++i)
454 {
455 const amber::ShaderInfo &shader = shaders[i];
456
457 if (!m_context.getBinaryCollection().contains(shader.shader_name))
458 continue;
459
460 size_t len = m_context.getBinaryCollection().get(shader.shader_name).getSize();
461 /* This is a compiled spir-v binary which must be made of 4-byte words. We
462 * are moving into a word sized vector so divide by 4
463 */
464 std::vector<uint32_t> data;
465 data.resize(len >> 2);
466 deMemcpy(data.data(), m_context.getBinaryCollection().get(shader.shader_name).getBinary(), len);
467
468 shaderMap[shader.shader_name] = data;
469 }
470
471 r = am.ExecuteWithShaderData(m_recipe, &amber_options, shaderMap);
472 if (!r.IsSuccess())
473 {
474 m_context.getTestContext().getLog() << tcu::TestLog::Message << r.Error() << "\n" << tcu::TestLog::EndMessage;
475 }
476
477 delete amber_options.config;
478
479 return r.IsSuccess() ? tcu::TestStatus::pass("Pass") : tcu::TestStatus::fail("Fail");
480 }
481
setSpirVAsmBuildOptions(const vk::SpirVAsmBuildOptions & asm_options)482 void AmberTestCase::setSpirVAsmBuildOptions(const vk::SpirVAsmBuildOptions &asm_options)
483 {
484 m_asm_options = asm_options;
485 }
486
addRequirement(const std::string & requirement)487 void AmberTestCase::addRequirement(const std::string &requirement)
488 {
489 if (requirement.find(".") != std::string::npos)
490 m_required_features.insert(requirement);
491 else
492 m_required_extensions.insert(requirement);
493 }
494
addImageRequirement(vk::VkImageCreateInfo info)495 void AmberTestCase::addImageRequirement(vk::VkImageCreateInfo info)
496 {
497 m_imageRequirements.push_back(info);
498 }
499
addBufferRequirement(BufferRequirement req)500 void AmberTestCase::addBufferRequirement(BufferRequirement req)
501 {
502 m_bufferRequirements.push_back(req);
503 }
504
validateRequirements()505 bool AmberTestCase::validateRequirements()
506 {
507 if (!parse(m_readFilename))
508 {
509 std::string message = "Failed to parse Amber file: " + m_readFilename;
510 m_testCtx.getLog() << tcu::TestLog::Message << message << tcu::TestLog::EndMessage;
511 return false;
512 }
513
514 // Check if the list of required CTS features and extensions matches the
515 // one in the recipe. Throw InternalError if they do not match.
516
517 const auto &deviceExtensions = m_recipe->GetRequiredInstanceExtensions();
518 const auto &instanceExtensions = m_recipe->GetRequiredDeviceExtensions();
519 auto requiredFeatures = m_recipe->GetRequiredFeatures();
520
521 for (auto &req : requiredFeatures)
522 {
523 if (req.find(".") == std::string::npos)
524 req = "Features." + req;
525 }
526
527 std::set<std::string> allRequirements;
528 allRequirements.insert(begin(deviceExtensions), end(deviceExtensions));
529 allRequirements.insert(begin(instanceExtensions), end(instanceExtensions));
530 allRequirements.insert(begin(requiredFeatures), end(requiredFeatures));
531
532 std::set<std::string> ctsRequirements = m_required_features;
533 ctsRequirements.insert(begin(m_required_extensions), end(m_required_extensions));
534
535 if (allRequirements != ctsRequirements)
536 {
537 auto &log = m_testCtx.getLog();
538 log << tcu::TestLog::Message << "ERROR: CTS and Amber test requirement mismatch." << tcu::TestLog::EndMessage;
539 log << tcu::TestLog::Message << "Amber filename: " << m_readFilename << tcu::TestLog::EndMessage;
540 log << tcu::TestLog::Message << "CTS requirements:" << tcu::TestLog::EndMessage;
541 for (const auto &ctsReq : ctsRequirements)
542 log << tcu::TestLog::Message << " " << ctsReq << tcu::TestLog::EndMessage;
543
544 log << tcu::TestLog::Message << "Amber requirements:" << tcu::TestLog::EndMessage;
545 for (const auto &amberReq : allRequirements)
546 log << tcu::TestLog::Message << " " << amberReq << tcu::TestLog::EndMessage;
547
548 // Repeat message for cerr so it's visible in console log.
549 std::cerr << "ERROR: CTS and Amber test requirement mismatch.\n";
550 std::cerr << "Amber filename: " << m_readFilename << "\n";
551 std::cerr << "CTS requirements:\n";
552 for (const auto &ctsReq : ctsRequirements)
553 std::cerr << " " << ctsReq << "\n";
554
555 std::cerr << "Amber requirements:\n";
556 for (const auto &amberReq : allRequirements)
557 std::cerr << " " << amberReq << "\n";
558
559 return false;
560 }
561 return true;
562 }
563
564 } // namespace cts_amber
565 } // namespace vkt
566