1 /*------------------------------------------------------------------------
2  * Vulkan Conformance Tests
3  * ------------------------
4  *
5  * Copyright (c) 2016 The Khronos Group Inc.
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  *
19  *//*!
20  * \file
21  * \brief Synchronization internally synchronized objects tests
22  *//*--------------------------------------------------------------------*/
23 
24 #include "vktSynchronizationInternallySynchronizedObjectsTests.hpp"
25 #include "vktTestCaseUtil.hpp"
26 #include "vktSynchronizationUtil.hpp"
27 #include "vktCustomInstancesDevices.hpp"
28 
29 #include "vkRef.hpp"
30 #include "tcuDefs.hpp"
31 #include "vkTypeUtil.hpp"
32 #include "vkBarrierUtil.hpp"
33 #include "vkPlatform.hpp"
34 #include "vkBuilderUtil.hpp"
35 #include "vkImageUtil.hpp"
36 #include "vkCmdUtil.hpp"
37 #include "vkObjUtil.hpp"
38 #include "vkDeviceUtil.hpp"
39 #include "vkSafetyCriticalUtil.hpp"
40 
41 #include "tcuResultCollector.hpp"
42 #include "tcuCommandLine.hpp"
43 
44 #include "deThread.hpp"
45 #include "deMutex.hpp"
46 #include "deSharedPtr.hpp"
47 #include "deSpinBarrier.hpp"
48 
49 #include <limits>
50 #include <iterator>
51 
52 namespace vkt
53 {
54 namespace synchronization
55 {
56 namespace
57 {
58 using namespace vk;
59 
60 using std::exception;
61 using std::map;
62 using std::ostringstream;
63 using std::string;
64 using std::vector;
65 
66 using tcu::ResultCollector;
67 using tcu::TestContext;
68 using tcu::TestException;
69 using tcu::TestStatus;
70 
71 using de::clamp;
72 using de::MovePtr;
73 using de::Mutex;
74 using de::SharedPtr;
75 using de::Thread;
76 using de::UniquePtr;
77 
78 template <typename T>
makeVkSharedPtr(Move<T> move)79 inline SharedPtr<Move<T>> makeVkSharedPtr(Move<T> move)
80 {
81     return SharedPtr<Move<T>>(new Move<T>(move));
82 }
83 
84 #ifndef CTS_USES_VULKANSC
85 enum
86 {
87     EXECUTION_PER_THREAD = 100,
88     BUFFER_ELEMENT_COUNT = 16,
89     BUFFER_SIZE          = BUFFER_ELEMENT_COUNT * 4
90 };
91 #else
92 enum
93 {
94     EXECUTION_PER_THREAD = 10,
95     BUFFER_ELEMENT_COUNT = 16,
96     BUFFER_SIZE          = BUFFER_ELEMENT_COUNT * 4
97 };
98 #endif // CTS_USES_VULKANSC
99 
100 class MultiQueues
101 {
102     typedef struct QueueType
103     {
104         vector<VkQueue> queues;
105         vector<bool> available;
106         vector<SharedPtr<Move<VkCommandPool>>> commandPools;
107     } Queues;
108 
109 public:
addQueueFamilyIndex(const uint32_t & queueFamilyIndex,const uint32_t & count)110     inline void addQueueFamilyIndex(const uint32_t &queueFamilyIndex, const uint32_t &count)
111     {
112         Queues temp;
113         vector<bool>::iterator it;
114         it = temp.available.begin();
115         temp.available.insert(it, count, false);
116 
117         temp.queues.resize(count);
118 
119         m_queues[queueFamilyIndex] = temp;
120     }
121 
getQueueFamilyIndex(const int index) const122     uint32_t getQueueFamilyIndex(const int index) const
123     {
124         map<uint32_t, Queues>::const_iterator it = begin(m_queues);
125         std::advance(it, index);
126         return it->first;
127     }
128 
countQueueFamilyIndex(void)129     inline size_t countQueueFamilyIndex(void)
130     {
131         return m_queues.size();
132     }
133 
getQueues(int index)134     Queues &getQueues(int index)
135     {
136         map<uint32_t, Queues>::iterator it = m_queues.begin();
137         advance(it, index);
138         return it->second;
139     }
140 
getFreeQueue(const DeviceInterface & vk,const VkDevice device,uint32_t & returnQueueFamilyIndex,VkQueue & returnQueues,Move<VkCommandBuffer> & commandBuffer,int & returnQueueIndex)141     bool getFreeQueue(const DeviceInterface &vk, const VkDevice device, uint32_t &returnQueueFamilyIndex,
142                       VkQueue &returnQueues, Move<VkCommandBuffer> &commandBuffer, int &returnQueueIndex)
143     {
144         for (int queueFamilyIndexNdx = 0; queueFamilyIndexNdx < static_cast<int>(m_queues.size());
145              ++queueFamilyIndexNdx)
146         {
147             Queues &queue = m_queues[getQueueFamilyIndex(queueFamilyIndexNdx)];
148             for (int queueNdx = 0; queueNdx < static_cast<int>(queue.queues.size()); ++queueNdx)
149             {
150                 m_mutex.lock();
151                 if (queue.available[queueNdx])
152                 {
153                     queue.available[queueNdx] = false;
154                     returnQueueFamilyIndex    = getQueueFamilyIndex(queueFamilyIndexNdx);
155                     returnQueues              = queue.queues[queueNdx];
156                     commandBuffer             = makeCommandBuffer(vk, device, queue.commandPools[queueNdx]->get());
157                     returnQueueIndex          = queueNdx;
158                     m_mutex.unlock();
159                     return true;
160                 }
161                 m_mutex.unlock();
162             }
163         }
164         return false;
165     }
166 
releaseQueue(const uint32_t & queueFamilyIndex,const int & queueIndex,Move<VkCommandBuffer> & commandBuffer)167     void releaseQueue(const uint32_t &queueFamilyIndex, const int &queueIndex, Move<VkCommandBuffer> &commandBuffer)
168     {
169         m_mutex.lock();
170         commandBuffer                                    = Move<VkCommandBuffer>();
171         m_queues[queueFamilyIndex].available[queueIndex] = true;
172         m_mutex.unlock();
173     }
174 
setDevice(Move<VkDevice> device,const Context & context)175     inline void setDevice(Move<VkDevice> device, const Context &context)
176     {
177         m_logicalDevice = device;
178 #ifndef CTS_USES_VULKANSC
179         m_deviceDriver = de::MovePtr<DeviceDriver>(
180             new DeviceDriver(context.getPlatformInterface(), context.getInstance(), *m_logicalDevice,
181                              context.getUsedApiVersion(), context.getTestContext().getCommandLine()));
182 #else
183         m_deviceDriver = de::MovePtr<DeviceDriverSC, DeinitDeviceDeleter>(
184             new DeviceDriverSC(context.getPlatformInterface(), context.getInstance(), *m_logicalDevice,
185                                context.getTestContext().getCommandLine(), context.getResourceInterface(),
186                                context.getDeviceVulkanSC10Properties(), context.getDeviceProperties(),
187                                context.getUsedApiVersion()),
188             vk::DeinitDeviceDeleter(context.getResourceInterface().get(), *m_logicalDevice));
189 #endif // CTS_USES_VULKANSC
190     }
191 
getDevice(void)192     inline VkDevice getDevice(void)
193     {
194         return *m_logicalDevice;
195     }
196 
getDeviceInterface(void)197     inline DeviceInterface &getDeviceInterface(void)
198     {
199         return *m_deviceDriver;
200     }
201 
202     MovePtr<Allocator> m_allocator;
203 
204 protected:
205     Move<VkDevice> m_logicalDevice;
206 #ifndef CTS_USES_VULKANSC
207     de::MovePtr<vk::DeviceDriver> m_deviceDriver;
208 #else
209     de::MovePtr<DeviceDriverSC, DeinitDeviceDeleter> m_deviceDriver;
210 #endif // CTS_USES_VULKANSC
211     map<uint32_t, Queues> m_queues;
212     Mutex m_mutex;
213 };
214 
createAllocator(const Context & context,const VkDevice & device)215 MovePtr<Allocator> createAllocator(const Context &context, const VkDevice &device)
216 {
217     const DeviceInterface &deviceInterface = context.getDeviceInterface();
218     const InstanceInterface &instance      = context.getInstanceInterface();
219     const VkPhysicalDevice physicalDevice  = context.getPhysicalDevice();
220     const VkPhysicalDeviceMemoryProperties deviceMemoryProperties =
221         getPhysicalDeviceMemoryProperties(instance, physicalDevice);
222 
223     // Create memory allocator for device
224     return MovePtr<Allocator>(new SimpleAllocator(deviceInterface, device, deviceMemoryProperties));
225 }
226 
checkQueueFlags(const VkQueueFlags & availableFlag,const VkQueueFlags & neededFlag)227 bool checkQueueFlags(const VkQueueFlags &availableFlag, const VkQueueFlags &neededFlag)
228 {
229     if (VK_QUEUE_TRANSFER_BIT == neededFlag)
230     {
231         if ((availableFlag & VK_QUEUE_GRAPHICS_BIT) == VK_QUEUE_GRAPHICS_BIT ||
232             (availableFlag & VK_QUEUE_COMPUTE_BIT) == VK_QUEUE_COMPUTE_BIT ||
233             (availableFlag & VK_QUEUE_TRANSFER_BIT) == VK_QUEUE_TRANSFER_BIT)
234             return true;
235     }
236     else if ((availableFlag & neededFlag) == neededFlag)
237     {
238         return true;
239     }
240     return false;
241 }
242 
createQueues(Context & context,const VkQueueFlags & queueFlag,const VkInstance & instance,const InstanceInterface & vki)243 MovePtr<MultiQueues> createQueues(Context &context, const VkQueueFlags &queueFlag, const VkInstance &instance,
244                                   const InstanceInterface &vki)
245 {
246     const VkPhysicalDevice physicalDevice = chooseDevice(vki, instance, context.getTestContext().getCommandLine());
247     MovePtr<MultiQueues> moveQueues(new MultiQueues());
248     MultiQueues &queues = *moveQueues;
249     VkDeviceCreateInfo deviceInfo;
250     VkPhysicalDeviceFeatures deviceFeatures;
251     vector<VkQueueFamilyProperties> queueFamilyProperties;
252     vector<float> queuePriorities;
253     vector<VkDeviceQueueCreateInfo> queueInfos;
254 
255     queueFamilyProperties = getPhysicalDeviceQueueFamilyProperties(vki, physicalDevice);
256 
257     for (uint32_t queuePropertiesNdx = 0; queuePropertiesNdx < queueFamilyProperties.size(); ++queuePropertiesNdx)
258     {
259         if (checkQueueFlags(queueFamilyProperties[queuePropertiesNdx].queueFlags, queueFlag))
260         {
261             queues.addQueueFamilyIndex(queuePropertiesNdx, queueFamilyProperties[queuePropertiesNdx].queueCount);
262         }
263     }
264 
265     if (queues.countQueueFamilyIndex() == 0)
266     {
267         TCU_THROW(NotSupportedError, "Queue not found");
268     }
269 
270     {
271         vector<float>::iterator it = queuePriorities.begin();
272         unsigned int maxQueueCount = 0;
273         for (int queueFamilyIndexNdx = 0; queueFamilyIndexNdx < static_cast<int>(queues.countQueueFamilyIndex());
274              ++queueFamilyIndexNdx)
275         {
276             if (queues.getQueues(queueFamilyIndexNdx).queues.size() > maxQueueCount)
277                 maxQueueCount = static_cast<unsigned int>(queues.getQueues(queueFamilyIndexNdx).queues.size());
278         }
279         queuePriorities.insert(it, maxQueueCount, 1.0);
280     }
281 
282     for (int queueFamilyIndexNdx = 0; queueFamilyIndexNdx < static_cast<int>(queues.countQueueFamilyIndex());
283          ++queueFamilyIndexNdx)
284     {
285         VkDeviceQueueCreateInfo queueInfo;
286         const uint32_t queueCount = static_cast<uint32_t>(queues.getQueues(queueFamilyIndexNdx).queues.size());
287 
288         deMemset(&queueInfo, 0, sizeof(queueInfo));
289 
290         queueInfo.sType            = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
291         queueInfo.pNext            = DE_NULL;
292         queueInfo.flags            = (VkDeviceQueueCreateFlags)0u;
293         queueInfo.queueFamilyIndex = queues.getQueueFamilyIndex(queueFamilyIndexNdx);
294         queueInfo.queueCount       = queueCount;
295         queueInfo.pQueuePriorities = &queuePriorities[0];
296 
297         queueInfos.push_back(queueInfo);
298     }
299 
300     deMemset(&deviceInfo, 0, sizeof(deviceInfo));
301     vki.getPhysicalDeviceFeatures(physicalDevice, &deviceFeatures);
302 
303     void *pNext = DE_NULL;
304 #ifdef CTS_USES_VULKANSC
305     VkDeviceObjectReservationCreateInfo memReservationInfo = context.getTestContext().getCommandLine().isSubProcess() ?
306                                                                  context.getResourceInterface()->getStatMax() :
307                                                                  resetDeviceObjectReservationCreateInfo();
308     memReservationInfo.pNext                               = pNext;
309     pNext                                                  = &memReservationInfo;
310 
311     VkPhysicalDeviceVulkanSC10Features sc10Features = createDefaultSC10Features();
312     sc10Features.pNext                              = pNext;
313     pNext                                           = &sc10Features;
314 
315     VkPipelineCacheCreateInfo pcCI;
316     std::vector<VkPipelinePoolSize> poolSizes;
317     if (context.getTestContext().getCommandLine().isSubProcess())
318     {
319         if (context.getResourceInterface()->getCacheDataSize() > 0)
320         {
321             pcCI = {
322                 VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, // VkStructureType sType;
323                 DE_NULL,                                      // const void* pNext;
324                 VK_PIPELINE_CACHE_CREATE_READ_ONLY_BIT |
325                     VK_PIPELINE_CACHE_CREATE_USE_APPLICATION_STORAGE_BIT, // VkPipelineCacheCreateFlags flags;
326                 context.getResourceInterface()->getCacheDataSize(),       // uintptr_t initialDataSize;
327                 context.getResourceInterface()->getCacheData()            // const void* pInitialData;
328             };
329             memReservationInfo.pipelineCacheCreateInfoCount = 1;
330             memReservationInfo.pPipelineCacheCreateInfos    = &pcCI;
331         }
332 
333         poolSizes = context.getResourceInterface()->getPipelinePoolSizes();
334         if (!poolSizes.empty())
335         {
336             memReservationInfo.pipelinePoolSizeCount = uint32_t(poolSizes.size());
337             memReservationInfo.pPipelinePoolSizes    = poolSizes.data();
338         }
339     }
340 #endif // CTS_USES_VULKANSC
341 
342     deviceInfo.sType                   = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO;
343     deviceInfo.pNext                   = pNext;
344     deviceInfo.enabledExtensionCount   = 0u;
345     deviceInfo.ppEnabledExtensionNames = DE_NULL;
346     deviceInfo.enabledLayerCount       = 0u;
347     deviceInfo.ppEnabledLayerNames     = DE_NULL;
348     deviceInfo.pEnabledFeatures        = &deviceFeatures;
349     deviceInfo.queueCreateInfoCount    = static_cast<uint32_t>(queues.countQueueFamilyIndex());
350     deviceInfo.pQueueCreateInfos       = &queueInfos[0];
351 
352     queues.setDevice(createCustomDevice(context.getTestContext().getCommandLine().isValidationEnabled(),
353                                         context.getPlatformInterface(), instance, vki, physicalDevice, &deviceInfo),
354                      context);
355     vk::DeviceInterface &vk = queues.getDeviceInterface();
356 
357     for (uint32_t queueFamilyIndex = 0; queueFamilyIndex < queues.countQueueFamilyIndex(); ++queueFamilyIndex)
358     {
359         for (uint32_t queueReqNdx = 0; queueReqNdx < queues.getQueues(queueFamilyIndex).queues.size(); ++queueReqNdx)
360         {
361             vk.getDeviceQueue(queues.getDevice(), queues.getQueueFamilyIndex(queueFamilyIndex), queueReqNdx,
362                               &queues.getQueues(queueFamilyIndex).queues[queueReqNdx]);
363             queues.getQueues(queueFamilyIndex).available[queueReqNdx] = true;
364             queues.getQueues(queueFamilyIndex)
365                 .commandPools.push_back(makeVkSharedPtr(
366                     createCommandPool(vk, queues.getDevice(), VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
367                                       queues.getQueueFamilyIndex(queueFamilyIndex))));
368         }
369     }
370 
371     queues.m_allocator = createAllocator(context, queues.getDevice());
372     return moveQueues;
373 }
374 
executeComputePipeline(const Context & context,const VkPipeline & pipeline,const VkPipelineLayout & pipelineLayout,const VkDescriptorSetLayout & descriptorSetLayout,MultiQueues & queues,const uint32_t & shadersExecutions)375 TestStatus executeComputePipeline(const Context &context, const VkPipeline &pipeline,
376                                   const VkPipelineLayout &pipelineLayout,
377                                   const VkDescriptorSetLayout &descriptorSetLayout, MultiQueues &queues,
378                                   const uint32_t &shadersExecutions)
379 {
380     DE_UNREF(context);
381     const DeviceInterface &vk = queues.getDeviceInterface();
382     const VkDevice device     = queues.getDevice();
383     uint32_t queueFamilyIndex;
384     VkQueue queue;
385     int queueIndex;
386     Move<VkCommandBuffer> cmdBuffer;
387     while (!queues.getFreeQueue(vk, device, queueFamilyIndex, queue, cmdBuffer, queueIndex))
388     {
389     }
390 
391     {
392         const Unique<VkDescriptorPool> descriptorPool(
393             DescriptorPoolBuilder()
394                 .addType(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER)
395                 .build(vk, device, VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT, 1u));
396         Buffer resultBuffer(vk, device, *queues.m_allocator,
397                             makeBufferCreateInfo(BUFFER_SIZE, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT),
398                             MemoryRequirement::HostVisible);
399         const VkBufferMemoryBarrier bufferBarrier = makeBufferMemoryBarrier(
400             VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT, *resultBuffer, 0ull, BUFFER_SIZE);
401 
402         {
403             const Allocation &alloc = resultBuffer.getAllocation();
404             deMemset(alloc.getHostPtr(), 0, BUFFER_SIZE);
405             flushAlloc(vk, device, alloc);
406         }
407 
408         // Start recording commands
409         beginCommandBuffer(vk, *cmdBuffer);
410 
411         vk.cmdBindPipeline(*cmdBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
412 
413         // Create descriptor set
414         const Unique<VkDescriptorSet> descriptorSet(
415             makeDescriptorSet(vk, device, *descriptorPool, descriptorSetLayout));
416 
417         const VkDescriptorBufferInfo resultDescriptorInfo = makeDescriptorBufferInfo(*resultBuffer, 0ull, BUFFER_SIZE);
418 
419         DescriptorSetUpdateBuilder()
420             .writeSingle(*descriptorSet, DescriptorSetUpdateBuilder::Location::binding(0u),
421                          VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, &resultDescriptorInfo)
422             .update(vk, device);
423 
424         vk.cmdBindDescriptorSets(*cmdBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0u, 1u,
425                                  &descriptorSet.get(), 0u, DE_NULL);
426 
427         // Dispatch indirect compute command
428         vk.cmdDispatch(*cmdBuffer, shadersExecutions, 1u, 1u);
429 
430         vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT,
431                               (VkDependencyFlags)0, 0, (const VkMemoryBarrier *)DE_NULL, 1, &bufferBarrier, 0,
432                               (const VkImageMemoryBarrier *)DE_NULL);
433 
434         // End recording commands
435         endCommandBuffer(vk, *cmdBuffer);
436 
437         // Wait for command buffer execution finish
438         submitCommandsAndWait(vk, device, queue, *cmdBuffer);
439         queues.releaseQueue(queueFamilyIndex, queueIndex, cmdBuffer);
440 
441         {
442             const Allocation &resultAlloc = resultBuffer.getAllocation();
443             invalidateAlloc(vk, device, resultAlloc);
444 
445             const int32_t *ptr = reinterpret_cast<int32_t *>(resultAlloc.getHostPtr());
446             for (int32_t ndx = 0; ndx < BUFFER_ELEMENT_COUNT; ++ndx)
447             {
448                 if (ptr[ndx] != ndx)
449                 {
450                     return TestStatus::fail("The data don't match");
451                 }
452             }
453         }
454         return TestStatus::pass("Passed");
455     }
456 }
457 
executeGraphicPipeline(const Context & context,const VkPipeline & pipeline,const VkPipelineLayout & pipelineLayout,const VkDescriptorSetLayout & descriptorSetLayout,MultiQueues & queues,const VkRenderPass & renderPass,const uint32_t shadersExecutions)458 TestStatus executeGraphicPipeline(const Context &context, const VkPipeline &pipeline,
459                                   const VkPipelineLayout &pipelineLayout,
460                                   const VkDescriptorSetLayout &descriptorSetLayout, MultiQueues &queues,
461                                   const VkRenderPass &renderPass, const uint32_t shadersExecutions)
462 {
463     DE_UNREF(context);
464     const DeviceInterface &vk = queues.getDeviceInterface();
465     const VkDevice device     = queues.getDevice();
466     uint32_t queueFamilyIndex;
467     VkQueue queue;
468     int queueIndex;
469     Move<VkCommandBuffer> cmdBuffer;
470     while (!queues.getFreeQueue(vk, device, queueFamilyIndex, queue, cmdBuffer, queueIndex))
471     {
472     }
473 
474     {
475         const Unique<VkDescriptorPool> descriptorPool(
476             DescriptorPoolBuilder()
477                 .addType(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER)
478                 .build(vk, device, VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT, 1u));
479         Move<VkDescriptorSet> descriptorSet = makeDescriptorSet(vk, device, *descriptorPool, descriptorSetLayout);
480         Buffer resultBuffer(vk, device, *queues.m_allocator,
481                             makeBufferCreateInfo(BUFFER_SIZE, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT),
482                             MemoryRequirement::HostVisible);
483         const VkBufferMemoryBarrier bufferBarrier = makeBufferMemoryBarrier(
484             VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT, *resultBuffer, 0ull, BUFFER_SIZE);
485         const VkFormat colorFormat        = VK_FORMAT_R8G8B8A8_UNORM;
486         const VkExtent3D colorImageExtent = makeExtent3D(1u, 1u, 1u);
487         const VkImageSubresourceRange colorImageSubresourceRange =
488             makeImageSubresourceRange(VK_IMAGE_ASPECT_COLOR_BIT, 0u, 1u, 0u, 1u);
489         de::MovePtr<Image> colorAttachmentImage = de::MovePtr<Image>(new Image(
490             vk, device, *queues.m_allocator,
491             makeImageCreateInfo(VK_IMAGE_TYPE_2D, colorImageExtent, colorFormat, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT,
492                                 VK_SAMPLE_COUNT_1_BIT, VK_IMAGE_TILING_OPTIMAL),
493             MemoryRequirement::Any));
494         Move<VkImageView> colorAttachmentView = makeImageView(vk, device, **colorAttachmentImage, VK_IMAGE_VIEW_TYPE_2D,
495                                                               colorFormat, colorImageSubresourceRange);
496         Move<VkFramebuffer> framebuffer       = makeFramebuffer(vk, device, renderPass, *colorAttachmentView,
497                                                                 colorImageExtent.width, colorImageExtent.height);
498         const VkDescriptorBufferInfo outputBufferDescriptorInfo =
499             makeDescriptorBufferInfo(*resultBuffer, 0ull, BUFFER_SIZE);
500 
501         DescriptorSetUpdateBuilder()
502             .writeSingle(*descriptorSet, DescriptorSetUpdateBuilder::Location::binding(0u),
503                          VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, &outputBufferDescriptorInfo)
504             .update(vk, device);
505 
506         {
507             const Allocation &alloc = resultBuffer.getAllocation();
508             deMemset(alloc.getHostPtr(), 0, BUFFER_SIZE);
509             flushAlloc(vk, device, alloc);
510         }
511 
512         // Start recording commands
513         beginCommandBuffer(vk, *cmdBuffer);
514         // Change color attachment image layout
515         {
516             const VkImageMemoryBarrier colorAttachmentLayoutBarrier = makeImageMemoryBarrier(
517                 (VkAccessFlags)0, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_IMAGE_LAYOUT_UNDEFINED,
518                 VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, **colorAttachmentImage, colorImageSubresourceRange);
519 
520             vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
521                                   VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, (VkDependencyFlags)0, 0u, DE_NULL, 0u,
522                                   DE_NULL, 1u, &colorAttachmentLayoutBarrier);
523         }
524 
525         {
526             const VkRect2D renderArea  = makeRect2D(1u, 1u);
527             const tcu::Vec4 clearColor = tcu::Vec4(0.0f, 0.0f, 0.0f, 1.0f);
528             beginRenderPass(vk, *cmdBuffer, renderPass, *framebuffer, renderArea, clearColor);
529         }
530 
531         vk.cmdBindPipeline(*cmdBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
532         vk.cmdBindDescriptorSets(*cmdBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0u, 1u,
533                                  &descriptorSet.get(), 0u, DE_NULL);
534 
535         vk.cmdDraw(*cmdBuffer, shadersExecutions, 1u, 0u, 0u);
536         endRenderPass(vk, *cmdBuffer);
537 
538         vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_VERTEX_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT,
539                               (VkDependencyFlags)0, 0, (const VkMemoryBarrier *)DE_NULL, 1, &bufferBarrier, 0,
540                               (const VkImageMemoryBarrier *)DE_NULL);
541 
542         // End recording commands
543         endCommandBuffer(vk, *cmdBuffer);
544 
545         // Wait for command buffer execution finish
546         submitCommandsAndWait(vk, device, queue, *cmdBuffer);
547         queues.releaseQueue(queueFamilyIndex, queueIndex, cmdBuffer);
548 
549         {
550             const Allocation &resultAlloc = resultBuffer.getAllocation();
551             invalidateAlloc(vk, device, resultAlloc);
552 
553             const int32_t *ptr = reinterpret_cast<int32_t *>(resultAlloc.getHostPtr());
554             for (int32_t ndx = 0; ndx < BUFFER_ELEMENT_COUNT; ++ndx)
555             {
556                 if (ptr[ndx] != ndx)
557                 {
558                     return TestStatus::fail("The data don't match");
559                 }
560             }
561         }
562         return TestStatus::pass("Passed");
563     }
564 }
565 
566 class ThreadGroupThread : private Thread
567 {
568 public:
ThreadGroupThread(const Context & context,VkPipelineCache pipelineCache,const VkPipelineLayout & pipelineLayout,const VkDescriptorSetLayout & descriptorSetLayout,MultiQueues & queues,const vector<uint32_t> & shadersExecutions)569     ThreadGroupThread(const Context &context, VkPipelineCache pipelineCache, const VkPipelineLayout &pipelineLayout,
570                       const VkDescriptorSetLayout &descriptorSetLayout, MultiQueues &queues,
571                       const vector<uint32_t> &shadersExecutions)
572         : m_context(context)
573         , m_pipelineCache(pipelineCache)
574         , m_pipelineLayout(pipelineLayout)
575         , m_descriptorSetLayout(descriptorSetLayout)
576         , m_queues(queues)
577         , m_shadersExecutions(shadersExecutions)
578         , m_barrier(DE_NULL)
579     {
580     }
581 
~ThreadGroupThread(void)582     virtual ~ThreadGroupThread(void)
583     {
584     }
585 
getResultCollector(void)586     ResultCollector &getResultCollector(void)
587     {
588         return m_resultCollector;
589     }
590 
591     void start(de::SpinBarrier *groupBarrier);
592     using Thread::join;
593 
594 protected:
595     virtual TestStatus runThread() = 0;
596     const Context &m_context;
597     VkPipelineCache m_pipelineCache;
598     const VkPipelineLayout &m_pipelineLayout;
599     const VkDescriptorSetLayout &m_descriptorSetLayout;
600     MultiQueues &m_queues;
601     const vector<uint32_t> &m_shadersExecutions;
602 
603     void barrier(void);
604 
605 private:
606     ThreadGroupThread(const ThreadGroupThread &);
607     ThreadGroupThread &operator=(const ThreadGroupThread &);
608 
run(void)609     void run(void)
610     {
611         try
612         {
613             TestStatus result = runThread();
614             m_resultCollector.addResult(result.getCode(), result.getDescription());
615         }
616         catch (const TestException &e)
617         {
618             m_resultCollector.addResult(e.getTestResult(), e.getMessage());
619         }
620         catch (const exception &e)
621         {
622             m_resultCollector.addResult(QP_TEST_RESULT_FAIL, e.what());
623         }
624         catch (...)
625         {
626             m_resultCollector.addResult(QP_TEST_RESULT_FAIL, "Exception");
627         }
628 
629         m_barrier->removeThread(de::SpinBarrier::WAIT_MODE_AUTO);
630     }
631 
632     ResultCollector m_resultCollector;
633     de::SpinBarrier *m_barrier;
634 };
635 
start(de::SpinBarrier * groupBarrier)636 void ThreadGroupThread::start(de::SpinBarrier *groupBarrier)
637 {
638     m_barrier = groupBarrier;
639     de::Thread::start();
640 }
641 
barrier(void)642 inline void ThreadGroupThread::barrier(void)
643 {
644     m_barrier->sync(de::SpinBarrier::WAIT_MODE_AUTO);
645 }
646 
647 class ThreadGroup
648 {
649     typedef vector<SharedPtr<ThreadGroupThread>> ThreadVector;
650 
651 public:
ThreadGroup(void)652     ThreadGroup(void) : m_barrier(1)
653     {
654     }
~ThreadGroup(void)655     ~ThreadGroup(void)
656     {
657     }
658 
add(MovePtr<ThreadGroupThread> thread)659     void add(MovePtr<ThreadGroupThread> thread)
660     {
661         m_threads.push_back(SharedPtr<ThreadGroupThread>(thread.release()));
662     }
663 
run(void)664     TestStatus run(void)
665     {
666         ResultCollector resultCollector;
667 
668         m_barrier.reset((int)m_threads.size());
669 
670         for (ThreadVector::iterator threadIter = m_threads.begin(); threadIter != m_threads.end(); ++threadIter)
671             (*threadIter)->start(&m_barrier);
672 
673         for (ThreadVector::iterator threadIter = m_threads.begin(); threadIter != m_threads.end(); ++threadIter)
674         {
675             ResultCollector &threadResult = (*threadIter)->getResultCollector();
676             (*threadIter)->join();
677             resultCollector.addResult(threadResult.getResult(), threadResult.getMessage());
678         }
679 
680         return TestStatus(resultCollector.getResult(), resultCollector.getMessage());
681     }
682 
683 private:
684     ThreadVector m_threads;
685     de::SpinBarrier m_barrier;
686 };
687 
688 class CreateComputeThread : public ThreadGroupThread
689 {
690 public:
CreateComputeThread(const Context & context,VkPipelineCache pipelineCache,vector<VkComputePipelineCreateInfo> & pipelineInfo,const VkPipelineLayout & pipelineLayout,const VkDescriptorSetLayout & descriptorSetLayout,MultiQueues & queues,const vector<uint32_t> & shadersExecutions)691     CreateComputeThread(const Context &context, VkPipelineCache pipelineCache,
692                         vector<VkComputePipelineCreateInfo> &pipelineInfo, const VkPipelineLayout &pipelineLayout,
693                         const VkDescriptorSetLayout &descriptorSetLayout, MultiQueues &queues,
694                         const vector<uint32_t> &shadersExecutions)
695         : ThreadGroupThread(context, pipelineCache, pipelineLayout, descriptorSetLayout, queues, shadersExecutions)
696         , m_pipelineInfo(pipelineInfo)
697     {
698     }
699 
runThread(void)700     TestStatus runThread(void)
701     {
702         ResultCollector resultCollector;
703         for (int executionNdx = 0; executionNdx < EXECUTION_PER_THREAD; ++executionNdx)
704         {
705             const int shaderNdx       = executionNdx % (int)m_pipelineInfo.size();
706             const DeviceInterface &vk = m_context.getDeviceInterface();
707             const VkDevice device     = m_queues.getDevice();
708             Move<VkPipeline> pipeline = createComputePipeline(vk, device, m_pipelineCache, &m_pipelineInfo[shaderNdx]);
709 
710             TestStatus result = executeComputePipeline(m_context, *pipeline, m_pipelineLayout, m_descriptorSetLayout,
711                                                        m_queues, m_shadersExecutions[shaderNdx]);
712 
713 #ifdef CTS_USES_VULKANSC
714             // While collecting pipelines, synchronize between all threads for each pipeline that gets
715             // created, so we will reserve the maximum amount of pipeline pool space that could need.
716             if (!m_context.getTestContext().getCommandLine().isSubProcess())
717             {
718                 barrier();
719             }
720 #endif
721 
722             resultCollector.addResult(result.getCode(), result.getDescription());
723         }
724         return TestStatus(resultCollector.getResult(), resultCollector.getMessage());
725     }
726 
727 private:
728     vector<VkComputePipelineCreateInfo> &m_pipelineInfo;
729 };
730 
731 class CreateGraphicThread : public ThreadGroupThread
732 {
733 public:
CreateGraphicThread(const Context & context,VkPipelineCache pipelineCache,vector<VkGraphicsPipelineCreateInfo> & pipelineInfo,const VkPipelineLayout & pipelineLayout,const VkDescriptorSetLayout & descriptorSetLayout,MultiQueues & queues,const VkRenderPass & renderPass,const vector<uint32_t> & shadersExecutions)734     CreateGraphicThread(const Context &context, VkPipelineCache pipelineCache,
735                         vector<VkGraphicsPipelineCreateInfo> &pipelineInfo, const VkPipelineLayout &pipelineLayout,
736                         const VkDescriptorSetLayout &descriptorSetLayout, MultiQueues &queues,
737                         const VkRenderPass &renderPass, const vector<uint32_t> &shadersExecutions)
738         : ThreadGroupThread(context, pipelineCache, pipelineLayout, descriptorSetLayout, queues, shadersExecutions)
739         , m_pipelineInfo(pipelineInfo)
740         , m_renderPass(renderPass)
741     {
742     }
743 
runThread(void)744     TestStatus runThread(void)
745     {
746         ResultCollector resultCollector;
747         for (int executionNdx = 0; executionNdx < EXECUTION_PER_THREAD; ++executionNdx)
748         {
749             const int shaderNdx       = executionNdx % (int)m_pipelineInfo.size();
750             const DeviceInterface &vk = m_context.getDeviceInterface();
751             const VkDevice device     = m_queues.getDevice();
752             Move<VkPipeline> pipeline = createGraphicsPipeline(vk, device, m_pipelineCache, &m_pipelineInfo[shaderNdx]);
753 
754             TestStatus result = executeGraphicPipeline(m_context, *pipeline, m_pipelineLayout, m_descriptorSetLayout,
755                                                        m_queues, m_renderPass, m_shadersExecutions[shaderNdx]);
756 
757 #ifdef CTS_USES_VULKANSC
758             // While collecting pipelines, synchronize between all threads for each pipeline that gets
759             // created, so we will reserve the maximum amount of pipeline pool space that could need.
760             if (!m_context.getTestContext().getCommandLine().isSubProcess())
761             {
762                 barrier();
763             }
764 #endif
765 
766             resultCollector.addResult(result.getCode(), result.getDescription());
767         }
768         return TestStatus(resultCollector.getResult(), resultCollector.getMessage());
769     }
770 
771 private:
772     vector<VkGraphicsPipelineCreateInfo> &m_pipelineInfo;
773     const VkRenderPass &m_renderPass;
774 };
775 
776 class PipelineCacheComputeTestInstance : public TestInstance
777 {
778     typedef vector<SharedPtr<Unique<VkShaderModule>>> ShaderModuleVector;
779 
780 public:
PipelineCacheComputeTestInstance(Context & context,const vector<uint32_t> & shadersExecutions)781     PipelineCacheComputeTestInstance(Context &context, const vector<uint32_t> &shadersExecutions)
782         : TestInstance(context)
783         , m_shadersExecutions(shadersExecutions)
784 
785     {
786     }
787 
iterate(void)788     TestStatus iterate(void)
789     {
790 #ifdef CTS_USES_VULKANSC
791         MultithreadedDestroyGuard mdGuard(m_context.getResourceInterface());
792 #endif // CTS_USES_VULKANSC
793         const CustomInstance instance(createCustomInstanceFromContext(m_context));
794         const InstanceDriver &instanceDriver(instance.getDriver());
795 
796         MovePtr<MultiQueues> queues          = createQueues(m_context, VK_QUEUE_COMPUTE_BIT, instance, instanceDriver);
797         const DeviceInterface &vk            = queues->getDeviceInterface();
798         const VkDevice device                = queues->getDevice();
799         ShaderModuleVector shaderCompModules = addShaderModules(device);
800         Buffer resultBuffer(vk, device, *queues->m_allocator,
801                             makeBufferCreateInfo(BUFFER_SIZE, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT),
802                             MemoryRequirement::HostVisible);
803         const Move<VkDescriptorSetLayout> descriptorSetLayout(
804             DescriptorSetLayoutBuilder()
805                 .addSingleBinding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_COMPUTE_BIT)
806                 .build(vk, device));
807         const Move<VkPipelineLayout> pipelineLayout(makePipelineLayout(vk, device, *descriptorSetLayout));
808         vector<VkPipelineShaderStageCreateInfo> shaderStageInfos = addShaderStageInfo(shaderCompModules);
809         vector<VkComputePipelineCreateInfo> pipelineInfo         = addPipelineInfo(*pipelineLayout, shaderStageInfos);
810         const VkPipelineCacheCreateInfo pipelineCacheInfo        = {
811             VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, // VkStructureType             sType;
812             DE_NULL,                                      // const void*                 pNext;
813 #ifndef CTS_USES_VULKANSC
814             0u,      // VkPipelineCacheCreateFlags  flags;
815             0u,      // uintptr_t                   initialDataSize;
816             DE_NULL, // const void*                 pInitialData;
817 #else
818             VK_PIPELINE_CACHE_CREATE_READ_ONLY_BIT |
819                 VK_PIPELINE_CACHE_CREATE_USE_APPLICATION_STORAGE_BIT, // VkPipelineCacheCreateFlags flags;
820             m_context.getResourceInterface()->getCacheDataSize(),     // uintptr_t initialDataSize;
821             m_context.getResourceInterface()->getCacheData()          // const void* pInitialData;
822 #endif // CTS_USES_VULKANSC
823         };
824         Move<VkPipelineCache> pipelineCache = createPipelineCache(vk, device, &pipelineCacheInfo);
825         Move<VkPipeline> pipeline           = createComputePipeline(vk, device, *pipelineCache, &pipelineInfo[0]);
826 #ifndef CTS_USES_VULKANSC
827         const uint32_t numThreads = clamp(deGetNumAvailableLogicalCores(), 4u, 32u);
828 #else
829         const uint32_t numThreads = 2u;
830 #endif // CTS_USES_VULKANSC
831         ThreadGroup threads;
832 
833         executeComputePipeline(m_context, *pipeline, *pipelineLayout, *descriptorSetLayout, *queues,
834                                m_shadersExecutions[0]);
835 
836         for (uint32_t ndx = 0; ndx < numThreads; ++ndx)
837             threads.add(MovePtr<ThreadGroupThread>(new CreateComputeThread(m_context, *pipelineCache, pipelineInfo,
838                                                                            *pipelineLayout, *descriptorSetLayout,
839                                                                            *queues, m_shadersExecutions)));
840 
841         {
842             TestStatus thread_result = threads.run();
843             if (thread_result.getCode())
844             {
845                 return thread_result;
846             }
847         }
848         return TestStatus::pass("Passed");
849     }
850 
851 private:
addShaderModules(const VkDevice & device)852     ShaderModuleVector addShaderModules(const VkDevice &device)
853     {
854         const DeviceInterface &vk = m_context.getDeviceInterface();
855         ShaderModuleVector shaderCompModules;
856         shaderCompModules.resize(m_shadersExecutions.size());
857         for (int shaderNdx = 0; shaderNdx < static_cast<int>(m_shadersExecutions.size()); ++shaderNdx)
858         {
859             ostringstream shaderName;
860             shaderName << "compute_" << shaderNdx;
861             shaderCompModules[shaderNdx] =
862                 SharedPtr<Unique<VkShaderModule>>(new Unique<VkShaderModule>(createShaderModule(
863                     vk, device, m_context.getBinaryCollection().get(shaderName.str()), (VkShaderModuleCreateFlags)0)));
864         }
865         return shaderCompModules;
866     }
867 
addShaderStageInfo(const ShaderModuleVector & shaderCompModules)868     vector<VkPipelineShaderStageCreateInfo> addShaderStageInfo(const ShaderModuleVector &shaderCompModules)
869     {
870         VkPipelineShaderStageCreateInfo shaderStageInfo;
871         vector<VkPipelineShaderStageCreateInfo> shaderStageInfos;
872         shaderStageInfo.sType               = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
873         shaderStageInfo.pNext               = DE_NULL;
874         shaderStageInfo.flags               = (VkPipelineShaderStageCreateFlags)0;
875         shaderStageInfo.stage               = VK_SHADER_STAGE_COMPUTE_BIT;
876         shaderStageInfo.pName               = "main";
877         shaderStageInfo.pSpecializationInfo = DE_NULL;
878 
879         for (int shaderNdx = 0; shaderNdx < static_cast<int>(m_shadersExecutions.size()); ++shaderNdx)
880         {
881             shaderStageInfo.module = *(*shaderCompModules[shaderNdx]);
882             shaderStageInfos.push_back(shaderStageInfo);
883         }
884         return shaderStageInfos;
885     }
886 
addPipelineInfo(VkPipelineLayout pipelineLayout,const vector<VkPipelineShaderStageCreateInfo> & shaderStageInfos)887     vector<VkComputePipelineCreateInfo> addPipelineInfo(VkPipelineLayout pipelineLayout,
888                                                         const vector<VkPipelineShaderStageCreateInfo> &shaderStageInfos)
889     {
890         vector<VkComputePipelineCreateInfo> pipelineInfos;
891         VkComputePipelineCreateInfo computePipelineInfo;
892         computePipelineInfo.sType              = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO;
893         computePipelineInfo.pNext              = DE_NULL;
894         computePipelineInfo.flags              = (VkPipelineCreateFlags)0;
895         computePipelineInfo.layout             = pipelineLayout;
896         computePipelineInfo.basePipelineHandle = DE_NULL;
897         computePipelineInfo.basePipelineIndex  = 0;
898 
899         for (int shaderNdx = 0; shaderNdx < static_cast<int>(m_shadersExecutions.size()); ++shaderNdx)
900         {
901             computePipelineInfo.stage = shaderStageInfos[shaderNdx];
902             pipelineInfos.push_back(computePipelineInfo);
903         }
904         return pipelineInfos;
905     }
906 
907     const vector<uint32_t> m_shadersExecutions;
908 };
909 
910 class PipelineCacheGraphicTestInstance : public TestInstance
911 {
912     typedef vector<SharedPtr<Unique<VkShaderModule>>> ShaderModuleVector;
913 
914 public:
PipelineCacheGraphicTestInstance(Context & context,const vector<uint32_t> & shadersExecutions)915     PipelineCacheGraphicTestInstance(Context &context, const vector<uint32_t> &shadersExecutions)
916         : TestInstance(context)
917         , m_shadersExecutions(shadersExecutions)
918 
919     {
920     }
921 
iterate(void)922     TestStatus iterate(void)
923     {
924 #ifdef CTS_USES_VULKANSC
925         MultithreadedDestroyGuard mdGuard(m_context.getResourceInterface());
926 #endif // CTS_USES_VULKANSC
927         const CustomInstance instance(createCustomInstanceFromContext(m_context));
928         const InstanceDriver &instanceDriver(instance.getDriver());
929         const VkPhysicalDevice physicalDevice =
930             chooseDevice(instanceDriver, instance, m_context.getTestContext().getCommandLine());
931         requireFeatures(instanceDriver, physicalDevice, FEATURE_VERTEX_PIPELINE_STORES_AND_ATOMICS);
932 
933         MovePtr<MultiQueues> queues   = createQueues(m_context, VK_QUEUE_GRAPHICS_BIT, instance, instanceDriver);
934         const DeviceInterface &vk     = m_context.getDeviceInterface();
935         const VkDevice device         = queues->getDevice();
936         VkFormat colorFormat          = VK_FORMAT_R8G8B8A8_UNORM;
937         Move<VkRenderPass> renderPass = makeRenderPass(vk, device, colorFormat);
938         const Move<VkDescriptorSetLayout> descriptorSetLayout(
939             DescriptorSetLayoutBuilder()
940                 .addSingleBinding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, VK_SHADER_STAGE_VERTEX_BIT)
941                 .build(vk, device));
942         ShaderModuleVector shaderGraphicModules = addShaderModules(device);
943         const Move<VkPipelineLayout> pipelineLayout(makePipelineLayout(vk, device, *descriptorSetLayout));
944         vector<VkPipelineShaderStageCreateInfo> shaderStageInfos = addShaderStageInfo(shaderGraphicModules);
945         vector<VkGraphicsPipelineCreateInfo> pipelineInfo =
946             addPipelineInfo(*pipelineLayout, shaderStageInfos, *renderPass);
947         const VkPipelineCacheCreateInfo pipelineCacheInfo = {
948             VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, // VkStructureType             sType;
949             DE_NULL,                                      // const void*                 pNext;
950 #ifndef CTS_USES_VULKANSC
951             0u,     // VkPipelineCacheCreateFlags  flags;
952             0u,     // uintptr_t                   initialDataSize;
953             DE_NULL // const void*                 pInitialData;
954 #else
955             VK_PIPELINE_CACHE_CREATE_READ_ONLY_BIT |
956                 VK_PIPELINE_CACHE_CREATE_USE_APPLICATION_STORAGE_BIT, // VkPipelineCacheCreateFlags flags;
957             m_context.getResourceInterface()->getCacheDataSize(),     // uintptr_t initialDataSize;
958             m_context.getResourceInterface()->getCacheData()          // const void* pInitialData;
959 #endif // CTS_USES_VULKANSC
960         };
961         Move<VkPipelineCache> pipelineCache = createPipelineCache(vk, device, &pipelineCacheInfo);
962         Move<VkPipeline> pipeline           = createGraphicsPipeline(vk, device, *pipelineCache, &pipelineInfo[0]);
963 #ifndef CTS_USES_VULKANSC
964         const uint32_t numThreads = clamp(deGetNumAvailableLogicalCores(), 4u, 32u);
965 #else
966         const uint32_t numThreads = 2u;
967 #endif // CTS_USES_VULKANSC
968         ThreadGroup threads;
969 
970         executeGraphicPipeline(m_context, *pipeline, *pipelineLayout, *descriptorSetLayout, *queues, *renderPass,
971                                m_shadersExecutions[0]);
972 
973         for (uint32_t ndx = 0; ndx < numThreads; ++ndx)
974             threads.add(MovePtr<ThreadGroupThread>(new CreateGraphicThread(m_context, *pipelineCache, pipelineInfo,
975                                                                            *pipelineLayout, *descriptorSetLayout,
976                                                                            *queues, *renderPass, m_shadersExecutions)));
977 
978         {
979             TestStatus thread_result = threads.run();
980             if (thread_result.getCode())
981             {
982                 return thread_result;
983             }
984         }
985         return TestStatus::pass("Passed");
986     }
987 
988 private:
addShaderModules(const VkDevice & device)989     ShaderModuleVector addShaderModules(const VkDevice &device)
990     {
991         const DeviceInterface &vk = m_context.getDeviceInterface();
992         ShaderModuleVector shaderModules;
993         shaderModules.resize(m_shadersExecutions.size() + 1);
994         for (int shaderNdx = 0; shaderNdx < static_cast<int>(m_shadersExecutions.size()); ++shaderNdx)
995         {
996             ostringstream shaderName;
997             shaderName << "vert_" << shaderNdx;
998             shaderModules[shaderNdx] = SharedPtr<Unique<VkShaderModule>>(new Unique<VkShaderModule>(createShaderModule(
999                 vk, device, m_context.getBinaryCollection().get(shaderName.str()), (VkShaderModuleCreateFlags)0)));
1000         }
1001         shaderModules[m_shadersExecutions.size()] = SharedPtr<Unique<VkShaderModule>>(new Unique<VkShaderModule>(
1002             createShaderModule(vk, device, m_context.getBinaryCollection().get("frag"), (VkShaderModuleCreateFlags)0)));
1003         return shaderModules;
1004     }
1005 
addShaderStageInfo(const ShaderModuleVector & shaderCompModules)1006     vector<VkPipelineShaderStageCreateInfo> addShaderStageInfo(const ShaderModuleVector &shaderCompModules)
1007     {
1008         VkPipelineShaderStageCreateInfo shaderStageInfo;
1009         vector<VkPipelineShaderStageCreateInfo> shaderStageInfos;
1010         shaderStageInfo.sType               = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
1011         shaderStageInfo.pNext               = DE_NULL;
1012         shaderStageInfo.flags               = (VkPipelineShaderStageCreateFlags)0;
1013         shaderStageInfo.pName               = "main";
1014         shaderStageInfo.pSpecializationInfo = DE_NULL;
1015 
1016         for (int shaderNdx = 0; shaderNdx < static_cast<int>(m_shadersExecutions.size()); ++shaderNdx)
1017         {
1018             shaderStageInfo.stage  = VK_SHADER_STAGE_VERTEX_BIT;
1019             shaderStageInfo.module = *(*shaderCompModules[shaderNdx]);
1020             shaderStageInfos.push_back(shaderStageInfo);
1021 
1022             shaderStageInfo.stage  = VK_SHADER_STAGE_FRAGMENT_BIT;
1023             shaderStageInfo.module = *(*shaderCompModules[m_shadersExecutions.size()]);
1024             shaderStageInfos.push_back(shaderStageInfo);
1025         }
1026         return shaderStageInfos;
1027     }
1028 
addPipelineInfo(VkPipelineLayout pipelineLayout,const vector<VkPipelineShaderStageCreateInfo> & shaderStageInfos,const VkRenderPass & renderPass)1029     vector<VkGraphicsPipelineCreateInfo> addPipelineInfo(
1030         VkPipelineLayout pipelineLayout, const vector<VkPipelineShaderStageCreateInfo> &shaderStageInfos,
1031         const VkRenderPass &renderPass)
1032     {
1033         VkExtent3D colorImageExtent = makeExtent3D(1u, 1u, 1u);
1034         vector<VkGraphicsPipelineCreateInfo> pipelineInfo;
1035 
1036         m_vertexInputStateParams.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO;
1037         m_vertexInputStateParams.pNext = DE_NULL;
1038         m_vertexInputStateParams.flags = 0u;
1039         m_vertexInputStateParams.vertexBindingDescriptionCount   = 0u;
1040         m_vertexInputStateParams.pVertexBindingDescriptions      = DE_NULL;
1041         m_vertexInputStateParams.vertexAttributeDescriptionCount = 0u;
1042         m_vertexInputStateParams.pVertexAttributeDescriptions    = DE_NULL;
1043 
1044         m_inputAssemblyStateParams.sType                  = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO;
1045         m_inputAssemblyStateParams.pNext                  = DE_NULL;
1046         m_inputAssemblyStateParams.flags                  = 0u;
1047         m_inputAssemblyStateParams.topology               = VK_PRIMITIVE_TOPOLOGY_POINT_LIST;
1048         m_inputAssemblyStateParams.primitiveRestartEnable = VK_FALSE;
1049 
1050         m_viewport.x        = 0.0f;
1051         m_viewport.y        = 0.0f;
1052         m_viewport.width    = (float)colorImageExtent.width;
1053         m_viewport.height   = (float)colorImageExtent.height;
1054         m_viewport.minDepth = 0.0f;
1055         m_viewport.maxDepth = 1.0f;
1056 
1057         //TODO
1058         m_scissor.offset.x      = 0;
1059         m_scissor.offset.y      = 0;
1060         m_scissor.extent.width  = colorImageExtent.width;
1061         m_scissor.extent.height = colorImageExtent.height;
1062 
1063         m_viewportStateParams.sType         = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO;
1064         m_viewportStateParams.pNext         = DE_NULL;
1065         m_viewportStateParams.flags         = 0u;
1066         m_viewportStateParams.viewportCount = 1u;
1067         m_viewportStateParams.pViewports    = &m_viewport;
1068         m_viewportStateParams.scissorCount  = 1u;
1069         m_viewportStateParams.pScissors     = &m_scissor;
1070 
1071         m_rasterStateParams.sType                   = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO;
1072         m_rasterStateParams.pNext                   = DE_NULL;
1073         m_rasterStateParams.flags                   = 0u;
1074         m_rasterStateParams.depthClampEnable        = VK_FALSE;
1075         m_rasterStateParams.rasterizerDiscardEnable = VK_FALSE;
1076         m_rasterStateParams.polygonMode             = VK_POLYGON_MODE_FILL;
1077         m_rasterStateParams.cullMode                = VK_CULL_MODE_NONE;
1078         m_rasterStateParams.frontFace               = VK_FRONT_FACE_COUNTER_CLOCKWISE;
1079         m_rasterStateParams.depthBiasEnable         = VK_FALSE;
1080         m_rasterStateParams.depthBiasConstantFactor = 0.0f;
1081         m_rasterStateParams.depthBiasClamp          = 0.0f;
1082         m_rasterStateParams.depthBiasSlopeFactor    = 0.0f;
1083         m_rasterStateParams.lineWidth               = 1.0f;
1084 
1085         m_colorBlendAttachmentState.blendEnable         = VK_FALSE;
1086         m_colorBlendAttachmentState.srcColorBlendFactor = VK_BLEND_FACTOR_ONE;
1087         m_colorBlendAttachmentState.dstColorBlendFactor = VK_BLEND_FACTOR_ZERO;
1088         m_colorBlendAttachmentState.colorBlendOp        = VK_BLEND_OP_ADD;
1089         m_colorBlendAttachmentState.srcAlphaBlendFactor = VK_BLEND_FACTOR_ONE;
1090         m_colorBlendAttachmentState.dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO;
1091         m_colorBlendAttachmentState.alphaBlendOp        = VK_BLEND_OP_ADD;
1092         m_colorBlendAttachmentState.colorWriteMask =
1093             VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
1094 
1095         m_colorBlendStateParams.sType             = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO;
1096         m_colorBlendStateParams.pNext             = DE_NULL;
1097         m_colorBlendStateParams.flags             = 0u;
1098         m_colorBlendStateParams.logicOpEnable     = VK_FALSE;
1099         m_colorBlendStateParams.logicOp           = VK_LOGIC_OP_COPY;
1100         m_colorBlendStateParams.attachmentCount   = 1u;
1101         m_colorBlendStateParams.pAttachments      = &m_colorBlendAttachmentState;
1102         m_colorBlendStateParams.blendConstants[0] = 0.0f;
1103         m_colorBlendStateParams.blendConstants[1] = 0.0f;
1104         m_colorBlendStateParams.blendConstants[2] = 0.0f;
1105         m_colorBlendStateParams.blendConstants[3] = 0.0f;
1106 
1107         m_multisampleStateParams.sType                 = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO;
1108         m_multisampleStateParams.pNext                 = DE_NULL;
1109         m_multisampleStateParams.flags                 = 0u;
1110         m_multisampleStateParams.rasterizationSamples  = VK_SAMPLE_COUNT_1_BIT;
1111         m_multisampleStateParams.sampleShadingEnable   = VK_FALSE;
1112         m_multisampleStateParams.minSampleShading      = 0.0f;
1113         m_multisampleStateParams.pSampleMask           = DE_NULL;
1114         m_multisampleStateParams.alphaToCoverageEnable = VK_FALSE;
1115         m_multisampleStateParams.alphaToOneEnable      = VK_FALSE;
1116 
1117         m_depthStencilStateParams.sType                 = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO;
1118         m_depthStencilStateParams.pNext                 = DE_NULL;
1119         m_depthStencilStateParams.flags                 = 0u;
1120         m_depthStencilStateParams.depthTestEnable       = VK_TRUE;
1121         m_depthStencilStateParams.depthWriteEnable      = VK_TRUE;
1122         m_depthStencilStateParams.depthCompareOp        = VK_COMPARE_OP_LESS_OR_EQUAL;
1123         m_depthStencilStateParams.depthBoundsTestEnable = VK_FALSE;
1124         m_depthStencilStateParams.stencilTestEnable     = VK_FALSE;
1125         m_depthStencilStateParams.front.failOp          = VK_STENCIL_OP_KEEP;
1126         m_depthStencilStateParams.front.passOp          = VK_STENCIL_OP_KEEP;
1127         m_depthStencilStateParams.front.depthFailOp     = VK_STENCIL_OP_KEEP;
1128         m_depthStencilStateParams.front.compareOp       = VK_COMPARE_OP_NEVER;
1129         m_depthStencilStateParams.front.compareMask     = 0u;
1130         m_depthStencilStateParams.front.writeMask       = 0u;
1131         m_depthStencilStateParams.front.reference       = 0u;
1132         m_depthStencilStateParams.back.failOp           = VK_STENCIL_OP_KEEP;
1133         m_depthStencilStateParams.back.passOp           = VK_STENCIL_OP_KEEP;
1134         m_depthStencilStateParams.back.depthFailOp      = VK_STENCIL_OP_KEEP;
1135         m_depthStencilStateParams.back.compareOp        = VK_COMPARE_OP_NEVER;
1136         m_depthStencilStateParams.back.compareMask      = 0u;
1137         m_depthStencilStateParams.back.writeMask        = 0u;
1138         m_depthStencilStateParams.back.reference        = 0u;
1139         m_depthStencilStateParams.minDepthBounds        = 0.0f;
1140         m_depthStencilStateParams.maxDepthBounds        = 1.0f;
1141 
1142         VkGraphicsPipelineCreateInfo graphicsPipelineParams = {
1143             VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, // VkStructureType sType;
1144             DE_NULL,                                         // const void* pNext;
1145             0u,                                              // VkPipelineCreateFlags flags;
1146             2u,                                              // uint32_t stageCount;
1147             DE_NULL,                                         // const VkPipelineShaderStageCreateInfo* pStages;
1148             &m_vertexInputStateParams,   // const VkPipelineVertexInputStateCreateInfo* pVertexInputState;
1149             &m_inputAssemblyStateParams, // const VkPipelineInputAssemblyStateCreateInfo* pInputAssemblyState;
1150             DE_NULL,                     // const VkPipelineTessellationStateCreateInfo* pTessellationState;
1151             &m_viewportStateParams,      // const VkPipelineViewportStateCreateInfo* pViewportState;
1152             &m_rasterStateParams,        // const VkPipelineRasterizationStateCreateInfo* pRasterState;
1153             &m_multisampleStateParams,   // const VkPipelineMultisampleStateCreateInfo* pMultisampleState;
1154             &m_depthStencilStateParams,  // const VkPipelineDepthStencilStateCreateInfo* pDepthStencilState;
1155             &m_colorBlendStateParams,    // const VkPipelineColorBlendStateCreateInfo* pColorBlendState;
1156             (const VkPipelineDynamicStateCreateInfo *)DE_NULL, // const VkPipelineDynamicStateCreateInfo* pDynamicState;
1157             pipelineLayout,                                    // VkPipelineLayout layout;
1158             renderPass,                                        // VkRenderPass renderPass;
1159             0u,                                                // uint32_t subpass;
1160             DE_NULL,                                           // VkPipeline basePipelineHandle;
1161             0,                                                 // int32_t basePipelineIndex;
1162         };
1163         for (int shaderNdx = 0; shaderNdx < static_cast<int>(m_shadersExecutions.size()) * 2; shaderNdx += 2)
1164         {
1165             graphicsPipelineParams.pStages = &shaderStageInfos[shaderNdx];
1166             pipelineInfo.push_back(graphicsPipelineParams);
1167         }
1168         return pipelineInfo;
1169     }
1170 
1171     const vector<uint32_t> m_shadersExecutions;
1172     VkPipelineVertexInputStateCreateInfo m_vertexInputStateParams;
1173     VkPipelineInputAssemblyStateCreateInfo m_inputAssemblyStateParams;
1174     VkViewport m_viewport;
1175     VkRect2D m_scissor;
1176     VkPipelineViewportStateCreateInfo m_viewportStateParams;
1177     VkPipelineRasterizationStateCreateInfo m_rasterStateParams;
1178     VkPipelineColorBlendAttachmentState m_colorBlendAttachmentState;
1179     VkPipelineColorBlendStateCreateInfo m_colorBlendStateParams;
1180     VkPipelineMultisampleStateCreateInfo m_multisampleStateParams;
1181     VkPipelineDepthStencilStateCreateInfo m_depthStencilStateParams;
1182 };
1183 
1184 class PipelineCacheComputeTest : public TestCase
1185 {
1186 public:
PipelineCacheComputeTest(TestContext & testCtx,const string & name)1187     PipelineCacheComputeTest(TestContext &testCtx, const string &name) : TestCase(testCtx, name)
1188     {
1189     }
1190 
initPrograms(SourceCollections & programCollection) const1191     void initPrograms(SourceCollections &programCollection) const
1192     {
1193         ostringstream buffer;
1194         buffer << "layout(set = 0, binding = 0, std430) buffer Output\n"
1195                << "{\n"
1196                << "    int result[];\n"
1197                << "} sb_out;\n";
1198         {
1199             ostringstream src;
1200             src << glu::getGLSLVersionDeclaration(glu::GLSL_VERSION_310_ES) << "\n"
1201                 << "\n"
1202                 << "layout(local_size_x = 1, local_size_y = 1, local_size_z = 1) in;\n"
1203                 << "\n"
1204                 << buffer.str() << "void main (void)\n"
1205                 << "{\n"
1206                 << "    highp uint ndx = gl_GlobalInvocationID.x;\n"
1207                 << "    sb_out.result[ndx] = int(ndx);\n"
1208                 << "}\n";
1209             programCollection.glslSources.add("compute_0") << glu::ComputeSource(src.str());
1210         }
1211         {
1212             ostringstream src;
1213             src << glu::getGLSLVersionDeclaration(glu::GLSL_VERSION_310_ES) << "\n"
1214                 << "\n"
1215                 << "layout(local_size_x = 1, local_size_y = 1, local_size_z = 1) in;\n"
1216                 << "\n"
1217                 << buffer.str() << "void main (void)\n"
1218                 << "{\n"
1219                 << "    for (highp uint ndx = 0u; ndx < " << BUFFER_ELEMENT_COUNT << "u; ndx++)\n"
1220                 << "    {\n"
1221                 << "        sb_out.result[ndx] = int(ndx);\n"
1222                 << "    }\n"
1223                 << "}\n";
1224             programCollection.glslSources.add("compute_1") << glu::ComputeSource(src.str());
1225         }
1226         {
1227             ostringstream src;
1228             src << glu::getGLSLVersionDeclaration(glu::GLSL_VERSION_310_ES) << "\n"
1229                 << "\n"
1230                 << "layout(local_size_x = " << BUFFER_ELEMENT_COUNT << ", local_size_y = 1, local_size_z = 1) in;\n"
1231                 << "\n"
1232                 << buffer.str() << "void main (void)\n"
1233                 << "{\n"
1234                 << "    highp uint ndx = gl_LocalInvocationID.x;\n"
1235                 << "    sb_out.result[ndx] = int(ndx);\n"
1236                 << "}\n";
1237             programCollection.glslSources.add("compute_2") << glu::ComputeSource(src.str());
1238         }
1239     }
1240 
createInstance(Context & context) const1241     TestInstance *createInstance(Context &context) const
1242     {
1243         vector<uint32_t> shadersExecutions;
1244         shadersExecutions.push_back(16u); //compute_0
1245         shadersExecutions.push_back(1u);  //compute_1
1246         shadersExecutions.push_back(1u);  //compute_2
1247         return new PipelineCacheComputeTestInstance(context, shadersExecutions);
1248     }
1249 };
1250 
1251 class PipelineCacheGraphicTest : public TestCase
1252 {
1253 public:
PipelineCacheGraphicTest(TestContext & testCtx,const string & name)1254     PipelineCacheGraphicTest(TestContext &testCtx, const string &name) : TestCase(testCtx, name)
1255     {
1256     }
1257 
initPrograms(SourceCollections & programCollection) const1258     void initPrograms(SourceCollections &programCollection) const
1259     {
1260         ostringstream buffer;
1261         buffer << "layout(set = 0, binding = 0, std430) buffer Output\n"
1262                << "{\n"
1263                << "    int result[];\n"
1264                << "} sb_out;\n";
1265 
1266         // Vertex
1267         {
1268             std::ostringstream src;
1269             src << glu::getGLSLVersionDeclaration(glu::GLSL_VERSION_440) << "\n"
1270                 << "\n"
1271                 << buffer.str() << "\n"
1272                 << "void main (void)\n"
1273                 << "{\n"
1274                 << "   sb_out.result[gl_VertexIndex] = int(gl_VertexIndex);\n"
1275                 << "   gl_PointSize = 1.0f;\n"
1276                 << "}\n";
1277             programCollection.glslSources.add("vert_0") << glu::VertexSource(src.str());
1278         }
1279         // Vertex
1280         {
1281             std::ostringstream src;
1282             src << glu::getGLSLVersionDeclaration(glu::GLSL_VERSION_440) << "\n"
1283                 << "\n"
1284                 << buffer.str() << "\n"
1285                 << "void main (void)\n"
1286                 << "{\n"
1287                 << "    for (highp uint ndx = 0u; ndx < " << BUFFER_ELEMENT_COUNT << "u; ndx++)\n"
1288                 << "    {\n"
1289                 << "        sb_out.result[ndx] = int(ndx);\n"
1290                 << "    }\n"
1291                 << "    gl_PointSize = 1.0f;\n"
1292                 << "}\n";
1293             programCollection.glslSources.add("vert_1") << glu::VertexSource(src.str());
1294         }
1295         // Vertex
1296         {
1297             std::ostringstream src;
1298             src << glu::getGLSLVersionDeclaration(glu::GLSL_VERSION_440) << "\n"
1299                 << "\n"
1300                 << buffer.str() << "\n"
1301                 << "void main (void)\n"
1302                 << "{\n"
1303                 << "    for (int ndx = " << BUFFER_ELEMENT_COUNT - 1 << "; ndx >= 0; ndx--)\n"
1304                 << "    {\n"
1305                 << "        sb_out.result[uint(ndx)] = ndx;\n"
1306                 << "    }\n"
1307                 << "    gl_PointSize = 1.0f;\n"
1308                 << "}\n";
1309             programCollection.glslSources.add("vert_2") << glu::VertexSource(src.str());
1310         }
1311         // Fragment
1312         {
1313             std::ostringstream src;
1314             src << glu::getGLSLVersionDeclaration(glu::GLSL_VERSION_440) << "\n"
1315                 << "\n"
1316                 << "layout(location = 0) out vec4 o_color;\n"
1317                 << "\n"
1318                 << "void main (void)\n"
1319                 << "{\n"
1320                 << "    o_color = vec4(1.0);\n"
1321                 << "}\n";
1322             programCollection.glslSources.add("frag") << glu::FragmentSource(src.str());
1323         }
1324     }
1325 
createInstance(Context & context) const1326     TestInstance *createInstance(Context &context) const
1327     {
1328         vector<uint32_t> shadersExecutions;
1329         shadersExecutions.push_back(16u); //vert_0
1330         shadersExecutions.push_back(1u);  //vert_1
1331         shadersExecutions.push_back(1u);  //vert_2
1332         return new PipelineCacheGraphicTestInstance(context, shadersExecutions);
1333     }
1334 };
1335 
1336 } // namespace
1337 
createInternallySynchronizedObjects(tcu::TestContext & testCtx)1338 tcu::TestCaseGroup *createInternallySynchronizedObjects(tcu::TestContext &testCtx)
1339 {
1340     // Internally synchronized objects
1341     de::MovePtr<tcu::TestCaseGroup> tests(new tcu::TestCaseGroup(testCtx, "internally_synchronized_objects"));
1342     // Internally synchronized object VkPipelineCache for compute pipeline is tested
1343     tests->addChild(new PipelineCacheComputeTest(testCtx, "pipeline_cache_compute"));
1344     // Internally synchronized object VkPipelineCache for graphics pipeline is tested
1345     tests->addChild(new PipelineCacheGraphicTest(testCtx, "pipeline_cache_graphics"));
1346     return tests.release();
1347 }
1348 
1349 } // namespace synchronization
1350 } // namespace vkt
1351