1 //
2 // Copyright 2024 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // RefCountedEvent:
7 // Manages reference count of VkEvent and its associated functions.
8 //
9
10 #include "libANGLE/renderer/vulkan/vk_ref_counted_event.h"
11 #include "libANGLE/renderer/vulkan/vk_helpers.h"
12 #include "libANGLE/renderer/vulkan/vk_renderer.h"
13
14 namespace rx
15 {
16 namespace vk
17 {
18 namespace
19 {
20 // Predefined VkPipelineStageFlags for RefCountedEvent
21 constexpr angle::PackedEnumMap<EventStage, VkPipelineStageFlags>
22 kEventStageAndPipelineStageFlagsMap = {
23 {EventStage::Transfer, VK_PIPELINE_STAGE_TRANSFER_BIT},
24 {EventStage::VertexShader, VK_PIPELINE_STAGE_VERTEX_SHADER_BIT},
25 {EventStage::FragmentShader, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT},
26 {EventStage::ComputeShader, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT},
27 {EventStage::AllShaders, kAllShadersPipelineStageFlags},
28 {EventStage::PreFragmentShaders, kPreFragmentStageFlags},
29 {EventStage::FragmentShadingRate,
30 VK_PIPELINE_STAGE_FRAGMENT_SHADING_RATE_ATTACHMENT_BIT_KHR},
31 {EventStage::ColorAttachmentOutput, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT},
32 {EventStage::ColorAttachmentOutputAndFragmentShader,
33 VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT},
34 {EventStage::ColorAttachmentOutputAndFragmentShaderAndTransfer,
35 VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT |
36 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT},
37 {EventStage::ColorAttachmentOutputAndAllShaders,
38 VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | kAllShadersPipelineStageFlags},
39 {EventStage::AllFragmentTest, kAllDepthStencilPipelineStageFlags},
40 {EventStage::AllFragmentTestAndFragmentShader,
41 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | kAllDepthStencilPipelineStageFlags},
42 {EventStage::AllFragmentTestAndAllShaders,
43 kAllShadersPipelineStageFlags | kAllDepthStencilPipelineStageFlags},
44 {EventStage::TransferAndComputeShader,
45 VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT}};
46
DestroyRefCountedEvents(VkDevice device,RefCountedEventCollector & events)47 void DestroyRefCountedEvents(VkDevice device, RefCountedEventCollector &events)
48 {
49 while (!events.empty())
50 {
51 events.back().destroy(device);
52 events.pop_back();
53 }
54 }
55 } // namespace
56
InitializeEventAndPipelineStagesMap(angle::PackedEnumMap<EventStage,VkPipelineStageFlags> * map,VkPipelineStageFlags supportedVulkanPipelineStageMask)57 void InitializeEventAndPipelineStagesMap(
58 angle::PackedEnumMap<EventStage, VkPipelineStageFlags> *map,
59 VkPipelineStageFlags supportedVulkanPipelineStageMask)
60 {
61 *map = kEventStageAndPipelineStageFlagsMap;
62 for (VkPipelineStageFlags &flag : *map)
63 {
64 flag &= supportedVulkanPipelineStageMask;
65 }
66 }
67
init(Context * context,EventStage eventStage)68 bool RefCountedEvent::init(Context *context, EventStage eventStage)
69 {
70 ASSERT(mHandle == nullptr);
71 ASSERT(eventStage != EventStage::InvalidEnum);
72
73 // First try with recycler. We must issue VkCmdResetEvent before VkCmdSetEvent
74 if (context->getRefCountedEventsGarbageRecycler()->fetch(context->getRenderer(), this))
75 {
76 ASSERT(valid());
77 ASSERT(!mHandle->isReferenced());
78 }
79 else
80 {
81 // If failed to fetch from recycler, then create a new event.
82 mHandle = new RefCounted<EventAndStage>;
83 VkEventCreateInfo createInfo = {};
84 createInfo.sType = VK_STRUCTURE_TYPE_EVENT_CREATE_INFO;
85 // Use device only for performance reasons.
86 createInfo.flags = context->getFeatures().supportsSynchronization2.enabled
87 ? VK_EVENT_CREATE_DEVICE_ONLY_BIT_KHR
88 : 0;
89 VkResult result = mHandle->get().event.init(context->getDevice(), createInfo);
90 if (result != VK_SUCCESS)
91 {
92 WARN() << "event.init failed. Clean up garbage and retry again";
93 // Proactively clean up garbage and retry
94 context->getRefCountedEventsGarbageRecycler()->cleanup(context->getRenderer());
95 result = mHandle->get().event.init(context->getDevice(), createInfo);
96 if (result != VK_SUCCESS)
97 {
98 // Drivers usually can allocate huge amount of VkEvents, and we should never use
99 // that many VkEvents under normal situation. If we failed to allocate, there is a
100 // high chance that we may have a leak somewhere. This macro should help us catch
101 // such potential bugs in the bots if that happens.
102 UNREACHABLE();
103 // If still fail to create, we just return. An invalid event will trigger
104 // pipelineBarrier code path
105 return false;
106 }
107 }
108 }
109
110 mHandle->addRef();
111 mHandle->get().eventStage = eventStage;
112 return true;
113 }
114
release(Context * context)115 void RefCountedEvent::release(Context *context)
116 {
117 if (mHandle != nullptr)
118 {
119 releaseImpl(context->getRenderer(), context->getRefCountedEventsGarbageRecycler());
120 }
121 }
122
release(Renderer * renderer)123 void RefCountedEvent::release(Renderer *renderer)
124 {
125 if (mHandle != nullptr)
126 {
127 releaseImpl(renderer, renderer->getRefCountedEventRecycler());
128 }
129 }
130
131 template <typename RecyclerT>
releaseImpl(Renderer * renderer,RecyclerT * recycler)132 void RefCountedEvent::releaseImpl(Renderer *renderer, RecyclerT *recycler)
133 {
134 ASSERT(mHandle != nullptr);
135 // This should never called from async submission thread since the refcount is not atomic. It is
136 // expected only called under context share lock.
137 ASSERT(std::this_thread::get_id() != renderer->getCommandProcessorThreadId());
138
139 const bool isLastReference = mHandle->getAndReleaseRef() == 1;
140 if (isLastReference)
141 {
142 // When async submission is enabled, recycler will be null when release call comes from
143 // CommandProcessor. But in that case it will not be the last reference since garbage
144 // collector should have one reference count and will never release that reference count
145 // until GPU finished.
146 ASSERT(recycler != nullptr);
147 recycler->recycle(std::move(*this));
148 ASSERT(mHandle == nullptr);
149 }
150 else
151 {
152 mHandle = nullptr;
153 }
154 }
155
destroy(VkDevice device)156 void RefCountedEvent::destroy(VkDevice device)
157 {
158 ASSERT(mHandle != nullptr);
159 ASSERT(!mHandle->isReferenced());
160 mHandle->get().event.destroy(device);
161 SafeDelete(mHandle);
162 }
163
164 // RefCountedEventsGarbage implementation.
destroy(Renderer * renderer)165 void RefCountedEventsGarbage::destroy(Renderer *renderer)
166 {
167 ASSERT(renderer->hasQueueSerialFinished(mQueueSerial));
168 while (!mRefCountedEvents.empty())
169 {
170 ASSERT(mRefCountedEvents.back().valid());
171 mRefCountedEvents.back().release(renderer);
172 mRefCountedEvents.pop_back();
173 }
174 }
175
releaseIfComplete(Renderer * renderer,RefCountedEventsGarbageRecycler * recycler)176 bool RefCountedEventsGarbage::releaseIfComplete(Renderer *renderer,
177 RefCountedEventsGarbageRecycler *recycler)
178 {
179 if (!renderer->hasQueueSerialFinished(mQueueSerial))
180 {
181 return false;
182 }
183
184 while (!mRefCountedEvents.empty())
185 {
186 ASSERT(mRefCountedEvents.back().valid());
187 mRefCountedEvents.back().releaseImpl(renderer, recycler);
188 ASSERT(!mRefCountedEvents.back().valid());
189 mRefCountedEvents.pop_back();
190 }
191 return true;
192 }
193
moveIfComplete(Renderer * renderer,std::deque<RefCountedEventCollector> * releasedBucket)194 bool RefCountedEventsGarbage::moveIfComplete(Renderer *renderer,
195 std::deque<RefCountedEventCollector> *releasedBucket)
196 {
197 if (!renderer->hasQueueSerialFinished(mQueueSerial))
198 {
199 return false;
200 }
201
202 releasedBucket->emplace_back(std::move(mRefCountedEvents));
203 return true;
204 }
205
206 // RefCountedEventRecycler implementation.
destroy(VkDevice device)207 void RefCountedEventRecycler::destroy(VkDevice device)
208 {
209 std::lock_guard<angle::SimpleMutex> lock(mMutex);
210
211 while (!mEventsToReset.empty())
212 {
213 DestroyRefCountedEvents(device, mEventsToReset.back());
214 mEventsToReset.pop_back();
215 }
216
217 ASSERT(mResettingQueue.empty());
218
219 while (!mEventsToReuse.empty())
220 {
221 DestroyRefCountedEvents(device, mEventsToReuse.back());
222 mEventsToReuse.pop_back();
223 }
224 }
225
resetEvents(Context * context,const QueueSerial queueSerial,PrimaryCommandBuffer * commandbuffer)226 void RefCountedEventRecycler::resetEvents(Context *context,
227 const QueueSerial queueSerial,
228 PrimaryCommandBuffer *commandbuffer)
229 {
230 std::lock_guard<angle::SimpleMutex> lock(mMutex);
231
232 if (mEventsToReset.empty())
233 {
234 return;
235 }
236
237 Renderer *renderer = context->getRenderer();
238 while (!mEventsToReset.empty())
239 {
240 RefCountedEventCollector &events = mEventsToReset.back();
241 ASSERT(!events.empty());
242 for (const RefCountedEvent &refCountedEvent : events)
243 {
244 VkPipelineStageFlags stageMask = renderer->getEventPipelineStageMask(refCountedEvent);
245 commandbuffer->resetEvent(refCountedEvent.getEvent().getHandle(), stageMask);
246 }
247 mResettingQueue.emplace(queueSerial, std::move(events));
248 mEventsToReset.pop_back();
249 }
250 }
251
cleanupResettingEvents(Renderer * renderer)252 size_t RefCountedEventRecycler::cleanupResettingEvents(Renderer *renderer)
253 {
254 size_t eventsReleased = 0;
255 std::lock_guard<angle::SimpleMutex> lock(mMutex);
256 while (!mResettingQueue.empty())
257 {
258 bool released = mResettingQueue.front().moveIfComplete(renderer, &mEventsToReuse);
259 if (released)
260 {
261 mResettingQueue.pop();
262 ++eventsReleased;
263 }
264 else
265 {
266 break;
267 }
268 }
269 return eventsReleased;
270 }
271
fetchEventsToReuse(RefCountedEventCollector * eventsToReuseOut)272 bool RefCountedEventRecycler::fetchEventsToReuse(RefCountedEventCollector *eventsToReuseOut)
273 {
274 ASSERT(eventsToReuseOut != nullptr);
275 ASSERT(eventsToReuseOut->empty());
276 std::lock_guard<angle::SimpleMutex> lock(mMutex);
277 if (mEventsToReuse.empty())
278 {
279 return false;
280 }
281 eventsToReuseOut->swap(mEventsToReuse.back());
282 mEventsToReuse.pop_back();
283 return true;
284 }
285
286 // RefCountedEventsGarbageRecycler implementation.
~RefCountedEventsGarbageRecycler()287 RefCountedEventsGarbageRecycler::~RefCountedEventsGarbageRecycler()
288 {
289 ASSERT(mEventsToReset.empty());
290 ASSERT(mGarbageQueue.empty());
291 ASSERT(mEventsToReuse.empty());
292 ASSERT(mGarbageCount == 0);
293 }
294
destroy(Renderer * renderer)295 void RefCountedEventsGarbageRecycler::destroy(Renderer *renderer)
296 {
297 VkDevice device = renderer->getDevice();
298 DestroyRefCountedEvents(device, mEventsToReset);
299 ASSERT(mGarbageQueue.empty());
300 ASSERT(mGarbageCount == 0);
301 mEventsToReuse.destroy(device);
302 }
303
cleanup(Renderer * renderer)304 void RefCountedEventsGarbageRecycler::cleanup(Renderer *renderer)
305 {
306 // First cleanup already completed events and add to mEventsToReset
307 while (!mGarbageQueue.empty())
308 {
309 size_t count = mGarbageQueue.front().size();
310 bool released = mGarbageQueue.front().releaseIfComplete(renderer, this);
311 if (released)
312 {
313 mGarbageCount -= count;
314 mGarbageQueue.pop();
315 }
316 else
317 {
318 break;
319 }
320 }
321
322 // Move mEventsToReset to the renderer so that it can be reset.
323 if (!mEventsToReset.empty())
324 {
325 renderer->getRefCountedEventRecycler()->recycle(std::move(mEventsToReset));
326 }
327 }
328
fetch(Renderer * renderer,RefCountedEvent * outObject)329 bool RefCountedEventsGarbageRecycler::fetch(Renderer *renderer, RefCountedEvent *outObject)
330 {
331 if (mEventsToReuse.empty())
332 {
333 // Retrieve a list of ready to reuse events from renderer.
334 RefCountedEventCollector events;
335 if (!renderer->getRefCountedEventRecycler()->fetchEventsToReuse(&events))
336 {
337 return false;
338 }
339 mEventsToReuse.refill(std::move(events));
340 ASSERT(!mEventsToReuse.empty());
341 }
342 mEventsToReuse.fetch(outObject);
343 return true;
344 }
345
346 // EventBarrier implementation.
addDiagnosticsString(std::ostringstream & out) const347 void EventBarrier::addDiagnosticsString(std::ostringstream &out) const
348 {
349 if (mMemoryBarrierSrcAccess != 0 || mMemoryBarrierDstAccess != 0)
350 {
351 out << "Src: 0x" << std::hex << mMemoryBarrierSrcAccess << " → Dst: 0x" << std::hex
352 << mMemoryBarrierDstAccess << std::endl;
353 }
354 }
355
execute(PrimaryCommandBuffer * primary)356 void EventBarrier::execute(PrimaryCommandBuffer *primary)
357 {
358 if (isEmpty())
359 {
360 return;
361 }
362 ASSERT(mEvent != VK_NULL_HANDLE);
363 ASSERT(mImageMemoryBarrierCount == 0 ||
364 (mImageMemoryBarrierCount == 1 && mImageMemoryBarrier.image != VK_NULL_HANDLE));
365
366 // Issue vkCmdWaitEvents call
367 VkMemoryBarrier memoryBarrier = {};
368 memoryBarrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
369 memoryBarrier.srcAccessMask = mMemoryBarrierSrcAccess;
370 memoryBarrier.dstAccessMask = mMemoryBarrierDstAccess;
371
372 primary->waitEvents(1, &mEvent, mSrcStageMask, mDstStageMask, 1, &memoryBarrier, 0, nullptr,
373 mImageMemoryBarrierCount,
374 mImageMemoryBarrierCount == 0 ? nullptr : &mImageMemoryBarrier);
375 }
376
377 // EventBarrierArray implementation.
addAdditionalStageAccess(const RefCountedEvent & waitEvent,VkPipelineStageFlags dstStageMask,VkAccessFlags dstAccess)378 void EventBarrierArray::addAdditionalStageAccess(const RefCountedEvent &waitEvent,
379 VkPipelineStageFlags dstStageMask,
380 VkAccessFlags dstAccess)
381 {
382 for (EventBarrier &barrier : mBarriers)
383 {
384 if (barrier.hasEvent(waitEvent.getEvent().getHandle()))
385 {
386 barrier.addAdditionalStageAccess(dstStageMask, dstAccess);
387 return;
388 }
389 }
390 UNREACHABLE();
391 }
392
addMemoryEvent(Renderer * renderer,const RefCountedEvent & waitEvent,VkPipelineStageFlags dstStageMask,VkAccessFlags dstAccess)393 void EventBarrierArray::addMemoryEvent(Renderer *renderer,
394 const RefCountedEvent &waitEvent,
395 VkPipelineStageFlags dstStageMask,
396 VkAccessFlags dstAccess)
397 {
398 ASSERT(waitEvent.valid());
399 VkPipelineStageFlags stageFlags = renderer->getEventPipelineStageMask(waitEvent);
400 // This should come down as WAW without layout change, dstStageMask should be the same as
401 // event's stageMask. Otherwise you should get into addImageEvent.
402 ASSERT(stageFlags == dstStageMask);
403 mBarriers.emplace_back(stageFlags, dstStageMask, dstAccess, dstAccess,
404 waitEvent.getEvent().getHandle());
405 }
406
addImageEvent(Renderer * renderer,const RefCountedEvent & waitEvent,VkPipelineStageFlags dstStageMask,const VkImageMemoryBarrier & imageMemoryBarrier)407 void EventBarrierArray::addImageEvent(Renderer *renderer,
408 const RefCountedEvent &waitEvent,
409 VkPipelineStageFlags dstStageMask,
410 const VkImageMemoryBarrier &imageMemoryBarrier)
411 {
412 ASSERT(waitEvent.valid());
413 VkPipelineStageFlags srcStageFlags = renderer->getEventPipelineStageMask(waitEvent);
414 mBarriers.emplace_back(srcStageFlags, dstStageMask, waitEvent.getEvent().getHandle(),
415 imageMemoryBarrier);
416 }
417
execute(Renderer * renderer,PrimaryCommandBuffer * primary)418 void EventBarrierArray::execute(Renderer *renderer, PrimaryCommandBuffer *primary)
419 {
420 while (!mBarriers.empty())
421 {
422 mBarriers.back().execute(primary);
423 mBarriers.pop_back();
424 }
425 reset();
426 }
427
addDiagnosticsString(std::ostringstream & out) const428 void EventBarrierArray::addDiagnosticsString(std::ostringstream &out) const
429 {
430 out << "Event Barrier: ";
431 for (const EventBarrier &barrier : mBarriers)
432 {
433 barrier.addDiagnosticsString(out);
434 }
435 out << "\\l";
436 }
437 } // namespace vk
438 } // namespace rx
439