1 //
2 // Copyright 2022 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // Suballocation.h:
7 // Defines class interface for BufferBlock and Suballocation and other related classes.
8 //
9
10 #ifndef LIBANGLE_RENDERER_VULKAN_SUBALLOCATION_H_
11 #define LIBANGLE_RENDERER_VULKAN_SUBALLOCATION_H_
12
13 #include "common/SimpleMutex.h"
14 #include "common/debug.h"
15 #include "libANGLE/angletypes.h"
16 #include "libANGLE/renderer/serial_utils.h"
17 #include "libANGLE/renderer/vulkan/vk_cache_utils.h"
18 #include "libANGLE/renderer/vulkan/vk_resource.h"
19 #include "libANGLE/renderer/vulkan/vk_utils.h"
20 #include "libANGLE/renderer/vulkan/vk_wrapper.h"
21
22 namespace rx
23 {
24 enum class MemoryAllocationType;
25
26 namespace vk
27 {
28 class Context;
29
30 // BufferBlock
31 class BufferBlock final : angle::NonCopyable
32 {
33 public:
34 BufferBlock();
35 BufferBlock(BufferBlock &&other);
36 ~BufferBlock();
37
38 void destroy(Renderer *renderer);
39 VkResult init(Context *context,
40 Buffer &buffer,
41 uint32_t memoryTypeIndex,
42 vma::VirtualBlockCreateFlags flags,
43 DeviceMemory &deviceMemory,
44 VkMemoryPropertyFlags memoryPropertyFlags,
45 VkDeviceSize size);
46 void initWithoutVirtualBlock(Context *context,
47 Buffer &buffer,
48 MemoryAllocationType memoryAllocationType,
49 uint32_t memoryTypeIndex,
50 DeviceMemory &deviceMemory,
51 VkMemoryPropertyFlags memoryPropertyFlags,
52 VkDeviceSize size,
53 VkDeviceSize allocatedBufferSize);
54
55 BufferBlock &operator=(BufferBlock &&other);
56
getBuffer()57 const Buffer &getBuffer() const { return mBuffer; }
getDeviceMemory()58 const DeviceMemory &getDeviceMemory() const { return mDeviceMemory; }
getDeviceMemory()59 DeviceMemory &getDeviceMemory() { return mDeviceMemory; }
getBufferSerial()60 BufferSerial getBufferSerial() const { return mSerial; }
61
62 VkMemoryPropertyFlags getMemoryPropertyFlags() const;
63 VkDeviceSize getMemorySize() const;
64
65 VkResult allocate(VkDeviceSize size,
66 VkDeviceSize alignment,
67 VmaVirtualAllocation *allocationOut,
68 VkDeviceSize *offsetOut);
69 void free(VmaVirtualAllocation allocation, VkDeviceSize offset);
70 VkBool32 isEmpty();
71
hasVirtualBlock()72 bool hasVirtualBlock() const { return mVirtualBlock.valid(); }
73 bool isHostVisible() const;
74 bool isCoherent() const;
75 bool isCached() const;
76 bool isMapped() const;
77 VkResult map(const VkDevice device);
78 void unmap(const VkDevice device);
79 uint8_t *getMappedMemory() const;
80
81 // This should be called whenever this found to be empty. The total number of count of empty is
82 // returned.
83 int32_t getAndIncrementEmptyCounter();
84 void calculateStats(vma::StatInfo *pStatInfo) const;
85
onNewDescriptorSet(const SharedDescriptorSetCacheKey & sharedCacheKey)86 void onNewDescriptorSet(const SharedDescriptorSetCacheKey &sharedCacheKey)
87 {
88 mDescriptorSetCacheManager.addKey(sharedCacheKey);
89 }
releaseAllCachedDescriptorSetCacheKeys(Renderer * renderer)90 void releaseAllCachedDescriptorSetCacheKeys(Renderer *renderer)
91 {
92 if (!mDescriptorSetCacheManager.empty())
93 {
94 mDescriptorSetCacheManager.releaseKeys(renderer);
95 }
96 }
97
98 private:
99 mutable angle::SimpleMutex mVirtualBlockMutex;
100 VirtualBlock mVirtualBlock;
101
102 Buffer mBuffer;
103 DeviceMemory mDeviceMemory;
104 VkMemoryPropertyFlags mMemoryPropertyFlags;
105
106 // Memory size that user of this object thinks we have.
107 VkDeviceSize mSize;
108 // Memory size that was actually allocated for this object.
109 VkDeviceSize mAllocatedBufferSize;
110 // Memory allocation type used for this object.
111 MemoryAllocationType mMemoryAllocationType;
112 // Memory type index used for the allocation. It can be used to determine the heap index.
113 uint32_t mMemoryTypeIndex;
114
115 uint8_t *mMappedMemory;
116 BufferSerial mSerial;
117 // Heuristic information for pruneEmptyBuffer. This tracks how many times (consecutively) this
118 // buffer block is found to be empty when pruneEmptyBuffer is called. This gets reset whenever
119 // it becomes non-empty.
120 int32_t mCountRemainsEmpty;
121 // Manages the descriptorSet cache that created with this BufferBlock.
122 DescriptorSetCacheManager mDescriptorSetCacheManager;
123 };
124 using BufferBlockPointer = std::unique_ptr<BufferBlock>;
125 using BufferBlockPointerVector = std::vector<BufferBlockPointer>;
126
127 class BufferBlockGarbageList final : angle::NonCopyable
128 {
129 public:
BufferBlockGarbageList()130 BufferBlockGarbageList() : mBufferBlockQueue(kInitialQueueCapacity) {}
~BufferBlockGarbageList()131 ~BufferBlockGarbageList() { ASSERT(mBufferBlockQueue.empty()); }
132
add(BufferBlock * bufferBlock)133 void add(BufferBlock *bufferBlock)
134 {
135 std::unique_lock<angle::SimpleMutex> lock(mMutex);
136 if (mBufferBlockQueue.full())
137 {
138 size_t newCapacity = mBufferBlockQueue.capacity() << 1;
139 mBufferBlockQueue.updateCapacity(newCapacity);
140 }
141 mBufferBlockQueue.push(bufferBlock);
142 }
143
144 // Number of buffer blocks destroyed is returned.
pruneEmptyBufferBlocks(Renderer * renderer)145 size_t pruneEmptyBufferBlocks(Renderer *renderer)
146 {
147 size_t blocksDestroyed = 0;
148 if (!mBufferBlockQueue.empty())
149 {
150 std::unique_lock<angle::SimpleMutex> lock(mMutex);
151 size_t count = mBufferBlockQueue.size();
152 for (size_t i = 0; i < count; i++)
153 {
154 BufferBlock *block = mBufferBlockQueue.front();
155 mBufferBlockQueue.pop();
156 if (block->isEmpty())
157 {
158 block->destroy(renderer);
159 ++blocksDestroyed;
160 }
161 else
162 {
163 mBufferBlockQueue.push(block);
164 }
165 }
166 }
167 return blocksDestroyed;
168 }
169
empty()170 bool empty() const { return mBufferBlockQueue.empty(); }
171
172 private:
173 static constexpr size_t kInitialQueueCapacity = 4;
174 angle::SimpleMutex mMutex;
175 angle::FixedQueue<BufferBlock *> mBufferBlockQueue;
176 };
177
178 // BufferSuballocation
179 class BufferSuballocation final : angle::NonCopyable
180 {
181 public:
182 BufferSuballocation();
183
184 BufferSuballocation(BufferSuballocation &&other);
185 BufferSuballocation &operator=(BufferSuballocation &&other);
186
187 void destroy(Renderer *renderer);
188
189 void init(BufferBlock *block,
190 VmaVirtualAllocation allocation,
191 VkDeviceSize offset,
192 VkDeviceSize size);
193 void initWithEntireBuffer(Context *context,
194 Buffer &buffer,
195 MemoryAllocationType memoryAllocationType,
196 uint32_t memoryTypeIndex,
197 DeviceMemory &deviceMemory,
198 VkMemoryPropertyFlags memoryPropertyFlags,
199 VkDeviceSize size,
200 VkDeviceSize allocatedBufferSize);
201
202 const Buffer &getBuffer() const;
203 VkDeviceSize getSize() const;
204 const DeviceMemory &getDeviceMemory() const;
205 VkMemoryMapFlags getMemoryPropertyFlags() const;
206 bool isHostVisible() const;
207 bool isCoherent() const;
208 bool isCached() const;
209 bool isMapped() const;
210 uint8_t *getMappedMemory() const;
211 void flush(const VkDevice &device);
212 void invalidate(const VkDevice &device);
213 VkDeviceSize getOffset() const;
214 bool valid() const;
215 VkResult map(Context *context);
216 BufferSerial getBlockSerial() const;
217 uint8_t *getBlockMemory() const;
218 VkDeviceSize getBlockMemorySize() const;
isSuballocated()219 bool isSuballocated() const { return mBufferBlock->hasVirtualBlock(); }
getBufferBlock()220 BufferBlock *getBufferBlock() const { return mBufferBlock; }
221
222 private:
223 // Only used by DynamicBuffer where DynamicBuffer does the actual suballocation and pass the
224 // offset/size to this object. Since DynamicBuffer does not have a VMA virtual allocator, they
225 // will be ignored at destroy time. The offset/size is set here mainly for easy retrieval when
226 // the BufferHelper object is passed around.
227 friend class BufferHelper;
228 void setOffsetAndSize(VkDeviceSize offset, VkDeviceSize size);
229
230 BufferBlock *mBufferBlock;
231 VmaVirtualAllocation mAllocation;
232 VkDeviceSize mOffset;
233 VkDeviceSize mSize;
234 };
235
236 class BufferSuballocationGarbage
237 {
238 public:
239 BufferSuballocationGarbage() = default;
BufferSuballocationGarbage(BufferSuballocationGarbage && other)240 BufferSuballocationGarbage(BufferSuballocationGarbage &&other)
241 : mLifetime(other.mLifetime),
242 mSuballocation(std::move(other.mSuballocation)),
243 mBuffer(std::move(other.mBuffer))
244 {}
245 BufferSuballocationGarbage &operator=(BufferSuballocationGarbage &&other)
246 {
247 mLifetime = other.mLifetime;
248 mSuballocation = std::move(other.mSuballocation);
249 mBuffer = std::move(other.mBuffer);
250 return *this;
251 }
BufferSuballocationGarbage(const ResourceUse & use,BufferSuballocation && suballocation,Buffer && buffer)252 BufferSuballocationGarbage(const ResourceUse &use,
253 BufferSuballocation &&suballocation,
254 Buffer &&buffer)
255 : mLifetime(use), mSuballocation(std::move(suballocation)), mBuffer(std::move(buffer))
256 {}
257 ~BufferSuballocationGarbage() = default;
258
259 bool destroyIfComplete(Renderer *renderer);
260 bool hasResourceUseSubmitted(Renderer *renderer) const;
getSize()261 VkDeviceSize getSize() const { return mSuballocation.getSize(); }
isSuballocated()262 bool isSuballocated() const { return mSuballocation.isSuballocated(); }
263
264 private:
265 ResourceUse mLifetime;
266 BufferSuballocation mSuballocation;
267 Buffer mBuffer;
268 };
269
270 // BufferBlock implementation.
getMemoryPropertyFlags()271 ANGLE_INLINE VkMemoryPropertyFlags BufferBlock::getMemoryPropertyFlags() const
272 {
273 return mMemoryPropertyFlags;
274 }
275
getMemorySize()276 ANGLE_INLINE VkDeviceSize BufferBlock::getMemorySize() const
277 {
278 return mSize;
279 }
280
isEmpty()281 ANGLE_INLINE VkBool32 BufferBlock::isEmpty()
282 {
283 std::unique_lock<angle::SimpleMutex> lock(mVirtualBlockMutex);
284 return vma::IsVirtualBlockEmpty(mVirtualBlock.getHandle());
285 }
286
isHostVisible()287 ANGLE_INLINE bool BufferBlock::isHostVisible() const
288 {
289 return (mMemoryPropertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0;
290 }
291
isCoherent()292 ANGLE_INLINE bool BufferBlock::isCoherent() const
293 {
294 return (mMemoryPropertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0;
295 }
296
isCached()297 ANGLE_INLINE bool BufferBlock::isCached() const
298 {
299 return (mMemoryPropertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0;
300 }
301
isMapped()302 ANGLE_INLINE bool BufferBlock::isMapped() const
303 {
304 return mMappedMemory != nullptr;
305 }
306
getMappedMemory()307 ANGLE_INLINE uint8_t *BufferBlock::getMappedMemory() const
308 {
309 ASSERT(mMappedMemory != nullptr);
310 return mMappedMemory;
311 }
312
313 // BufferSuballocation implementation.
BufferSuballocation()314 ANGLE_INLINE BufferSuballocation::BufferSuballocation()
315 : mBufferBlock(nullptr), mAllocation(VK_NULL_HANDLE), mOffset(0), mSize(0)
316 {}
317
BufferSuballocation(BufferSuballocation && other)318 ANGLE_INLINE BufferSuballocation::BufferSuballocation(BufferSuballocation &&other)
319 : BufferSuballocation()
320 {
321 *this = std::move(other);
322 }
323
324 ANGLE_INLINE BufferSuballocation &BufferSuballocation::operator=(BufferSuballocation &&other)
325 {
326 std::swap(mBufferBlock, other.mBufferBlock);
327 std::swap(mSize, other.mSize);
328 std::swap(mAllocation, other.mAllocation);
329 std::swap(mOffset, other.mOffset);
330 return *this;
331 }
332
valid()333 ANGLE_INLINE bool BufferSuballocation::valid() const
334 {
335 return mBufferBlock != nullptr;
336 }
337
destroy(Renderer * renderer)338 ANGLE_INLINE void BufferSuballocation::destroy(Renderer *renderer)
339 {
340 if (valid())
341 {
342 ASSERT(mBufferBlock);
343 if (mBufferBlock->hasVirtualBlock())
344 {
345 mBufferBlock->free(mAllocation, mOffset);
346 mBufferBlock = nullptr;
347 }
348 else
349 {
350 // When virtual block is invalid, this is the standalone buffer that are created by
351 // BufferSuballocation::initWithEntireBuffer call. In this case, vmaBufferSuballocation
352 // owns block, we must properly delete the block object.
353 mBufferBlock->destroy(renderer);
354 SafeDelete(mBufferBlock);
355 }
356 mAllocation = VK_NULL_HANDLE;
357 mOffset = 0;
358 mSize = 0;
359 }
360 }
361
init(BufferBlock * block,VmaVirtualAllocation allocation,VkDeviceSize offset,VkDeviceSize size)362 ANGLE_INLINE void BufferSuballocation::init(BufferBlock *block,
363 VmaVirtualAllocation allocation,
364 VkDeviceSize offset,
365 VkDeviceSize size)
366 {
367 ASSERT(!valid());
368 ASSERT(block != nullptr);
369 ASSERT(allocation != VK_NULL_HANDLE);
370 ASSERT(offset != VK_WHOLE_SIZE);
371 mBufferBlock = block;
372 mAllocation = allocation;
373 mOffset = offset;
374 mSize = size;
375 }
376
initWithEntireBuffer(Context * context,Buffer & buffer,MemoryAllocationType memoryAllocationType,uint32_t memoryTypeIndex,DeviceMemory & deviceMemory,VkMemoryPropertyFlags memoryPropertyFlags,VkDeviceSize size,VkDeviceSize allocatedBufferSize)377 ANGLE_INLINE void BufferSuballocation::initWithEntireBuffer(
378 Context *context,
379 Buffer &buffer,
380 MemoryAllocationType memoryAllocationType,
381 uint32_t memoryTypeIndex,
382 DeviceMemory &deviceMemory,
383 VkMemoryPropertyFlags memoryPropertyFlags,
384 VkDeviceSize size,
385 VkDeviceSize allocatedBufferSize)
386 {
387 ASSERT(!valid());
388
389 std::unique_ptr<BufferBlock> block = std::make_unique<BufferBlock>();
390 block->initWithoutVirtualBlock(context, buffer, memoryAllocationType, memoryTypeIndex,
391 deviceMemory, memoryPropertyFlags, size, allocatedBufferSize);
392
393 mBufferBlock = block.release();
394 mAllocation = VK_NULL_HANDLE;
395 mOffset = 0;
396 mSize = mBufferBlock->getMemorySize();
397 }
398
getBuffer()399 ANGLE_INLINE const Buffer &BufferSuballocation::getBuffer() const
400 {
401 return mBufferBlock->getBuffer();
402 }
403
getSize()404 ANGLE_INLINE VkDeviceSize BufferSuballocation::getSize() const
405 {
406 return mSize;
407 }
408
getDeviceMemory()409 ANGLE_INLINE const DeviceMemory &BufferSuballocation::getDeviceMemory() const
410 {
411 return mBufferBlock->getDeviceMemory();
412 }
413
getMemoryPropertyFlags()414 ANGLE_INLINE VkMemoryMapFlags BufferSuballocation::getMemoryPropertyFlags() const
415 {
416 return mBufferBlock->getMemoryPropertyFlags();
417 }
418
isHostVisible()419 ANGLE_INLINE bool BufferSuballocation::isHostVisible() const
420 {
421 return mBufferBlock->isHostVisible();
422 }
isCoherent()423 ANGLE_INLINE bool BufferSuballocation::isCoherent() const
424 {
425 return mBufferBlock->isCoherent();
426 }
isCached()427 ANGLE_INLINE bool BufferSuballocation::isCached() const
428 {
429 return mBufferBlock->isCached();
430 }
isMapped()431 ANGLE_INLINE bool BufferSuballocation::isMapped() const
432 {
433 return mBufferBlock->isMapped();
434 }
getMappedMemory()435 ANGLE_INLINE uint8_t *BufferSuballocation::getMappedMemory() const
436 {
437 return mBufferBlock->getMappedMemory() + getOffset();
438 }
439
flush(const VkDevice & device)440 ANGLE_INLINE void BufferSuballocation::flush(const VkDevice &device)
441 {
442 if (!isCoherent())
443 {
444 VkMappedMemoryRange mappedRange = {};
445 mappedRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
446 mappedRange.memory = mBufferBlock->getDeviceMemory().getHandle();
447 mappedRange.offset = getOffset();
448 mappedRange.size = mSize;
449 mBufferBlock->getDeviceMemory().flush(device, mappedRange);
450 }
451 }
452
invalidate(const VkDevice & device)453 ANGLE_INLINE void BufferSuballocation::invalidate(const VkDevice &device)
454 {
455 if (!isCoherent())
456 {
457 VkMappedMemoryRange mappedRange = {};
458 mappedRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE;
459 mappedRange.memory = mBufferBlock->getDeviceMemory().getHandle();
460 mappedRange.offset = getOffset();
461 mappedRange.size = mSize;
462 mBufferBlock->getDeviceMemory().invalidate(device, mappedRange);
463 }
464 }
465
getOffset()466 ANGLE_INLINE VkDeviceSize BufferSuballocation::getOffset() const
467 {
468 return mOffset;
469 }
470
setOffsetAndSize(VkDeviceSize offset,VkDeviceSize size)471 ANGLE_INLINE void BufferSuballocation::setOffsetAndSize(VkDeviceSize offset, VkDeviceSize size)
472 {
473 mOffset = offset;
474 mSize = size;
475 }
476
getBlockMemory()477 ANGLE_INLINE uint8_t *BufferSuballocation::getBlockMemory() const
478 {
479 return mBufferBlock->getMappedMemory();
480 }
getBlockMemorySize()481 ANGLE_INLINE VkDeviceSize BufferSuballocation::getBlockMemorySize() const
482 {
483 return mBufferBlock->getMemorySize();
484 }
getBlockSerial()485 ANGLE_INLINE BufferSerial BufferSuballocation::getBlockSerial() const
486 {
487 ASSERT(valid());
488 return mBufferBlock->getBufferSerial();
489 }
490 } // namespace vk
491 } // namespace rx
492
493 #endif // LIBANGLE_RENDERER_VULKAN_SUBALLOCATION_H_
494