1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * SPDX-License-Identifier: MIT
9 */
10
11 #include "radv_device_memory.h"
12 #include "radv_android.h"
13 #include "radv_buffer.h"
14 #include "radv_entrypoints.h"
15 #include "radv_image.h"
16 #include "radv_rmv.h"
17
18 #include "vk_log.h"
19
20 void
radv_device_memory_init(struct radv_device_memory * mem,struct radv_device * device,struct radeon_winsys_bo * bo)21 radv_device_memory_init(struct radv_device_memory *mem, struct radv_device *device, struct radeon_winsys_bo *bo)
22 {
23 memset(mem, 0, sizeof(*mem));
24 vk_object_base_init(&device->vk, &mem->base, VK_OBJECT_TYPE_DEVICE_MEMORY);
25
26 mem->bo = bo;
27 }
28
29 void
radv_device_memory_finish(struct radv_device_memory * mem)30 radv_device_memory_finish(struct radv_device_memory *mem)
31 {
32 vk_object_base_finish(&mem->base);
33 }
34
35 void
radv_free_memory(struct radv_device * device,const VkAllocationCallbacks * pAllocator,struct radv_device_memory * mem)36 radv_free_memory(struct radv_device *device, const VkAllocationCallbacks *pAllocator, struct radv_device_memory *mem)
37 {
38 if (mem == NULL)
39 return;
40
41 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
42 if (mem->android_hardware_buffer)
43 AHardwareBuffer_release(mem->android_hardware_buffer);
44 #endif
45
46 if (mem->bo) {
47 if (device->overallocation_disallowed) {
48 mtx_lock(&device->overallocation_mutex);
49 device->allocated_memory_size[mem->heap_index] -= mem->alloc_size;
50 mtx_unlock(&device->overallocation_mutex);
51 }
52
53 if (device->use_global_bo_list)
54 device->ws->buffer_make_resident(device->ws, mem->bo, false);
55 radv_bo_destroy(device, &mem->base, mem->bo);
56 mem->bo = NULL;
57 }
58
59 radv_rmv_log_resource_destroy(device, (uint64_t)radv_device_memory_to_handle(mem));
60 radv_device_memory_finish(mem);
61 vk_free2(&device->vk.alloc, pAllocator, mem);
62 }
63
64 VkResult
radv_alloc_memory(struct radv_device * device,const VkMemoryAllocateInfo * pAllocateInfo,const VkAllocationCallbacks * pAllocator,VkDeviceMemory * pMem,bool is_internal)65 radv_alloc_memory(struct radv_device *device, const VkMemoryAllocateInfo *pAllocateInfo,
66 const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMem, bool is_internal)
67 {
68 struct radv_device_memory *mem;
69 VkResult result;
70 enum radeon_bo_domain domain;
71 uint32_t flags = 0;
72
73 assert(pAllocateInfo->sType == VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO);
74
75 const VkImportMemoryFdInfoKHR *import_info = vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_FD_INFO_KHR);
76 const VkMemoryDedicatedAllocateInfo *dedicate_info =
77 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_DEDICATED_ALLOCATE_INFO);
78 const VkExportMemoryAllocateInfo *export_info =
79 vk_find_struct_const(pAllocateInfo->pNext, EXPORT_MEMORY_ALLOCATE_INFO);
80 const struct VkImportAndroidHardwareBufferInfoANDROID *ahb_import_info =
81 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID);
82 const VkImportMemoryHostPointerInfoEXT *host_ptr_info =
83 vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_HOST_POINTER_INFO_EXT);
84 const struct VkMemoryAllocateFlagsInfo *flags_info =
85 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_ALLOCATE_FLAGS_INFO);
86
87 const struct wsi_memory_allocate_info *wsi_info =
88 vk_find_struct_const(pAllocateInfo->pNext, WSI_MEMORY_ALLOCATE_INFO_MESA);
89
90 if (pAllocateInfo->allocationSize == 0 && !ahb_import_info &&
91 !(export_info &&
92 (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID))) {
93 /* Apparently, this is allowed */
94 *pMem = VK_NULL_HANDLE;
95 return VK_SUCCESS;
96 }
97
98 mem = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*mem), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
99 if (mem == NULL)
100 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
101
102 radv_device_memory_init(mem, device, NULL);
103
104 if (dedicate_info) {
105 mem->image = radv_image_from_handle(dedicate_info->image);
106 mem->buffer = radv_buffer_from_handle(dedicate_info->buffer);
107 } else {
108 mem->image = NULL;
109 mem->buffer = NULL;
110 }
111
112 if (wsi_info && wsi_info->implicit_sync) {
113 flags |= RADEON_FLAG_IMPLICIT_SYNC;
114
115 /* Mark the linear prime buffer (aka the destination of the prime blit
116 * as uncached.
117 */
118 if (mem->buffer)
119 flags |= RADEON_FLAG_VA_UNCACHED;
120 }
121
122 float priority_float = 0.5;
123 const struct VkMemoryPriorityAllocateInfoEXT *priority_ext =
124 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_PRIORITY_ALLOCATE_INFO_EXT);
125 if (priority_ext)
126 priority_float = priority_ext->priority;
127
128 uint64_t replay_address = 0;
129 const VkMemoryOpaqueCaptureAddressAllocateInfo *replay_info =
130 vk_find_struct_const(pAllocateInfo->pNext, MEMORY_OPAQUE_CAPTURE_ADDRESS_ALLOCATE_INFO);
131 if (replay_info && replay_info->opaqueCaptureAddress)
132 replay_address = replay_info->opaqueCaptureAddress;
133
134 unsigned priority =
135 MIN2(RADV_BO_PRIORITY_APPLICATION_MAX - 1, (int)(priority_float * RADV_BO_PRIORITY_APPLICATION_MAX));
136
137 mem->user_ptr = NULL;
138
139 #if RADV_SUPPORT_ANDROID_HARDWARE_BUFFER
140 mem->android_hardware_buffer = NULL;
141 #endif
142
143 if (ahb_import_info) {
144 result = radv_import_ahb_memory(device, mem, priority, ahb_import_info);
145 if (result != VK_SUCCESS)
146 goto fail;
147 } else if (export_info &&
148 (export_info->handleTypes & VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID)) {
149 result = radv_create_ahb_memory(device, mem, priority, pAllocateInfo);
150 if (result != VK_SUCCESS)
151 goto fail;
152 } else if (import_info) {
153 assert(import_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
154 import_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
155 result = device->ws->buffer_from_fd(device->ws, import_info->fd, priority, &mem->bo, NULL);
156 if (result != VK_SUCCESS) {
157 goto fail;
158 } else {
159 close(import_info->fd);
160 }
161
162 if (mem->image && mem->image->plane_count == 1 && !vk_format_is_depth_or_stencil(mem->image->vk.format) &&
163 mem->image->vk.samples == 1 && mem->image->vk.tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
164 struct radeon_bo_metadata metadata;
165 device->ws->buffer_get_metadata(device->ws, mem->bo, &metadata);
166
167 struct radv_image_create_info create_info = {.no_metadata_planes = true, .bo_metadata = &metadata};
168
169 /* This gives a basic ability to import radeonsi images
170 * that don't have DCC. This is not guaranteed by any
171 * spec and can be removed after we support modifiers. */
172 result = radv_image_create_layout(device, create_info, NULL, NULL, mem->image);
173 if (result != VK_SUCCESS) {
174 radv_bo_destroy(device, &mem->base, mem->bo);
175 goto fail;
176 }
177 }
178 } else if (host_ptr_info) {
179 assert(host_ptr_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT);
180 result = device->ws->buffer_from_ptr(device->ws, host_ptr_info->pHostPointer, pAllocateInfo->allocationSize,
181 priority, &mem->bo);
182 if (result != VK_SUCCESS) {
183 goto fail;
184 } else {
185 mem->user_ptr = host_ptr_info->pHostPointer;
186 }
187 } else {
188 const struct radv_physical_device *pdev = radv_device_physical(device);
189 const struct radv_instance *instance = radv_physical_device_instance(pdev);
190 uint64_t alloc_size = align64(pAllocateInfo->allocationSize, 4096);
191 uint32_t heap_index;
192
193 heap_index = pdev->memory_properties.memoryTypes[pAllocateInfo->memoryTypeIndex].heapIndex;
194 domain = pdev->memory_domains[pAllocateInfo->memoryTypeIndex];
195 flags |= pdev->memory_flags[pAllocateInfo->memoryTypeIndex];
196
197 if (export_info && export_info->handleTypes) {
198 /* Setting RADEON_FLAG_GTT_WC in case the bo is spilled to GTT. This is important when the
199 * foreign queue is the display engine of iGPU. The carveout of iGPU can be tiny and the
200 * kernel driver refuses to spill without the flag.
201 *
202 * This covers any external memory user, including WSI.
203 */
204 if (domain == RADEON_DOMAIN_VRAM)
205 flags |= RADEON_FLAG_GTT_WC;
206 } else if (!import_info) {
207 /* neither export nor import */
208 flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING;
209 if (device->use_global_bo_list) {
210 flags |= RADEON_FLAG_PREFER_LOCAL_BO;
211 }
212 }
213
214 if (flags_info && flags_info->flags & VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT)
215 flags |= RADEON_FLAG_REPLAYABLE;
216
217 if (instance->drirc.zero_vram)
218 flags |= RADEON_FLAG_ZERO_VRAM;
219
220 if (device->overallocation_disallowed) {
221 uint64_t total_size = pdev->memory_properties.memoryHeaps[heap_index].size;
222
223 mtx_lock(&device->overallocation_mutex);
224 if (device->allocated_memory_size[heap_index] + alloc_size > total_size) {
225 mtx_unlock(&device->overallocation_mutex);
226 result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
227 goto fail;
228 }
229 device->allocated_memory_size[heap_index] += alloc_size;
230 mtx_unlock(&device->overallocation_mutex);
231 }
232
233 result = radv_bo_create(device, &mem->base, alloc_size, pdev->info.max_alignment, domain, flags, priority,
234 replay_address, is_internal, &mem->bo);
235
236 if (result != VK_SUCCESS) {
237 if (device->overallocation_disallowed) {
238 mtx_lock(&device->overallocation_mutex);
239 device->allocated_memory_size[heap_index] -= alloc_size;
240 mtx_unlock(&device->overallocation_mutex);
241 }
242 goto fail;
243 }
244
245 mem->heap_index = heap_index;
246 mem->alloc_size = alloc_size;
247 }
248
249 if (!wsi_info) {
250 if (device->use_global_bo_list) {
251 result = device->ws->buffer_make_resident(device->ws, mem->bo, true);
252 if (result != VK_SUCCESS)
253 goto fail;
254 }
255 }
256
257 *pMem = radv_device_memory_to_handle(mem);
258 radv_rmv_log_heap_create(device, *pMem, is_internal, flags_info ? flags_info->flags : 0);
259
260 return VK_SUCCESS;
261
262 fail:
263 radv_free_memory(device, pAllocator, mem);
264
265 return result;
266 }
267
268 VKAPI_ATTR VkResult VKAPI_CALL
radv_AllocateMemory(VkDevice _device,const VkMemoryAllocateInfo * pAllocateInfo,const VkAllocationCallbacks * pAllocator,VkDeviceMemory * pMem)269 radv_AllocateMemory(VkDevice _device, const VkMemoryAllocateInfo *pAllocateInfo,
270 const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMem)
271 {
272 VK_FROM_HANDLE(radv_device, device, _device);
273 return radv_alloc_memory(device, pAllocateInfo, pAllocator, pMem, false);
274 }
275
276 VKAPI_ATTR void VKAPI_CALL
radv_FreeMemory(VkDevice _device,VkDeviceMemory _mem,const VkAllocationCallbacks * pAllocator)277 radv_FreeMemory(VkDevice _device, VkDeviceMemory _mem, const VkAllocationCallbacks *pAllocator)
278 {
279 VK_FROM_HANDLE(radv_device, device, _device);
280 VK_FROM_HANDLE(radv_device_memory, mem, _mem);
281
282 radv_free_memory(device, pAllocator, mem);
283 }
284
285 VKAPI_ATTR VkResult VKAPI_CALL
radv_MapMemory2KHR(VkDevice _device,const VkMemoryMapInfoKHR * pMemoryMapInfo,void ** ppData)286 radv_MapMemory2KHR(VkDevice _device, const VkMemoryMapInfoKHR *pMemoryMapInfo, void **ppData)
287 {
288 VK_FROM_HANDLE(radv_device, device, _device);
289 VK_FROM_HANDLE(radv_device_memory, mem, pMemoryMapInfo->memory);
290 void *fixed_address = NULL;
291 bool use_fixed_address = false;
292
293 if (pMemoryMapInfo->flags & VK_MEMORY_MAP_PLACED_BIT_EXT) {
294 const VkMemoryMapPlacedInfoEXT *placed_info =
295 vk_find_struct_const(pMemoryMapInfo->pNext, MEMORY_MAP_PLACED_INFO_EXT);
296 if (placed_info) {
297 fixed_address = placed_info->pPlacedAddress;
298 use_fixed_address = true;
299 }
300 }
301
302 if (mem->user_ptr)
303 *ppData = mem->user_ptr;
304 else
305 *ppData = device->ws->buffer_map(device->ws, mem->bo, use_fixed_address, fixed_address);
306
307 if (*ppData) {
308 vk_rmv_log_cpu_map(&device->vk, mem->bo->va, false);
309 *ppData = (uint8_t *)*ppData + pMemoryMapInfo->offset;
310 return VK_SUCCESS;
311 }
312
313 return vk_error(device, VK_ERROR_MEMORY_MAP_FAILED);
314 }
315
316 VKAPI_ATTR VkResult VKAPI_CALL
radv_UnmapMemory2KHR(VkDevice _device,const VkMemoryUnmapInfoKHR * pMemoryUnmapInfo)317 radv_UnmapMemory2KHR(VkDevice _device, const VkMemoryUnmapInfoKHR *pMemoryUnmapInfo)
318 {
319 VK_FROM_HANDLE(radv_device, device, _device);
320 VK_FROM_HANDLE(radv_device_memory, mem, pMemoryUnmapInfo->memory);
321
322 vk_rmv_log_cpu_map(&device->vk, mem->bo->va, true);
323 if (mem->user_ptr == NULL)
324 device->ws->buffer_unmap(device->ws, mem->bo, (pMemoryUnmapInfo->flags & VK_MEMORY_UNMAP_RESERVE_BIT_EXT));
325
326 return VK_SUCCESS;
327 }
328
329 VKAPI_ATTR VkResult VKAPI_CALL
radv_FlushMappedMemoryRanges(VkDevice _device,uint32_t memoryRangeCount,const VkMappedMemoryRange * pMemoryRanges)330 radv_FlushMappedMemoryRanges(VkDevice _device, uint32_t memoryRangeCount, const VkMappedMemoryRange *pMemoryRanges)
331 {
332 return VK_SUCCESS;
333 }
334
335 VKAPI_ATTR VkResult VKAPI_CALL
radv_InvalidateMappedMemoryRanges(VkDevice _device,uint32_t memoryRangeCount,const VkMappedMemoryRange * pMemoryRanges)336 radv_InvalidateMappedMemoryRanges(VkDevice _device, uint32_t memoryRangeCount, const VkMappedMemoryRange *pMemoryRanges)
337 {
338 return VK_SUCCESS;
339 }
340
341 VKAPI_ATTR uint64_t VKAPI_CALL
radv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device,const VkDeviceMemoryOpaqueCaptureAddressInfo * pInfo)342 radv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device, const VkDeviceMemoryOpaqueCaptureAddressInfo *pInfo)
343 {
344 VK_FROM_HANDLE(radv_device_memory, mem, pInfo->memory);
345 return radv_buffer_get_va(mem->bo);
346 }
347
348 VKAPI_ATTR void VKAPI_CALL
radv_GetDeviceMemoryCommitment(VkDevice device,VkDeviceMemory memory,VkDeviceSize * pCommittedMemoryInBytes)349 radv_GetDeviceMemoryCommitment(VkDevice device, VkDeviceMemory memory, VkDeviceSize *pCommittedMemoryInBytes)
350 {
351 *pCommittedMemoryInBytes = 0;
352 }
353