1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * SPDX-License-Identifier: MIT
6 */
7 #include <assert.h>
8 #include <fcntl.h>
9 #include <stdbool.h>
10 #include <string.h>
11
12 #include "ac_descriptors.h"
13 #include "radv_buffer.h"
14 #include "radv_buffer_view.h"
15 #include "radv_cmd_buffer.h"
16 #include "radv_descriptor_set.h"
17 #include "radv_entrypoints.h"
18 #include "radv_image.h"
19 #include "radv_image_view.h"
20 #include "radv_rmv.h"
21 #include "radv_sampler.h"
22 #include "sid.h"
23 #include "vk_acceleration_structure.h"
24 #include "vk_descriptors.h"
25 #include "vk_format.h"
26 #include "vk_log.h"
27 #include "vk_util.h"
28 #include "vk_ycbcr_conversion.h"
29
30 static unsigned
radv_descriptor_type_buffer_count(VkDescriptorType type)31 radv_descriptor_type_buffer_count(VkDescriptorType type)
32 {
33 switch (type) {
34 case VK_DESCRIPTOR_TYPE_SAMPLER:
35 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK:
36 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
37 return 0;
38 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
39 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
40 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
41 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
42 case VK_DESCRIPTOR_TYPE_MUTABLE_EXT:
43 return 3;
44 default:
45 return 1;
46 }
47 }
48
49 static bool
has_equal_immutable_samplers(const VkSampler * samplers,uint32_t count)50 has_equal_immutable_samplers(const VkSampler *samplers, uint32_t count)
51 {
52 if (!samplers)
53 return false;
54 for (uint32_t i = 1; i < count; ++i) {
55 if (memcmp(radv_sampler_from_handle(samplers[0])->state, radv_sampler_from_handle(samplers[i])->state, 16)) {
56 return false;
57 }
58 }
59 return true;
60 }
61
62 static uint32_t
radv_descriptor_alignment(VkDescriptorType type)63 radv_descriptor_alignment(VkDescriptorType type)
64 {
65 switch (type) {
66 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
67 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
68 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
69 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
70 case VK_DESCRIPTOR_TYPE_SAMPLER:
71 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK:
72 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
73 return 16;
74 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
75 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
76 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
77 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
78 case VK_DESCRIPTOR_TYPE_MUTABLE_EXT:
79 return 32;
80 default:
81 return 1;
82 }
83 }
84
85 static bool
radv_mutable_descriptor_type_size_alignment(const VkMutableDescriptorTypeListEXT * list,uint64_t * out_size,uint64_t * out_align)86 radv_mutable_descriptor_type_size_alignment(const VkMutableDescriptorTypeListEXT *list, uint64_t *out_size,
87 uint64_t *out_align)
88 {
89 uint32_t max_size = 0;
90 uint32_t max_align = 0;
91
92 for (uint32_t i = 0; i < list->descriptorTypeCount; i++) {
93 uint32_t size = 0;
94 uint32_t align = radv_descriptor_alignment(list->pDescriptorTypes[i]);
95
96 switch (list->pDescriptorTypes[i]) {
97 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
98 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
99 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
100 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
101 case VK_DESCRIPTOR_TYPE_SAMPLER:
102 size = 16;
103 break;
104 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
105 size = 32;
106 break;
107 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
108 size = 64;
109 break;
110 default:
111 return false;
112 }
113
114 max_size = MAX2(max_size, size);
115 max_align = MAX2(max_align, align);
116 }
117
118 *out_size = max_size;
119 *out_align = max_align;
120 return true;
121 }
122
123 VKAPI_ATTR VkResult VKAPI_CALL
radv_CreateDescriptorSetLayout(VkDevice _device,const VkDescriptorSetLayoutCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorSetLayout * pSetLayout)124 radv_CreateDescriptorSetLayout(VkDevice _device, const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
125 const VkAllocationCallbacks *pAllocator, VkDescriptorSetLayout *pSetLayout)
126 {
127 VK_FROM_HANDLE(radv_device, device, _device);
128 struct radv_descriptor_set_layout *set_layout;
129
130 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO);
131 const VkDescriptorSetLayoutBindingFlagsCreateInfo *variable_flags =
132 vk_find_struct_const(pCreateInfo->pNext, DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO);
133 const VkMutableDescriptorTypeCreateInfoEXT *mutable_info =
134 vk_find_struct_const(pCreateInfo->pNext, MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_EXT);
135
136 uint32_t num_bindings = 0;
137 uint32_t immutable_sampler_count = 0;
138 uint32_t ycbcr_sampler_count = 0;
139 for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
140 num_bindings = MAX2(num_bindings, pCreateInfo->pBindings[j].binding + 1);
141 if ((pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
142 pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
143 pCreateInfo->pBindings[j].pImmutableSamplers) {
144 immutable_sampler_count += pCreateInfo->pBindings[j].descriptorCount;
145
146 bool has_ycbcr_sampler = false;
147 for (unsigned i = 0; i < pCreateInfo->pBindings[j].descriptorCount; ++i) {
148 if (radv_sampler_from_handle(pCreateInfo->pBindings[j].pImmutableSamplers[i])->vk.ycbcr_conversion)
149 has_ycbcr_sampler = true;
150 }
151
152 if (has_ycbcr_sampler)
153 ycbcr_sampler_count += pCreateInfo->pBindings[j].descriptorCount;
154 }
155 }
156
157 uint32_t samplers_offset = offsetof(struct radv_descriptor_set_layout, binding[num_bindings]);
158 size_t size = samplers_offset + immutable_sampler_count * 4 * sizeof(uint32_t);
159 if (ycbcr_sampler_count > 0) {
160 /* Store block of offsets first, followed by the conversion descriptors (padded to the struct
161 * alignment) */
162 size += num_bindings * sizeof(uint32_t);
163 size = align_uintptr(size, alignof(struct vk_ycbcr_conversion_state));
164 size += ycbcr_sampler_count * sizeof(struct vk_ycbcr_conversion_state);
165 }
166
167 /* We need to allocate descriptor set layouts off the device allocator with DEVICE scope because
168 * they are reference counted and may not be destroyed when vkDestroyDescriptorSetLayout is
169 * called.
170 */
171 set_layout = vk_descriptor_set_layout_zalloc(&device->vk, size);
172 if (!set_layout)
173 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
174
175 set_layout->flags = pCreateInfo->flags;
176
177 /* We just allocate all the samplers at the end of the struct */
178 uint32_t *samplers = (uint32_t *)&set_layout->binding[num_bindings];
179 struct vk_ycbcr_conversion_state *ycbcr_samplers = NULL;
180 uint32_t *ycbcr_sampler_offsets = NULL;
181
182 if (ycbcr_sampler_count > 0) {
183 ycbcr_sampler_offsets = samplers + 4 * immutable_sampler_count;
184 set_layout->ycbcr_sampler_offsets_offset = (char *)ycbcr_sampler_offsets - (char *)set_layout;
185
186 uintptr_t first_ycbcr_sampler_offset = (uintptr_t)ycbcr_sampler_offsets + sizeof(uint32_t) * num_bindings;
187 first_ycbcr_sampler_offset = align_uintptr(first_ycbcr_sampler_offset, alignof(struct vk_ycbcr_conversion_state));
188 ycbcr_samplers = (struct vk_ycbcr_conversion_state *)first_ycbcr_sampler_offset;
189 } else
190 set_layout->ycbcr_sampler_offsets_offset = 0;
191
192 VkDescriptorSetLayoutBinding *bindings = NULL;
193 VkResult result = vk_create_sorted_bindings(pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings);
194 if (result != VK_SUCCESS) {
195 vk_descriptor_set_layout_unref(&device->vk, &set_layout->vk);
196 return vk_error(device, result);
197 }
198
199 set_layout->binding_count = num_bindings;
200 set_layout->dynamic_shader_stages = 0;
201 set_layout->has_immutable_samplers = false;
202 set_layout->size = 0;
203
204 uint32_t buffer_count = 0;
205 uint32_t dynamic_offset_count = 0;
206
207 uint32_t first_alignment = 32;
208 if (pCreateInfo->bindingCount > 0) {
209 uint32_t last_alignment = radv_descriptor_alignment(bindings[pCreateInfo->bindingCount - 1].descriptorType);
210 if (bindings[pCreateInfo->bindingCount - 1].descriptorType == VK_DESCRIPTOR_TYPE_MUTABLE_EXT) {
211 uint64_t mutable_size = 0, mutable_align = 0;
212 radv_mutable_descriptor_type_size_alignment(
213 &mutable_info->pMutableDescriptorTypeLists[pCreateInfo->bindingCount - 1], &mutable_size, &mutable_align);
214 last_alignment = mutable_align;
215 }
216
217 first_alignment = last_alignment == 32 ? 16 : 32;
218 }
219
220 for (unsigned pass = 0; pass < 2; ++pass) {
221 for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
222 const VkDescriptorSetLayoutBinding *binding = bindings + j;
223 uint32_t b = binding->binding;
224 uint32_t alignment = radv_descriptor_alignment(binding->descriptorType);
225 unsigned binding_buffer_count = radv_descriptor_type_buffer_count(binding->descriptorType);
226 uint32_t descriptor_count = binding->descriptorCount;
227 bool has_ycbcr_sampler = false;
228
229 /* main image + fmask */
230 uint32_t max_sampled_image_descriptors = 2;
231
232 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER && binding->pImmutableSamplers) {
233 for (unsigned i = 0; i < binding->descriptorCount; ++i) {
234 struct vk_ycbcr_conversion *conversion =
235 radv_sampler_from_handle(binding->pImmutableSamplers[i])->vk.ycbcr_conversion;
236
237 if (conversion) {
238 has_ycbcr_sampler = true;
239 max_sampled_image_descriptors =
240 MAX2(max_sampled_image_descriptors, vk_format_get_plane_count(conversion->state.format));
241 }
242 }
243 }
244
245 switch (binding->descriptorType) {
246 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
247 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
248 assert(!(pCreateInfo->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
249 set_layout->binding[b].dynamic_offset_count = 1;
250 set_layout->dynamic_shader_stages |= binding->stageFlags;
251 if (binding->stageFlags & RADV_RT_STAGE_BITS)
252 set_layout->dynamic_shader_stages |= VK_SHADER_STAGE_COMPUTE_BIT;
253 set_layout->binding[b].size = 0;
254 break;
255 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
256 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
257 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
258 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
259 set_layout->binding[b].size = 16;
260 break;
261 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
262 set_layout->binding[b].size = 32;
263 break;
264 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
265 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
266 /* main descriptor + fmask descriptor */
267 set_layout->binding[b].size = 64;
268 break;
269 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
270 /* main descriptor + fmask descriptor + sampler */
271 set_layout->binding[b].size = 96;
272 break;
273 case VK_DESCRIPTOR_TYPE_SAMPLER:
274 set_layout->binding[b].size = 16;
275 break;
276 case VK_DESCRIPTOR_TYPE_MUTABLE_EXT: {
277 uint64_t mutable_size = 0, mutable_align = 0;
278 radv_mutable_descriptor_type_size_alignment(&mutable_info->pMutableDescriptorTypeLists[j], &mutable_size,
279 &mutable_align);
280 assert(mutable_size && mutable_align);
281 set_layout->binding[b].size = mutable_size;
282 alignment = mutable_align;
283 break;
284 }
285 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK:
286 set_layout->binding[b].size = descriptor_count;
287 descriptor_count = 1;
288 break;
289 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
290 set_layout->binding[b].size = 16;
291 break;
292 default:
293 break;
294 }
295
296 if ((pass == 0 && alignment != first_alignment) || (pass == 1 && alignment == first_alignment))
297 continue;
298
299 set_layout->size = align(set_layout->size, alignment);
300 set_layout->binding[b].type = binding->descriptorType;
301 set_layout->binding[b].array_size = descriptor_count;
302 set_layout->binding[b].offset = set_layout->size;
303 set_layout->binding[b].buffer_offset = buffer_count;
304 set_layout->binding[b].dynamic_offset_offset = dynamic_offset_count;
305
306 if (variable_flags && binding->binding < variable_flags->bindingCount &&
307 (variable_flags->pBindingFlags[binding->binding] & VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT)) {
308 assert(!binding->pImmutableSamplers); /* Terribly ill defined how many samplers are valid */
309 assert(binding->binding == num_bindings - 1);
310
311 set_layout->has_variable_descriptors = true;
312 }
313
314 if ((binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
315 binding->descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
316 binding->pImmutableSamplers) {
317 set_layout->binding[b].immutable_samplers_offset = samplers_offset;
318 set_layout->has_immutable_samplers = true;
319
320 /* Do not optimize space for descriptor buffers and embedded samplers, otherwise the set
321 * layout size/offset are incorrect.
322 */
323 if (!(pCreateInfo->flags & (VK_DESCRIPTOR_SET_LAYOUT_CREATE_DESCRIPTOR_BUFFER_BIT_EXT |
324 VK_DESCRIPTOR_SET_LAYOUT_CREATE_EMBEDDED_IMMUTABLE_SAMPLERS_BIT_EXT))) {
325 set_layout->binding[b].immutable_samplers_equal =
326 has_equal_immutable_samplers(binding->pImmutableSamplers, binding->descriptorCount);
327 }
328
329 for (uint32_t i = 0; i < binding->descriptorCount; i++)
330 memcpy(samplers + 4 * i, &radv_sampler_from_handle(binding->pImmutableSamplers[i])->state, 16);
331
332 /* Don't reserve space for the samplers if they're not accessed. */
333 if (set_layout->binding[b].immutable_samplers_equal) {
334 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER &&
335 max_sampled_image_descriptors <= 2)
336 set_layout->binding[b].size -= 32;
337 else if (binding->descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER)
338 set_layout->binding[b].size -= 16;
339 }
340 samplers += 4 * binding->descriptorCount;
341 samplers_offset += 4 * sizeof(uint32_t) * binding->descriptorCount;
342
343 if (has_ycbcr_sampler) {
344 ycbcr_sampler_offsets[b] = (const char *)ycbcr_samplers - (const char *)set_layout;
345 for (uint32_t i = 0; i < binding->descriptorCount; i++) {
346 if (radv_sampler_from_handle(binding->pImmutableSamplers[i])->vk.ycbcr_conversion)
347 ycbcr_samplers[i] =
348 radv_sampler_from_handle(binding->pImmutableSamplers[i])->vk.ycbcr_conversion->state;
349 else
350 ycbcr_samplers[i].format = VK_FORMAT_UNDEFINED;
351 }
352 ycbcr_samplers += binding->descriptorCount;
353 }
354 }
355
356 set_layout->size += descriptor_count * set_layout->binding[b].size;
357 buffer_count += descriptor_count * binding_buffer_count;
358 dynamic_offset_count += descriptor_count * set_layout->binding[b].dynamic_offset_count;
359 }
360 }
361
362 free(bindings);
363
364 set_layout->buffer_count = buffer_count;
365 set_layout->dynamic_offset_count = dynamic_offset_count;
366
367 /* Hash the entire set layout except vk_descriptor_set_layout. The rest of the set layout is
368 * carefully constructed to not have pointers so a full hash instead of a per-field hash
369 * should be ok.
370 */
371 uint32_t hash_offset = offsetof(struct radv_descriptor_set_layout, hash) + sizeof(set_layout->hash);
372 _mesa_blake3_compute((const char *)set_layout + hash_offset, size - hash_offset, set_layout->hash);
373
374 *pSetLayout = radv_descriptor_set_layout_to_handle(set_layout);
375
376 return VK_SUCCESS;
377 }
378
379 VKAPI_ATTR void VKAPI_CALL
radv_GetDescriptorSetLayoutSupport(VkDevice device,const VkDescriptorSetLayoutCreateInfo * pCreateInfo,VkDescriptorSetLayoutSupport * pSupport)380 radv_GetDescriptorSetLayoutSupport(VkDevice device, const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
381 VkDescriptorSetLayoutSupport *pSupport)
382 {
383 VkDescriptorSetLayoutBinding *bindings = NULL;
384 VkResult result = vk_create_sorted_bindings(pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings);
385 if (result != VK_SUCCESS) {
386 pSupport->supported = false;
387 return;
388 }
389
390 const VkDescriptorSetLayoutBindingFlagsCreateInfo *variable_flags =
391 vk_find_struct_const(pCreateInfo->pNext, DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO);
392 VkDescriptorSetVariableDescriptorCountLayoutSupport *variable_count =
393 vk_find_struct(pSupport->pNext, DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_LAYOUT_SUPPORT);
394 const VkMutableDescriptorTypeCreateInfoEXT *mutable_info =
395 vk_find_struct_const(pCreateInfo->pNext, MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_EXT);
396 if (variable_count) {
397 variable_count->maxVariableDescriptorCount = 0;
398 }
399
400 uint32_t first_alignment = 32;
401 if (pCreateInfo->bindingCount > 0) {
402 uint32_t last_alignment = radv_descriptor_alignment(bindings[pCreateInfo->bindingCount - 1].descriptorType);
403 if (bindings[pCreateInfo->bindingCount - 1].descriptorType == VK_DESCRIPTOR_TYPE_MUTABLE_EXT) {
404 uint64_t mutable_size = 0, mutable_align = 0;
405 radv_mutable_descriptor_type_size_alignment(
406 &mutable_info->pMutableDescriptorTypeLists[pCreateInfo->bindingCount - 1], &mutable_size, &mutable_align);
407 last_alignment = mutable_align;
408 }
409
410 first_alignment = last_alignment == 32 ? 16 : 32;
411 }
412
413 bool supported = true;
414 uint64_t size = 0;
415 for (unsigned pass = 0; pass < 2; ++pass) {
416 for (uint32_t i = 0; i < pCreateInfo->bindingCount; i++) {
417 const VkDescriptorSetLayoutBinding *binding = bindings + i;
418
419 uint64_t descriptor_size = 0;
420 uint64_t descriptor_alignment = radv_descriptor_alignment(binding->descriptorType);
421 uint32_t descriptor_count = binding->descriptorCount;
422 switch (binding->descriptorType) {
423 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
424 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
425 break;
426 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
427 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
428 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
429 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
430 descriptor_size = 16;
431 break;
432 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
433 descriptor_size = 32;
434 break;
435 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
436 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
437 descriptor_size = 64;
438 break;
439 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
440 if (!has_equal_immutable_samplers(binding->pImmutableSamplers, descriptor_count)) {
441 descriptor_size = 64;
442 } else {
443 descriptor_size = 96;
444 }
445 break;
446 case VK_DESCRIPTOR_TYPE_SAMPLER:
447 if (!has_equal_immutable_samplers(binding->pImmutableSamplers, descriptor_count)) {
448 descriptor_size = 16;
449 }
450 break;
451 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK:
452 descriptor_size = descriptor_count;
453 descriptor_count = 1;
454 break;
455 case VK_DESCRIPTOR_TYPE_MUTABLE_EXT:
456 if (!radv_mutable_descriptor_type_size_alignment(&mutable_info->pMutableDescriptorTypeLists[i],
457 &descriptor_size, &descriptor_alignment)) {
458 supported = false;
459 }
460 break;
461 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
462 descriptor_size = 16;
463 break;
464 default:
465 break;
466 }
467
468 if ((pass == 0 && descriptor_alignment != first_alignment) ||
469 (pass == 1 && descriptor_alignment == first_alignment))
470 continue;
471
472 if (size && !align64(size, descriptor_alignment)) {
473 supported = false;
474 }
475 size = align64(size, descriptor_alignment);
476
477 uint64_t max_count = INT32_MAX;
478 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK)
479 max_count = INT32_MAX - size;
480 else if (descriptor_size)
481 max_count = (INT32_MAX - size) / descriptor_size;
482
483 if (max_count < descriptor_count) {
484 supported = false;
485 }
486 if (variable_flags && binding->binding < variable_flags->bindingCount && variable_count &&
487 (variable_flags->pBindingFlags[binding->binding] & VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT)) {
488 variable_count->maxVariableDescriptorCount = MIN2(UINT32_MAX, max_count);
489 }
490 size += descriptor_count * descriptor_size;
491 }
492 }
493
494 free(bindings);
495
496 pSupport->supported = supported;
497 }
498
499 /*
500 * Pipeline layouts. These have nothing to do with the pipeline. They are
501 * just multiple descriptor set layouts pasted together.
502 */
503 void
radv_pipeline_layout_init(struct radv_device * device,struct radv_pipeline_layout * layout,bool independent_sets)504 radv_pipeline_layout_init(struct radv_device *device, struct radv_pipeline_layout *layout, bool independent_sets)
505 {
506 memset(layout, 0, sizeof(*layout));
507
508 vk_object_base_init(&device->vk, &layout->base, VK_OBJECT_TYPE_PIPELINE_LAYOUT);
509
510 layout->independent_sets = independent_sets;
511 }
512
513 void
radv_pipeline_layout_add_set(struct radv_pipeline_layout * layout,uint32_t set_idx,struct radv_descriptor_set_layout * set_layout)514 radv_pipeline_layout_add_set(struct radv_pipeline_layout *layout, uint32_t set_idx,
515 struct radv_descriptor_set_layout *set_layout)
516 {
517 if (layout->set[set_idx].layout)
518 return;
519
520 layout->num_sets = MAX2(set_idx + 1, layout->num_sets);
521
522 layout->set[set_idx].layout = set_layout;
523 vk_descriptor_set_layout_ref(&set_layout->vk);
524
525 layout->set[set_idx].dynamic_offset_start = layout->dynamic_offset_count;
526
527 layout->dynamic_offset_count += set_layout->dynamic_offset_count;
528 layout->dynamic_shader_stages |= set_layout->dynamic_shader_stages;
529 }
530
531 void
radv_pipeline_layout_hash(struct radv_pipeline_layout * layout)532 radv_pipeline_layout_hash(struct radv_pipeline_layout *layout)
533 {
534 struct mesa_blake3 ctx;
535
536 _mesa_blake3_init(&ctx);
537 for (uint32_t i = 0; i < layout->num_sets; i++) {
538 struct radv_descriptor_set_layout *set_layout = layout->set[i].layout;
539
540 if (!set_layout)
541 continue;
542
543 _mesa_blake3_update(&ctx, set_layout->hash, sizeof(set_layout->hash));
544 }
545 _mesa_blake3_update(&ctx, &layout->push_constant_size, sizeof(layout->push_constant_size));
546 _mesa_blake3_final(&ctx, layout->hash);
547 }
548
549 void
radv_pipeline_layout_finish(struct radv_device * device,struct radv_pipeline_layout * layout)550 radv_pipeline_layout_finish(struct radv_device *device, struct radv_pipeline_layout *layout)
551 {
552 for (uint32_t i = 0; i < layout->num_sets; i++) {
553 if (!layout->set[i].layout)
554 continue;
555
556 vk_descriptor_set_layout_unref(&device->vk, &layout->set[i].layout->vk);
557 }
558
559 vk_object_base_finish(&layout->base);
560 }
561
562 VKAPI_ATTR VkResult VKAPI_CALL
radv_CreatePipelineLayout(VkDevice _device,const VkPipelineLayoutCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkPipelineLayout * pPipelineLayout)563 radv_CreatePipelineLayout(VkDevice _device, const VkPipelineLayoutCreateInfo *pCreateInfo,
564 const VkAllocationCallbacks *pAllocator, VkPipelineLayout *pPipelineLayout)
565 {
566 VK_FROM_HANDLE(radv_device, device, _device);
567 struct radv_pipeline_layout *layout;
568
569 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO);
570
571 layout = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*layout), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
572 if (layout == NULL)
573 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
574
575 radv_pipeline_layout_init(device, layout, pCreateInfo->flags & VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT);
576
577 layout->num_sets = pCreateInfo->setLayoutCount;
578
579 for (uint32_t set = 0; set < pCreateInfo->setLayoutCount; set++) {
580 VK_FROM_HANDLE(radv_descriptor_set_layout, set_layout, pCreateInfo->pSetLayouts[set]);
581
582 if (set_layout == NULL) {
583 layout->set[set].layout = NULL;
584 continue;
585 }
586
587 radv_pipeline_layout_add_set(layout, set, set_layout);
588 }
589
590 layout->push_constant_size = 0;
591
592 for (unsigned i = 0; i < pCreateInfo->pushConstantRangeCount; ++i) {
593 const VkPushConstantRange *range = pCreateInfo->pPushConstantRanges + i;
594 layout->push_constant_size = MAX2(layout->push_constant_size, range->offset + range->size);
595 }
596
597 layout->push_constant_size = align(layout->push_constant_size, 16);
598
599 radv_pipeline_layout_hash(layout);
600
601 *pPipelineLayout = radv_pipeline_layout_to_handle(layout);
602
603 return VK_SUCCESS;
604 }
605
606 VKAPI_ATTR void VKAPI_CALL
radv_DestroyPipelineLayout(VkDevice _device,VkPipelineLayout _pipelineLayout,const VkAllocationCallbacks * pAllocator)607 radv_DestroyPipelineLayout(VkDevice _device, VkPipelineLayout _pipelineLayout, const VkAllocationCallbacks *pAllocator)
608 {
609 VK_FROM_HANDLE(radv_device, device, _device);
610 VK_FROM_HANDLE(radv_pipeline_layout, pipeline_layout, _pipelineLayout);
611
612 if (!pipeline_layout)
613 return;
614
615 radv_pipeline_layout_finish(device, pipeline_layout);
616
617 vk_free2(&device->vk.alloc, pAllocator, pipeline_layout);
618 }
619
620 static VkResult
radv_descriptor_set_create(struct radv_device * device,struct radv_descriptor_pool * pool,struct radv_descriptor_set_layout * layout,const uint32_t * variable_count,struct radv_descriptor_set ** out_set)621 radv_descriptor_set_create(struct radv_device *device, struct radv_descriptor_pool *pool,
622 struct radv_descriptor_set_layout *layout, const uint32_t *variable_count,
623 struct radv_descriptor_set **out_set)
624 {
625 if (pool->entry_count == pool->max_entry_count)
626 return VK_ERROR_OUT_OF_POOL_MEMORY;
627
628 struct radv_descriptor_set *set;
629 uint32_t buffer_count = layout->buffer_count;
630 if (variable_count) {
631 unsigned stride = radv_descriptor_type_buffer_count(layout->binding[layout->binding_count - 1].type);
632 buffer_count = layout->binding[layout->binding_count - 1].buffer_offset + *variable_count * stride;
633 }
634 unsigned range_offset = sizeof(struct radv_descriptor_set_header) + sizeof(struct radeon_winsys_bo *) * buffer_count;
635 const unsigned dynamic_offset_count = layout->dynamic_offset_count;
636 unsigned mem_size = range_offset + sizeof(struct radv_descriptor_range) * dynamic_offset_count;
637
638 if (pool->host_memory_base) {
639 if (pool->host_memory_end - pool->host_memory_ptr < mem_size)
640 return VK_ERROR_OUT_OF_POOL_MEMORY;
641
642 set = (struct radv_descriptor_set *)pool->host_memory_ptr;
643 pool->host_memory_ptr += mem_size;
644 } else {
645 set = vk_alloc2(&device->vk.alloc, NULL, mem_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
646
647 if (!set)
648 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
649 }
650
651 memset(set, 0, mem_size);
652
653 vk_object_base_init(&device->vk, &set->header.base, VK_OBJECT_TYPE_DESCRIPTOR_SET);
654
655 if (dynamic_offset_count) {
656 set->header.dynamic_descriptors = (struct radv_descriptor_range *)((uint8_t *)set + range_offset);
657 }
658
659 set->header.layout = layout;
660 set->header.buffer_count = buffer_count;
661 uint32_t layout_size = layout->size;
662 if (variable_count) {
663 uint32_t stride = layout->binding[layout->binding_count - 1].size;
664 if (layout->binding[layout->binding_count - 1].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK)
665 stride = 1;
666
667 layout_size = layout->binding[layout->binding_count - 1].offset + *variable_count * stride;
668 }
669 layout_size = align(layout_size, 32);
670 set->header.size = layout_size;
671
672 /* try to allocate linearly first, so that we don't spend
673 * time looking for gaps if the app only allocates &
674 * resets via the pool. */
675 if (pool->current_offset + layout_size <= pool->size) {
676 set->header.bo = pool->bo;
677 set->header.mapped_ptr = (uint32_t *)(pool->mapped_ptr + pool->current_offset);
678 set->header.va = pool->bo ? (radv_buffer_get_va(set->header.bo) + pool->current_offset) : 0;
679
680 if (!pool->host_memory_base) {
681 pool->entries[pool->entry_count].offset = pool->current_offset;
682 pool->entries[pool->entry_count].size = layout_size;
683 pool->entries[pool->entry_count].set = set;
684 } else {
685 pool->sets[pool->entry_count] = set;
686 }
687
688 pool->current_offset += layout_size;
689 } else if (!pool->host_memory_base) {
690 uint64_t offset = 0;
691 int index;
692
693 for (index = 0; index < pool->entry_count; ++index) {
694 if (pool->entries[index].offset - offset >= layout_size)
695 break;
696 offset = pool->entries[index].offset + pool->entries[index].size;
697 }
698
699 if (pool->size - offset < layout_size) {
700 vk_free2(&device->vk.alloc, NULL, set);
701 return VK_ERROR_OUT_OF_POOL_MEMORY;
702 }
703 set->header.bo = pool->bo;
704 set->header.mapped_ptr = (uint32_t *)(pool->mapped_ptr + offset);
705 set->header.va = pool->bo ? (radv_buffer_get_va(set->header.bo) + offset) : 0;
706 memmove(&pool->entries[index + 1], &pool->entries[index], sizeof(pool->entries[0]) * (pool->entry_count - index));
707 pool->entries[index].offset = offset;
708 pool->entries[index].size = layout_size;
709 pool->entries[index].set = set;
710 } else
711 return VK_ERROR_OUT_OF_POOL_MEMORY;
712
713 if (layout->has_immutable_samplers) {
714 for (unsigned i = 0; i < layout->binding_count; ++i) {
715 if (!layout->binding[i].immutable_samplers_offset || layout->binding[i].immutable_samplers_equal)
716 continue;
717
718 unsigned offset = layout->binding[i].offset / 4;
719 if (layout->binding[i].type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
720 offset += radv_combined_image_descriptor_sampler_offset(layout->binding + i) / 4;
721
722 const uint32_t *samplers =
723 (const uint32_t *)((const char *)layout + layout->binding[i].immutable_samplers_offset);
724 for (unsigned j = 0; j < layout->binding[i].array_size; ++j) {
725 memcpy(set->header.mapped_ptr + offset, samplers + 4 * j, 16);
726 offset += layout->binding[i].size / 4;
727 }
728 }
729 }
730
731 pool->entry_count++;
732 vk_descriptor_set_layout_ref(&layout->vk);
733 *out_set = set;
734 return VK_SUCCESS;
735 }
736
737 static void
radv_descriptor_set_destroy(struct radv_device * device,struct radv_descriptor_pool * pool,struct radv_descriptor_set * set,bool free_bo)738 radv_descriptor_set_destroy(struct radv_device *device, struct radv_descriptor_pool *pool,
739 struct radv_descriptor_set *set, bool free_bo)
740 {
741 assert(!pool->host_memory_base);
742
743 vk_descriptor_set_layout_unref(&device->vk, &set->header.layout->vk);
744
745 if (free_bo && !pool->host_memory_base) {
746 for (int i = 0; i < pool->entry_count; ++i) {
747 if (pool->entries[i].set == set) {
748 memmove(&pool->entries[i], &pool->entries[i + 1], sizeof(pool->entries[i]) * (pool->entry_count - i - 1));
749 --pool->entry_count;
750 break;
751 }
752 }
753 }
754 vk_object_base_finish(&set->header.base);
755 vk_free2(&device->vk.alloc, NULL, set);
756 }
757
758 static void
radv_destroy_descriptor_pool(struct radv_device * device,const VkAllocationCallbacks * pAllocator,struct radv_descriptor_pool * pool)759 radv_destroy_descriptor_pool(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
760 struct radv_descriptor_pool *pool)
761 {
762
763 if (!pool->host_memory_base) {
764 for (uint32_t i = 0; i < pool->entry_count; ++i) {
765 radv_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
766 }
767 } else {
768 for (uint32_t i = 0; i < pool->entry_count; ++i) {
769 vk_descriptor_set_layout_unref(&device->vk, &pool->sets[i]->header.layout->vk);
770 vk_object_base_finish(&pool->sets[i]->header.base);
771 }
772 }
773
774 if (pool->bo)
775 radv_bo_destroy(device, &pool->base, pool->bo);
776 if (pool->host_bo)
777 vk_free2(&device->vk.alloc, pAllocator, pool->host_bo);
778
779 radv_rmv_log_resource_destroy(device, (uint64_t)radv_descriptor_pool_to_handle(pool));
780 vk_object_base_finish(&pool->base);
781 vk_free2(&device->vk.alloc, pAllocator, pool);
782 }
783
784 static VkResult
radv_create_descriptor_pool(struct radv_device * device,const VkDescriptorPoolCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorPool * pDescriptorPool)785 radv_create_descriptor_pool(struct radv_device *device, const VkDescriptorPoolCreateInfo *pCreateInfo,
786 const VkAllocationCallbacks *pAllocator, VkDescriptorPool *pDescriptorPool)
787 {
788 struct radv_descriptor_pool *pool;
789 uint64_t size = sizeof(struct radv_descriptor_pool);
790 uint64_t bo_size = 0, bo_count = 0, range_count = 0;
791
792 const VkMutableDescriptorTypeCreateInfoEXT *mutable_info =
793 vk_find_struct_const(pCreateInfo->pNext, MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_EXT);
794
795 vk_foreach_struct_const (ext, pCreateInfo->pNext) {
796 switch (ext->sType) {
797 case VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_INLINE_UNIFORM_BLOCK_CREATE_INFO: {
798 const VkDescriptorPoolInlineUniformBlockCreateInfo *info =
799 (const VkDescriptorPoolInlineUniformBlockCreateInfo *)ext;
800 /* the sizes are 4 aligned, and we need to align to at
801 * most 32, which needs at most 28 bytes extra per
802 * binding. */
803 bo_size += 28llu * info->maxInlineUniformBlockBindings;
804 break;
805 }
806 default:
807 break;
808 }
809 }
810
811 uint64_t num_16byte_descriptors = 0;
812 for (unsigned i = 0; i < pCreateInfo->poolSizeCount; ++i) {
813 bo_count += radv_descriptor_type_buffer_count(pCreateInfo->pPoolSizes[i].type) *
814 pCreateInfo->pPoolSizes[i].descriptorCount;
815
816 switch (pCreateInfo->pPoolSizes[i].type) {
817 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
818 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
819 range_count += pCreateInfo->pPoolSizes[i].descriptorCount;
820 break;
821 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
822 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
823 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
824 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
825 case VK_DESCRIPTOR_TYPE_SAMPLER:
826 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
827 bo_size += 16 * pCreateInfo->pPoolSizes[i].descriptorCount;
828 num_16byte_descriptors += pCreateInfo->pPoolSizes[i].descriptorCount;
829 break;
830 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
831 bo_size += 32 * pCreateInfo->pPoolSizes[i].descriptorCount;
832 break;
833 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
834 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
835 bo_size += 64 * pCreateInfo->pPoolSizes[i].descriptorCount;
836 break;
837 case VK_DESCRIPTOR_TYPE_MUTABLE_EXT:
838 /* Per spec, if a mutable descriptor type list is provided for the pool entry, we
839 * allocate enough memory to hold any subset of that list.
840 * If there is no mutable descriptor type list available,
841 * we must allocate enough for any supported mutable descriptor type, i.e. 64 bytes. */
842 if (mutable_info && i < mutable_info->mutableDescriptorTypeListCount) {
843 uint64_t mutable_size, mutable_alignment;
844 if (radv_mutable_descriptor_type_size_alignment(&mutable_info->pMutableDescriptorTypeLists[i],
845 &mutable_size, &mutable_alignment)) {
846 /* 32 as we may need to align for images */
847 mutable_size = align(mutable_size, 32);
848 bo_size += mutable_size * pCreateInfo->pPoolSizes[i].descriptorCount;
849 if (mutable_size < 32)
850 num_16byte_descriptors += pCreateInfo->pPoolSizes[i].descriptorCount;
851 }
852 } else {
853 bo_size += 64 * pCreateInfo->pPoolSizes[i].descriptorCount;
854 }
855 break;
856 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
857 bo_size += 96 * pCreateInfo->pPoolSizes[i].descriptorCount;
858 break;
859 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK:
860 bo_size += pCreateInfo->pPoolSizes[i].descriptorCount;
861 break;
862 default:
863 break;
864 }
865 }
866
867 if (num_16byte_descriptors) {
868 /* Reserve space to align before image descriptors. Our layout code ensures at most one gap
869 * per set. */
870 bo_size += 16 * MIN2(num_16byte_descriptors, pCreateInfo->maxSets);
871 }
872
873 uint64_t sets_size = 0;
874
875 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
876 size += pCreateInfo->maxSets * sizeof(struct radv_descriptor_set);
877 size += sizeof(struct radeon_winsys_bo *) * bo_count;
878 size += sizeof(struct radv_descriptor_range) * range_count;
879
880 sets_size = sizeof(struct radv_descriptor_set *) * pCreateInfo->maxSets;
881 size += sets_size;
882 } else {
883 size += sizeof(struct radv_descriptor_pool_entry) * pCreateInfo->maxSets;
884 }
885
886 pool = vk_alloc2(&device->vk.alloc, pAllocator, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
887 if (!pool)
888 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
889
890 memset(pool, 0, sizeof(*pool));
891
892 vk_object_base_init(&device->vk, &pool->base, VK_OBJECT_TYPE_DESCRIPTOR_POOL);
893
894 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
895 pool->host_memory_base = (uint8_t *)pool + sizeof(struct radv_descriptor_pool) + sets_size;
896 pool->host_memory_ptr = pool->host_memory_base;
897 pool->host_memory_end = (uint8_t *)pool + size;
898 }
899
900 if (bo_size) {
901 const struct radv_physical_device *pdev = radv_device_physical(device);
902 const struct radv_instance *instance = radv_physical_device_instance(pdev);
903
904 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_HOST_ONLY_BIT_EXT)) {
905 enum radeon_bo_flag flags = RADEON_FLAG_NO_INTERPROCESS_SHARING | RADEON_FLAG_READ_ONLY | RADEON_FLAG_32BIT;
906
907 if (instance->drirc.zero_vram)
908 flags |= RADEON_FLAG_ZERO_VRAM;
909
910 VkResult result = radv_bo_create(device, &pool->base, bo_size, 32, RADEON_DOMAIN_VRAM, flags,
911 RADV_BO_PRIORITY_DESCRIPTOR, 0, false, &pool->bo);
912 if (result != VK_SUCCESS) {
913 radv_destroy_descriptor_pool(device, pAllocator, pool);
914 return vk_error(device, result);
915 }
916 pool->mapped_ptr = (uint8_t *)radv_buffer_map(device->ws, pool->bo);
917 if (!pool->mapped_ptr) {
918 radv_destroy_descriptor_pool(device, pAllocator, pool);
919 return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
920 }
921 } else {
922 pool->host_bo = vk_alloc2(&device->vk.alloc, pAllocator, bo_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
923 if (!pool->host_bo) {
924 radv_destroy_descriptor_pool(device, pAllocator, pool);
925 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
926 }
927 pool->mapped_ptr = pool->host_bo;
928 }
929 }
930 pool->size = bo_size;
931 pool->max_entry_count = pCreateInfo->maxSets;
932
933 *pDescriptorPool = radv_descriptor_pool_to_handle(pool);
934 radv_rmv_log_descriptor_pool_create(device, pCreateInfo, *pDescriptorPool);
935 return VK_SUCCESS;
936 }
937
938 VKAPI_ATTR VkResult VKAPI_CALL
radv_CreateDescriptorPool(VkDevice _device,const VkDescriptorPoolCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorPool * pDescriptorPool)939 radv_CreateDescriptorPool(VkDevice _device, const VkDescriptorPoolCreateInfo *pCreateInfo,
940 const VkAllocationCallbacks *pAllocator, VkDescriptorPool *pDescriptorPool)
941 {
942 VK_FROM_HANDLE(radv_device, device, _device);
943 return radv_create_descriptor_pool(device, pCreateInfo, pAllocator, pDescriptorPool);
944 }
945
946 VKAPI_ATTR void VKAPI_CALL
radv_DestroyDescriptorPool(VkDevice _device,VkDescriptorPool _pool,const VkAllocationCallbacks * pAllocator)947 radv_DestroyDescriptorPool(VkDevice _device, VkDescriptorPool _pool, const VkAllocationCallbacks *pAllocator)
948 {
949 VK_FROM_HANDLE(radv_device, device, _device);
950 VK_FROM_HANDLE(radv_descriptor_pool, pool, _pool);
951
952 if (!pool)
953 return;
954
955 radv_destroy_descriptor_pool(device, pAllocator, pool);
956 }
957
958 VKAPI_ATTR VkResult VKAPI_CALL
radv_ResetDescriptorPool(VkDevice _device,VkDescriptorPool descriptorPool,VkDescriptorPoolResetFlags flags)959 radv_ResetDescriptorPool(VkDevice _device, VkDescriptorPool descriptorPool, VkDescriptorPoolResetFlags flags)
960 {
961 VK_FROM_HANDLE(radv_device, device, _device);
962 VK_FROM_HANDLE(radv_descriptor_pool, pool, descriptorPool);
963
964 if (!pool->host_memory_base) {
965 for (uint32_t i = 0; i < pool->entry_count; ++i) {
966 radv_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
967 }
968 } else {
969 for (uint32_t i = 0; i < pool->entry_count; ++i) {
970 vk_descriptor_set_layout_unref(&device->vk, &pool->sets[i]->header.layout->vk);
971 vk_object_base_finish(&pool->sets[i]->header.base);
972 }
973 }
974
975 pool->entry_count = 0;
976
977 pool->current_offset = 0;
978 pool->host_memory_ptr = pool->host_memory_base;
979
980 return VK_SUCCESS;
981 }
982
983 VKAPI_ATTR VkResult VKAPI_CALL
radv_AllocateDescriptorSets(VkDevice _device,const VkDescriptorSetAllocateInfo * pAllocateInfo,VkDescriptorSet * pDescriptorSets)984 radv_AllocateDescriptorSets(VkDevice _device, const VkDescriptorSetAllocateInfo *pAllocateInfo,
985 VkDescriptorSet *pDescriptorSets)
986 {
987 VK_FROM_HANDLE(radv_device, device, _device);
988 VK_FROM_HANDLE(radv_descriptor_pool, pool, pAllocateInfo->descriptorPool);
989
990 VkResult result = VK_SUCCESS;
991 uint32_t i;
992 struct radv_descriptor_set *set = NULL;
993
994 const VkDescriptorSetVariableDescriptorCountAllocateInfo *variable_counts =
995 vk_find_struct_const(pAllocateInfo->pNext, DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_ALLOCATE_INFO);
996 const uint32_t zero = 0;
997
998 /* allocate a set of buffers for each shader to contain descriptors */
999 for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
1000 VK_FROM_HANDLE(radv_descriptor_set_layout, layout, pAllocateInfo->pSetLayouts[i]);
1001
1002 const uint32_t *variable_count = NULL;
1003 if (layout->has_variable_descriptors && variable_counts) {
1004 if (i < variable_counts->descriptorSetCount)
1005 variable_count = variable_counts->pDescriptorCounts + i;
1006 else
1007 variable_count = &zero;
1008 }
1009
1010 assert(!(layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
1011
1012 result = radv_descriptor_set_create(device, pool, layout, variable_count, &set);
1013 if (result != VK_SUCCESS)
1014 break;
1015
1016 pDescriptorSets[i] = radv_descriptor_set_to_handle(set);
1017 }
1018
1019 if (result != VK_SUCCESS) {
1020 radv_FreeDescriptorSets(_device, pAllocateInfo->descriptorPool, i, pDescriptorSets);
1021 for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
1022 pDescriptorSets[i] = VK_NULL_HANDLE;
1023 }
1024 }
1025 return result;
1026 }
1027
1028 VKAPI_ATTR VkResult VKAPI_CALL
radv_FreeDescriptorSets(VkDevice _device,VkDescriptorPool descriptorPool,uint32_t count,const VkDescriptorSet * pDescriptorSets)1029 radv_FreeDescriptorSets(VkDevice _device, VkDescriptorPool descriptorPool, uint32_t count,
1030 const VkDescriptorSet *pDescriptorSets)
1031 {
1032 VK_FROM_HANDLE(radv_device, device, _device);
1033 VK_FROM_HANDLE(radv_descriptor_pool, pool, descriptorPool);
1034
1035 for (uint32_t i = 0; i < count; i++) {
1036 VK_FROM_HANDLE(radv_descriptor_set, set, pDescriptorSets[i]);
1037
1038 if (set && !pool->host_memory_base)
1039 radv_descriptor_set_destroy(device, pool, set, true);
1040 }
1041 return VK_SUCCESS;
1042 }
1043
1044 static ALWAYS_INLINE void
write_texel_buffer_descriptor(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,unsigned * dst,struct radeon_winsys_bo ** buffer_list,const VkBufferView _buffer_view)1045 write_texel_buffer_descriptor(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer, unsigned *dst,
1046 struct radeon_winsys_bo **buffer_list, const VkBufferView _buffer_view)
1047 {
1048 VK_FROM_HANDLE(radv_buffer_view, buffer_view, _buffer_view);
1049
1050 if (!buffer_view) {
1051 memset(dst, 0, 4 * 4);
1052 if (!cmd_buffer)
1053 *buffer_list = NULL;
1054 return;
1055 }
1056
1057 memcpy(dst, buffer_view->state, 4 * 4);
1058
1059 if (device->use_global_bo_list)
1060 return;
1061
1062 if (cmd_buffer)
1063 radv_cs_add_buffer(device->ws, cmd_buffer->cs, buffer_view->bo);
1064 else
1065 *buffer_list = buffer_view->bo;
1066 }
1067
1068 static ALWAYS_INLINE void
write_buffer_descriptor(struct radv_device * device,unsigned * dst,uint64_t va,uint64_t range)1069 write_buffer_descriptor(struct radv_device *device, unsigned *dst, uint64_t va, uint64_t range)
1070 {
1071 const struct radv_physical_device *pdev = radv_device_physical(device);
1072
1073 if (!va) {
1074 memset(dst, 0, 4 * 4);
1075 return;
1076 }
1077
1078 /* robustBufferAccess is relaxed enough to allow this (in combination with the alignment/size
1079 * we return from vkGetBufferMemoryRequirements) and this allows the shader compiler to create
1080 * more efficient 8/16-bit buffer accesses.
1081 */
1082 ac_build_raw_buffer_descriptor(pdev->info.gfx_level, va, align(range, 4), dst);
1083 }
1084
1085 static ALWAYS_INLINE void
write_buffer_descriptor_impl(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,unsigned * dst,struct radeon_winsys_bo ** buffer_list,const VkDescriptorBufferInfo * buffer_info)1086 write_buffer_descriptor_impl(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer, unsigned *dst,
1087 struct radeon_winsys_bo **buffer_list, const VkDescriptorBufferInfo *buffer_info)
1088 {
1089 VK_FROM_HANDLE(radv_buffer, buffer, buffer_info->buffer);
1090 uint64_t va = 0, range = 0;
1091
1092 if (buffer) {
1093 va = radv_buffer_get_va(buffer->bo) + buffer_info->offset + buffer->offset;
1094
1095 range = vk_buffer_range(&buffer->vk, buffer_info->offset, buffer_info->range);
1096 assert(buffer->vk.size > 0 && range > 0);
1097 }
1098
1099 write_buffer_descriptor(device, dst, va, range);
1100
1101 if (device->use_global_bo_list)
1102 return;
1103
1104 if (!buffer) {
1105 if (!cmd_buffer)
1106 *buffer_list = NULL;
1107 return;
1108 }
1109
1110 if (cmd_buffer)
1111 radv_cs_add_buffer(device->ws, cmd_buffer->cs, buffer->bo);
1112 else
1113 *buffer_list = buffer->bo;
1114 }
1115
1116 static ALWAYS_INLINE void
write_block_descriptor(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,void * dst,const VkWriteDescriptorSet * writeset)1117 write_block_descriptor(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer, void *dst,
1118 const VkWriteDescriptorSet *writeset)
1119 {
1120 const VkWriteDescriptorSetInlineUniformBlock *inline_ub =
1121 vk_find_struct_const(writeset->pNext, WRITE_DESCRIPTOR_SET_INLINE_UNIFORM_BLOCK);
1122
1123 memcpy(dst, inline_ub->pData, inline_ub->dataSize);
1124 }
1125
1126 static ALWAYS_INLINE void
write_dynamic_buffer_descriptor(struct radv_device * device,struct radv_descriptor_range * range,struct radeon_winsys_bo ** buffer_list,const VkDescriptorBufferInfo * buffer_info)1127 write_dynamic_buffer_descriptor(struct radv_device *device, struct radv_descriptor_range *range,
1128 struct radeon_winsys_bo **buffer_list, const VkDescriptorBufferInfo *buffer_info)
1129 {
1130 VK_FROM_HANDLE(radv_buffer, buffer, buffer_info->buffer);
1131 uint64_t va;
1132 unsigned size;
1133
1134 if (!buffer) {
1135 range->va = 0;
1136 *buffer_list = NULL;
1137 return;
1138 }
1139
1140 va = radv_buffer_get_va(buffer->bo);
1141
1142 size = vk_buffer_range(&buffer->vk, buffer_info->offset, buffer_info->range);
1143 assert(buffer->vk.size > 0 && size > 0);
1144
1145 /* robustBufferAccess is relaxed enough to allow this (in combination
1146 * with the alignment/size we return from vkGetBufferMemoryRequirements)
1147 * and this allows the shader compiler to create more efficient 8/16-bit
1148 * buffer accesses. */
1149 size = align(size, 4);
1150
1151 va += buffer_info->offset + buffer->offset;
1152 range->va = va;
1153 range->size = size;
1154
1155 *buffer_list = buffer->bo;
1156 }
1157
1158 static ALWAYS_INLINE void
write_image_descriptor(unsigned * dst,unsigned size,VkDescriptorType descriptor_type,const VkDescriptorImageInfo * image_info)1159 write_image_descriptor(unsigned *dst, unsigned size, VkDescriptorType descriptor_type,
1160 const VkDescriptorImageInfo *image_info)
1161 {
1162 struct radv_image_view *iview = NULL;
1163 union radv_descriptor *descriptor;
1164
1165 if (image_info)
1166 iview = radv_image_view_from_handle(image_info->imageView);
1167
1168 if (!iview) {
1169 memset(dst, 0, size);
1170 return;
1171 }
1172
1173 if (descriptor_type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) {
1174 descriptor = &iview->storage_descriptor;
1175 } else {
1176 descriptor = &iview->descriptor;
1177 }
1178 assert(size > 0);
1179
1180 /* Encourage compilers to inline memcpy for combined image/sampler descriptors. */
1181 switch (size) {
1182 case 32:
1183 memcpy(dst, descriptor, 32);
1184 break;
1185 case 64:
1186 memcpy(dst, descriptor, 64);
1187 break;
1188 case 80:
1189 memcpy(dst, descriptor, 80);
1190 break;
1191 case 96:
1192 memcpy(dst, descriptor, 96);
1193 break;
1194 default:
1195 unreachable("Invalid size");
1196 }
1197 }
1198
1199 static ALWAYS_INLINE void
write_image_descriptor_impl(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,unsigned size,unsigned * dst,struct radeon_winsys_bo ** buffer_list,VkDescriptorType descriptor_type,const VkDescriptorImageInfo * image_info)1200 write_image_descriptor_impl(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer, unsigned size,
1201 unsigned *dst, struct radeon_winsys_bo **buffer_list, VkDescriptorType descriptor_type,
1202 const VkDescriptorImageInfo *image_info)
1203 {
1204 VK_FROM_HANDLE(radv_image_view, iview, image_info->imageView);
1205
1206 write_image_descriptor(dst, size, descriptor_type, image_info);
1207
1208 if (device->use_global_bo_list)
1209 return;
1210
1211 if (!iview) {
1212 if (!cmd_buffer)
1213 *buffer_list = NULL;
1214 return;
1215 }
1216
1217 const uint32_t max_bindings = sizeof(iview->image->bindings) / sizeof(iview->image->bindings[0]);
1218 for (uint32_t b = 0; b < max_bindings; b++) {
1219 if (cmd_buffer) {
1220 if (iview->image->bindings[b].bo)
1221 radv_cs_add_buffer(device->ws, cmd_buffer->cs, iview->image->bindings[b].bo);
1222 } else {
1223 *buffer_list = iview->image->bindings[b].bo;
1224 buffer_list++;
1225 }
1226 }
1227 }
1228
1229 static ALWAYS_INLINE void
write_combined_image_sampler_descriptor(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,unsigned sampler_offset,unsigned * dst,struct radeon_winsys_bo ** buffer_list,VkDescriptorType descriptor_type,const VkDescriptorImageInfo * image_info,bool has_sampler)1230 write_combined_image_sampler_descriptor(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
1231 unsigned sampler_offset, unsigned *dst, struct radeon_winsys_bo **buffer_list,
1232 VkDescriptorType descriptor_type, const VkDescriptorImageInfo *image_info,
1233 bool has_sampler)
1234 {
1235 write_image_descriptor_impl(device, cmd_buffer, sampler_offset, dst, buffer_list, descriptor_type, image_info);
1236 /* copy over sampler state */
1237 if (has_sampler) {
1238 VK_FROM_HANDLE(radv_sampler, sampler, image_info->sampler);
1239 memcpy(dst + sampler_offset / sizeof(*dst), sampler->state, 16);
1240 }
1241 }
1242
1243 static ALWAYS_INLINE void
write_sampler_descriptor(unsigned * dst,VkSampler _sampler)1244 write_sampler_descriptor(unsigned *dst, VkSampler _sampler)
1245 {
1246 VK_FROM_HANDLE(radv_sampler, sampler, _sampler);
1247 memcpy(dst, sampler->state, 16);
1248 }
1249
1250 static ALWAYS_INLINE void
write_accel_struct(struct radv_device * device,void * ptr,VkDeviceAddress va)1251 write_accel_struct(struct radv_device *device, void *ptr, VkDeviceAddress va)
1252 {
1253 if (!va) {
1254 VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, device->meta_state.accel_struct_build.null.accel_struct);
1255 va = vk_acceleration_structure_get_va(accel_struct);
1256 }
1257
1258 memcpy(ptr, &va, sizeof(va));
1259 }
1260
1261 static ALWAYS_INLINE void
radv_update_descriptor_sets_impl(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,VkDescriptorSet dstSetOverride,uint32_t descriptorWriteCount,const VkWriteDescriptorSet * pDescriptorWrites,uint32_t descriptorCopyCount,const VkCopyDescriptorSet * pDescriptorCopies)1262 radv_update_descriptor_sets_impl(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
1263 VkDescriptorSet dstSetOverride, uint32_t descriptorWriteCount,
1264 const VkWriteDescriptorSet *pDescriptorWrites, uint32_t descriptorCopyCount,
1265 const VkCopyDescriptorSet *pDescriptorCopies)
1266 {
1267 uint32_t i, j;
1268 for (i = 0; i < descriptorWriteCount; i++) {
1269 const VkWriteDescriptorSet *writeset = &pDescriptorWrites[i];
1270 VK_FROM_HANDLE(radv_descriptor_set, set, dstSetOverride ? dstSetOverride : writeset->dstSet);
1271 const struct radv_descriptor_set_binding_layout *binding_layout =
1272 set->header.layout->binding + writeset->dstBinding;
1273 uint32_t *ptr = set->header.mapped_ptr;
1274 struct radeon_winsys_bo **buffer_list = set->descriptors;
1275 /* Immutable samplers are not copied into push descriptors when they are
1276 * allocated, so if we are writing push descriptors we have to copy the
1277 * immutable samplers into them now.
1278 */
1279 const bool copy_immutable_samplers =
1280 cmd_buffer && binding_layout->immutable_samplers_offset && !binding_layout->immutable_samplers_equal;
1281 const uint32_t *samplers = radv_immutable_samplers(set->header.layout, binding_layout);
1282 const VkWriteDescriptorSetAccelerationStructureKHR *accel_structs = NULL;
1283
1284 ptr += binding_layout->offset / 4;
1285
1286 if (writeset->descriptorType == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) {
1287 write_block_descriptor(device, cmd_buffer, (uint8_t *)ptr + writeset->dstArrayElement, writeset);
1288 continue;
1289 } else if (writeset->descriptorType == VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR) {
1290 accel_structs = vk_find_struct_const(writeset->pNext, WRITE_DESCRIPTOR_SET_ACCELERATION_STRUCTURE_KHR);
1291 }
1292
1293 ptr += binding_layout->size * writeset->dstArrayElement / 4;
1294 buffer_list += binding_layout->buffer_offset;
1295 buffer_list += writeset->dstArrayElement * radv_descriptor_type_buffer_count(writeset->descriptorType);
1296 for (j = 0; j < writeset->descriptorCount; ++j) {
1297 switch (writeset->descriptorType) {
1298 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1299 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
1300 unsigned idx = writeset->dstArrayElement + j;
1301 idx += binding_layout->dynamic_offset_offset;
1302 assert(!(set->header.layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
1303 write_dynamic_buffer_descriptor(device, set->header.dynamic_descriptors + idx, buffer_list,
1304 writeset->pBufferInfo + j);
1305 break;
1306 }
1307 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
1308 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
1309 write_buffer_descriptor_impl(device, cmd_buffer, ptr, buffer_list, writeset->pBufferInfo + j);
1310 break;
1311 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
1312 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
1313 write_texel_buffer_descriptor(device, cmd_buffer, ptr, buffer_list, writeset->pTexelBufferView[j]);
1314 break;
1315 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
1316 write_image_descriptor_impl(device, cmd_buffer, 32, ptr, buffer_list, writeset->descriptorType,
1317 writeset->pImageInfo + j);
1318 break;
1319 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
1320 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
1321 write_image_descriptor_impl(device, cmd_buffer, 64, ptr, buffer_list, writeset->descriptorType,
1322 writeset->pImageInfo + j);
1323 break;
1324 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: {
1325 unsigned sampler_offset = radv_combined_image_descriptor_sampler_offset(binding_layout);
1326 write_combined_image_sampler_descriptor(device, cmd_buffer, sampler_offset, ptr, buffer_list,
1327 writeset->descriptorType, writeset->pImageInfo + j,
1328 !binding_layout->immutable_samplers_offset);
1329 if (copy_immutable_samplers) {
1330 const unsigned idx = writeset->dstArrayElement + j;
1331 memcpy((char *)ptr + sampler_offset, samplers + 4 * idx, 16);
1332 }
1333 break;
1334 }
1335 case VK_DESCRIPTOR_TYPE_SAMPLER:
1336 if (!binding_layout->immutable_samplers_offset) {
1337 const VkDescriptorImageInfo *pImageInfo = writeset->pImageInfo + j;
1338 write_sampler_descriptor(ptr, pImageInfo->sampler);
1339 } else if (copy_immutable_samplers) {
1340 unsigned idx = writeset->dstArrayElement + j;
1341 memcpy(ptr, samplers + 4 * idx, 16);
1342 }
1343 break;
1344 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: {
1345 VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, accel_structs->pAccelerationStructures[j]);
1346
1347 write_accel_struct(device, ptr, accel_struct ? vk_acceleration_structure_get_va(accel_struct) : 0);
1348 break;
1349 }
1350 default:
1351 break;
1352 }
1353 ptr += binding_layout->size / 4;
1354 buffer_list += radv_descriptor_type_buffer_count(writeset->descriptorType);
1355 }
1356 }
1357
1358 for (i = 0; i < descriptorCopyCount; i++) {
1359 const VkCopyDescriptorSet *copyset = &pDescriptorCopies[i];
1360 VK_FROM_HANDLE(radv_descriptor_set, src_set, copyset->srcSet);
1361 VK_FROM_HANDLE(radv_descriptor_set, dst_set, copyset->dstSet);
1362 const struct radv_descriptor_set_binding_layout *src_binding_layout =
1363 src_set->header.layout->binding + copyset->srcBinding;
1364 const struct radv_descriptor_set_binding_layout *dst_binding_layout =
1365 dst_set->header.layout->binding + copyset->dstBinding;
1366 uint32_t *src_ptr = src_set->header.mapped_ptr;
1367 uint32_t *dst_ptr = dst_set->header.mapped_ptr;
1368 struct radeon_winsys_bo **src_buffer_list = src_set->descriptors;
1369 struct radeon_winsys_bo **dst_buffer_list = dst_set->descriptors;
1370
1371 src_ptr += src_binding_layout->offset / 4;
1372 dst_ptr += dst_binding_layout->offset / 4;
1373
1374 if (src_binding_layout->type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) {
1375 src_ptr += copyset->srcArrayElement / 4;
1376 dst_ptr += copyset->dstArrayElement / 4;
1377
1378 memcpy(dst_ptr, src_ptr, copyset->descriptorCount);
1379 continue;
1380 }
1381
1382 src_ptr += src_binding_layout->size * copyset->srcArrayElement / 4;
1383 dst_ptr += dst_binding_layout->size * copyset->dstArrayElement / 4;
1384
1385 src_buffer_list += src_binding_layout->buffer_offset;
1386 src_buffer_list += copyset->srcArrayElement;
1387
1388 dst_buffer_list += dst_binding_layout->buffer_offset;
1389 dst_buffer_list += copyset->dstArrayElement;
1390
1391 /* In case of copies between mutable descriptor types
1392 * and non-mutable descriptor types. */
1393 size_t copy_size = MIN2(src_binding_layout->size, dst_binding_layout->size);
1394
1395 for (j = 0; j < copyset->descriptorCount; ++j) {
1396 switch (src_binding_layout->type) {
1397 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1398 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
1399 unsigned src_idx = copyset->srcArrayElement + j;
1400 unsigned dst_idx = copyset->dstArrayElement + j;
1401 struct radv_descriptor_range *src_range, *dst_range;
1402 src_idx += src_binding_layout->dynamic_offset_offset;
1403 dst_idx += dst_binding_layout->dynamic_offset_offset;
1404
1405 src_range = src_set->header.dynamic_descriptors + src_idx;
1406 dst_range = dst_set->header.dynamic_descriptors + dst_idx;
1407 *dst_range = *src_range;
1408 break;
1409 }
1410 default:
1411 memcpy(dst_ptr, src_ptr, copy_size);
1412 }
1413 src_ptr += src_binding_layout->size / 4;
1414 dst_ptr += dst_binding_layout->size / 4;
1415
1416 unsigned src_buffer_count = radv_descriptor_type_buffer_count(src_binding_layout->type);
1417 unsigned dst_buffer_count = radv_descriptor_type_buffer_count(dst_binding_layout->type);
1418 for (unsigned k = 0; k < dst_buffer_count; k++) {
1419 if (k < src_buffer_count)
1420 dst_buffer_list[k] = src_buffer_list[k];
1421 else
1422 dst_buffer_list[k] = NULL;
1423 }
1424
1425 dst_buffer_list += dst_buffer_count;
1426 src_buffer_list += src_buffer_count;
1427 }
1428 }
1429 }
1430
1431 VKAPI_ATTR void VKAPI_CALL
radv_UpdateDescriptorSets(VkDevice _device,uint32_t descriptorWriteCount,const VkWriteDescriptorSet * pDescriptorWrites,uint32_t descriptorCopyCount,const VkCopyDescriptorSet * pDescriptorCopies)1432 radv_UpdateDescriptorSets(VkDevice _device, uint32_t descriptorWriteCount,
1433 const VkWriteDescriptorSet *pDescriptorWrites, uint32_t descriptorCopyCount,
1434 const VkCopyDescriptorSet *pDescriptorCopies)
1435 {
1436 VK_FROM_HANDLE(radv_device, device, _device);
1437
1438 radv_update_descriptor_sets_impl(device, NULL, VK_NULL_HANDLE, descriptorWriteCount, pDescriptorWrites,
1439 descriptorCopyCount, pDescriptorCopies);
1440 }
1441
1442 void
radv_cmd_update_descriptor_sets(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,VkDescriptorSet dstSetOverride,uint32_t descriptorWriteCount,const VkWriteDescriptorSet * pDescriptorWrites,uint32_t descriptorCopyCount,const VkCopyDescriptorSet * pDescriptorCopies)1443 radv_cmd_update_descriptor_sets(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
1444 VkDescriptorSet dstSetOverride, uint32_t descriptorWriteCount,
1445 const VkWriteDescriptorSet *pDescriptorWrites, uint32_t descriptorCopyCount,
1446 const VkCopyDescriptorSet *pDescriptorCopies)
1447 {
1448 /* Assume cmd_buffer != NULL to optimize out cmd_buffer checks in generic code above. */
1449 assume(cmd_buffer != NULL);
1450 radv_update_descriptor_sets_impl(device, cmd_buffer, dstSetOverride, descriptorWriteCount, pDescriptorWrites,
1451 descriptorCopyCount, pDescriptorCopies);
1452 }
1453
1454 VKAPI_ATTR VkResult VKAPI_CALL
radv_CreateDescriptorUpdateTemplate(VkDevice _device,const VkDescriptorUpdateTemplateCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorUpdateTemplate * pDescriptorUpdateTemplate)1455 radv_CreateDescriptorUpdateTemplate(VkDevice _device, const VkDescriptorUpdateTemplateCreateInfo *pCreateInfo,
1456 const VkAllocationCallbacks *pAllocator,
1457 VkDescriptorUpdateTemplate *pDescriptorUpdateTemplate)
1458 {
1459 VK_FROM_HANDLE(radv_device, device, _device);
1460 const uint32_t entry_count = pCreateInfo->descriptorUpdateEntryCount;
1461 const size_t size = sizeof(struct radv_descriptor_update_template) +
1462 sizeof(struct radv_descriptor_update_template_entry) * entry_count;
1463 struct radv_descriptor_set_layout *set_layout = NULL;
1464 struct radv_descriptor_update_template *templ;
1465 uint32_t i;
1466
1467 templ = vk_alloc2(&device->vk.alloc, pAllocator, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1468 if (!templ)
1469 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1470
1471 vk_object_base_init(&device->vk, &templ->base, VK_OBJECT_TYPE_DESCRIPTOR_UPDATE_TEMPLATE);
1472
1473 templ->entry_count = entry_count;
1474
1475 if (pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR) {
1476 VK_FROM_HANDLE(radv_pipeline_layout, pipeline_layout, pCreateInfo->pipelineLayout);
1477
1478 /* descriptorSetLayout should be ignored for push descriptors
1479 * and instead it refers to pipelineLayout and set.
1480 */
1481 assert(pCreateInfo->set < MAX_SETS);
1482 set_layout = pipeline_layout->set[pCreateInfo->set].layout;
1483
1484 templ->bind_point = pCreateInfo->pipelineBindPoint;
1485 } else {
1486 assert(pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET);
1487 set_layout = radv_descriptor_set_layout_from_handle(pCreateInfo->descriptorSetLayout);
1488 }
1489
1490 for (i = 0; i < entry_count; i++) {
1491 const VkDescriptorUpdateTemplateEntry *entry = &pCreateInfo->pDescriptorUpdateEntries[i];
1492 const struct radv_descriptor_set_binding_layout *binding_layout = set_layout->binding + entry->dstBinding;
1493 const uint32_t buffer_offset = binding_layout->buffer_offset + entry->dstArrayElement;
1494 const uint32_t *immutable_samplers = NULL;
1495 uint32_t dst_offset;
1496 uint32_t dst_stride;
1497
1498 /* dst_offset is an offset into dynamic_descriptors when the descriptor
1499 is dynamic, and an offset into mapped_ptr otherwise */
1500 switch (entry->descriptorType) {
1501 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1502 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
1503 assert(pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET);
1504 dst_offset = binding_layout->dynamic_offset_offset + entry->dstArrayElement;
1505 dst_stride = 0; /* Not used */
1506 break;
1507 default:
1508 switch (entry->descriptorType) {
1509 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1510 case VK_DESCRIPTOR_TYPE_SAMPLER:
1511 /* Immutable samplers are copied into push descriptors when they are pushed */
1512 if (pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR &&
1513 binding_layout->immutable_samplers_offset && !binding_layout->immutable_samplers_equal) {
1514 immutable_samplers = radv_immutable_samplers(set_layout, binding_layout) + entry->dstArrayElement * 4;
1515 }
1516 break;
1517 default:
1518 break;
1519 }
1520 dst_offset = binding_layout->offset / 4;
1521 if (entry->descriptorType == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK)
1522 dst_offset += entry->dstArrayElement / 4;
1523 else
1524 dst_offset += binding_layout->size * entry->dstArrayElement / 4;
1525
1526 dst_stride = binding_layout->size / 4;
1527 break;
1528 }
1529
1530 templ->entry[i] = (struct radv_descriptor_update_template_entry){
1531 .descriptor_type = entry->descriptorType,
1532 .descriptor_count = entry->descriptorCount,
1533 .src_offset = entry->offset,
1534 .src_stride = entry->stride,
1535 .dst_offset = dst_offset,
1536 .dst_stride = dst_stride,
1537 .buffer_offset = buffer_offset,
1538 .has_sampler = !binding_layout->immutable_samplers_offset,
1539 .sampler_offset = radv_combined_image_descriptor_sampler_offset(binding_layout),
1540 .immutable_samplers = immutable_samplers};
1541 }
1542
1543 *pDescriptorUpdateTemplate = radv_descriptor_update_template_to_handle(templ);
1544 return VK_SUCCESS;
1545 }
1546
1547 VKAPI_ATTR void VKAPI_CALL
radv_DestroyDescriptorUpdateTemplate(VkDevice _device,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const VkAllocationCallbacks * pAllocator)1548 radv_DestroyDescriptorUpdateTemplate(VkDevice _device, VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1549 const VkAllocationCallbacks *pAllocator)
1550 {
1551 VK_FROM_HANDLE(radv_device, device, _device);
1552 VK_FROM_HANDLE(radv_descriptor_update_template, templ, descriptorUpdateTemplate);
1553
1554 if (!templ)
1555 return;
1556
1557 vk_object_base_finish(&templ->base);
1558 vk_free2(&device->vk.alloc, pAllocator, templ);
1559 }
1560
1561 static ALWAYS_INLINE void
radv_update_descriptor_set_with_template_impl(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,struct radv_descriptor_set * set,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const void * pData)1562 radv_update_descriptor_set_with_template_impl(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
1563 struct radv_descriptor_set *set,
1564 VkDescriptorUpdateTemplate descriptorUpdateTemplate, const void *pData)
1565 {
1566 VK_FROM_HANDLE(radv_descriptor_update_template, templ, descriptorUpdateTemplate);
1567 uint32_t i;
1568
1569 for (i = 0; i < templ->entry_count; ++i) {
1570 struct radeon_winsys_bo **buffer_list = set->descriptors + templ->entry[i].buffer_offset;
1571 uint32_t *pDst = set->header.mapped_ptr + templ->entry[i].dst_offset;
1572 const uint8_t *pSrc = ((const uint8_t *)pData) + templ->entry[i].src_offset;
1573 uint32_t j;
1574
1575 if (templ->entry[i].descriptor_type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) {
1576 memcpy((uint8_t *)pDst, pSrc, templ->entry[i].descriptor_count);
1577 continue;
1578 }
1579
1580 for (j = 0; j < templ->entry[i].descriptor_count; ++j) {
1581 switch (templ->entry[i].descriptor_type) {
1582 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1583 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
1584 const unsigned idx = templ->entry[i].dst_offset + j;
1585 assert(!(set->header.layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
1586 write_dynamic_buffer_descriptor(device, set->header.dynamic_descriptors + idx, buffer_list,
1587 (struct VkDescriptorBufferInfo *)pSrc);
1588 break;
1589 }
1590 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
1591 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
1592 write_buffer_descriptor_impl(device, cmd_buffer, pDst, buffer_list, (struct VkDescriptorBufferInfo *)pSrc);
1593 break;
1594 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
1595 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
1596 write_texel_buffer_descriptor(device, cmd_buffer, pDst, buffer_list, *(VkBufferView *)pSrc);
1597 break;
1598 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
1599 write_image_descriptor_impl(device, cmd_buffer, 32, pDst, buffer_list, templ->entry[i].descriptor_type,
1600 (struct VkDescriptorImageInfo *)pSrc);
1601 break;
1602 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
1603 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
1604 write_image_descriptor_impl(device, cmd_buffer, 64, pDst, buffer_list, templ->entry[i].descriptor_type,
1605 (struct VkDescriptorImageInfo *)pSrc);
1606 break;
1607 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1608 write_combined_image_sampler_descriptor(device, cmd_buffer, templ->entry[i].sampler_offset, pDst,
1609 buffer_list, templ->entry[i].descriptor_type,
1610 (struct VkDescriptorImageInfo *)pSrc, templ->entry[i].has_sampler);
1611 if (cmd_buffer && templ->entry[i].immutable_samplers) {
1612 memcpy((char *)pDst + templ->entry[i].sampler_offset, templ->entry[i].immutable_samplers + 4 * j, 16);
1613 }
1614 break;
1615 case VK_DESCRIPTOR_TYPE_SAMPLER:
1616 if (templ->entry[i].has_sampler) {
1617 const VkDescriptorImageInfo *pImageInfo = (struct VkDescriptorImageInfo *)pSrc;
1618 write_sampler_descriptor(pDst, pImageInfo->sampler);
1619 } else if (cmd_buffer && templ->entry[i].immutable_samplers)
1620 memcpy(pDst, templ->entry[i].immutable_samplers + 4 * j, 16);
1621 break;
1622 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: {
1623 VK_FROM_HANDLE(vk_acceleration_structure, accel_struct, *(const VkAccelerationStructureKHR *)pSrc);
1624 write_accel_struct(device, pDst, accel_struct ? vk_acceleration_structure_get_va(accel_struct) : 0);
1625 break;
1626 }
1627 default:
1628 break;
1629 }
1630 pSrc += templ->entry[i].src_stride;
1631 pDst += templ->entry[i].dst_stride;
1632
1633 buffer_list += radv_descriptor_type_buffer_count(templ->entry[i].descriptor_type);
1634 }
1635 }
1636 }
1637
1638 void
radv_cmd_update_descriptor_set_with_template(struct radv_device * device,struct radv_cmd_buffer * cmd_buffer,struct radv_descriptor_set * set,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const void * pData)1639 radv_cmd_update_descriptor_set_with_template(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
1640 struct radv_descriptor_set *set,
1641 VkDescriptorUpdateTemplate descriptorUpdateTemplate, const void *pData)
1642 {
1643 /* Assume cmd_buffer != NULL to optimize out cmd_buffer checks in generic code above. */
1644 assume(cmd_buffer != NULL);
1645 radv_update_descriptor_set_with_template_impl(device, cmd_buffer, set, descriptorUpdateTemplate, pData);
1646 }
1647
1648 VKAPI_ATTR void VKAPI_CALL
radv_UpdateDescriptorSetWithTemplate(VkDevice _device,VkDescriptorSet descriptorSet,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const void * pData)1649 radv_UpdateDescriptorSetWithTemplate(VkDevice _device, VkDescriptorSet descriptorSet,
1650 VkDescriptorUpdateTemplate descriptorUpdateTemplate, const void *pData)
1651 {
1652 VK_FROM_HANDLE(radv_device, device, _device);
1653 VK_FROM_HANDLE(radv_descriptor_set, set, descriptorSet);
1654
1655 radv_update_descriptor_set_with_template_impl(device, NULL, set, descriptorUpdateTemplate, pData);
1656 }
1657
1658 VKAPI_ATTR void VKAPI_CALL
radv_GetDescriptorSetLayoutHostMappingInfoVALVE(VkDevice _device,const VkDescriptorSetBindingReferenceVALVE * pBindingReference,VkDescriptorSetLayoutHostMappingInfoVALVE * pHostMapping)1659 radv_GetDescriptorSetLayoutHostMappingInfoVALVE(VkDevice _device,
1660 const VkDescriptorSetBindingReferenceVALVE *pBindingReference,
1661 VkDescriptorSetLayoutHostMappingInfoVALVE *pHostMapping)
1662 {
1663 struct radv_descriptor_set_layout *set_layout =
1664 radv_descriptor_set_layout_from_handle(pBindingReference->descriptorSetLayout);
1665
1666 const struct radv_descriptor_set_binding_layout *binding_layout = set_layout->binding + pBindingReference->binding;
1667
1668 pHostMapping->descriptorOffset = binding_layout->offset;
1669 pHostMapping->descriptorSize = binding_layout->size;
1670 }
1671
1672 VKAPI_ATTR void VKAPI_CALL
radv_GetDescriptorSetHostMappingVALVE(VkDevice _device,VkDescriptorSet descriptorSet,void ** ppData)1673 radv_GetDescriptorSetHostMappingVALVE(VkDevice _device, VkDescriptorSet descriptorSet, void **ppData)
1674 {
1675 VK_FROM_HANDLE(radv_descriptor_set, set, descriptorSet);
1676 *ppData = set->header.mapped_ptr;
1677 }
1678
1679 /* VK_EXT_descriptor_buffer */
1680 VKAPI_ATTR void VKAPI_CALL
radv_GetDescriptorSetLayoutSizeEXT(VkDevice device,VkDescriptorSetLayout layout,VkDeviceSize * pLayoutSizeInBytes)1681 radv_GetDescriptorSetLayoutSizeEXT(VkDevice device, VkDescriptorSetLayout layout, VkDeviceSize *pLayoutSizeInBytes)
1682 {
1683 VK_FROM_HANDLE(radv_descriptor_set_layout, set_layout, layout);
1684 *pLayoutSizeInBytes = set_layout->size;
1685 }
1686
1687 VKAPI_ATTR void VKAPI_CALL
radv_GetDescriptorSetLayoutBindingOffsetEXT(VkDevice device,VkDescriptorSetLayout layout,uint32_t binding,VkDeviceSize * pOffset)1688 radv_GetDescriptorSetLayoutBindingOffsetEXT(VkDevice device, VkDescriptorSetLayout layout, uint32_t binding,
1689 VkDeviceSize *pOffset)
1690 {
1691 VK_FROM_HANDLE(radv_descriptor_set_layout, set_layout, layout);
1692 *pOffset = set_layout->binding[binding].offset;
1693 }
1694
1695 VKAPI_ATTR void VKAPI_CALL
radv_GetDescriptorEXT(VkDevice _device,const VkDescriptorGetInfoEXT * pDescriptorInfo,size_t dataSize,void * pDescriptor)1696 radv_GetDescriptorEXT(VkDevice _device, const VkDescriptorGetInfoEXT *pDescriptorInfo, size_t dataSize,
1697 void *pDescriptor)
1698 {
1699 VK_FROM_HANDLE(radv_device, device, _device);
1700
1701 switch (pDescriptorInfo->type) {
1702 case VK_DESCRIPTOR_TYPE_SAMPLER: {
1703 write_sampler_descriptor(pDescriptor, *pDescriptorInfo->data.pSampler);
1704 break;
1705 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1706 write_image_descriptor(pDescriptor, 64, pDescriptorInfo->type, pDescriptorInfo->data.pCombinedImageSampler);
1707 if (pDescriptorInfo->data.pCombinedImageSampler) {
1708 write_sampler_descriptor((uint32_t *)pDescriptor + 20, pDescriptorInfo->data.pCombinedImageSampler->sampler);
1709 }
1710 break;
1711 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
1712 write_image_descriptor(pDescriptor, 64, pDescriptorInfo->type, pDescriptorInfo->data.pInputAttachmentImage);
1713 break;
1714 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
1715 write_image_descriptor(pDescriptor, 64, pDescriptorInfo->type, pDescriptorInfo->data.pSampledImage);
1716 break;
1717 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
1718 write_image_descriptor(pDescriptor, 32, pDescriptorInfo->type, pDescriptorInfo->data.pStorageImage);
1719 break;
1720 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: {
1721 const VkDescriptorAddressInfoEXT *addr_info = pDescriptorInfo->data.pUniformBuffer;
1722
1723 write_buffer_descriptor(device, pDescriptor, addr_info ? addr_info->address : 0,
1724 addr_info ? addr_info->range : 0);
1725 break;
1726 }
1727 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: {
1728 const VkDescriptorAddressInfoEXT *addr_info = pDescriptorInfo->data.pStorageBuffer;
1729
1730 write_buffer_descriptor(device, pDescriptor, addr_info ? addr_info->address : 0,
1731 addr_info ? addr_info->range : 0);
1732 break;
1733 }
1734 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: {
1735 const VkDescriptorAddressInfoEXT *addr_info = pDescriptorInfo->data.pUniformTexelBuffer;
1736
1737 if (addr_info && addr_info->address) {
1738 radv_make_texel_buffer_descriptor(device, addr_info->address, addr_info->format, 0, addr_info->range,
1739 pDescriptor);
1740 } else {
1741 memset(pDescriptor, 0, 4 * 4);
1742 }
1743 break;
1744 }
1745 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: {
1746 const VkDescriptorAddressInfoEXT *addr_info = pDescriptorInfo->data.pStorageTexelBuffer;
1747
1748 if (addr_info && addr_info->address) {
1749 radv_make_texel_buffer_descriptor(device, addr_info->address, addr_info->format, 0, addr_info->range,
1750 pDescriptor);
1751 } else {
1752 memset(pDescriptor, 0, 4 * 4);
1753 }
1754 break;
1755 }
1756 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR:
1757 write_accel_struct(device, pDescriptor, pDescriptorInfo->data.accelerationStructure);
1758 break;
1759 }
1760 default:
1761 unreachable("invalid descriptor type");
1762 }
1763 }
1764