1 /*
2 * Copyright © 2022 Collabora Ltd. and Red Hat Inc.
3 * SPDX-License-Identifier: MIT
4 */
5 #include "nvk_image.h"
6
7 #include "nvk_device.h"
8 #include "nvk_device_memory.h"
9 #include "nvk_entrypoints.h"
10 #include "nvk_format.h"
11 #include "nvk_physical_device.h"
12 #include "nvkmd/nvkmd.h"
13
14 #include "vk_enum_to_str.h"
15 #include "vk_format.h"
16 #include "nil.h"
17 #include "vk_enum_defines.h"
18 #include "vk_format.h"
19
20 #include "clb097.h"
21 #include "clb197.h"
22 #include "clc097.h"
23 #include "clc597.h"
24
25 static VkFormatFeatureFlags2
nvk_get_image_plane_format_features(struct nvk_physical_device * pdev,VkFormat vk_format,VkImageTiling tiling,uint64_t drm_format_mod)26 nvk_get_image_plane_format_features(struct nvk_physical_device *pdev,
27 VkFormat vk_format, VkImageTiling tiling,
28 uint64_t drm_format_mod)
29 {
30 VkFormatFeatureFlags2 features = 0;
31
32 if (tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT &&
33 drm_format_mod != DRM_FORMAT_MOD_LINEAR &&
34 !fourcc_mod_is_vendor(drm_format_mod, NVIDIA))
35 return 0;
36
37 enum pipe_format p_format = vk_format_to_pipe_format(vk_format);
38 if (p_format == PIPE_FORMAT_NONE)
39 return 0;
40
41 /* You can't tile a non-power-of-two */
42 if (!util_is_power_of_two_nonzero(util_format_get_blocksize(p_format)))
43 return 0;
44
45 if (nil_format_supports_texturing(&pdev->info, p_format)) {
46 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT;
47 features |= VK_FORMAT_FEATURE_2_BLIT_SRC_BIT;
48 }
49
50 if (nil_format_supports_filtering(&pdev->info, p_format)) {
51 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_LINEAR_BIT;
52 if (pdev->info.cls_eng3d >= MAXWELL_B)
53 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_MINMAX_BIT;
54 }
55
56 /* TODO: VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT */
57 if (vk_format_has_depth(vk_format)) {
58 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT;
59 }
60
61 if (nil_format_supports_color_targets(&pdev->info, p_format) &&
62 tiling != VK_IMAGE_TILING_LINEAR) {
63 features |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT;
64 if (nil_format_supports_blending(&pdev->info, p_format))
65 features |= VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BLEND_BIT;
66 features |= VK_FORMAT_FEATURE_2_BLIT_DST_BIT;
67 }
68
69 if (vk_format_is_depth_or_stencil(vk_format)) {
70 if (!nil_format_supports_depth_stencil(&pdev->info, p_format) ||
71 tiling == VK_IMAGE_TILING_LINEAR)
72 return 0;
73
74 features |= VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT;
75 }
76
77 if (nil_format_supports_storage(&pdev->info, p_format)) {
78 features |= VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT |
79 VK_FORMAT_FEATURE_2_STORAGE_WRITE_WITHOUT_FORMAT_BIT;
80 if (pdev->info.cls_eng3d >= MAXWELL_A)
81 features |= VK_FORMAT_FEATURE_2_STORAGE_READ_WITHOUT_FORMAT_BIT;
82 }
83
84 if (p_format == PIPE_FORMAT_R32_UINT || p_format == PIPE_FORMAT_R32_SINT ||
85 p_format == PIPE_FORMAT_R64_UINT || p_format == PIPE_FORMAT_R64_SINT)
86 features |= VK_FORMAT_FEATURE_2_STORAGE_IMAGE_ATOMIC_BIT;
87
88 if (features != 0) {
89 features |= VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT;
90 features |= VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT;
91 }
92
93 return features;
94 }
95
96 VkFormatFeatureFlags2
nvk_get_image_format_features(struct nvk_physical_device * pdev,VkFormat vk_format,VkImageTiling tiling,uint64_t drm_format_mod)97 nvk_get_image_format_features(struct nvk_physical_device *pdev,
98 VkFormat vk_format, VkImageTiling tiling,
99 uint64_t drm_format_mod)
100 {
101 const struct vk_format_ycbcr_info *ycbcr_info =
102 vk_format_get_ycbcr_info(vk_format);
103 if (ycbcr_info == NULL) {
104 return nvk_get_image_plane_format_features(pdev, vk_format, tiling,
105 drm_format_mod);
106 }
107
108 /* For multi-plane, we get the feature flags of each plane separately,
109 * then take their intersection as the overall format feature flags
110 */
111 VkFormatFeatureFlags2 features = ~0ull;
112 bool cosited_chroma = false;
113 for (uint8_t plane = 0; plane < ycbcr_info->n_planes; plane++) {
114 const struct vk_format_ycbcr_plane *plane_info = &ycbcr_info->planes[plane];
115 features &= nvk_get_image_plane_format_features(pdev, plane_info->format,
116 tiling, drm_format_mod);
117 if (plane_info->denominator_scales[0] > 1 ||
118 plane_info->denominator_scales[1] > 1)
119 cosited_chroma = true;
120 }
121 if (features == 0)
122 return 0;
123
124 /* Uh... We really should be able to sample from YCbCr */
125 assert(features & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT);
126 assert(features & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_LINEAR_BIT);
127
128 /* These aren't allowed for YCbCr formats */
129 features &= ~(VK_FORMAT_FEATURE_2_BLIT_SRC_BIT |
130 VK_FORMAT_FEATURE_2_BLIT_DST_BIT |
131 VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT |
132 VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BLEND_BIT |
133 VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
134
135 /* This is supported on all YCbCr formats */
136 features |= VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_YCBCR_CONVERSION_LINEAR_FILTER_BIT;
137
138 if (ycbcr_info->n_planes > 1) {
139 /* DISJOINT_BIT implies that each plane has its own separate binding,
140 * while SEPARATE_RECONSTRUCTION_FILTER_BIT implies that luma and chroma
141 * each have their own, separate filters, so these two bits make sense
142 * for multi-planar formats only.
143 *
144 * For MIDPOINT_CHROMA_SAMPLES_BIT, NVIDIA HW on single-plane interleaved
145 * YCbCr defaults to COSITED_EVEN, which is inaccurate and fails tests.
146 * This can be fixed with a NIR tweak but for now, we only enable this bit
147 * for multi-plane formats. See Issue #9525 on the mesa/main tracker.
148 */
149 features |= VK_FORMAT_FEATURE_DISJOINT_BIT |
150 VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_YCBCR_CONVERSION_SEPARATE_RECONSTRUCTION_FILTER_BIT |
151 VK_FORMAT_FEATURE_2_MIDPOINT_CHROMA_SAMPLES_BIT;
152 }
153
154 if (cosited_chroma)
155 features |= VK_FORMAT_FEATURE_COSITED_CHROMA_SAMPLES_BIT;
156
157 return features;
158 }
159
160 void
nvk_get_drm_format_modifier_properties_list(struct nvk_physical_device * pdev,VkFormat vk_format,VkBaseOutStructure * ext)161 nvk_get_drm_format_modifier_properties_list(struct nvk_physical_device *pdev,
162 VkFormat vk_format,
163 VkBaseOutStructure *ext)
164 {
165 assert(ext->sType == VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT ||
166 ext->sType == VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_2_EXT);
167
168 /* The two top-level data structures are the same. It's only when
169 * you get to walking the actual list of modifier properties that
170 * they differ.
171 */
172 VkDrmFormatModifierPropertiesListEXT *p = (void *)ext;
173
174 /* We don't support modifiers for YCbCr images */
175 if (vk_format_get_ycbcr_info(vk_format) != NULL) {
176 p->drmFormatModifierCount = 0;
177 return;
178 }
179
180 /* Check that we actually support the format so we don't try to query
181 * modifiers for formats NIL doesn't support.
182 */
183 const VkFormatFeatureFlags2 tiled_features =
184 nvk_get_image_plane_format_features(pdev, vk_format,
185 VK_IMAGE_TILING_OPTIMAL,
186 DRM_FORMAT_MOD_INVALID);
187 if (tiled_features == 0) {
188 p->drmFormatModifierCount = 0;
189 return;
190 }
191
192 uint64_t mods[NIL_MAX_DRM_FORMAT_MODS];
193 size_t mod_count = NIL_MAX_DRM_FORMAT_MODS;
194 enum pipe_format p_format = vk_format_to_pipe_format(vk_format);
195 nil_drm_format_mods_for_format(&pdev->info, nil_format(p_format),
196 &mod_count, &mods);
197 if (mod_count == 0) {
198 p->drmFormatModifierCount = 0;
199 return;
200 }
201
202 switch (ext->sType) {
203 case VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT: {
204 VK_OUTARRAY_MAKE_TYPED(VkDrmFormatModifierPropertiesEXT, out,
205 p->pDrmFormatModifierProperties,
206 &p->drmFormatModifierCount);
207
208 for (uint32_t i = 0; i < mod_count; i++) {
209 const VkFormatFeatureFlags2 features2 =
210 nvk_get_image_format_features(pdev, vk_format,
211 VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
212 mods[i]);
213 if (features2 != 0) {
214 vk_outarray_append_typed(VkDrmFormatModifierPropertiesEXT, &out, mp) {
215 mp->drmFormatModifier = mods[i];
216 mp->drmFormatModifierPlaneCount = 1;
217 mp->drmFormatModifierTilingFeatures =
218 vk_format_features2_to_features(features2);
219 }
220 }
221 }
222 break;
223 }
224
225 case VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_2_EXT: {
226 VkDrmFormatModifierPropertiesList2EXT *p2 = (void *)p;
227 VK_OUTARRAY_MAKE_TYPED(VkDrmFormatModifierProperties2EXT, out,
228 p2->pDrmFormatModifierProperties,
229 &p2->drmFormatModifierCount);
230
231 for (uint32_t i = 0; i < mod_count; i++) {
232 const VkFormatFeatureFlags2 features2 =
233 nvk_get_image_format_features(pdev, vk_format,
234 VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
235 mods[i]);
236 if (features2 != 0) {
237 vk_outarray_append_typed(VkDrmFormatModifierProperties2EXT, &out, mp) {
238 mp->drmFormatModifier = mods[i];
239 mp->drmFormatModifierPlaneCount = 1;
240 mp->drmFormatModifierTilingFeatures = features2;
241 }
242 }
243 }
244 break;
245 }
246
247 default:
248 unreachable("Invalid structure type");
249 }
250 }
251
252 static VkFormatFeatureFlags2
vk_image_usage_to_format_features(VkImageUsageFlagBits usage_flag)253 vk_image_usage_to_format_features(VkImageUsageFlagBits usage_flag)
254 {
255 assert(util_bitcount(usage_flag) == 1);
256 switch (usage_flag) {
257 case VK_IMAGE_USAGE_TRANSFER_SRC_BIT:
258 return VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT |
259 VK_FORMAT_FEATURE_BLIT_SRC_BIT;
260 case VK_IMAGE_USAGE_TRANSFER_DST_BIT:
261 return VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT |
262 VK_FORMAT_FEATURE_BLIT_DST_BIT;
263 case VK_IMAGE_USAGE_SAMPLED_BIT:
264 return VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT;
265 case VK_IMAGE_USAGE_STORAGE_BIT:
266 return VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT;
267 case VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT:
268 return VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT;
269 case VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT:
270 return VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT;
271 case VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT:
272 return VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT |
273 VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT;
274 default:
275 return 0;
276 }
277 }
278
279 uint32_t
nvk_image_max_dimension(const struct nv_device_info * info,VkImageType image_type)280 nvk_image_max_dimension(const struct nv_device_info *info,
281 VkImageType image_type)
282 {
283 switch (image_type) {
284 case VK_IMAGE_TYPE_1D:
285 case VK_IMAGE_TYPE_2D:
286 return info->cls_eng3d >= PASCAL_A ? 0x8000 : 0x4000;
287 case VK_IMAGE_TYPE_3D:
288 return 0x4000;
289 default:
290 unreachable("Invalid image type");
291 }
292 }
293
294 static uint64_t
get_explicit_drm_format_mod(const void * pNext)295 get_explicit_drm_format_mod(const void *pNext)
296 {
297 const VkPhysicalDeviceImageDrmFormatModifierInfoEXT *drm_format_mod_info =
298 vk_find_struct_const(pNext,
299 PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT);
300 if (drm_format_mod_info)
301 return drm_format_mod_info->drmFormatModifier;
302 else
303 return DRM_FORMAT_MOD_INVALID;
304 }
305
306 VKAPI_ATTR VkResult VKAPI_CALL
nvk_GetPhysicalDeviceImageFormatProperties2(VkPhysicalDevice physicalDevice,const VkPhysicalDeviceImageFormatInfo2 * pImageFormatInfo,VkImageFormatProperties2 * pImageFormatProperties)307 nvk_GetPhysicalDeviceImageFormatProperties2(
308 VkPhysicalDevice physicalDevice,
309 const VkPhysicalDeviceImageFormatInfo2 *pImageFormatInfo,
310 VkImageFormatProperties2 *pImageFormatProperties)
311 {
312 VK_FROM_HANDLE(nvk_physical_device, pdev, physicalDevice);
313
314 const VkPhysicalDeviceExternalImageFormatInfo *external_info =
315 vk_find_struct_const(pImageFormatInfo->pNext,
316 PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO);
317
318 /* Initialize to zero in case we return VK_ERROR_FORMAT_NOT_SUPPORTED */
319 memset(&pImageFormatProperties->imageFormatProperties, 0,
320 sizeof(pImageFormatProperties->imageFormatProperties));
321
322 uint64_t drm_format_mod =
323 get_explicit_drm_format_mod(pImageFormatInfo->pNext);
324 const struct vk_format_ycbcr_info *ycbcr_info =
325 vk_format_get_ycbcr_info(pImageFormatInfo->format);
326
327 /* For the purposes of these checks, we don't care about all the extra
328 * YCbCr features and we just want the accumulation of features available
329 * to all planes of the given format.
330 */
331 VkFormatFeatureFlags2 features;
332 if (ycbcr_info == NULL) {
333 features = nvk_get_image_plane_format_features(
334 pdev, pImageFormatInfo->format, pImageFormatInfo->tiling,
335 drm_format_mod);
336 } else {
337 features = ~0ull;
338 assert(ycbcr_info->n_planes > 0);
339 for (uint8_t plane = 0; plane < ycbcr_info->n_planes; plane++) {
340 const VkFormat plane_format = ycbcr_info->planes[plane].format;
341 features &= nvk_get_image_plane_format_features(
342 pdev, plane_format, pImageFormatInfo->tiling, drm_format_mod);
343 }
344 }
345
346 if (features == 0)
347 return VK_ERROR_FORMAT_NOT_SUPPORTED;
348
349 if (pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR &&
350 pImageFormatInfo->type != VK_IMAGE_TYPE_2D)
351 return VK_ERROR_FORMAT_NOT_SUPPORTED;
352
353 if (ycbcr_info && pImageFormatInfo->type != VK_IMAGE_TYPE_2D)
354 return VK_ERROR_FORMAT_NOT_SUPPORTED;
355
356 /* Maxwell B and earlier don't support sparse residency */
357 if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) &&
358 pdev->info.cls_eng3d < MAXWELL_B)
359 return VK_ERROR_FORMAT_NOT_SUPPORTED;
360
361 /* Don't allow sparse on D32S8 cube maps. The hardware doesn't seem to
362 * handle these correctly and hard-faults instead of the expected soft
363 * fault when there's sparse VA.
364 */
365 if (pImageFormatInfo->format == VK_FORMAT_D32_SFLOAT_S8_UINT &&
366 (pImageFormatInfo->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) &&
367 (pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) &&
368 pdev->info.cls_eng3d < TURING_A)
369 return VK_ERROR_FORMAT_NOT_SUPPORTED;
370
371 /* From the Vulkan 1.3.279 spec:
372 *
373 * VUID-VkImageCreateInfo-tiling-04121
374 *
375 * "If tiling is VK_IMAGE_TILING_LINEAR, flags must not contain
376 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
377 *
378 * VUID-VkImageCreateInfo-imageType-00970
379 *
380 * "If imageType is VK_IMAGE_TYPE_1D, flags must not contain
381 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
382 */
383 if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) &&
384 (pImageFormatInfo->type == VK_IMAGE_TYPE_1D ||
385 pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR))
386 return VK_ERROR_FORMAT_NOT_SUPPORTED;
387
388 /* From the Vulkan 1.3.279 spec:
389 *
390 * VUID-VkImageCreateInfo-flags-09403
391 *
392 * "If flags contains VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT, flags
393 * must not include VK_IMAGE_CREATE_SPARSE_ALIASED_BIT,
394 * VK_IMAGE_CREATE_SPARSE_BINDING_BIT, or
395 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT"
396 */
397 if ((pImageFormatInfo->flags & VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT) &&
398 (pImageFormatInfo->flags & (VK_IMAGE_CREATE_SPARSE_ALIASED_BIT |
399 VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
400 VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)))
401 return VK_ERROR_FORMAT_NOT_SUPPORTED;
402
403 const uint32_t max_dim =
404 nvk_image_max_dimension(&pdev->info, VK_IMAGE_TYPE_1D);
405 VkExtent3D maxExtent;
406 uint32_t maxArraySize;
407 switch (pImageFormatInfo->type) {
408 case VK_IMAGE_TYPE_1D:
409 maxExtent = (VkExtent3D) { max_dim, 1, 1 };
410 maxArraySize = 2048;
411 break;
412 case VK_IMAGE_TYPE_2D:
413 maxExtent = (VkExtent3D) { max_dim, max_dim, 1 };
414 maxArraySize = 2048;
415 break;
416 case VK_IMAGE_TYPE_3D:
417 maxExtent = (VkExtent3D) { max_dim, max_dim, max_dim };
418 maxArraySize = 1;
419 break;
420 default:
421 unreachable("Invalid image type");
422 }
423 if (pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR)
424 maxArraySize = 1;
425
426 assert(util_is_power_of_two_nonzero(max_dim));
427 uint32_t maxMipLevels = util_logbase2(max_dim) + 1;
428 if (ycbcr_info != NULL || pImageFormatInfo->tiling == VK_IMAGE_TILING_LINEAR)
429 maxMipLevels = 1;
430
431 VkSampleCountFlags sampleCounts = VK_SAMPLE_COUNT_1_BIT;
432 if (pImageFormatInfo->tiling == VK_IMAGE_TILING_OPTIMAL &&
433 pImageFormatInfo->type == VK_IMAGE_TYPE_2D &&
434 ycbcr_info == NULL &&
435 (features & (VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT |
436 VK_FORMAT_FEATURE_2_DEPTH_STENCIL_ATTACHMENT_BIT)) &&
437 !(pImageFormatInfo->flags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT)) {
438 sampleCounts = VK_SAMPLE_COUNT_1_BIT |
439 VK_SAMPLE_COUNT_2_BIT |
440 VK_SAMPLE_COUNT_4_BIT |
441 VK_SAMPLE_COUNT_8_BIT;
442 }
443
444 /* From the Vulkan 1.2.199 spec:
445 *
446 * "VK_IMAGE_CREATE_EXTENDED_USAGE_BIT specifies that the image can be
447 * created with usage flags that are not supported for the format the
448 * image is created with but are supported for at least one format a
449 * VkImageView created from the image can have."
450 *
451 * If VK_IMAGE_CREATE_EXTENDED_USAGE_BIT is set, views can be created with
452 * different usage than the image so we can't always filter on usage.
453 * There is one exception to this below for storage.
454 */
455 const VkImageUsageFlags image_usage = pImageFormatInfo->usage;
456 VkImageUsageFlags view_usage = image_usage;
457 if (pImageFormatInfo->flags & VK_IMAGE_CREATE_EXTENDED_USAGE_BIT)
458 view_usage = 0;
459
460 u_foreach_bit(b, view_usage) {
461 VkFormatFeatureFlags2 usage_features =
462 vk_image_usage_to_format_features(1 << b);
463 if (usage_features && !(features & usage_features))
464 return VK_ERROR_FORMAT_NOT_SUPPORTED;
465 }
466
467 const VkExternalMemoryProperties *ext_mem_props = NULL;
468 if (external_info != NULL && external_info->handleType != 0) {
469 bool tiling_has_explicit_layout;
470 switch (pImageFormatInfo->tiling) {
471 case VK_IMAGE_TILING_LINEAR:
472 case VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT:
473 tiling_has_explicit_layout = true;
474 break;
475 case VK_IMAGE_TILING_OPTIMAL:
476 tiling_has_explicit_layout = false;
477 break;
478 default:
479 unreachable("Unsupported VkImageTiling");
480 }
481
482 switch (external_info->handleType) {
483 case VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT:
484 /* No special restrictions */
485 if (tiling_has_explicit_layout) {
486 /* With an explicit memory layout, we don't care which type of
487 * fd the image belongs too. Both OPAQUE_FD and DMA_BUF are
488 * interchangeable here.
489 */
490 ext_mem_props = &nvk_dma_buf_mem_props;
491 } else {
492 ext_mem_props = &nvk_opaque_fd_mem_props;
493 }
494 break;
495
496 case VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT:
497 if (!tiling_has_explicit_layout) {
498 return vk_errorf(pdev, VK_ERROR_FORMAT_NOT_SUPPORTED,
499 "VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT "
500 "requires VK_IMAGE_TILING_LINEAR or "
501 "VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT");
502 }
503 ext_mem_props = &nvk_dma_buf_mem_props;
504 break;
505
506 default:
507 /* From the Vulkan 1.3.256 spec:
508 *
509 * "If handleType is not compatible with the [parameters] in
510 * VkPhysicalDeviceImageFormatInfo2, then
511 * vkGetPhysicalDeviceImageFormatProperties2 returns
512 * VK_ERROR_FORMAT_NOT_SUPPORTED."
513 */
514 return vk_errorf(pdev, VK_ERROR_FORMAT_NOT_SUPPORTED,
515 "unsupported VkExternalMemoryHandleTypeFlagBits: %s ",
516 vk_ExternalMemoryHandleTypeFlagBits_to_str(external_info->handleType));
517 }
518 }
519
520 const unsigned plane_count =
521 vk_format_get_plane_count(pImageFormatInfo->format);
522
523 /* From the Vulkan 1.3.259 spec, VkImageCreateInfo:
524 *
525 * VUID-VkImageCreateInfo-imageCreateFormatFeatures-02260
526 *
527 * "If format is a multi-planar format, and if imageCreateFormatFeatures
528 * (as defined in Image Creation Limits) does not contain
529 * VK_FORMAT_FEATURE_DISJOINT_BIT, then flags must not contain
530 * VK_IMAGE_CREATE_DISJOINT_BIT"
531 *
532 * This is satisfied trivially because we support DISJOINT on all
533 * multi-plane formats. Also,
534 *
535 * VUID-VkImageCreateInfo-format-01577
536 *
537 * "If format is not a multi-planar format, and flags does not include
538 * VK_IMAGE_CREATE_ALIAS_BIT, flags must not contain
539 * VK_IMAGE_CREATE_DISJOINT_BIT"
540 */
541 if (plane_count == 1 &&
542 !(pImageFormatInfo->flags & VK_IMAGE_CREATE_ALIAS_BIT) &&
543 (pImageFormatInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT))
544 return VK_ERROR_FORMAT_NOT_SUPPORTED;
545
546 if (ycbcr_info &&
547 ((pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) ||
548 (pImageFormatInfo->flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT)))
549 return VK_ERROR_FORMAT_NOT_SUPPORTED;
550
551 pImageFormatProperties->imageFormatProperties = (VkImageFormatProperties) {
552 .maxExtent = maxExtent,
553 .maxMipLevels = maxMipLevels,
554 .maxArrayLayers = maxArraySize,
555 .sampleCounts = sampleCounts,
556 .maxResourceSize = UINT32_MAX, /* TODO */
557 };
558
559 vk_foreach_struct(s, pImageFormatProperties->pNext) {
560 switch (s->sType) {
561 case VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES: {
562 VkExternalImageFormatProperties *p = (void *)s;
563 /* From the Vulkan 1.3.256 spec:
564 *
565 * "If handleType is 0, vkGetPhysicalDeviceImageFormatProperties2
566 * will behave as if VkPhysicalDeviceExternalImageFormatInfo was
567 * not present, and VkExternalImageFormatProperties will be
568 * ignored."
569 *
570 * This is true if and only if ext_mem_props == NULL
571 */
572 if (ext_mem_props != NULL)
573 p->externalMemoryProperties = *ext_mem_props;
574 break;
575 }
576 case VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_IMAGE_FORMAT_PROPERTIES: {
577 VkSamplerYcbcrConversionImageFormatProperties *ycbcr_props = (void *) s;
578 ycbcr_props->combinedImageSamplerDescriptorCount = plane_count;
579 break;
580 }
581 default:
582 vk_debug_ignored_stype(s->sType);
583 break;
584 }
585 }
586
587 return VK_SUCCESS;
588 }
589
590 static enum nil_image_dim
vk_image_type_to_nil_dim(VkImageType type)591 vk_image_type_to_nil_dim(VkImageType type)
592 {
593 switch (type) {
594 case VK_IMAGE_TYPE_1D: return NIL_IMAGE_DIM_1D;
595 case VK_IMAGE_TYPE_2D: return NIL_IMAGE_DIM_2D;
596 case VK_IMAGE_TYPE_3D: return NIL_IMAGE_DIM_3D;
597 default:
598 unreachable("Invalid image type");
599 }
600 }
601
602 static VkSparseImageFormatProperties
nvk_fill_sparse_image_fmt_props(VkImageAspectFlags aspects,const enum pipe_format format,const enum nil_image_dim dim,const enum nil_sample_layout sample_layout)603 nvk_fill_sparse_image_fmt_props(VkImageAspectFlags aspects,
604 const enum pipe_format format,
605 const enum nil_image_dim dim,
606 const enum nil_sample_layout sample_layout)
607 {
608 struct nil_Extent4D_Pixels sparse_block_extent_px =
609 nil_sparse_block_extent_px(nil_format(format), dim, sample_layout);
610
611 assert(sparse_block_extent_px.array_len == 1);
612
613 VkSparseImageFormatProperties sparse_format_props = {
614 .aspectMask = aspects,
615 .flags = VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT,
616 .imageGranularity = {
617 .width = sparse_block_extent_px.width,
618 .height = sparse_block_extent_px.height,
619 .depth = sparse_block_extent_px.depth,
620 },
621 };
622
623 return sparse_format_props;
624 }
625
626 VKAPI_ATTR void VKAPI_CALL
nvk_GetPhysicalDeviceSparseImageFormatProperties2(VkPhysicalDevice physicalDevice,const VkPhysicalDeviceSparseImageFormatInfo2 * pFormatInfo,uint32_t * pPropertyCount,VkSparseImageFormatProperties2 * pProperties)627 nvk_GetPhysicalDeviceSparseImageFormatProperties2(
628 VkPhysicalDevice physicalDevice,
629 const VkPhysicalDeviceSparseImageFormatInfo2* pFormatInfo,
630 uint32_t *pPropertyCount,
631 VkSparseImageFormatProperties2 *pProperties)
632 {
633 VkResult result;
634
635 /* Check if the given format info is valid first before returning sparse
636 * props. The easiest way to do this is to just call
637 * nvk_GetPhysicalDeviceImageFormatProperties2()
638 */
639 const VkPhysicalDeviceImageFormatInfo2 img_fmt_info = {
640 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
641 .format = pFormatInfo->format,
642 .type = pFormatInfo->type,
643 .tiling = pFormatInfo->tiling,
644 .usage = pFormatInfo->usage,
645 .flags = VK_IMAGE_CREATE_SPARSE_BINDING_BIT |
646 VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT,
647 };
648
649 VkImageFormatProperties2 img_fmt_props2 = {
650 .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
651 .pNext = NULL,
652 };
653
654 result = nvk_GetPhysicalDeviceImageFormatProperties2(physicalDevice,
655 &img_fmt_info,
656 &img_fmt_props2);
657 if (result != VK_SUCCESS) {
658 *pPropertyCount = 0;
659 return;
660 }
661
662 const VkImageFormatProperties *props = &img_fmt_props2.imageFormatProperties;
663 if (!(pFormatInfo->samples & props->sampleCounts)) {
664 *pPropertyCount = 0;
665 return;
666 }
667
668 VK_OUTARRAY_MAKE_TYPED(VkSparseImageFormatProperties2, out,
669 pProperties, pPropertyCount);
670
671 VkImageAspectFlags aspects = vk_format_aspects(pFormatInfo->format);
672 const enum pipe_format pipe_format =
673 vk_format_to_pipe_format(pFormatInfo->format);
674 const enum nil_image_dim dim = vk_image_type_to_nil_dim(pFormatInfo->type);
675 const enum nil_sample_layout sample_layout =
676 nil_choose_sample_layout(pFormatInfo->samples);
677
678 vk_outarray_append_typed(VkSparseImageFormatProperties2, &out, props) {
679 props->properties = nvk_fill_sparse_image_fmt_props(aspects, pipe_format,
680 dim, sample_layout);
681 }
682 }
683
684 static VkResult
nvk_image_init(struct nvk_device * dev,struct nvk_image * image,const VkImageCreateInfo * pCreateInfo)685 nvk_image_init(struct nvk_device *dev,
686 struct nvk_image *image,
687 const VkImageCreateInfo *pCreateInfo)
688 {
689 struct nvk_physical_device *pdev = nvk_device_physical(dev);
690
691 vk_image_init(&dev->vk, &image->vk, pCreateInfo);
692
693 if ((image->vk.usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
694 VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT)) &&
695 image->vk.samples > 1) {
696 image->vk.usage |= VK_IMAGE_USAGE_SAMPLED_BIT;
697 image->vk.stencil_usage |= VK_IMAGE_USAGE_SAMPLED_BIT;
698 }
699
700 if (image->vk.usage & VK_IMAGE_USAGE_TRANSFER_SRC_BIT)
701 image->vk.usage |= VK_IMAGE_USAGE_SAMPLED_BIT;
702 if (image->vk.usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT)
703 image->vk.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
704
705 nil_image_usage_flags usage = 0;
706 if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR)
707 usage |= NIL_IMAGE_USAGE_LINEAR_BIT;
708 if (pCreateInfo->flags & VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT)
709 usage |= NIL_IMAGE_USAGE_2D_VIEW_BIT;
710 if (pCreateInfo->flags & VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT)
711 usage |= NIL_IMAGE_USAGE_2D_VIEW_BIT;
712
713 /* In order to be able to clear 3D depth/stencil images, we need to bind
714 * them as 2D arrays. Fortunately, 3D depth/stencil shouldn't be common.
715 */
716 if ((image->vk.aspects & (VK_IMAGE_ASPECT_DEPTH_BIT |
717 VK_IMAGE_ASPECT_STENCIL_BIT)) &&
718 pCreateInfo->imageType == VK_IMAGE_TYPE_3D)
719 usage |= NIL_IMAGE_USAGE_2D_VIEW_BIT;
720
721 image->plane_count = vk_format_get_plane_count(pCreateInfo->format);
722 image->disjoint = image->plane_count > 1 &&
723 (pCreateInfo->flags & VK_IMAGE_CREATE_DISJOINT_BIT);
724
725 if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT) {
726 /* Sparse multiplane is not supported */
727 assert(image->plane_count == 1);
728 usage |= NIL_IMAGE_USAGE_SPARSE_RESIDENCY_BIT;
729 }
730
731 uint32_t explicit_row_stride_B = 0;
732 if (image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
733 /* Modifiers are not supported with YCbCr */
734 assert(image->plane_count == 1);
735
736 const struct VkImageDrmFormatModifierExplicitCreateInfoEXT *mod_explicit_info =
737 vk_find_struct_const(pCreateInfo->pNext,
738 IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT);
739 if (mod_explicit_info) {
740 image->vk.drm_format_mod = mod_explicit_info->drmFormatModifier;
741 /* Normally with explicit modifiers, the client specifies all strides,
742 * however in our case, we can only really make use of this in the linear
743 * case, and we can only create 2D non-array linear images, so ultimately
744 * we only care about the row stride.
745 */
746 explicit_row_stride_B = mod_explicit_info->pPlaneLayouts->rowPitch;
747 } else {
748 const struct VkImageDrmFormatModifierListCreateInfoEXT *mod_list_info =
749 vk_find_struct_const(pCreateInfo->pNext,
750 IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
751
752 enum pipe_format p_format =
753 vk_format_to_pipe_format(pCreateInfo->format);
754 image->vk.drm_format_mod =
755 nil_select_best_drm_format_mod(&pdev->info, nil_format(p_format),
756 mod_list_info->drmFormatModifierCount,
757 mod_list_info->pDrmFormatModifiers);
758 assert(image->vk.drm_format_mod != DRM_FORMAT_MOD_INVALID);
759 }
760
761 if (image->vk.drm_format_mod == DRM_FORMAT_MOD_LINEAR) {
762 /* We only have one shadow plane per nvk_image */
763 assert(image->plane_count == 1);
764
765 struct nil_image_init_info tiled_shadow_nil_info = {
766 .dim = vk_image_type_to_nil_dim(pCreateInfo->imageType),
767 .format = nil_format(vk_format_to_pipe_format(image->vk.format)),
768 .modifier = DRM_FORMAT_MOD_INVALID,
769 .extent_px = {
770 .width = pCreateInfo->extent.width,
771 .height = pCreateInfo->extent.height,
772 .depth = pCreateInfo->extent.depth,
773 .array_len = pCreateInfo->arrayLayers,
774 },
775 .levels = pCreateInfo->mipLevels,
776 .samples = pCreateInfo->samples,
777 .usage = usage & ~NIL_IMAGE_USAGE_LINEAR_BIT,
778 .explicit_row_stride_B = 0,
779 };
780 image->linear_tiled_shadow.nil =
781 nil_image_new(&pdev->info, &tiled_shadow_nil_info);
782 }
783 }
784
785 const struct vk_format_ycbcr_info *ycbcr_info =
786 vk_format_get_ycbcr_info(pCreateInfo->format);
787 for (uint8_t plane = 0; plane < image->plane_count; plane++) {
788 VkFormat format = ycbcr_info ?
789 ycbcr_info->planes[plane].format : pCreateInfo->format;
790 const uint8_t width_scale = ycbcr_info ?
791 ycbcr_info->planes[plane].denominator_scales[0] : 1;
792 const uint8_t height_scale = ycbcr_info ?
793 ycbcr_info->planes[plane].denominator_scales[1] : 1;
794 struct nil_image_init_info nil_info = {
795 .dim = vk_image_type_to_nil_dim(pCreateInfo->imageType),
796 .format = nil_format(vk_format_to_pipe_format(format)),
797 .modifier = image->vk.drm_format_mod,
798 .extent_px = {
799 .width = pCreateInfo->extent.width / width_scale,
800 .height = pCreateInfo->extent.height / height_scale,
801 .depth = pCreateInfo->extent.depth,
802 .array_len = pCreateInfo->arrayLayers,
803 },
804 .levels = pCreateInfo->mipLevels,
805 .samples = pCreateInfo->samples,
806 .usage = usage,
807 .explicit_row_stride_B = explicit_row_stride_B,
808 };
809
810 image->planes[plane].nil = nil_image_new(&pdev->info, &nil_info);
811 }
812
813 if (image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
814 struct nil_image_init_info stencil_nil_info = {
815 .dim = vk_image_type_to_nil_dim(pCreateInfo->imageType),
816 .format = nil_format(PIPE_FORMAT_R32_UINT),
817 .modifier = DRM_FORMAT_MOD_INVALID,
818 .extent_px = {
819 .width = pCreateInfo->extent.width,
820 .height = pCreateInfo->extent.height,
821 .depth = pCreateInfo->extent.depth,
822 .array_len = pCreateInfo->arrayLayers,
823 },
824 .levels = pCreateInfo->mipLevels,
825 .samples = pCreateInfo->samples,
826 .usage = usage,
827 .explicit_row_stride_B = 0,
828 };
829
830 image->stencil_copy_temp.nil =
831 nil_image_new(&pdev->info, &stencil_nil_info);
832 }
833
834 return VK_SUCCESS;
835 }
836
837 static void
nvk_image_plane_size_align_B(struct nvk_device * dev,const struct nvk_image * image,const struct nvk_image_plane * plane,uint64_t * size_B_out,uint64_t * align_B_out)838 nvk_image_plane_size_align_B(struct nvk_device *dev,
839 const struct nvk_image *image,
840 const struct nvk_image_plane *plane,
841 uint64_t *size_B_out, uint64_t *align_B_out)
842 {
843 struct nvk_physical_device *pdev = nvk_device_physical(dev);
844 const bool sparse_bound =
845 image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT;
846
847 assert(util_is_power_of_two_or_zero64(plane->nil.align_B));
848 if (sparse_bound || plane->nil.pte_kind) {
849 *align_B_out = MAX2(plane->nil.align_B, pdev->nvkmd->bind_align_B);
850 } else {
851 *align_B_out = plane->nil.align_B;
852 }
853 *size_B_out = align64(plane->nil.size_B, *align_B_out);
854 }
855
856 static VkResult
nvk_image_plane_alloc_va(struct nvk_device * dev,const struct nvk_image * image,struct nvk_image_plane * plane)857 nvk_image_plane_alloc_va(struct nvk_device *dev,
858 const struct nvk_image *image,
859 struct nvk_image_plane *plane)
860 {
861 VkResult result;
862
863 const bool sparse_bound =
864 image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT;
865 const bool sparse_resident =
866 image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
867 assert(sparse_bound || !sparse_resident);
868
869 if (sparse_bound || plane->nil.pte_kind) {
870 enum nvkmd_va_flags va_flags = 0;
871 if (sparse_resident)
872 va_flags |= NVKMD_VA_SPARSE;
873
874 uint64_t va_size_B, va_align_B;
875 nvk_image_plane_size_align_B(dev, image, plane, &va_size_B, &va_align_B);
876
877 result = nvkmd_dev_alloc_va(dev->nvkmd, &dev->vk.base,
878 va_flags, plane->nil.pte_kind,
879 va_size_B, va_align_B,
880 0 /* fixed_addr */, &plane->va);
881 if (result != VK_SUCCESS)
882 return result;
883
884 plane->addr = plane->va->addr;
885 }
886
887 return VK_SUCCESS;
888 }
889
890 static void
nvk_image_plane_finish(struct nvk_device * dev,struct nvk_image_plane * plane,VkImageCreateFlags create_flags,const VkAllocationCallbacks * pAllocator)891 nvk_image_plane_finish(struct nvk_device *dev,
892 struct nvk_image_plane *plane,
893 VkImageCreateFlags create_flags,
894 const VkAllocationCallbacks *pAllocator)
895 {
896 if (plane->va != NULL)
897 nvkmd_va_free(plane->va);
898 }
899
900 static void
nvk_image_finish(struct nvk_device * dev,struct nvk_image * image,const VkAllocationCallbacks * pAllocator)901 nvk_image_finish(struct nvk_device *dev, struct nvk_image *image,
902 const VkAllocationCallbacks *pAllocator)
903 {
904 for (uint8_t plane = 0; plane < image->plane_count; plane++) {
905 nvk_image_plane_finish(dev, &image->planes[plane],
906 image->vk.create_flags, pAllocator);
907 }
908
909 if (image->stencil_copy_temp.nil.size_B > 0) {
910 nvk_image_plane_finish(dev, &image->stencil_copy_temp,
911 image->vk.create_flags, pAllocator);
912 }
913
914 assert(image->linear_tiled_shadow.va == NULL);
915 if (image->linear_tiled_shadow_mem != NULL)
916 nvkmd_mem_unref(image->linear_tiled_shadow_mem);
917
918 vk_image_finish(&image->vk);
919 }
920
921 VKAPI_ATTR VkResult VKAPI_CALL
nvk_CreateImage(VkDevice _device,const VkImageCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkImage * pImage)922 nvk_CreateImage(VkDevice _device,
923 const VkImageCreateInfo *pCreateInfo,
924 const VkAllocationCallbacks *pAllocator,
925 VkImage *pImage)
926 {
927 VK_FROM_HANDLE(nvk_device, dev, _device);
928 struct nvk_physical_device *pdev = nvk_device_physical(dev);
929 struct nvk_image *image;
930 VkResult result;
931
932 #ifdef NVK_USE_WSI_PLATFORM
933 /* Ignore swapchain creation info on Android. Since we don't have an
934 * implementation in Mesa, we're guaranteed to access an Android object
935 * incorrectly.
936 */
937 const VkImageSwapchainCreateInfoKHR *swapchain_info =
938 vk_find_struct_const(pCreateInfo->pNext, IMAGE_SWAPCHAIN_CREATE_INFO_KHR);
939 if (swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE) {
940 return wsi_common_create_swapchain_image(&pdev->wsi_device,
941 pCreateInfo,
942 swapchain_info->swapchain,
943 pImage);
944 }
945 #endif
946
947 image = vk_zalloc2(&dev->vk.alloc, pAllocator, sizeof(*image), 8,
948 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
949 if (!image)
950 return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
951
952 result = nvk_image_init(dev, image, pCreateInfo);
953 if (result != VK_SUCCESS) {
954 vk_free2(&dev->vk.alloc, pAllocator, image);
955 return result;
956 }
957
958 for (uint8_t plane = 0; plane < image->plane_count; plane++) {
959 result = nvk_image_plane_alloc_va(dev, image, &image->planes[plane]);
960 if (result != VK_SUCCESS) {
961 nvk_image_finish(dev, image, pAllocator);
962 vk_free2(&dev->vk.alloc, pAllocator, image);
963 return result;
964 }
965 }
966
967 if (image->stencil_copy_temp.nil.size_B > 0) {
968 result = nvk_image_plane_alloc_va(dev, image, &image->stencil_copy_temp);
969 if (result != VK_SUCCESS) {
970 nvk_image_finish(dev, image, pAllocator);
971 vk_free2(&dev->vk.alloc, pAllocator, image);
972 return result;
973 }
974 }
975
976 if (image->linear_tiled_shadow.nil.size_B > 0) {
977 struct nvk_image_plane *shadow = &image->linear_tiled_shadow;
978 result = nvkmd_dev_alloc_tiled_mem(dev->nvkmd, &dev->vk.base,
979 shadow->nil.size_B, shadow->nil.align_B,
980 shadow->nil.pte_kind, shadow->nil.tile_mode,
981 NVKMD_MEM_LOCAL,
982 &image->linear_tiled_shadow_mem);
983 if (result != VK_SUCCESS) {
984 nvk_image_finish(dev, image, pAllocator);
985 vk_free2(&dev->vk.alloc, pAllocator, image);
986 return result;
987 }
988 shadow->addr = image->linear_tiled_shadow_mem->va->addr;
989 }
990
991 *pImage = nvk_image_to_handle(image);
992
993 return VK_SUCCESS;
994 }
995
996 VKAPI_ATTR void VKAPI_CALL
nvk_DestroyImage(VkDevice device,VkImage _image,const VkAllocationCallbacks * pAllocator)997 nvk_DestroyImage(VkDevice device,
998 VkImage _image,
999 const VkAllocationCallbacks *pAllocator)
1000 {
1001 VK_FROM_HANDLE(nvk_device, dev, device);
1002 VK_FROM_HANDLE(nvk_image, image, _image);
1003
1004 if (!image)
1005 return;
1006
1007 nvk_image_finish(dev, image, pAllocator);
1008 vk_free2(&dev->vk.alloc, pAllocator, image);
1009 }
1010
1011 static void
nvk_image_plane_add_req(struct nvk_device * dev,const struct nvk_image * image,const struct nvk_image_plane * plane,uint64_t * size_B,uint32_t * align_B)1012 nvk_image_plane_add_req(struct nvk_device *dev,
1013 const struct nvk_image *image,
1014 const struct nvk_image_plane *plane,
1015 uint64_t *size_B, uint32_t *align_B)
1016 {
1017 assert(util_is_power_of_two_or_zero64(*align_B));
1018 uint64_t plane_size_B, plane_align_B;
1019 nvk_image_plane_size_align_B(dev, image, plane,
1020 &plane_size_B, &plane_align_B);
1021
1022 *align_B = MAX2(*align_B, plane_align_B);
1023 *size_B = align64(*size_B, plane_align_B);
1024 *size_B += plane_size_B;
1025 }
1026
1027 static void
nvk_get_image_memory_requirements(struct nvk_device * dev,struct nvk_image * image,VkImageAspectFlags aspects,VkMemoryRequirements2 * pMemoryRequirements)1028 nvk_get_image_memory_requirements(struct nvk_device *dev,
1029 struct nvk_image *image,
1030 VkImageAspectFlags aspects,
1031 VkMemoryRequirements2 *pMemoryRequirements)
1032 {
1033 struct nvk_physical_device *pdev = nvk_device_physical(dev);
1034 uint32_t memory_types = (1 << pdev->mem_type_count) - 1;
1035
1036 // TODO hope for the best?
1037
1038 uint64_t size_B = 0;
1039 uint32_t align_B = 0;
1040 if (image->disjoint) {
1041 uint8_t plane = nvk_image_memory_aspects_to_plane(image, aspects);
1042 nvk_image_plane_add_req(dev, image, &image->planes[plane],
1043 &size_B, &align_B);
1044 } else {
1045 for (unsigned plane = 0; plane < image->plane_count; plane++) {
1046 nvk_image_plane_add_req(dev, image, &image->planes[plane],
1047 &size_B, &align_B);
1048 }
1049 }
1050
1051 if (image->stencil_copy_temp.nil.size_B > 0) {
1052 nvk_image_plane_add_req(dev, image, &image->stencil_copy_temp,
1053 &size_B, &align_B);
1054 }
1055
1056 pMemoryRequirements->memoryRequirements.memoryTypeBits = memory_types;
1057 pMemoryRequirements->memoryRequirements.alignment = align_B;
1058 pMemoryRequirements->memoryRequirements.size = size_B;
1059
1060 vk_foreach_struct_const(ext, pMemoryRequirements->pNext) {
1061 switch (ext->sType) {
1062 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
1063 VkMemoryDedicatedRequirements *dedicated = (void *)ext;
1064 dedicated->prefersDedicatedAllocation =
1065 image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
1066 dedicated->requiresDedicatedAllocation =
1067 image->vk.tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
1068 break;
1069 }
1070 default:
1071 vk_debug_ignored_stype(ext->sType);
1072 break;
1073 }
1074 }
1075 }
1076
1077 VKAPI_ATTR void VKAPI_CALL
nvk_GetImageMemoryRequirements2(VkDevice device,const VkImageMemoryRequirementsInfo2 * pInfo,VkMemoryRequirements2 * pMemoryRequirements)1078 nvk_GetImageMemoryRequirements2(VkDevice device,
1079 const VkImageMemoryRequirementsInfo2 *pInfo,
1080 VkMemoryRequirements2 *pMemoryRequirements)
1081 {
1082 VK_FROM_HANDLE(nvk_device, dev, device);
1083 VK_FROM_HANDLE(nvk_image, image, pInfo->image);
1084
1085 const VkImagePlaneMemoryRequirementsInfo *plane_info =
1086 vk_find_struct_const(pInfo->pNext, IMAGE_PLANE_MEMORY_REQUIREMENTS_INFO);
1087 const VkImageAspectFlags aspects =
1088 image->disjoint ? plane_info->planeAspect : image->vk.aspects;
1089
1090 nvk_get_image_memory_requirements(dev, image, aspects,
1091 pMemoryRequirements);
1092 }
1093
1094 VKAPI_ATTR void VKAPI_CALL
nvk_GetDeviceImageMemoryRequirements(VkDevice device,const VkDeviceImageMemoryRequirements * pInfo,VkMemoryRequirements2 * pMemoryRequirements)1095 nvk_GetDeviceImageMemoryRequirements(VkDevice device,
1096 const VkDeviceImageMemoryRequirements *pInfo,
1097 VkMemoryRequirements2 *pMemoryRequirements)
1098 {
1099 VK_FROM_HANDLE(nvk_device, dev, device);
1100 ASSERTED VkResult result;
1101 struct nvk_image image = {0};
1102
1103 result = nvk_image_init(dev, &image, pInfo->pCreateInfo);
1104 assert(result == VK_SUCCESS);
1105
1106 const VkImageAspectFlags aspects =
1107 image.disjoint ? pInfo->planeAspect : image.vk.aspects;
1108
1109 nvk_get_image_memory_requirements(dev, &image, aspects,
1110 pMemoryRequirements);
1111
1112 nvk_image_finish(dev, &image, NULL);
1113 }
1114
1115 static VkSparseImageMemoryRequirements
nvk_fill_sparse_image_memory_reqs(const struct nil_image * nil,const struct nil_image * stencil_tmp,VkImageAspectFlags aspects)1116 nvk_fill_sparse_image_memory_reqs(const struct nil_image *nil,
1117 const struct nil_image *stencil_tmp,
1118 VkImageAspectFlags aspects)
1119 {
1120 VkSparseImageFormatProperties sparse_format_props =
1121 nvk_fill_sparse_image_fmt_props(aspects, nil->format.p_format,
1122 nil->dim, nil->sample_layout);
1123
1124 assert(nil->mip_tail_first_lod <= nil->num_levels);
1125 VkSparseImageMemoryRequirements sparse_memory_reqs = {
1126 .formatProperties = sparse_format_props,
1127 .imageMipTailFirstLod = nil->mip_tail_first_lod,
1128 .imageMipTailStride = 0,
1129 };
1130
1131 if (nil->mip_tail_first_lod == 0) {
1132 sparse_memory_reqs.imageMipTailSize = nil->size_B;
1133 sparse_memory_reqs.imageMipTailOffset = 0;
1134 } else if (nil->mip_tail_first_lod < nil->num_levels) {
1135 sparse_memory_reqs.imageMipTailSize =
1136 nil_image_mip_tail_size_B(nil) * nil->extent_px.array_len;
1137 sparse_memory_reqs.imageMipTailOffset = NVK_MIP_TAIL_START_OFFSET;
1138 } else {
1139 sparse_memory_reqs.imageMipTailSize = 0;
1140 sparse_memory_reqs.imageMipTailOffset = NVK_MIP_TAIL_START_OFFSET;
1141 }
1142
1143 if (stencil_tmp != NULL)
1144 sparse_memory_reqs.imageMipTailSize += stencil_tmp->size_B;
1145
1146 return sparse_memory_reqs;
1147 }
1148
1149 static void
nvk_get_image_sparse_memory_requirements(struct nvk_device * dev,struct nvk_image * image,VkImageAspectFlags aspects,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)1150 nvk_get_image_sparse_memory_requirements(
1151 struct nvk_device *dev,
1152 struct nvk_image *image,
1153 VkImageAspectFlags aspects,
1154 uint32_t *pSparseMemoryRequirementCount,
1155 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
1156 {
1157 VK_OUTARRAY_MAKE_TYPED(VkSparseImageMemoryRequirements2, out,
1158 pSparseMemoryRequirements,
1159 pSparseMemoryRequirementCount);
1160
1161 /* From the Vulkan 1.3.279 spec:
1162 *
1163 * "The sparse image must have been created using the
1164 * VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT flag to retrieve valid sparse
1165 * image memory requirements."
1166 */
1167 if (!(image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT))
1168 return;
1169
1170 /* We don't support multiplane sparse for now */
1171 if (image->plane_count > 1)
1172 return;
1173
1174 const struct nil_image *stencil_tmp = NULL;
1175 if (image->stencil_copy_temp.nil.size_B > 0)
1176 stencil_tmp = &image->stencil_copy_temp.nil;
1177
1178 vk_outarray_append_typed(VkSparseImageMemoryRequirements2, &out, reqs) {
1179 reqs->memoryRequirements =
1180 nvk_fill_sparse_image_memory_reqs(&image->planes[0].nil,
1181 stencil_tmp, aspects);
1182 };
1183 }
1184
1185 VKAPI_ATTR void VKAPI_CALL
nvk_GetImageSparseMemoryRequirements2(VkDevice device,const VkImageSparseMemoryRequirementsInfo2 * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)1186 nvk_GetImageSparseMemoryRequirements2(
1187 VkDevice device,
1188 const VkImageSparseMemoryRequirementsInfo2* pInfo,
1189 uint32_t* pSparseMemoryRequirementCount,
1190 VkSparseImageMemoryRequirements2* pSparseMemoryRequirements)
1191 {
1192 VK_FROM_HANDLE(nvk_device, dev, device);
1193 VK_FROM_HANDLE(nvk_image, image, pInfo->image);
1194
1195 const VkImageAspectFlags aspects = image->vk.aspects;
1196
1197 nvk_get_image_sparse_memory_requirements(dev, image, aspects,
1198 pSparseMemoryRequirementCount,
1199 pSparseMemoryRequirements);
1200 }
1201
1202 VKAPI_ATTR void VKAPI_CALL
nvk_GetDeviceImageSparseMemoryRequirements(VkDevice device,const VkDeviceImageMemoryRequirements * pInfo,uint32_t * pSparseMemoryRequirementCount,VkSparseImageMemoryRequirements2 * pSparseMemoryRequirements)1203 nvk_GetDeviceImageSparseMemoryRequirements(
1204 VkDevice device,
1205 const VkDeviceImageMemoryRequirements* pInfo,
1206 uint32_t *pSparseMemoryRequirementCount,
1207 VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
1208 {
1209 VK_FROM_HANDLE(nvk_device, dev, device);
1210 ASSERTED VkResult result;
1211 struct nvk_image image = {0};
1212
1213 result = nvk_image_init(dev, &image, pInfo->pCreateInfo);
1214 assert(result == VK_SUCCESS);
1215
1216 const VkImageAspectFlags aspects =
1217 image.disjoint ? pInfo->planeAspect : image.vk.aspects;
1218
1219 nvk_get_image_sparse_memory_requirements(dev, &image, aspects,
1220 pSparseMemoryRequirementCount,
1221 pSparseMemoryRequirements);
1222
1223 nvk_image_finish(dev, &image, NULL);
1224 }
1225
1226 static void
nvk_get_image_subresource_layout(struct nvk_device * dev,struct nvk_image * image,const VkImageSubresource2KHR * pSubresource,VkSubresourceLayout2KHR * pLayout)1227 nvk_get_image_subresource_layout(struct nvk_device *dev,
1228 struct nvk_image *image,
1229 const VkImageSubresource2KHR *pSubresource,
1230 VkSubresourceLayout2KHR *pLayout)
1231 {
1232 const VkImageSubresource *isr = &pSubresource->imageSubresource;
1233
1234 const uint8_t p = nvk_image_memory_aspects_to_plane(image, isr->aspectMask);
1235 const struct nvk_image_plane *plane = &image->planes[p];
1236
1237 uint64_t offset_B = 0;
1238 if (!image->disjoint) {
1239 uint32_t align_B = 0;
1240 for (unsigned plane = 0; plane < p; plane++) {
1241 nvk_image_plane_add_req(dev, image, &image->planes[plane],
1242 &offset_B, &align_B);
1243 }
1244 }
1245 offset_B += nil_image_level_layer_offset_B(&plane->nil, isr->mipLevel,
1246 isr->arrayLayer);
1247
1248 pLayout->subresourceLayout = (VkSubresourceLayout) {
1249 .offset = offset_B,
1250 .size = nil_image_level_size_B(&plane->nil, isr->mipLevel),
1251 .rowPitch = plane->nil.levels[isr->mipLevel].row_stride_B,
1252 .arrayPitch = plane->nil.array_stride_B,
1253 .depthPitch = nil_image_level_depth_stride_B(&plane->nil, isr->mipLevel),
1254 };
1255 }
1256
1257 VKAPI_ATTR void VKAPI_CALL
nvk_GetImageSubresourceLayout2KHR(VkDevice device,VkImage _image,const VkImageSubresource2KHR * pSubresource,VkSubresourceLayout2KHR * pLayout)1258 nvk_GetImageSubresourceLayout2KHR(VkDevice device,
1259 VkImage _image,
1260 const VkImageSubresource2KHR *pSubresource,
1261 VkSubresourceLayout2KHR *pLayout)
1262 {
1263 VK_FROM_HANDLE(nvk_device, dev, device);
1264 VK_FROM_HANDLE(nvk_image, image, _image);
1265
1266 nvk_get_image_subresource_layout(dev, image, pSubresource, pLayout);
1267 }
1268
1269 VKAPI_ATTR void VKAPI_CALL
nvk_GetDeviceImageSubresourceLayoutKHR(VkDevice device,const VkDeviceImageSubresourceInfoKHR * pInfo,VkSubresourceLayout2KHR * pLayout)1270 nvk_GetDeviceImageSubresourceLayoutKHR(
1271 VkDevice device,
1272 const VkDeviceImageSubresourceInfoKHR *pInfo,
1273 VkSubresourceLayout2KHR *pLayout)
1274 {
1275 VK_FROM_HANDLE(nvk_device, dev, device);
1276 ASSERTED VkResult result;
1277 struct nvk_image image = {0};
1278
1279 result = nvk_image_init(dev, &image, pInfo->pCreateInfo);
1280 assert(result == VK_SUCCESS);
1281
1282 nvk_get_image_subresource_layout(dev, &image, pInfo->pSubresource, pLayout);
1283
1284 nvk_image_finish(dev, &image, NULL);
1285 }
1286
1287 static VkResult
nvk_image_plane_bind(struct nvk_device * dev,struct nvk_image * image,struct nvk_image_plane * plane,struct nvk_device_memory * mem,uint64_t * offset_B)1288 nvk_image_plane_bind(struct nvk_device *dev,
1289 struct nvk_image *image,
1290 struct nvk_image_plane *plane,
1291 struct nvk_device_memory *mem,
1292 uint64_t *offset_B)
1293 {
1294 uint64_t plane_size_B, plane_align_B;
1295 nvk_image_plane_size_align_B(dev, image, plane,
1296 &plane_size_B, &plane_align_B);
1297 *offset_B = align64(*offset_B, plane_align_B);
1298
1299 if (plane->va != NULL) {
1300 VkResult result = nvkmd_va_bind_mem(plane->va, &image->vk.base, 0,
1301 mem->mem, *offset_B,
1302 plane->va->size_B);
1303 if (result != VK_SUCCESS)
1304 return result;
1305 } else {
1306 assert(plane->nil.pte_kind == 0);
1307 plane->addr = mem->mem->va->addr + *offset_B;
1308 }
1309
1310 *offset_B += plane_size_B;
1311
1312 return VK_SUCCESS;
1313 }
1314
1315 static VkResult
nvk_bind_image_memory(struct nvk_device * dev,const VkBindImageMemoryInfo * info)1316 nvk_bind_image_memory(struct nvk_device *dev,
1317 const VkBindImageMemoryInfo *info)
1318 {
1319 VK_FROM_HANDLE(nvk_device_memory, mem, info->memory);
1320 VK_FROM_HANDLE(nvk_image, image, info->image);
1321 VkResult result;
1322
1323 /* Ignore this struct on Android, we cannot access swapchain structures there. */
1324 #ifdef NVK_USE_WSI_PLATFORM
1325 const VkBindImageMemorySwapchainInfoKHR *swapchain_info =
1326 vk_find_struct_const(info->pNext, BIND_IMAGE_MEMORY_SWAPCHAIN_INFO_KHR);
1327
1328 if (swapchain_info && swapchain_info->swapchain != VK_NULL_HANDLE) {
1329 VkImage _wsi_image = wsi_common_get_image(swapchain_info->swapchain,
1330 swapchain_info->imageIndex);
1331 VK_FROM_HANDLE(nvk_image, wsi_img, _wsi_image);
1332
1333 assert(image->plane_count == 1);
1334 assert(wsi_img->plane_count == 1);
1335
1336 struct nvk_image_plane *plane = &image->planes[0];
1337 struct nvk_image_plane *swapchain_plane = &wsi_img->planes[0];
1338
1339 /* Copy memory binding information from swapchain image to the current image's plane. */
1340 plane->addr = swapchain_plane->addr;
1341
1342 return VK_SUCCESS;
1343 }
1344 #endif
1345
1346 uint64_t offset_B = info->memoryOffset;
1347 if (image->disjoint) {
1348 const VkBindImagePlaneMemoryInfo *plane_info =
1349 vk_find_struct_const(info->pNext, BIND_IMAGE_PLANE_MEMORY_INFO);
1350 const uint8_t plane =
1351 nvk_image_memory_aspects_to_plane(image, plane_info->planeAspect);
1352 result = nvk_image_plane_bind(dev, image, &image->planes[plane],
1353 mem, &offset_B);
1354 if (result != VK_SUCCESS)
1355 return result;
1356 } else {
1357 for (unsigned plane = 0; plane < image->plane_count; plane++) {
1358 result = nvk_image_plane_bind(dev, image, &image->planes[plane],
1359 mem, &offset_B);
1360 if (result != VK_SUCCESS)
1361 return result;
1362 }
1363 }
1364
1365 if (image->stencil_copy_temp.nil.size_B > 0) {
1366 result = nvk_image_plane_bind(dev, image, &image->stencil_copy_temp,
1367 mem, &offset_B);
1368 if (result != VK_SUCCESS)
1369 return result;
1370 }
1371
1372 return VK_SUCCESS;
1373 }
1374
1375 VKAPI_ATTR VkResult VKAPI_CALL
nvk_BindImageMemory2(VkDevice device,uint32_t bindInfoCount,const VkBindImageMemoryInfo * pBindInfos)1376 nvk_BindImageMemory2(VkDevice device,
1377 uint32_t bindInfoCount,
1378 const VkBindImageMemoryInfo *pBindInfos)
1379 {
1380 VK_FROM_HANDLE(nvk_device, dev, device);
1381 VkResult first_error_or_success = VK_SUCCESS;
1382
1383 for (uint32_t i = 0; i < bindInfoCount; ++i) {
1384 VkResult result = nvk_bind_image_memory(dev, &pBindInfos[i]);
1385
1386 const VkBindMemoryStatusKHR *status =
1387 vk_find_struct_const(pBindInfos[i].pNext, BIND_MEMORY_STATUS_KHR);
1388 if (status != NULL && status->pResult != NULL)
1389 *status->pResult = VK_SUCCESS;
1390
1391 if (first_error_or_success == VK_SUCCESS)
1392 first_error_or_success = result;
1393 }
1394
1395 return first_error_or_success;
1396 }
1397
1398
1399 static VkResult
queue_image_plane_bind(struct nvk_queue * queue,const struct nvk_image_plane * plane,const VkSparseImageMemoryBind * bind)1400 queue_image_plane_bind(struct nvk_queue *queue,
1401 const struct nvk_image_plane *plane,
1402 const VkSparseImageMemoryBind *bind)
1403 {
1404 VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory);
1405 uint64_t image_bind_offset_B;
1406
1407 const uint64_t mem_bind_offset_B = bind->memoryOffset;
1408 const uint32_t layer = bind->subresource.arrayLayer;
1409 const uint32_t level = bind->subresource.mipLevel;
1410
1411 const struct nil_tiling plane_tiling = plane->nil.levels[level].tiling;
1412 const uint32_t tile_size_B = nil_tiling_size_B(&plane_tiling);
1413
1414 const struct nil_Extent4D_Pixels bind_extent_px = {
1415 .width = bind->extent.width,
1416 .height = bind->extent.height,
1417 .depth = bind->extent.depth,
1418 .array_len = 1,
1419 };
1420 const struct nil_Offset4D_Pixels bind_offset_px = {
1421 .x = bind->offset.x,
1422 .y = bind->offset.y,
1423 .z = bind->offset.z,
1424 .a = layer,
1425 };
1426
1427 const struct nil_Extent4D_Pixels level_extent_px =
1428 nil_image_level_extent_px(&plane->nil, level);
1429 const struct nil_Extent4D_Tiles level_extent_tl =
1430 nil_extent4d_px_to_tl(level_extent_px, &plane_tiling,
1431 plane->nil.format,
1432 plane->nil.sample_layout);
1433
1434 /* Convert the extent and offset to tiles */
1435 const struct nil_Extent4D_Tiles bind_extent_tl =
1436 nil_extent4d_px_to_tl(bind_extent_px, &plane_tiling,
1437 plane->nil.format,
1438 plane->nil.sample_layout);
1439 const struct nil_Offset4D_Tiles bind_offset_tl =
1440 nil_offset4d_px_to_tl(bind_offset_px, &plane_tiling,
1441 plane->nil.format,
1442 plane->nil.sample_layout);
1443
1444 image_bind_offset_B =
1445 nil_image_level_layer_offset_B(&plane->nil, level, layer);
1446
1447 /* We can only bind contiguous ranges, so we'll split the image into rows
1448 * of tiles that are guaranteed to be contiguous, and bind in terms of
1449 * these rows
1450 */
1451
1452 /* First, get the size of the bind. Since we have the extent in terms of
1453 * tiles already, we just need to multiply that by the tile size to get
1454 * the size in bytes
1455 */
1456 uint64_t row_bind_size_B = bind_extent_tl.width * tile_size_B;
1457
1458 const uint32_t nvkmd_bind_count = bind_extent_tl.depth *
1459 bind_extent_tl.height;
1460 STACK_ARRAY(struct nvkmd_ctx_bind, nvkmd_binds, nvkmd_bind_count);
1461 uint32_t nvkmd_bind_idx = 0;
1462
1463 /* Second, start walking the binding region in units of tiles, starting
1464 * from the third dimension
1465 */
1466 for (uint32_t z_tl = 0; z_tl < bind_extent_tl.depth; z_tl++) {
1467 /* Start walking the rows to be bound */
1468 for (uint32_t y_tl = 0; y_tl < bind_extent_tl.height; y_tl++) {
1469 /* For the bind offset, get a memory offset to the start of the row
1470 * in terms of the bind extent
1471 */
1472 const uint64_t mem_row_start_tl =
1473 y_tl * bind_extent_tl.width +
1474 z_tl * bind_extent_tl.width * bind_extent_tl.height;
1475
1476 const uint32_t image_x_tl = bind_offset_tl.x;
1477 const uint32_t image_y_tl = bind_offset_tl.y + y_tl;
1478 const uint32_t image_z_tl = bind_offset_tl.z + z_tl;
1479
1480 /* The image offset is calculated in terms of the level extent */
1481 const uint64_t image_row_start_tl =
1482 image_x_tl +
1483 image_y_tl * level_extent_tl.width +
1484 image_z_tl * level_extent_tl.width * level_extent_tl.height;
1485
1486 nvkmd_binds[nvkmd_bind_idx++] = (struct nvkmd_ctx_bind) {
1487 .op = mem ? NVKMD_BIND_OP_BIND : NVKMD_BIND_OP_UNBIND,
1488 .va = plane->va,
1489 .va_offset_B = image_bind_offset_B +
1490 image_row_start_tl * tile_size_B,
1491 .mem = mem ? mem->mem : NULL,
1492 .mem_offset_B = mem_bind_offset_B +
1493 mem_row_start_tl * tile_size_B,
1494 .range_B = row_bind_size_B,
1495 };
1496 }
1497 }
1498
1499 assert(nvkmd_bind_idx == nvkmd_bind_count);
1500 VkResult result = nvkmd_ctx_bind(queue->bind_ctx, &queue->vk.base,
1501 nvkmd_bind_count, nvkmd_binds);
1502
1503 STACK_ARRAY_FINISH(nvkmd_binds);
1504
1505 return result;
1506 }
1507
1508 VkResult
nvk_queue_image_bind(struct nvk_queue * queue,const VkSparseImageMemoryBindInfo * bind_info)1509 nvk_queue_image_bind(struct nvk_queue *queue,
1510 const VkSparseImageMemoryBindInfo *bind_info)
1511 {
1512 VK_FROM_HANDLE(nvk_image, image, bind_info->image);
1513 VkResult result;
1514
1515 /* Sparse residency with multiplane is currently not supported */
1516 assert(image->plane_count == 1);
1517
1518 for (unsigned i = 0; i < bind_info->bindCount; i++) {
1519 result = queue_image_plane_bind(queue, &image->planes[0],
1520 &bind_info->pBinds[i]);
1521 if (result != VK_SUCCESS)
1522 return result;
1523 }
1524
1525 return VK_SUCCESS;
1526 }
1527
1528 static bool
next_opaque_bind_plane(const VkSparseMemoryBind * bind,uint64_t size_B,uint32_t align_B,uint64_t * plane_offset_B,uint64_t * mem_offset_B,uint64_t * bind_size_B,uint64_t * image_plane_offset_B_iter)1529 next_opaque_bind_plane(const VkSparseMemoryBind *bind,
1530 uint64_t size_B, uint32_t align_B,
1531 uint64_t *plane_offset_B,
1532 uint64_t *mem_offset_B,
1533 uint64_t *bind_size_B,
1534 uint64_t *image_plane_offset_B_iter)
1535 {
1536 /* Figure out the offset to thise plane and increment _iter up-front so
1537 * that we're free to early return elsewhere in the function.
1538 */
1539 *image_plane_offset_B_iter = align64(*image_plane_offset_B_iter, align_B);
1540 const uint64_t image_plane_offset_B = *image_plane_offset_B_iter;
1541 *image_plane_offset_B_iter += size_B;
1542
1543 /* Offset into the image or image mip tail, as appropriate */
1544 uint64_t bind_offset_B = bind->resourceOffset;
1545 if (bind_offset_B >= NVK_MIP_TAIL_START_OFFSET)
1546 bind_offset_B -= NVK_MIP_TAIL_START_OFFSET;
1547
1548 if (bind_offset_B < image_plane_offset_B) {
1549 /* The offset of the plane within the bind */
1550 const uint64_t bind_plane_offset_B =
1551 image_plane_offset_B - bind_offset_B;
1552
1553 /* If this plane lies above the bound range, skip this plane */
1554 if (bind_plane_offset_B >= bind->size)
1555 return false;
1556
1557 *plane_offset_B = 0;
1558 *mem_offset_B = bind->memoryOffset + bind_plane_offset_B;
1559 *bind_size_B = MIN2(bind->size - bind_plane_offset_B, size_B);
1560 } else {
1561 /* The offset of the bind within the plane */
1562 const uint64_t plane_bind_offset_B =
1563 bind_offset_B - image_plane_offset_B;
1564
1565 /* If this plane lies below the bound range, skip this plane */
1566 if (plane_bind_offset_B >= size_B)
1567 return false;
1568
1569 *plane_offset_B = plane_bind_offset_B;
1570 *mem_offset_B = bind->memoryOffset;
1571 *bind_size_B = MIN2(bind->size, size_B - plane_bind_offset_B);
1572 }
1573
1574 return true;
1575 }
1576
1577 static VkResult
queue_image_plane_opaque_bind(struct nvk_queue * queue,struct nvk_image * image,struct nvk_image_plane * plane,const VkSparseMemoryBind * bind,uint64_t * image_plane_offset_B)1578 queue_image_plane_opaque_bind(struct nvk_queue *queue,
1579 struct nvk_image *image,
1580 struct nvk_image_plane *plane,
1581 const VkSparseMemoryBind *bind,
1582 uint64_t *image_plane_offset_B)
1583 {
1584 uint64_t plane_size_B, plane_align_B;
1585 nvk_image_plane_size_align_B(nvk_queue_device(queue), image, plane,
1586 &plane_size_B, &plane_align_B);
1587
1588 uint64_t plane_offset_B, mem_offset_B, bind_size_B;
1589 if (!next_opaque_bind_plane(bind, plane_size_B, plane_align_B,
1590 &plane_offset_B, &mem_offset_B, &bind_size_B,
1591 image_plane_offset_B))
1592 return VK_SUCCESS;
1593
1594 VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory);
1595
1596 assert(plane_offset_B + bind_size_B <= plane->va->size_B);
1597 assert(!mem || mem_offset_B + bind_size_B <= mem->vk.size);
1598
1599 const struct nvkmd_ctx_bind nvkmd_bind = {
1600 .op = mem ? NVKMD_BIND_OP_BIND : NVKMD_BIND_OP_UNBIND,
1601 .va = plane->va,
1602 .va_offset_B = plane_offset_B,
1603 .mem = mem ? mem->mem : NULL,
1604 .mem_offset_B = mem_offset_B,
1605 .range_B = bind_size_B,
1606 };
1607 return nvkmd_ctx_bind(queue->bind_ctx, &queue->vk.base, 1, &nvkmd_bind);
1608 }
1609
1610 static VkResult
queue_image_plane_bind_mip_tail(struct nvk_queue * queue,struct nvk_image * image,struct nvk_image_plane * plane,const VkSparseMemoryBind * bind,uint64_t * image_plane_offset_B)1611 queue_image_plane_bind_mip_tail(struct nvk_queue *queue,
1612 struct nvk_image *image,
1613 struct nvk_image_plane *plane,
1614 const VkSparseMemoryBind *bind,
1615 uint64_t *image_plane_offset_B)
1616 {
1617 uint64_t plane_size_B, plane_align_B;
1618 nvk_image_plane_size_align_B(nvk_queue_device(queue), image, plane,
1619 &plane_size_B, &plane_align_B);
1620
1621 const uint64_t mip_tail_offset_B =
1622 nil_image_mip_tail_offset_B(&plane->nil);
1623 const uint64_t mip_tail_size_B =
1624 nil_image_mip_tail_size_B(&plane->nil);
1625 const uint64_t mip_tail_stride_B = plane->nil.array_stride_B;
1626
1627 const uint64_t whole_mip_tail_size_B =
1628 mip_tail_size_B * plane->nil.extent_px.array_len;
1629
1630 uint64_t plane_offset_B, mem_offset_B, bind_size_B;
1631 if (!next_opaque_bind_plane(bind, whole_mip_tail_size_B, plane_align_B,
1632 &plane_offset_B, &mem_offset_B, &bind_size_B,
1633 image_plane_offset_B))
1634 return VK_SUCCESS;
1635
1636 VK_FROM_HANDLE(nvk_device_memory, mem, bind->memory);
1637
1638 /* Range within the virtual mip_tail space */
1639 const uint64_t mip_bind_start_B = plane_offset_B;
1640 const uint64_t mip_bind_end_B = mip_bind_start_B + bind_size_B;
1641
1642 /* Range of array slices covered by this bind */
1643 const uint32_t start_a = mip_bind_start_B / mip_tail_size_B;
1644 const uint32_t end_a = DIV_ROUND_UP(mip_bind_end_B, mip_tail_size_B);
1645
1646 const uint32_t nvkmd_bind_count = end_a - start_a;
1647 STACK_ARRAY(struct nvkmd_ctx_bind, nvkmd_binds, nvkmd_bind_count);
1648 uint32_t nvkmd_bind_idx = 0;
1649
1650 for (uint32_t a = start_a; a < end_a; a++) {
1651 /* Range within the virtual mip_tail space of this array slice */
1652 const uint64_t a_mip_bind_start_B =
1653 MAX2(a * mip_tail_size_B, mip_bind_start_B);
1654 const uint64_t a_mip_bind_end_B =
1655 MIN2((a + 1) * mip_tail_size_B, mip_bind_end_B);
1656
1657 /* Offset and range within this mip_tail slice */
1658 const uint64_t a_offset_B = a_mip_bind_start_B - a * mip_tail_size_B;
1659 const uint64_t a_range_B = a_mip_bind_end_B - a_mip_bind_start_B;
1660
1661 /* Offset within the current bind operation */
1662 const uint64_t a_bind_offset_B =
1663 a_mip_bind_start_B - mip_bind_start_B;
1664
1665 /* Offset within the image */
1666 const uint64_t a_image_offset_B =
1667 mip_tail_offset_B + (a * mip_tail_stride_B) + a_offset_B;
1668
1669 nvkmd_binds[nvkmd_bind_idx++] = (struct nvkmd_ctx_bind) {
1670 .op = mem ? NVKMD_BIND_OP_BIND : NVKMD_BIND_OP_UNBIND,
1671 .va = plane->va,
1672 .va_offset_B = a_image_offset_B,
1673 .mem = mem ? mem->mem : NULL,
1674 .mem_offset_B = mem_offset_B + a_bind_offset_B,
1675 .range_B = a_range_B,
1676 };
1677 }
1678
1679 assert(nvkmd_bind_idx == nvkmd_bind_count);
1680 VkResult result = nvkmd_ctx_bind(queue->bind_ctx, &queue->vk.base,
1681 nvkmd_bind_count, nvkmd_binds);
1682
1683 STACK_ARRAY_FINISH(nvkmd_binds);
1684
1685 return result;
1686 }
1687
1688 VkResult
nvk_queue_image_opaque_bind(struct nvk_queue * queue,const VkSparseImageOpaqueMemoryBindInfo * bind_info)1689 nvk_queue_image_opaque_bind(struct nvk_queue *queue,
1690 const VkSparseImageOpaqueMemoryBindInfo *bind_info)
1691 {
1692 VK_FROM_HANDLE(nvk_image, image, bind_info->image);
1693 VkResult result;
1694
1695 for (unsigned i = 0; i < bind_info->bindCount; i++) {
1696 const VkSparseMemoryBind *bind = &bind_info->pBinds[i];
1697
1698 uint64_t image_plane_offset_B = 0;
1699 for (unsigned plane = 0; plane < image->plane_count; plane++) {
1700 if (bind->resourceOffset >= NVK_MIP_TAIL_START_OFFSET) {
1701 result = queue_image_plane_bind_mip_tail(queue, image,
1702 &image->planes[plane],
1703 bind,
1704 &image_plane_offset_B);
1705 } else {
1706 result = queue_image_plane_opaque_bind(queue, image,
1707 &image->planes[plane],
1708 bind,
1709 &image_plane_offset_B);
1710 }
1711 if (result != VK_SUCCESS)
1712 return result;
1713 }
1714 if (image->stencil_copy_temp.nil.size_B > 0) {
1715 result = queue_image_plane_opaque_bind(queue, image,
1716 &image->stencil_copy_temp,
1717 bind,
1718 &image_plane_offset_B);
1719 if (result != VK_SUCCESS)
1720 return result;
1721 }
1722 }
1723
1724 return VK_SUCCESS;
1725 }
1726
1727 VKAPI_ATTR VkResult VKAPI_CALL
nvk_GetImageOpaqueCaptureDescriptorDataEXT(VkDevice _device,const VkImageCaptureDescriptorDataInfoEXT * pInfo,void * pData)1728 nvk_GetImageOpaqueCaptureDescriptorDataEXT(
1729 VkDevice _device,
1730 const VkImageCaptureDescriptorDataInfoEXT *pInfo,
1731 void *pData)
1732 {
1733 return VK_SUCCESS;
1734 }
1735