1 /*
2 * Copyright © 2022 Collabora Ltd
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "vk_meta_private.h"
25
26 #include "vk_command_buffer.h"
27 #include "vk_device.h"
28 #include "vk_format.h"
29 #include "vk_pipeline.h"
30
31 #include "nir_builder.h"
32
33 struct vk_meta_blit_key {
34 enum vk_meta_object_key_type key_type;
35 enum glsl_sampler_dim dim;
36 VkSampleCountFlagBits src_samples;
37 VkResolveModeFlagBits resolve_mode;
38 VkResolveModeFlagBits stencil_resolve_mode;
39 bool stencil_as_discard;
40 VkFormat dst_format;
41 VkImageAspectFlags aspects;
42 };
43
44 static enum glsl_sampler_dim
vk_image_sampler_dim(const struct vk_image * image)45 vk_image_sampler_dim(const struct vk_image *image)
46 {
47 switch (image->image_type) {
48 case VK_IMAGE_TYPE_1D: return GLSL_SAMPLER_DIM_1D;
49 case VK_IMAGE_TYPE_2D:
50 if (image->samples > 1)
51 return GLSL_SAMPLER_DIM_MS;
52 else
53 return GLSL_SAMPLER_DIM_2D;
54 case VK_IMAGE_TYPE_3D: return GLSL_SAMPLER_DIM_3D;
55 default: unreachable("Invalid image type");
56 }
57 }
58
59 enum blit_desc_binding {
60 BLIT_DESC_BINDING_SAMPLER,
61 BLIT_DESC_BINDING_COLOR,
62 BLIT_DESC_BINDING_DEPTH,
63 BLIT_DESC_BINDING_STENCIL,
64 };
65
66 static enum blit_desc_binding
aspect_to_tex_binding(VkImageAspectFlagBits aspect)67 aspect_to_tex_binding(VkImageAspectFlagBits aspect)
68 {
69 switch (aspect) {
70 case VK_IMAGE_ASPECT_COLOR_BIT: return BLIT_DESC_BINDING_COLOR;
71 case VK_IMAGE_ASPECT_DEPTH_BIT: return BLIT_DESC_BINDING_DEPTH;
72 case VK_IMAGE_ASPECT_STENCIL_BIT: return BLIT_DESC_BINDING_STENCIL;
73 default: unreachable("Unsupported aspect");
74 }
75 }
76
77 struct vk_meta_blit_push_data {
78 float x_off, y_off, x_scale, y_scale;
79 float z_off, z_scale;
80 int32_t arr_delta;
81 uint32_t stencil_bit;
82 };
83
84 static inline void
compute_off_scale(uint32_t src_level_size,uint32_t src0,uint32_t src1,uint32_t dst0,uint32_t dst1,uint32_t * dst0_out,uint32_t * dst1_out,float * off_out,float * scale_out)85 compute_off_scale(uint32_t src_level_size,
86 uint32_t src0, uint32_t src1,
87 uint32_t dst0, uint32_t dst1,
88 uint32_t *dst0_out, uint32_t *dst1_out,
89 float *off_out, float *scale_out)
90 {
91 assert(src0 <= src_level_size && src1 <= src_level_size);
92
93 if (dst0 < dst1) {
94 *dst0_out = dst0;
95 *dst1_out = dst1;
96 } else {
97 *dst0_out = dst1;
98 *dst1_out = dst0;
99
100 /* Flip the source region */
101 uint32_t tmp = src0;
102 src0 = src1;
103 src1 = tmp;
104 }
105
106 double src_region_size = (double)src1 - (double)src0;
107 assert(src_region_size != 0);
108
109 double dst_region_size = (double)*dst1_out - (double)*dst0_out;
110 assert(dst_region_size > 0);
111
112 double src_offset = src0 / (double)src_level_size;
113 double dst_scale = src_region_size / (src_level_size * dst_region_size);
114 double dst_offset = (double)*dst0_out * dst_scale;
115
116 *off_out = src_offset - dst_offset;
117 *scale_out = dst_scale;
118 }
119
120 static inline nir_def *
load_struct_var(nir_builder * b,nir_variable * var,uint32_t field)121 load_struct_var(nir_builder *b, nir_variable *var, uint32_t field)
122 {
123 nir_deref_instr *deref =
124 nir_build_deref_struct(b, nir_build_deref_var(b, var), field);
125 return nir_load_deref(b, deref);
126 }
127
128 static nir_def *
build_tex_resolve(nir_builder * b,nir_deref_instr * t,nir_def * coord,VkSampleCountFlagBits samples,VkResolveModeFlagBits resolve_mode)129 build_tex_resolve(nir_builder *b, nir_deref_instr *t,
130 nir_def *coord,
131 VkSampleCountFlagBits samples,
132 VkResolveModeFlagBits resolve_mode)
133 {
134 nir_def *accum = nir_txf_ms_deref(b, t, coord, nir_imm_int(b, 0));
135 if (resolve_mode == VK_RESOLVE_MODE_SAMPLE_ZERO_BIT)
136 return accum;
137
138 const enum glsl_base_type base_type =
139 glsl_get_sampler_result_type(t->type);
140
141 for (unsigned i = 1; i < samples; i++) {
142 nir_def *val = nir_txf_ms_deref(b, t, coord, nir_imm_int(b, i));
143 switch (resolve_mode) {
144 case VK_RESOLVE_MODE_AVERAGE_BIT:
145 assert(base_type == GLSL_TYPE_FLOAT);
146 accum = nir_fadd(b, accum, val);
147 break;
148
149 case VK_RESOLVE_MODE_MIN_BIT:
150 switch (base_type) {
151 case GLSL_TYPE_UINT:
152 accum = nir_umin(b, accum, val);
153 break;
154 case GLSL_TYPE_INT:
155 accum = nir_imin(b, accum, val);
156 break;
157 case GLSL_TYPE_FLOAT:
158 accum = nir_fmin(b, accum, val);
159 break;
160 default:
161 unreachable("Invalid sample result type");
162 }
163 break;
164
165 case VK_RESOLVE_MODE_MAX_BIT:
166 switch (base_type) {
167 case GLSL_TYPE_UINT:
168 accum = nir_umax(b, accum, val);
169 break;
170 case GLSL_TYPE_INT:
171 accum = nir_imax(b, accum, val);
172 break;
173 case GLSL_TYPE_FLOAT:
174 accum = nir_fmax(b, accum, val);
175 break;
176 default:
177 unreachable("Invalid sample result type");
178 }
179 break;
180
181 default:
182 unreachable("Unsupported resolve mode");
183 }
184 }
185
186 if (resolve_mode == VK_RESOLVE_MODE_AVERAGE_BIT)
187 accum = nir_fmul_imm(b, accum, 1.0 / samples);
188
189 return accum;
190 }
191
192 static nir_shader *
build_blit_shader(const struct vk_meta_blit_key * key)193 build_blit_shader(const struct vk_meta_blit_key *key)
194 {
195 nir_builder build;
196 if (key->resolve_mode || key->stencil_resolve_mode) {
197 build = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL,
198 "vk-meta-resolve");
199 } else {
200 build = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT,
201 NULL, "vk-meta-blit");
202 }
203 nir_builder *b = &build;
204
205 struct glsl_struct_field push_fields[] = {
206 { .type = glsl_vec4_type(), .name = "xy_xform", .offset = 0 },
207 { .type = glsl_vec4_type(), .name = "z_xform", .offset = 16 },
208 };
209 const struct glsl_type *push_iface_type =
210 glsl_interface_type(push_fields, ARRAY_SIZE(push_fields),
211 GLSL_INTERFACE_PACKING_STD140,
212 false /* row_major */, "push");
213 nir_variable *push = nir_variable_create(b->shader, nir_var_mem_push_const,
214 push_iface_type, "push");
215
216 nir_def *xy_xform = load_struct_var(b, push, 0);
217 nir_def *xy_off = nir_channels(b, xy_xform, 3 << 0);
218 nir_def *xy_scale = nir_channels(b, xy_xform, 3 << 2);
219
220 nir_def *out_coord_xy = nir_load_frag_coord(b);
221 out_coord_xy = nir_trim_vector(b, out_coord_xy, 2);
222 nir_def *src_coord_xy = nir_ffma(b, out_coord_xy, xy_scale, xy_off);
223
224 nir_def *z_xform = load_struct_var(b, push, 1);
225 nir_def *out_layer = nir_load_layer_id(b);
226 nir_def *src_coord;
227 if (key->dim == GLSL_SAMPLER_DIM_3D) {
228 nir_def *z_off = nir_channel(b, z_xform, 0);
229 nir_def *z_scale = nir_channel(b, z_xform, 1);
230 nir_def *out_coord_z = nir_fadd_imm(b, nir_u2f32(b, out_layer), 0.5);
231 nir_def *src_coord_z = nir_ffma(b, out_coord_z, z_scale, z_off);
232 src_coord = nir_vec3(b, nir_channel(b, src_coord_xy, 0),
233 nir_channel(b, src_coord_xy, 1),
234 src_coord_z);
235 } else {
236 nir_def *arr_delta = nir_channel(b, z_xform, 2);
237 nir_def *in_layer = nir_iadd(b, out_layer, arr_delta);
238 if (key->dim == GLSL_SAMPLER_DIM_1D) {
239 src_coord = nir_vec2(b, nir_channel(b, src_coord_xy, 0),
240 nir_u2f32(b, in_layer));
241 } else {
242 assert(key->dim == GLSL_SAMPLER_DIM_2D ||
243 key->dim == GLSL_SAMPLER_DIM_MS);
244 src_coord = nir_vec3(b, nir_channel(b, src_coord_xy, 0),
245 nir_channel(b, src_coord_xy, 1),
246 nir_u2f32(b, in_layer));
247 }
248 }
249
250 nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform,
251 glsl_bare_sampler_type(), NULL);
252 sampler->data.descriptor_set = 0;
253 sampler->data.binding = BLIT_DESC_BINDING_SAMPLER;
254 nir_deref_instr *s = nir_build_deref_var(b, sampler);
255
256 u_foreach_bit(a, key->aspects) {
257 VkImageAspectFlagBits aspect = (1 << a);
258
259 enum glsl_base_type base_type;
260 unsigned out_location, out_comps;
261 const char *tex_name, *out_name;
262 VkResolveModeFlagBits resolve_mode;
263 switch (aspect) {
264 case VK_IMAGE_ASPECT_COLOR_BIT:
265 tex_name = "color_tex";
266 if (vk_format_is_sint(key->dst_format))
267 base_type = GLSL_TYPE_INT;
268 else if (vk_format_is_uint(key->dst_format))
269 base_type = GLSL_TYPE_UINT;
270 else
271 base_type = GLSL_TYPE_FLOAT;
272 resolve_mode = key->resolve_mode;
273 out_name = "gl_FragData[0]";
274 out_location = FRAG_RESULT_DATA0;
275 out_comps = 4;
276 break;
277 case VK_IMAGE_ASPECT_DEPTH_BIT:
278 tex_name = "depth_tex";
279 base_type = GLSL_TYPE_FLOAT;
280 resolve_mode = key->resolve_mode;
281 out_name = "gl_FragDepth";
282 out_location = FRAG_RESULT_DEPTH;
283 out_comps = 1;
284 break;
285 case VK_IMAGE_ASPECT_STENCIL_BIT:
286 tex_name = "stencil_tex";
287 base_type = GLSL_TYPE_UINT;
288 resolve_mode = key->stencil_resolve_mode;
289 out_name = "gl_FragStencilRef";
290 out_location = FRAG_RESULT_STENCIL;
291 out_comps = 1;
292 break;
293 default:
294 unreachable("Unsupported aspect");
295 }
296
297 const bool is_array = key->dim != GLSL_SAMPLER_DIM_3D;
298 const struct glsl_type *texture_type =
299 glsl_sampler_type(key->dim, false, is_array, base_type);
300 nir_variable *texture = nir_variable_create(b->shader, nir_var_uniform,
301 texture_type, tex_name);
302 texture->data.descriptor_set = 0;
303 texture->data.binding = aspect_to_tex_binding(aspect);
304 nir_deref_instr *t = nir_build_deref_var(b, texture);
305
306 nir_def *val;
307 if (resolve_mode == VK_RESOLVE_MODE_NONE) {
308 val = nir_txl_deref(b, t, s, src_coord, nir_imm_float(b, 0));
309 } else {
310 val = build_tex_resolve(b, t, nir_f2u32(b, src_coord),
311 key->src_samples, resolve_mode);
312 }
313 val = nir_trim_vector(b, val, out_comps);
314
315 if (key->stencil_as_discard) {
316 assert(key->aspects == VK_IMAGE_ASPECT_STENCIL_BIT);
317 nir_def *stencil_bit = nir_channel(b, z_xform, 3);
318 nir_demote_if(b, nir_ieq(b, nir_iand(b, val, stencil_bit),
319 nir_imm_int(b, 0)));
320 } else {
321 const struct glsl_type *out_type =
322 glsl_vector_type(base_type, out_comps);
323 nir_variable *out = nir_variable_create(b->shader, nir_var_shader_out,
324 out_type, out_name);
325 out->data.location = out_location;
326
327 nir_store_var(b, out, val, BITFIELD_MASK(out_comps));
328 }
329 }
330
331 return b->shader;
332 }
333
334 static VkResult
get_blit_pipeline_layout(struct vk_device * device,struct vk_meta_device * meta,VkPipelineLayout * layout_out)335 get_blit_pipeline_layout(struct vk_device *device,
336 struct vk_meta_device *meta,
337 VkPipelineLayout *layout_out)
338 {
339 const char key[] = "vk-meta-blit-pipeline-layout";
340
341 const VkDescriptorSetLayoutBinding bindings[] = {{
342 .binding = BLIT_DESC_BINDING_SAMPLER,
343 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER,
344 .descriptorCount = 1,
345 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
346 }, {
347 .binding = BLIT_DESC_BINDING_COLOR,
348 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
349 .descriptorCount = 1,
350 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
351 }, {
352 .binding = BLIT_DESC_BINDING_DEPTH,
353 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
354 .descriptorCount = 1,
355 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
356 }, {
357 .binding = BLIT_DESC_BINDING_STENCIL,
358 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
359 .descriptorCount = 1,
360 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
361 }};
362
363 const VkDescriptorSetLayoutCreateInfo desc_info = {
364 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
365 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
366 .bindingCount = ARRAY_SIZE(bindings),
367 .pBindings = bindings,
368 };
369
370 const VkPushConstantRange push_range = {
371 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
372 .offset = 0,
373 .size = sizeof(struct vk_meta_blit_push_data),
374 };
375
376 return vk_meta_get_pipeline_layout(device, meta, &desc_info, &push_range,
377 key, sizeof(key), layout_out);
378 }
379
380 static VkResult
get_blit_pipeline(struct vk_device * device,struct vk_meta_device * meta,const struct vk_meta_blit_key * key,VkPipelineLayout layout,VkPipeline * pipeline_out)381 get_blit_pipeline(struct vk_device *device,
382 struct vk_meta_device *meta,
383 const struct vk_meta_blit_key *key,
384 VkPipelineLayout layout,
385 VkPipeline *pipeline_out)
386 {
387 VkPipeline from_cache = vk_meta_lookup_pipeline(meta, key, sizeof(*key));
388 if (from_cache != VK_NULL_HANDLE) {
389 *pipeline_out = from_cache;
390 return VK_SUCCESS;
391 }
392
393 const VkPipelineShaderStageNirCreateInfoMESA fs_nir_info = {
394 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_NIR_CREATE_INFO_MESA,
395 .nir = build_blit_shader(key),
396 };
397 const VkPipelineShaderStageCreateInfo fs_info = {
398 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
399 .pNext = &fs_nir_info,
400 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
401 .pName = "main",
402 };
403
404 VkPipelineDepthStencilStateCreateInfo ds_info = {
405 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
406 };
407 VkDynamicState dyn_tmp;
408 VkPipelineDynamicStateCreateInfo dyn_info = {
409 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
410 };
411 struct vk_meta_rendering_info render = {
412 .samples = 1,
413 };
414 if (key->aspects & VK_IMAGE_ASPECT_COLOR_BIT) {
415 render.color_attachment_count = 1;
416 render.color_attachment_formats[0] = key->dst_format;
417 render.color_attachment_write_masks[0] =
418 VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
419 VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
420 }
421 if (key->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
422 ds_info.depthTestEnable = VK_TRUE;
423 ds_info.depthWriteEnable = VK_TRUE;
424 ds_info.depthCompareOp = VK_COMPARE_OP_ALWAYS;
425 render.depth_attachment_format = key->dst_format;
426 }
427 if (key->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
428 ds_info.stencilTestEnable = VK_TRUE;
429 ds_info.front.compareOp = VK_COMPARE_OP_ALWAYS;
430 ds_info.front.passOp = VK_STENCIL_OP_REPLACE;
431 ds_info.front.compareMask = ~0u;
432 ds_info.front.writeMask = ~0u;
433 ds_info.front.reference = ~0;
434 ds_info.back = ds_info.front;
435 if (key->stencil_as_discard) {
436 dyn_tmp = VK_DYNAMIC_STATE_STENCIL_WRITE_MASK;
437 dyn_info.dynamicStateCount = 1;
438 dyn_info.pDynamicStates = &dyn_tmp;
439 }
440 render.stencil_attachment_format = key->dst_format;
441 }
442
443 const VkGraphicsPipelineCreateInfo info = {
444 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
445 .stageCount = 1,
446 .pStages = &fs_info,
447 .pDepthStencilState = &ds_info,
448 .pDynamicState = &dyn_info,
449 .layout = layout,
450 };
451
452 VkResult result = vk_meta_create_graphics_pipeline(device, meta, &info,
453 &render,
454 key, sizeof(*key),
455 pipeline_out);
456 ralloc_free(fs_nir_info.nir);
457
458 return result;
459 }
460
461 static VkResult
get_blit_sampler(struct vk_device * device,struct vk_meta_device * meta,VkFilter filter,VkSampler * sampler_out)462 get_blit_sampler(struct vk_device *device,
463 struct vk_meta_device *meta,
464 VkFilter filter,
465 VkSampler *sampler_out)
466 {
467 struct {
468 enum vk_meta_object_key_type key_type;
469 VkFilter filter;
470 } key;
471
472 memset(&key, 0, sizeof(key));
473 key.key_type = VK_META_OBJECT_KEY_BLIT_SAMPLER;
474 key.filter = filter;
475
476 VkSampler from_cache = vk_meta_lookup_sampler(meta, &key, sizeof(key));
477 if (from_cache != VK_NULL_HANDLE) {
478 *sampler_out = from_cache;
479 return VK_SUCCESS;
480 }
481
482 const VkSamplerCreateInfo info = {
483 .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
484 .magFilter = filter,
485 .minFilter = filter,
486 .mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST,
487 .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
488 .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
489 .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
490 .unnormalizedCoordinates = VK_FALSE,
491 };
492
493 return vk_meta_create_sampler(device, meta, &info,
494 &key, sizeof(key), sampler_out);
495 }
496
497 static void
do_blit(struct vk_command_buffer * cmd,struct vk_meta_device * meta,struct vk_image * src_image,VkFormat src_format,VkImageLayout src_image_layout,VkImageSubresourceLayers src_subres,struct vk_image * dst_image,VkFormat dst_format,VkImageLayout dst_image_layout,VkImageSubresourceLayers dst_subres,VkSampler sampler,struct vk_meta_blit_key * key,struct vk_meta_blit_push_data * push,const struct vk_meta_rect * dst_rect,uint32_t dst_layer_count)498 do_blit(struct vk_command_buffer *cmd,
499 struct vk_meta_device *meta,
500 struct vk_image *src_image,
501 VkFormat src_format,
502 VkImageLayout src_image_layout,
503 VkImageSubresourceLayers src_subres,
504 struct vk_image *dst_image,
505 VkFormat dst_format,
506 VkImageLayout dst_image_layout,
507 VkImageSubresourceLayers dst_subres,
508 VkSampler sampler,
509 struct vk_meta_blit_key *key,
510 struct vk_meta_blit_push_data *push,
511 const struct vk_meta_rect *dst_rect,
512 uint32_t dst_layer_count)
513 {
514 struct vk_device *device = cmd->base.device;
515 const struct vk_device_dispatch_table *disp = &device->dispatch_table;
516 VkResult result;
517
518 VkPipelineLayout pipeline_layout;
519 result = get_blit_pipeline_layout(device, meta, &pipeline_layout);
520 if (unlikely(result != VK_SUCCESS)) {
521 vk_command_buffer_set_error(cmd, result);
522 return;
523 }
524
525 uint32_t desc_count = 0;
526 VkDescriptorImageInfo image_infos[3];
527 VkWriteDescriptorSet desc_writes[3];
528
529 if (sampler != VK_NULL_HANDLE) {
530 image_infos[desc_count] = (VkDescriptorImageInfo) {
531 .sampler = sampler,
532 };
533 desc_writes[desc_count] = (VkWriteDescriptorSet) {
534 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
535 .dstBinding = BLIT_DESC_BINDING_SAMPLER,
536 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER,
537 .descriptorCount = 1,
538 .pImageInfo = &image_infos[desc_count],
539 };
540 desc_count++;
541 }
542
543 u_foreach_bit(a, src_subres.aspectMask) {
544 VkImageAspectFlagBits aspect = (1 << a);
545
546 VkImageView src_view;
547 const VkImageViewUsageCreateInfo src_view_usage = {
548 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO,
549 .usage = VK_IMAGE_USAGE_SAMPLED_BIT,
550 };
551 const VkImageViewCreateInfo src_view_info = {
552 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
553 .pNext = &src_view_usage,
554 .image = vk_image_to_handle(src_image),
555 .viewType = vk_image_sampled_view_type(src_image),
556 .format = src_format,
557 .subresourceRange = {
558 .aspectMask = aspect,
559 .baseMipLevel = src_subres.mipLevel,
560 .levelCount = 1,
561 .baseArrayLayer = src_subres.baseArrayLayer,
562 .layerCount = src_subres.layerCount,
563 },
564 };
565 result = vk_meta_create_image_view(cmd, meta, &src_view_info,
566 &src_view);
567 if (unlikely(result != VK_SUCCESS)) {
568 vk_command_buffer_set_error(cmd, result);
569 return;
570 }
571
572 assert(desc_count < ARRAY_SIZE(image_infos));
573 assert(desc_count < ARRAY_SIZE(desc_writes));
574 image_infos[desc_count] = (VkDescriptorImageInfo) {
575 .imageView = src_view,
576 };
577 desc_writes[desc_count] = (VkWriteDescriptorSet) {
578 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
579 .dstBinding = aspect_to_tex_binding(aspect),
580 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE,
581 .descriptorCount = 1,
582 .pImageInfo = &image_infos[desc_count],
583 };
584 desc_count++;
585 }
586
587 disp->CmdPushDescriptorSetKHR(vk_command_buffer_to_handle(cmd),
588 VK_PIPELINE_BIND_POINT_GRAPHICS,
589 pipeline_layout, 0,
590 desc_count, desc_writes);
591
592 assert(dst_subres.aspectMask == src_subres.aspectMask);
593 VkImageAspectFlags aspects_left = dst_subres.aspectMask;
594
595 while (aspects_left) {
596 key->aspects = aspects_left;
597
598 /* If we need to write stencil via iterative discard, it has to be
599 * written by itself because otherwise the discards would also throw
600 * away color or depth data.
601 */
602 if ((key->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
603 key->aspects != VK_IMAGE_ASPECT_STENCIL_BIT &&
604 !meta->use_stencil_export)
605 key->aspects &= ~VK_IMAGE_ASPECT_STENCIL_BIT;
606
607 key->stencil_as_discard = key->aspects == VK_IMAGE_ASPECT_STENCIL_BIT &&
608 !meta->use_stencil_export;
609
610 VkImageView dst_view;
611 const VkImageViewUsageCreateInfo dst_view_usage = {
612 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO,
613 .usage = (key->aspects & VK_IMAGE_ASPECT_COLOR_BIT) ?
614 VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT :
615 VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT,
616 };
617 const VkImageViewCreateInfo dst_view_info = {
618 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
619 .pNext = &dst_view_usage,
620 .image = vk_image_to_handle(dst_image),
621 .viewType = vk_image_sampled_view_type(dst_image),
622 .format = dst_format,
623 .subresourceRange = {
624 .aspectMask = dst_subres.aspectMask,
625 .baseMipLevel = dst_subres.mipLevel,
626 .levelCount = 1,
627 .baseArrayLayer = dst_subres.baseArrayLayer,
628 .layerCount = dst_subres.layerCount,
629 },
630 };
631 result = vk_meta_create_image_view(cmd, meta, &dst_view_info,
632 &dst_view);
633 if (unlikely(result != VK_SUCCESS)) {
634 vk_command_buffer_set_error(cmd, result);
635 return;
636 }
637
638 const VkRenderingAttachmentInfo vk_att = {
639 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
640 .imageView = dst_view,
641 .imageLayout = dst_image_layout,
642 .loadOp = key->stencil_as_discard ? VK_ATTACHMENT_LOAD_OP_CLEAR :
643 VK_ATTACHMENT_LOAD_OP_DONT_CARE,
644 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
645 };
646 VkRenderingInfo vk_render = {
647 .sType = VK_STRUCTURE_TYPE_RENDERING_INFO,
648 .renderArea = {
649 .offset = {
650 dst_rect->x0,
651 dst_rect->y0
652 },
653 .extent = {
654 dst_rect->x1 - dst_rect->x0,
655 dst_rect->y1 - dst_rect->y0
656 },
657 },
658 .layerCount = dst_rect->layer + dst_layer_count,
659 };
660
661 if (key->aspects & VK_IMAGE_ASPECT_COLOR_BIT) {
662 vk_render.colorAttachmentCount = 1;
663 vk_render.pColorAttachments = &vk_att;
664 }
665 if (key->aspects & VK_IMAGE_ASPECT_DEPTH_BIT)
666 vk_render.pDepthAttachment = &vk_att;
667 if (key->aspects & VK_IMAGE_ASPECT_STENCIL_BIT)
668 vk_render.pStencilAttachment = &vk_att;
669
670 disp->CmdBeginRendering(vk_command_buffer_to_handle(cmd), &vk_render);
671
672 VkPipeline pipeline;
673 result = get_blit_pipeline(device, meta, key,
674 pipeline_layout, &pipeline);
675 if (unlikely(result != VK_SUCCESS)) {
676 vk_command_buffer_set_error(cmd, result);
677 return;
678 }
679
680 disp->CmdBindPipeline(vk_command_buffer_to_handle(cmd),
681 VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
682
683 if (key->stencil_as_discard) {
684 for (uint32_t i = 0; i < 8; i++) {
685 push->stencil_bit = BITFIELD_BIT(i);
686 disp->CmdPushConstants(vk_command_buffer_to_handle(cmd),
687 pipeline_layout,
688 VK_SHADER_STAGE_FRAGMENT_BIT,
689 0, sizeof(*push), push);
690
691 disp->CmdSetStencilWriteMask(vk_command_buffer_to_handle(cmd),
692 VK_STENCIL_FACE_FRONT_AND_BACK,
693 push->stencil_bit);
694
695 meta->cmd_draw_volume(cmd, meta, dst_rect, dst_layer_count);
696 }
697 } else {
698 disp->CmdPushConstants(vk_command_buffer_to_handle(cmd),
699 pipeline_layout,
700 VK_SHADER_STAGE_FRAGMENT_BIT,
701 0, sizeof(*push), push);
702
703 meta->cmd_draw_volume(cmd, meta, dst_rect, dst_layer_count);
704 }
705
706 disp->CmdEndRendering(vk_command_buffer_to_handle(cmd));
707
708 aspects_left &= ~key->aspects;
709 }
710 }
711
712 void
vk_meta_blit_image(struct vk_command_buffer * cmd,struct vk_meta_device * meta,struct vk_image * src_image,VkFormat src_format,VkImageLayout src_image_layout,struct vk_image * dst_image,VkFormat dst_format,VkImageLayout dst_image_layout,uint32_t region_count,const VkImageBlit2 * regions,VkFilter filter)713 vk_meta_blit_image(struct vk_command_buffer *cmd,
714 struct vk_meta_device *meta,
715 struct vk_image *src_image,
716 VkFormat src_format,
717 VkImageLayout src_image_layout,
718 struct vk_image *dst_image,
719 VkFormat dst_format,
720 VkImageLayout dst_image_layout,
721 uint32_t region_count,
722 const VkImageBlit2 *regions,
723 VkFilter filter)
724 {
725 struct vk_device *device = cmd->base.device;
726 VkResult result;
727
728 VkSampler sampler;
729 result = get_blit_sampler(device, meta, filter, &sampler);
730 if (unlikely(result != VK_SUCCESS)) {
731 vk_command_buffer_set_error(cmd, result);
732 return;
733 }
734
735 struct vk_meta_blit_key key;
736 memset(&key, 0, sizeof(key));
737 key.key_type = VK_META_OBJECT_KEY_BLIT_PIPELINE;
738 key.src_samples = src_image->samples;
739 key.dim = vk_image_sampler_dim(src_image);
740 key.dst_format = dst_format;
741
742 for (uint32_t r = 0; r < region_count; r++) {
743 struct vk_meta_blit_push_data push = {0};
744 struct vk_meta_rect dst_rect = {0};
745
746 uint32_t src_level = regions[r].srcSubresource.mipLevel;
747 VkExtent3D src_extent = vk_image_mip_level_extent(src_image, src_level);
748
749 compute_off_scale(src_extent.width,
750 regions[r].srcOffsets[0].x,
751 regions[r].srcOffsets[1].x,
752 regions[r].dstOffsets[0].x,
753 regions[r].dstOffsets[1].x,
754 &dst_rect.x0, &dst_rect.x1,
755 &push.x_off, &push.x_scale);
756 compute_off_scale(src_extent.height,
757 regions[r].srcOffsets[0].y,
758 regions[r].srcOffsets[1].y,
759 regions[r].dstOffsets[0].y,
760 regions[r].dstOffsets[1].y,
761 &dst_rect.y0, &dst_rect.y1,
762 &push.y_off, &push.y_scale);
763
764 VkImageSubresourceLayers src_subres = regions[r].srcSubresource;
765 src_subres.layerCount =
766 vk_image_subresource_layer_count(src_image, &src_subres);
767
768 VkImageSubresourceLayers dst_subres = regions[r].dstSubresource;
769 dst_subres.layerCount =
770 vk_image_subresource_layer_count(dst_image, &dst_subres);
771
772 uint32_t dst_layer_count;
773 if (src_image->image_type == VK_IMAGE_TYPE_3D) {
774 uint32_t layer0, layer1;
775 compute_off_scale(src_extent.depth,
776 regions[r].srcOffsets[0].z,
777 regions[r].srcOffsets[1].z,
778 regions[r].dstOffsets[0].z,
779 regions[r].dstOffsets[1].z,
780 &layer0, &layer1,
781 &push.z_off, &push.z_scale);
782 dst_rect.layer = layer0;
783 dst_layer_count = layer1 - layer0;
784 } else {
785 assert(src_subres.layerCount == dst_subres.layerCount);
786 dst_layer_count = dst_subres.layerCount;
787 push.arr_delta = dst_subres.baseArrayLayer -
788 src_subres.baseArrayLayer;
789 }
790
791 do_blit(cmd, meta,
792 src_image, src_format, src_image_layout, src_subres,
793 dst_image, dst_format, dst_image_layout, dst_subres,
794 sampler, &key, &push, &dst_rect, dst_layer_count);
795 }
796 }
797
798 void
vk_meta_blit_image2(struct vk_command_buffer * cmd,struct vk_meta_device * meta,const VkBlitImageInfo2 * blit)799 vk_meta_blit_image2(struct vk_command_buffer *cmd,
800 struct vk_meta_device *meta,
801 const VkBlitImageInfo2 *blit)
802 {
803 VK_FROM_HANDLE(vk_image, src_image, blit->srcImage);
804 VK_FROM_HANDLE(vk_image, dst_image, blit->dstImage);
805
806 vk_meta_blit_image(cmd, meta,
807 src_image, src_image->format, blit->srcImageLayout,
808 dst_image, dst_image->format, blit->dstImageLayout,
809 blit->regionCount, blit->pRegions, blit->filter);
810 }
811
812 void
vk_meta_resolve_image(struct vk_command_buffer * cmd,struct vk_meta_device * meta,struct vk_image * src_image,VkFormat src_format,VkImageLayout src_image_layout,struct vk_image * dst_image,VkFormat dst_format,VkImageLayout dst_image_layout,uint32_t region_count,const VkImageResolve2 * regions,VkResolveModeFlagBits resolve_mode,VkResolveModeFlagBits stencil_resolve_mode)813 vk_meta_resolve_image(struct vk_command_buffer *cmd,
814 struct vk_meta_device *meta,
815 struct vk_image *src_image,
816 VkFormat src_format,
817 VkImageLayout src_image_layout,
818 struct vk_image *dst_image,
819 VkFormat dst_format,
820 VkImageLayout dst_image_layout,
821 uint32_t region_count,
822 const VkImageResolve2 *regions,
823 VkResolveModeFlagBits resolve_mode,
824 VkResolveModeFlagBits stencil_resolve_mode)
825 {
826 struct vk_meta_blit_key key;
827 memset(&key, 0, sizeof(key));
828 key.key_type = VK_META_OBJECT_KEY_BLIT_PIPELINE;
829 key.dim = vk_image_sampler_dim(src_image);
830 key.src_samples = src_image->samples;
831 key.resolve_mode = resolve_mode;
832 key.stencil_resolve_mode = stencil_resolve_mode;
833 key.dst_format = dst_format;
834
835 for (uint32_t r = 0; r < region_count; r++) {
836 struct vk_meta_blit_push_data push = {
837 .x_off = regions[r].srcOffset.x - regions[r].dstOffset.x,
838 .y_off = regions[r].srcOffset.y - regions[r].dstOffset.y,
839 .x_scale = 1,
840 .y_scale = 1,
841 };
842 struct vk_meta_rect dst_rect = {
843 .x0 = regions[r].dstOffset.x,
844 .y0 = regions[r].dstOffset.y,
845 .x1 = regions[r].dstOffset.x + regions[r].extent.width,
846 .y1 = regions[r].dstOffset.y + regions[r].extent.height,
847 };
848
849 VkImageSubresourceLayers src_subres = regions[r].srcSubresource;
850 src_subres.layerCount =
851 vk_image_subresource_layer_count(src_image, &src_subres);
852
853 VkImageSubresourceLayers dst_subres = regions[r].dstSubresource;
854 dst_subres.layerCount =
855 vk_image_subresource_layer_count(dst_image, &dst_subres);
856
857 do_blit(cmd, meta,
858 src_image, src_format, src_image_layout, src_subres,
859 dst_image, dst_format, dst_image_layout, dst_subres,
860 VK_NULL_HANDLE, &key, &push, &dst_rect,
861 dst_subres.layerCount);
862 }
863 }
864
865 void
vk_meta_resolve_image2(struct vk_command_buffer * cmd,struct vk_meta_device * meta,const VkResolveImageInfo2 * resolve)866 vk_meta_resolve_image2(struct vk_command_buffer *cmd,
867 struct vk_meta_device *meta,
868 const VkResolveImageInfo2 *resolve)
869 {
870 VK_FROM_HANDLE(vk_image, src_image, resolve->srcImage);
871 VK_FROM_HANDLE(vk_image, dst_image, resolve->dstImage);
872
873 VkResolveModeFlagBits resolve_mode = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT;
874 if (vk_format_is_color(src_image->format) &&
875 !vk_format_is_int(src_image->format))
876 resolve_mode = VK_RESOLVE_MODE_AVERAGE_BIT;
877
878 vk_meta_resolve_image(cmd, meta,
879 src_image, src_image->format, resolve->srcImageLayout,
880 dst_image, dst_image->format, resolve->dstImageLayout,
881 resolve->regionCount, resolve->pRegions,
882 resolve_mode, VK_RESOLVE_MODE_SAMPLE_ZERO_BIT);
883 }
884
885 static void
vk_meta_resolve_attachment(struct vk_command_buffer * cmd,struct vk_meta_device * meta,struct vk_image_view * src_view,VkImageLayout src_image_layout,struct vk_image_view * dst_view,VkImageLayout dst_image_layout,VkImageAspectFlags resolve_aspects,VkResolveModeFlagBits resolve_mode,VkResolveModeFlagBits stencil_resolve_mode,VkRect2D area,uint32_t layer_count,uint32_t view_mask)886 vk_meta_resolve_attachment(struct vk_command_buffer *cmd,
887 struct vk_meta_device *meta,
888 struct vk_image_view *src_view,
889 VkImageLayout src_image_layout,
890 struct vk_image_view *dst_view,
891 VkImageLayout dst_image_layout,
892 VkImageAspectFlags resolve_aspects,
893 VkResolveModeFlagBits resolve_mode,
894 VkResolveModeFlagBits stencil_resolve_mode,
895 VkRect2D area, uint32_t layer_count,
896 uint32_t view_mask)
897 {
898 VkImageResolve2 region = {
899 .sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2,
900 .srcSubresource = {
901 .aspectMask = resolve_aspects,
902 .mipLevel = src_view->base_mip_level,
903 },
904 .srcOffset = { area.offset.x, area.offset.y, 0},
905 .dstSubresource = {
906 .aspectMask = resolve_aspects,
907 .mipLevel = dst_view->base_mip_level,
908 },
909 .dstOffset = { area.offset.x, area.offset.y, 0},
910 .extent = { area.extent.width, area.extent.height, 1},
911 };
912
913 if (view_mask) {
914 u_foreach_bit(v, view_mask) {
915 region.srcSubresource.baseArrayLayer = src_view->base_array_layer + v;
916 region.srcSubresource.layerCount = 1;
917 region.dstSubresource.baseArrayLayer = dst_view->base_array_layer + v;
918 region.dstSubresource.layerCount = 1;
919
920 vk_meta_resolve_image(cmd, meta,
921 src_view->image, src_view->format,
922 src_image_layout,
923 dst_view->image, dst_view->format,
924 dst_image_layout,
925 1, ®ion, resolve_mode, stencil_resolve_mode);
926 }
927 } else {
928 region.srcSubresource.baseArrayLayer = src_view->base_array_layer;
929 region.srcSubresource.layerCount = layer_count;
930 region.dstSubresource.baseArrayLayer = dst_view->base_array_layer;
931 region.dstSubresource.layerCount = layer_count;
932
933 vk_meta_resolve_image(cmd, meta,
934 src_view->image, src_view->format,
935 src_image_layout,
936 dst_view->image, dst_view->format,
937 dst_image_layout,
938 1, ®ion, resolve_mode, stencil_resolve_mode);
939 }
940 }
941
942 void
vk_meta_resolve_rendering(struct vk_command_buffer * cmd,struct vk_meta_device * meta,const VkRenderingInfo * pRenderingInfo)943 vk_meta_resolve_rendering(struct vk_command_buffer *cmd,
944 struct vk_meta_device *meta,
945 const VkRenderingInfo *pRenderingInfo)
946 {
947 for (uint32_t c = 0; c < pRenderingInfo->colorAttachmentCount; c++) {
948 const VkRenderingAttachmentInfo *att =
949 &pRenderingInfo->pColorAttachments[c];
950 if (att->resolveMode == VK_RESOLVE_MODE_NONE)
951 continue;
952
953 VK_FROM_HANDLE(vk_image_view, view, att->imageView);
954 VK_FROM_HANDLE(vk_image_view, res_view, att->resolveImageView);
955
956 vk_meta_resolve_attachment(cmd, meta, view, att->imageLayout,
957 res_view, att->resolveImageLayout,
958 VK_IMAGE_ASPECT_COLOR_BIT,
959 att->resolveMode, VK_RESOLVE_MODE_NONE,
960 pRenderingInfo->renderArea,
961 pRenderingInfo->layerCount,
962 pRenderingInfo->viewMask);
963 }
964
965 const VkRenderingAttachmentInfo *d_att = pRenderingInfo->pDepthAttachment;
966 if (d_att && d_att->resolveMode == VK_RESOLVE_MODE_NONE)
967 d_att = NULL;
968
969 const VkRenderingAttachmentInfo *s_att = pRenderingInfo->pStencilAttachment;
970 if (s_att && s_att->resolveMode == VK_RESOLVE_MODE_NONE)
971 s_att = NULL;
972
973 if (s_att != NULL || d_att != NULL) {
974 if (s_att != NULL && d_att != NULL &&
975 s_att->imageView == d_att->imageView &&
976 s_att->resolveImageView == d_att->resolveImageView) {
977 VK_FROM_HANDLE(vk_image_view, view, d_att->imageView);
978 VK_FROM_HANDLE(vk_image_view, res_view, d_att->resolveImageView);
979
980 vk_meta_resolve_attachment(cmd, meta, view, d_att->imageLayout,
981 res_view, d_att->resolveImageLayout,
982 VK_IMAGE_ASPECT_DEPTH_BIT |
983 VK_IMAGE_ASPECT_STENCIL_BIT,
984 d_att->resolveMode, s_att->resolveMode,
985 pRenderingInfo->renderArea,
986 pRenderingInfo->layerCount,
987 pRenderingInfo->viewMask);
988 } else {
989 if (d_att != NULL) {
990 VK_FROM_HANDLE(vk_image_view, view, d_att->imageView);
991 VK_FROM_HANDLE(vk_image_view, res_view, d_att->resolveImageView);
992
993 vk_meta_resolve_attachment(cmd, meta, view, d_att->imageLayout,
994 res_view, d_att->resolveImageLayout,
995 VK_IMAGE_ASPECT_DEPTH_BIT,
996 d_att->resolveMode, VK_RESOLVE_MODE_NONE,
997 pRenderingInfo->renderArea,
998 pRenderingInfo->layerCount,
999 pRenderingInfo->viewMask);
1000 }
1001
1002 if (s_att != NULL) {
1003 VK_FROM_HANDLE(vk_image_view, view, s_att->imageView);
1004 VK_FROM_HANDLE(vk_image_view, res_view, s_att->resolveImageView);
1005
1006 vk_meta_resolve_attachment(cmd, meta, view, s_att->imageLayout,
1007 res_view, s_att->resolveImageLayout,
1008 VK_IMAGE_ASPECT_STENCIL_BIT,
1009 VK_RESOLVE_MODE_NONE, s_att->resolveMode,
1010 pRenderingInfo->renderArea,
1011 pRenderingInfo->layerCount,
1012 pRenderingInfo->viewMask);
1013 }
1014 }
1015 }
1016 }
1017