1 /*
2 * Copyright © 2023 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Mike Blumenkrantz <[email protected]>
25 */
26
27 #include "zink_batch.h"
28 #include "zink_context.h"
29 #include "zink_descriptors.h"
30 #include "zink_resource.h"
31 #include "zink_screen.h"
32
33
34 static VkAccessFlags
access_src_flags(VkImageLayout layout)35 access_src_flags(VkImageLayout layout)
36 {
37 switch (layout) {
38 case VK_IMAGE_LAYOUT_UNDEFINED:
39 return VK_ACCESS_NONE;
40
41 case VK_IMAGE_LAYOUT_GENERAL:
42 return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
43
44 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
45 case VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT:
46 return VK_ACCESS_COLOR_ATTACHMENT_READ_BIT;
47 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
48 return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT;
49
50 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
51 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
52 return VK_ACCESS_SHADER_READ_BIT;
53
54 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
55 return VK_ACCESS_TRANSFER_READ_BIT;
56
57 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
58 return VK_ACCESS_TRANSFER_WRITE_BIT;
59
60 case VK_IMAGE_LAYOUT_PREINITIALIZED:
61 return VK_ACCESS_HOST_WRITE_BIT;
62
63 case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR:
64 return VK_ACCESS_NONE;
65
66 default:
67 unreachable("unexpected layout");
68 }
69 }
70
71 static VkAccessFlags
access_dst_flags(VkImageLayout layout)72 access_dst_flags(VkImageLayout layout)
73 {
74 switch (layout) {
75 case VK_IMAGE_LAYOUT_UNDEFINED:
76 return VK_ACCESS_NONE;
77
78 case VK_IMAGE_LAYOUT_GENERAL:
79 return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
80
81 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
82 case VK_IMAGE_LAYOUT_ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT:
83 return VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
84 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
85 return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
86
87 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
88 return VK_ACCESS_SHADER_READ_BIT;
89
90 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
91 return VK_ACCESS_TRANSFER_READ_BIT;
92
93 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
94 return VK_ACCESS_SHADER_READ_BIT;
95
96 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
97 return VK_ACCESS_TRANSFER_WRITE_BIT;
98
99 case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR:
100 return VK_ACCESS_NONE;
101
102 default:
103 unreachable("unexpected layout");
104 }
105 }
106
107 static VkPipelineStageFlags
pipeline_dst_stage(VkImageLayout layout)108 pipeline_dst_stage(VkImageLayout layout)
109 {
110 switch (layout) {
111 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
112 return VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
113 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
114 return VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
115
116 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
117 return VK_PIPELINE_STAGE_TRANSFER_BIT;
118 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
119 return VK_PIPELINE_STAGE_TRANSFER_BIT;
120
121 case VK_IMAGE_LAYOUT_GENERAL:
122 return VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
123
124 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
125 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
126 return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
127
128 default:
129 return VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
130 }
131 }
132
133 #define ALL_READ_ACCESS_FLAGS \
134 (VK_ACCESS_INDIRECT_COMMAND_READ_BIT | \
135 VK_ACCESS_INDEX_READ_BIT | \
136 VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | \
137 VK_ACCESS_UNIFORM_READ_BIT | \
138 VK_ACCESS_INPUT_ATTACHMENT_READ_BIT | \
139 VK_ACCESS_SHADER_READ_BIT | \
140 VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | \
141 VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | \
142 VK_ACCESS_TRANSFER_READ_BIT |\
143 VK_ACCESS_HOST_READ_BIT |\
144 VK_ACCESS_MEMORY_READ_BIT |\
145 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT |\
146 VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT |\
147 VK_ACCESS_COLOR_ATTACHMENT_READ_NONCOHERENT_BIT_EXT |\
148 VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\
149 VK_ACCESS_FRAGMENT_SHADING_RATE_ATTACHMENT_READ_BIT_KHR |\
150 VK_ACCESS_FRAGMENT_DENSITY_MAP_READ_BIT_EXT |\
151 VK_ACCESS_COMMAND_PREPROCESS_READ_BIT_NV |\
152 VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\
153 VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR)
154
155
156 bool
zink_resource_access_is_write(VkAccessFlags flags)157 zink_resource_access_is_write(VkAccessFlags flags)
158 {
159 return (flags & ~ALL_READ_ACCESS_FLAGS) > 0;
160 }
161
162 static bool
zink_resource_image_needs_barrier(struct zink_resource * res,VkImageLayout new_layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)163 zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
164 {
165 return res->layout != new_layout || (res->obj->access_stage & pipeline) != pipeline ||
166 (res->obj->access & flags) != flags ||
167 zink_resource_access_is_write(res->obj->access) ||
168 zink_resource_access_is_write(flags);
169 }
170
171 void
zink_resource_image_barrier_init(VkImageMemoryBarrier * imb,struct zink_resource * res,VkImageLayout new_layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)172 zink_resource_image_barrier_init(VkImageMemoryBarrier *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
173 {
174 if (!pipeline)
175 pipeline = pipeline_dst_stage(new_layout);
176 if (!flags)
177 flags = access_dst_flags(new_layout);
178
179 VkImageSubresourceRange isr = {
180 res->aspect,
181 0, VK_REMAINING_MIP_LEVELS,
182 0, VK_REMAINING_ARRAY_LAYERS
183 };
184 *imb = VkImageMemoryBarrier {
185 VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
186 NULL,
187 res->obj->access ? res->obj->access : access_src_flags(res->layout),
188 flags,
189 res->layout,
190 new_layout,
191 VK_QUEUE_FAMILY_IGNORED,
192 VK_QUEUE_FAMILY_IGNORED,
193 res->obj->image,
194 isr
195 };
196 }
197
198 void
zink_resource_image_barrier2_init(VkImageMemoryBarrier2 * imb,struct zink_resource * res,VkImageLayout new_layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)199 zink_resource_image_barrier2_init(VkImageMemoryBarrier2 *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
200 {
201 if (!pipeline)
202 pipeline = pipeline_dst_stage(new_layout);
203 if (!flags)
204 flags = access_dst_flags(new_layout);
205
206 VkImageSubresourceRange isr = {
207 res->aspect,
208 0, VK_REMAINING_MIP_LEVELS,
209 0, VK_REMAINING_ARRAY_LAYERS
210 };
211 *imb = VkImageMemoryBarrier2 {
212 VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
213 NULL,
214 res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
215 res->obj->access ? res->obj->access : access_src_flags(res->layout),
216 pipeline,
217 flags,
218 res->layout,
219 new_layout,
220 VK_QUEUE_FAMILY_IGNORED,
221 VK_QUEUE_FAMILY_IGNORED,
222 res->obj->image,
223 isr
224 };
225 }
226
227 static inline bool
is_shader_pipline_stage(VkPipelineStageFlags pipeline)228 is_shader_pipline_stage(VkPipelineStageFlags pipeline)
229 {
230 return pipeline & GFX_SHADER_BITS;
231 }
232
233 static void
resource_check_defer_buffer_barrier(struct zink_context * ctx,struct zink_resource * res,VkPipelineStageFlags pipeline)234 resource_check_defer_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkPipelineStageFlags pipeline)
235 {
236 assert(res->obj->is_buffer);
237 if (res->bind_count[0] - res->so_bind_count > 0) {
238 if ((res->vbo_bind_mask && !(pipeline & VK_PIPELINE_STAGE_VERTEX_INPUT_BIT)) ||
239 (util_bitcount(res->vbo_bind_mask) != res->bind_count[0] && !is_shader_pipline_stage(pipeline)))
240 /* gfx rebind */
241 _mesa_set_add(ctx->need_barriers[0], res);
242 }
243 if (res->bind_count[1] && !(pipeline & VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT))
244 /* compute rebind */
245 _mesa_set_add(ctx->need_barriers[1], res);
246 }
247
248 static inline bool
unordered_res_exec(const struct zink_context * ctx,const struct zink_resource * res,bool is_write)249 unordered_res_exec(const struct zink_context *ctx, const struct zink_resource *res, bool is_write)
250 {
251 /* if all usage is unordered, keep unordered */
252 if (res->obj->unordered_read && res->obj->unordered_write)
253 return true;
254 /* if testing write access but have any ordered read access, cannot promote */
255 if (is_write && zink_batch_usage_matches(res->obj->bo->reads.u, ctx->bs) && !res->obj->unordered_read)
256 return false;
257 /* if write access is unordered or nonexistent, always promote */
258 return res->obj->unordered_write || !zink_batch_usage_matches(res->obj->bo->writes.u, ctx->bs);
259 }
260
261 static ALWAYS_INLINE bool
check_unordered_exec(struct zink_context * ctx,struct zink_resource * res,bool is_write)262 check_unordered_exec(struct zink_context *ctx, struct zink_resource *res, bool is_write)
263 {
264 if (res) {
265 if (!res->obj->is_buffer) {
266 /* TODO: figure out how to link up unordered layout -> ordered layout and delete this conditionals */
267 if (zink_resource_usage_is_unflushed(res) && !res->obj->unordered_read && !res->obj->unordered_write)
268 return false;
269 }
270 return unordered_res_exec(ctx, res, is_write);
271 }
272 return true;
273 }
274
275 VkCommandBuffer
zink_get_cmdbuf(struct zink_context * ctx,struct zink_resource * src,struct zink_resource * dst)276 zink_get_cmdbuf(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst)
277 {
278 bool unordered_exec = !ctx->no_reorder;
279
280 unordered_exec &= check_unordered_exec(ctx, src, false) &&
281 check_unordered_exec(ctx, dst, true);
282
283 if (src)
284 src->obj->unordered_read = unordered_exec;
285 if (dst)
286 dst->obj->unordered_write = unordered_exec;
287
288 if (!unordered_exec || ctx->unordered_blitting)
289 zink_batch_no_rp(ctx);
290
291 if (unordered_exec) {
292 ctx->bs->has_reordered_work = true;
293 return ctx->bs->reordered_cmdbuf;
294 }
295 ctx->bs->has_work = true;
296 return ctx->bs->cmdbuf;
297 }
298
299 static void
resource_check_defer_image_barrier(struct zink_context * ctx,struct zink_resource * res,VkImageLayout layout,VkPipelineStageFlags pipeline)300 resource_check_defer_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout layout, VkPipelineStageFlags pipeline)
301 {
302 assert(!res->obj->is_buffer);
303 assert(!ctx->blitting);
304
305 bool is_compute = pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
306 /* if this is a non-shader barrier and there are binds, always queue a shader barrier */
307 bool is_shader = is_shader_pipline_stage(pipeline);
308 if ((is_shader || !res->bind_count[is_compute]) &&
309 /* if no layout change is needed between gfx and compute, do nothing */
310 !res->bind_count[!is_compute] && (!is_compute || !res->fb_bind_count))
311 return;
312
313 if (res->bind_count[!is_compute] && is_shader) {
314 /* if the layout is the same between gfx and compute, do nothing */
315 if (layout == zink_descriptor_util_image_layout_eval(ctx, res, !is_compute))
316 return;
317 }
318 /* queue a layout change if a layout change will be needed */
319 if (res->bind_count[!is_compute])
320 _mesa_set_add(ctx->need_barriers[!is_compute], res);
321 /* also queue a layout change if this is a non-shader layout */
322 if (res->bind_count[is_compute] && !is_shader)
323 _mesa_set_add(ctx->need_barriers[is_compute], res);
324 }
325
326 enum barrier_type {
327 barrier_default,
328 barrier_KHR_synchronzation2
329 };
330
331 template <barrier_type BARRIER_API>
332 struct emit_memory_barrier {
for_imageemit_memory_barrier333 static void for_image(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout,
334 VkAccessFlags flags, VkPipelineStageFlags pipeline, bool completed, VkCommandBuffer cmdbuf,
335 bool *queue_import)
336 {
337 VkImageMemoryBarrier imb;
338 zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline);
339 if (!res->obj->access_stage || completed)
340 imb.srcAccessMask = 0;
341 if (res->obj->needs_zs_evaluate)
342 imb.pNext = &res->obj->zs_evaluate;
343 res->obj->needs_zs_evaluate = false;
344 if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) {
345 imb.srcQueueFamilyIndex = res->queue;
346 imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue;
347 res->queue = VK_QUEUE_FAMILY_IGNORED;
348 *queue_import = true;
349 }
350 VKCTX(CmdPipelineBarrier)(
351 cmdbuf,
352 res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
353 pipeline,
354 0,
355 0, NULL,
356 0, NULL,
357 1, &imb
358 );
359 }
360
for_bufferemit_memory_barrier361 static void for_buffer(struct zink_context *ctx, struct zink_resource *res,
362 VkPipelineStageFlags pipeline,
363 VkAccessFlags flags,
364 bool unordered,
365 bool usage_matches,
366 VkPipelineStageFlags stages,
367 VkCommandBuffer cmdbuf)
368 {
369 VkMemoryBarrier bmb;
370 bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
371 bmb.pNext = NULL;
372 if (unordered) {
373 stages = usage_matches ? res->obj->unordered_access_stage : stages;
374 bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access;
375 } else {
376 bmb.srcAccessMask = res->obj->access;
377 }
378 bmb.dstAccessMask = flags;
379 VKCTX(CmdPipelineBarrier)(
380 cmdbuf,
381 stages,
382 pipeline,
383 0,
384 1, &bmb,
385 0, NULL,
386 0, NULL);
387 }
388 };
389
390
391 template <>
392 struct emit_memory_barrier<barrier_KHR_synchronzation2> {
for_imageemit_memory_barrier393 static void for_image(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout,
394 VkAccessFlags flags, VkPipelineStageFlags pipeline, bool completed, VkCommandBuffer cmdbuf,
395 bool *queue_import)
396 {
397 VkImageMemoryBarrier2 imb;
398 zink_resource_image_barrier2_init(&imb, res, new_layout, flags, pipeline);
399 if (!res->obj->access_stage || completed)
400 imb.srcAccessMask = 0;
401 if (res->obj->needs_zs_evaluate)
402 imb.pNext = &res->obj->zs_evaluate;
403 res->obj->needs_zs_evaluate = false;
404 if (res->queue != zink_screen(ctx->base.screen)->gfx_queue && res->queue != VK_QUEUE_FAMILY_IGNORED) {
405 imb.srcQueueFamilyIndex = res->queue;
406 imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue;
407 res->queue = VK_QUEUE_FAMILY_IGNORED;
408 *queue_import = true;
409 }
410 VkDependencyInfo dep = {
411 VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
412 NULL,
413 0,
414 0,
415 NULL,
416 0,
417 NULL,
418 1,
419 &imb
420 };
421 VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep);
422 }
423
for_bufferemit_memory_barrier424 static void for_buffer(struct zink_context *ctx, struct zink_resource *res,
425 VkPipelineStageFlags pipeline,
426 VkAccessFlags flags,
427 bool unordered,
428 bool usage_matches,
429 VkPipelineStageFlags stages,
430 VkCommandBuffer cmdbuf)
431 {
432 VkMemoryBarrier2 bmb;
433 bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2;
434 bmb.pNext = NULL;
435 if (unordered) {
436 bmb.srcStageMask = usage_matches ? res->obj->unordered_access_stage : stages;
437 bmb.srcAccessMask = usage_matches ? res->obj->unordered_access : res->obj->access;
438 } else {
439 bmb.srcStageMask = stages;
440 bmb.srcAccessMask = res->obj->access;
441 }
442 bmb.dstStageMask = pipeline;
443 bmb.dstAccessMask = flags;
444 VkDependencyInfo dep = {
445 VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
446 NULL,
447 0,
448 1,
449 &bmb,
450 0,
451 NULL,
452 0,
453 NULL
454 };
455 VKCTX(CmdPipelineBarrier2)(cmdbuf, &dep);
456 }
457 };
458
459 template <bool UNSYNCHRONIZED>
460 struct update_unordered_access_and_get_cmdbuf {
461 /* use base template to make the cases for true and false more explicite below */
462 };
463
464 template <>
465 struct update_unordered_access_and_get_cmdbuf<true> {
applyupdate_unordered_access_and_get_cmdbuf466 static VkCommandBuffer apply(struct zink_context *ctx, struct zink_resource *res, bool usage_matches, bool is_write)
467 {
468 assert(!usage_matches);
469 res->obj->unordered_write = true;
470 res->obj->unordered_read = true;
471 ctx->bs->has_unsync = true;
472 return ctx->bs->unsynchronized_cmdbuf;
473 }
474 };
475
476 template <>
477 struct update_unordered_access_and_get_cmdbuf<false> {
applyupdate_unordered_access_and_get_cmdbuf478 static VkCommandBuffer apply(struct zink_context *ctx, struct zink_resource *res, bool usage_matches, bool is_write)
479 {
480 VkCommandBuffer cmdbuf;
481 if (!usage_matches) {
482 res->obj->unordered_write = true;
483 if (is_write || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, ZINK_RESOURCE_ACCESS_RW))
484 res->obj->unordered_read = true;
485 }
486 if (zink_resource_usage_matches(res, ctx->bs) && !ctx->unordered_blitting &&
487 /* if current batch usage exists with ordered non-transfer access, never promote
488 * this avoids layout dsync
489 */
490 (!res->obj->unordered_read || !res->obj->unordered_write)) {
491 cmdbuf = ctx->bs->cmdbuf;
492 res->obj->unordered_write = false;
493 res->obj->unordered_read = false;
494 /* it's impossible to detect this from the caller
495 * there should be no valid case where this barrier can occur inside a renderpass
496 */
497 zink_batch_no_rp(ctx);
498 } else {
499 cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL);
500 /* force subsequent barriers to be ordered to avoid layout desync */
501 if (cmdbuf != ctx->bs->reordered_cmdbuf) {
502 res->obj->unordered_write = false;
503 res->obj->unordered_read = false;
504 }
505 }
506 return cmdbuf;
507 }
508 };
509
510 template <barrier_type BARRIER_API, bool UNSYNCHRONIZED>
511 void
zink_resource_image_barrier(struct zink_context * ctx,struct zink_resource * res,VkImageLayout new_layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)512 zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
513 {
514 if (!pipeline)
515 pipeline = pipeline_dst_stage(new_layout);
516 if (!flags)
517 flags = access_dst_flags(new_layout);
518
519 bool is_write = zink_resource_access_is_write(flags);
520 if (is_write && zink_is_swapchain(res))
521 zink_kopper_set_readback_needs_update(res);
522 if (!res->obj->needs_zs_evaluate && !zink_resource_image_needs_barrier(res, new_layout, flags, pipeline) &&
523 (res->queue == zink_screen(ctx->base.screen)->gfx_queue || res->queue == VK_QUEUE_FAMILY_IGNORED))
524 return;
525 enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE;
526 bool completed = zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw);
527 bool usage_matches = !completed && zink_resource_usage_matches(res, ctx->bs);
528 VkCommandBuffer cmdbuf = update_unordered_access_and_get_cmdbuf<UNSYNCHRONIZED>::apply(ctx, res, usage_matches, is_write);
529
530 assert(new_layout);
531 bool marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "image_barrier(%s->%s)", vk_ImageLayout_to_str(res->layout), vk_ImageLayout_to_str(new_layout));
532 bool queue_import = false;
533 emit_memory_barrier<BARRIER_API>::for_image(ctx, res, new_layout, flags, pipeline, completed, cmdbuf, &queue_import);
534 zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
535
536 if (!UNSYNCHRONIZED)
537 resource_check_defer_image_barrier(ctx, res, new_layout, pipeline);
538
539 if (is_write)
540 res->obj->last_write = flags;
541
542 res->obj->access = flags;
543 res->obj->access_stage = pipeline;
544 res->layout = new_layout;
545
546 if (new_layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
547 zink_resource_copies_reset(res);
548
549 if (res->obj->exportable)
550 simple_mtx_lock(&ctx->bs->exportable_lock);
551 if (res->obj->dt) {
552 struct kopper_displaytarget *cdt = res->obj->dt;
553 if (cdt->swapchain->num_acquires && res->obj->dt_idx != UINT32_MAX) {
554 cdt->swapchain->images[res->obj->dt_idx].layout = res->layout;
555 }
556 } else if (res->obj->exportable) {
557 struct pipe_resource *pres = NULL;
558 bool found = false;
559 _mesa_set_search_or_add(&ctx->bs->dmabuf_exports, res, &found);
560 if (!found) {
561 pipe_resource_reference(&pres, &res->base.b);
562 }
563 }
564 if (res->obj->exportable && queue_import) {
565 for (struct zink_resource *r = res; r; r = zink_resource(r->base.b.next)) {
566 VkSemaphore sem = zink_screen_export_dmabuf_semaphore(zink_screen(ctx->base.screen), r);
567 if (sem)
568 util_dynarray_append(&ctx->bs->fd_wait_semaphores, VkSemaphore, sem);
569 }
570 }
571 if (res->obj->exportable)
572 simple_mtx_unlock(&ctx->bs->exportable_lock);
573 }
574
575 bool
zink_check_unordered_transfer_access(struct zink_resource * res,unsigned level,const struct pipe_box * box)576 zink_check_unordered_transfer_access(struct zink_resource *res, unsigned level, const struct pipe_box *box)
577 {
578 /* always barrier against previous non-transfer writes */
579 bool non_transfer_write = res->obj->last_write && res->obj->last_write != VK_ACCESS_TRANSFER_WRITE_BIT;
580 /* must barrier if clobbering a previous write */
581 bool transfer_clobber = res->obj->last_write == VK_ACCESS_TRANSFER_WRITE_BIT && zink_resource_copy_box_intersects(res, level, box);
582 return non_transfer_write || transfer_clobber;
583 }
584
585 bool
zink_check_valid_buffer_src_access(struct zink_context * ctx,struct zink_resource * res,unsigned offset,unsigned size)586 zink_check_valid_buffer_src_access(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size)
587 {
588 return res->obj->access && util_ranges_intersect(&res->valid_buffer_range, offset, offset + size) && !unordered_res_exec(ctx, res, false);
589 }
590
591 void
zink_resource_image_transfer_dst_barrier(struct zink_context * ctx,struct zink_resource * res,unsigned level,const struct pipe_box * box,bool unsync)592 zink_resource_image_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned level, const struct pipe_box *box, bool unsync)
593 {
594 if (res->obj->copies_need_reset)
595 zink_resource_copies_reset(res);
596 /* skip TRANSFER_DST barrier if no intersection from previous copies */
597 if (res->layout != VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL ||
598 zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics ||
599 zink_check_unordered_transfer_access(res, level, box)) {
600 if (unsync)
601 zink_screen(ctx->base.screen)->image_barrier_unsync(ctx, res, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
602 else
603 zink_screen(ctx->base.screen)->image_barrier(ctx, res, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
604 } else {
605 res->obj->access = VK_ACCESS_TRANSFER_WRITE_BIT;
606 res->obj->last_write = VK_ACCESS_TRANSFER_WRITE_BIT;
607 res->obj->access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT;
608 }
609 zink_resource_copy_box_add(ctx, res, level, box);
610 }
611
612 bool
zink_resource_buffer_transfer_dst_barrier(struct zink_context * ctx,struct zink_resource * res,unsigned offset,unsigned size)613 zink_resource_buffer_transfer_dst_barrier(struct zink_context *ctx, struct zink_resource *res, unsigned offset, unsigned size)
614 {
615 if (res->obj->copies_need_reset)
616 zink_resource_copies_reset(res);
617 bool unordered = true;
618 struct pipe_box box;
619 u_box_3d((int)offset, 0, 0, (int)size, 0, 0, &box);
620 bool can_unordered_write = unordered_res_exec(ctx, res, true);
621 /* must barrier if something read the valid buffer range */
622 bool valid_read = (res->obj->access || res->obj->unordered_access) &&
623 util_ranges_intersect(&res->valid_buffer_range, offset, offset + size) && !can_unordered_write;
624 if (valid_read || zink_screen(ctx->base.screen)->driver_workarounds.broken_cache_semantics || zink_check_unordered_transfer_access(res, 0, &box)) {
625 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
626 unordered = res->obj->unordered_write;
627 } else {
628 res->obj->unordered_access = VK_ACCESS_TRANSFER_WRITE_BIT;
629 res->obj->last_write = VK_ACCESS_TRANSFER_WRITE_BIT;
630 res->obj->unordered_access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT;
631
632 ctx->bs->unordered_write_access |= VK_ACCESS_TRANSFER_WRITE_BIT;
633 ctx->bs->unordered_write_stages |= VK_PIPELINE_STAGE_TRANSFER_BIT;
634 if (!zink_resource_usage_matches(res, ctx->bs)) {
635 res->obj->access = VK_ACCESS_TRANSFER_WRITE_BIT;
636 res->obj->access_stage = VK_PIPELINE_STAGE_TRANSFER_BIT;
637 res->obj->ordered_access_is_copied = true;
638 }
639 }
640 zink_resource_copy_box_add(ctx, res, 0, &box);
641 /* this return value implies that the caller could do an unordered op on this resource */
642 return unordered;
643 }
644
645 VkPipelineStageFlags
zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage)646 zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage)
647 {
648 switch (stage) {
649 case VK_SHADER_STAGE_VERTEX_BIT:
650 return VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
651 case VK_SHADER_STAGE_FRAGMENT_BIT:
652 return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
653 case VK_SHADER_STAGE_GEOMETRY_BIT:
654 return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT;
655 case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT:
656 return VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT;
657 case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT:
658 return VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT;
659 case VK_SHADER_STAGE_COMPUTE_BIT:
660 return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
661 default:
662 unreachable("unknown shader stage bit");
663 }
664 }
665
666 ALWAYS_INLINE static VkPipelineStageFlags
pipeline_access_stage(VkAccessFlags flags)667 pipeline_access_stage(VkAccessFlags flags)
668 {
669 if (flags & (VK_ACCESS_UNIFORM_READ_BIT |
670 VK_ACCESS_SHADER_READ_BIT |
671 VK_ACCESS_SHADER_WRITE_BIT))
672 return VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
673 VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT |
674 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
675 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT |
676 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT |
677 VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
678 return VK_PIPELINE_STAGE_TRANSFER_BIT;
679 }
680
681 ALWAYS_INLINE static bool
buffer_needs_barrier(struct zink_resource * res,VkAccessFlags flags,VkPipelineStageFlags pipeline,bool unordered)682 buffer_needs_barrier(struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline, bool unordered)
683 {
684 return zink_resource_access_is_write(unordered ? res->obj->unordered_access : res->obj->access) ||
685 zink_resource_access_is_write(flags) ||
686 ((unordered ? res->obj->unordered_access_stage : res->obj->access_stage) & pipeline) != pipeline ||
687 ((unordered ? res->obj->unordered_access : res->obj->access) & flags) != flags;
688 }
689
690
691
692 template <barrier_type BARRIER_API>
693 void
zink_resource_buffer_barrier(struct zink_context * ctx,struct zink_resource * res,VkAccessFlags flags,VkPipelineStageFlags pipeline)694 zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline)
695 {
696 if (!pipeline)
697 pipeline = pipeline_access_stage(flags);
698
699 bool is_write = zink_resource_access_is_write(flags);
700 enum zink_resource_access rw = is_write ? ZINK_RESOURCE_ACCESS_RW : ZINK_RESOURCE_ACCESS_WRITE;
701 bool completed = zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, rw);
702 bool usage_matches = !completed && zink_resource_usage_matches(res, ctx->bs);
703 if (!usage_matches) {
704 res->obj->unordered_write = true;
705 if (is_write || zink_resource_usage_check_completion_fast(zink_screen(ctx->base.screen), res, ZINK_RESOURCE_ACCESS_RW))
706 res->obj->unordered_read = true;
707 }
708 bool unordered_usage_matches = res->obj->unordered_access && usage_matches;
709 bool unordered = unordered_res_exec(ctx, res, is_write);
710 if (!buffer_needs_barrier(res, flags, pipeline, unordered))
711 return;
712 if (completed) {
713 /* reset access on complete */
714 res->obj->access = VK_ACCESS_NONE;
715 res->obj->access_stage = VK_PIPELINE_STAGE_NONE;
716 res->obj->last_write = VK_ACCESS_NONE;
717 } else if (unordered && unordered_usage_matches && res->obj->ordered_access_is_copied) {
718 /* always reset propagated access to avoid weirdness */
719 res->obj->access = VK_ACCESS_NONE;
720 res->obj->access_stage = VK_PIPELINE_STAGE_NONE;
721 } else if (!unordered && !unordered_usage_matches) {
722 /* reset unordered access on first ordered barrier */
723 res->obj->unordered_access = VK_ACCESS_NONE;
724 res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE;
725 }
726 if (!usage_matches) {
727 /* reset unordered on first new cmdbuf barrier */
728 res->obj->unordered_access = VK_ACCESS_NONE;
729 res->obj->unordered_access_stage = VK_PIPELINE_STAGE_NONE;
730 res->obj->ordered_access_is_copied = false;
731 }
732 /* unordered barriers can be skipped when:
733 * - there is no current-batch unordered access AND previous batch usage is not write access
734 * - there is current-batch unordered access AND the unordered access is not write access
735 */
736 bool can_skip_unordered = !unordered ? false : !zink_resource_access_is_write(!unordered_usage_matches ? res->obj->access : res->obj->unordered_access);
737 /* ordered barriers can be skipped if both:
738 * - there is no current access
739 * - there is no current-batch unordered access
740 */
741 bool can_skip_ordered = unordered ? false : (!res->obj->access && !unordered_usage_matches);
742 if (ctx->no_reorder)
743 can_skip_unordered = can_skip_ordered = false;
744
745 if (!can_skip_unordered && !can_skip_ordered) {
746 VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL);
747 bool marker = false;
748 if (unlikely(zink_tracing)) {
749 char buf[4096];
750 zink_string_vkflags_unroll(buf, sizeof(buf), flags, (zink_vkflags_func)vk_AccessFlagBits_to_str);
751 marker = zink_cmd_debug_marker_begin(ctx, cmdbuf, "buffer_barrier(%s)", buf);
752 }
753
754 VkPipelineStageFlags stages = res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access);;
755 emit_memory_barrier<BARRIER_API>::for_buffer(ctx, res, pipeline, flags, unordered,usage_matches, stages, cmdbuf);
756
757 zink_cmd_debug_marker_end(ctx, cmdbuf, marker);
758 }
759
760 resource_check_defer_buffer_barrier(ctx, res, pipeline);
761
762 if (is_write)
763 res->obj->last_write = flags;
764 if (unordered) {
765 /* these should get automatically emitted during submission */
766 res->obj->unordered_access = flags;
767 res->obj->unordered_access_stage = pipeline;
768 if (is_write) {
769 ctx->bs->unordered_write_access |= flags;
770 ctx->bs->unordered_write_stages |= pipeline;
771 }
772 }
773 if (!unordered || !usage_matches || res->obj->ordered_access_is_copied) {
774 res->obj->access = flags;
775 res->obj->access_stage = pipeline;
776 res->obj->ordered_access_is_copied = unordered;
777 }
778 if (pipeline != VK_PIPELINE_STAGE_TRANSFER_BIT && is_write)
779 zink_resource_copies_reset(res);
780 }
781
782 void
zink_synchronization_init(struct zink_screen * screen)783 zink_synchronization_init(struct zink_screen *screen)
784 {
785 if (screen->info.have_vulkan13 || screen->info.have_KHR_synchronization2) {
786 screen->buffer_barrier = zink_resource_buffer_barrier<barrier_KHR_synchronzation2>;
787 screen->image_barrier = zink_resource_image_barrier<barrier_KHR_synchronzation2, false>;
788 screen->image_barrier_unsync = zink_resource_image_barrier<barrier_KHR_synchronzation2, true>;
789 } else {
790 screen->buffer_barrier = zink_resource_buffer_barrier<barrier_default>;
791 screen->image_barrier = zink_resource_image_barrier<barrier_default, false>;
792 screen->image_barrier_unsync = zink_resource_image_barrier<barrier_default, true>;
793 }
794 }
795