1 #include "zink_batch.h"
2 #include "zink_compiler.h"
3 #include "zink_context.h"
4 #include "zink_descriptors.h"
5 #include "zink_program.h"
6 #include "zink_program_state.hpp"
7 #include "zink_query.h"
8 #include "zink_resource.h"
9 #include "zink_screen.h"
10 #include "zink_state.h"
11 #include "zink_surface.h"
12 #include "zink_inlines.h"
13
14 #include "util/hash_table.h"
15 #include "util/u_cpu_detect.h"
16 #include "util/u_debug.h"
17 #include "util/u_helpers.h"
18 #include "util/u_inlines.h"
19 #include "util/u_prim.h"
20 #include "util/u_prim_restart.h"
21
22 static void
zink_emit_xfb_counter_barrier(struct zink_context * ctx)23 zink_emit_xfb_counter_barrier(struct zink_context *ctx)
24 {
25 for (unsigned i = 0; i < ctx->num_so_targets; i++) {
26 struct zink_so_target *t = zink_so_target(ctx->so_targets[i]);
27 if (!t)
28 continue;
29 struct zink_resource *res = zink_resource(t->counter_buffer);
30 VkAccessFlags access = VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT;
31 VkPipelineStageFlags stage = VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT;
32 if (t->counter_buffer_valid) {
33 /* Between the pause and resume there needs to be a memory barrier for the counter buffers
34 * with a source access of VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT
35 * at pipeline stage VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT
36 * to a destination access of VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT
37 * at pipeline stage VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT.
38 *
39 * - from VK_EXT_transform_feedback spec
40 */
41 access |= VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT;
42 stage |= VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT;
43 }
44 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, access, stage);
45 if (!ctx->unordered_blitting)
46 res->obj->unordered_read = false;
47 }
48 }
49
50 static void
zink_emit_stream_output_targets(struct pipe_context * pctx)51 zink_emit_stream_output_targets(struct pipe_context *pctx)
52 {
53 struct zink_context *ctx = zink_context(pctx);
54 VkBuffer buffers[PIPE_MAX_SO_BUFFERS] = {0};
55 VkDeviceSize buffer_offsets[PIPE_MAX_SO_BUFFERS] = {0};
56 VkDeviceSize buffer_sizes[PIPE_MAX_SO_BUFFERS] = {0};
57
58 for (unsigned i = 0; i < ctx->num_so_targets; i++) {
59 struct zink_so_target *t = (struct zink_so_target *)ctx->so_targets[i];
60 if (!t) {
61 /* no need to reference this or anything */
62 buffers[i] = zink_resource(ctx->dummy_xfb_buffer)->obj->buffer;
63 buffer_offsets[i] = 0;
64 buffer_sizes[i] = sizeof(uint8_t);
65 continue;
66 }
67 struct zink_resource *res = zink_resource(t->base.buffer);
68 if (!res->so_valid)
69 /* resource has been rebound */
70 t->counter_buffer_valid = false;
71 buffers[i] = res->obj->buffer;
72 zink_batch_reference_resource_rw(ctx, res, true);
73 buffer_offsets[i] = t->base.buffer_offset;
74 buffer_sizes[i] = t->base.buffer_size;
75 res->so_valid = true;
76 if (!ctx->unordered_blitting) {
77 res->obj->unordered_read = res->obj->unordered_write = false;
78 res->obj->access = VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT;
79 res->obj->access_stage = VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT;
80 }
81 util_range_add(t->base.buffer, &res->valid_buffer_range, t->base.buffer_offset,
82 t->base.buffer_offset + t->base.buffer_size);
83 }
84
85 VKCTX(CmdBindTransformFeedbackBuffersEXT)(ctx->bs->cmdbuf, 0, ctx->num_so_targets,
86 buffers, buffer_offsets,
87 buffer_sizes);
88 ctx->dirty_so_targets = false;
89 }
90
91 ALWAYS_INLINE static void
check_buffer_barrier(struct zink_context * ctx,struct pipe_resource * pres,VkAccessFlags flags,VkPipelineStageFlags pipeline)92 check_buffer_barrier(struct zink_context *ctx, struct pipe_resource *pres, VkAccessFlags flags, VkPipelineStageFlags pipeline)
93 {
94 struct zink_resource *res = zink_resource(pres);
95 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, flags, pipeline);
96 if (!ctx->unordered_blitting)
97 res->obj->unordered_read = false;
98 }
99
100 ALWAYS_INLINE static void
barrier_draw_buffers(struct zink_context * ctx,const struct pipe_draw_info * dinfo,const struct pipe_draw_indirect_info * dindirect,struct pipe_resource * index_buffer)101 barrier_draw_buffers(struct zink_context *ctx, const struct pipe_draw_info *dinfo,
102 const struct pipe_draw_indirect_info *dindirect, struct pipe_resource *index_buffer)
103 {
104 if (index_buffer)
105 check_buffer_barrier(ctx, index_buffer, VK_ACCESS_INDEX_READ_BIT, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT);
106 if (dindirect && dindirect->buffer) {
107 check_buffer_barrier(ctx, dindirect->buffer,
108 VK_ACCESS_INDIRECT_COMMAND_READ_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT);
109 if (dindirect->indirect_draw_count)
110 check_buffer_barrier(ctx, dindirect->indirect_draw_count,
111 VK_ACCESS_INDIRECT_COMMAND_READ_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT);
112 }
113 }
114
115 template <zink_dynamic_state DYNAMIC_STATE>
116 static void
zink_bind_vertex_buffers(struct zink_context * ctx)117 zink_bind_vertex_buffers(struct zink_context *ctx)
118 {
119 VkBuffer buffers[PIPE_MAX_ATTRIBS];
120 VkDeviceSize buffer_offsets[PIPE_MAX_ATTRIBS];
121 struct zink_vertex_elements_state *elems = ctx->element_state;
122 struct zink_screen *screen = zink_screen(ctx->base.screen);
123
124 for (unsigned i = 0; i < elems->hw_state.num_bindings; i++) {
125 struct pipe_vertex_buffer *vb = ctx->vertex_buffers + elems->hw_state.binding_map[i];
126 assert(vb);
127 if (vb->buffer.resource) {
128 struct zink_resource *res = zink_resource(vb->buffer.resource);
129 assert(res->obj->buffer);
130 buffers[i] = res->obj->buffer;
131 buffer_offsets[i] = vb->buffer_offset;
132 } else {
133 buffers[i] = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer;
134 buffer_offsets[i] = 0;
135 }
136 }
137
138 if (DYNAMIC_STATE != ZINK_NO_DYNAMIC_STATE &&
139 DYNAMIC_STATE != ZINK_DYNAMIC_VERTEX_INPUT2 &&
140 DYNAMIC_STATE != ZINK_DYNAMIC_VERTEX_INPUT) {
141 if (elems->hw_state.num_bindings)
142 VKCTX(CmdBindVertexBuffers2)(ctx->bs->cmdbuf, 0,
143 elems->hw_state.num_bindings,
144 buffers, buffer_offsets, NULL, elems->hw_state.b.strides);
145 } else if (elems->hw_state.num_bindings)
146 VKSCR(CmdBindVertexBuffers)(ctx->bs->cmdbuf, 0,
147 elems->hw_state.num_bindings,
148 buffers, buffer_offsets);
149
150 if (DYNAMIC_STATE == ZINK_DYNAMIC_VERTEX_INPUT2 || DYNAMIC_STATE == ZINK_DYNAMIC_VERTEX_INPUT)
151 VKCTX(CmdSetVertexInputEXT)(ctx->bs->cmdbuf,
152 elems->hw_state.num_bindings, elems->hw_state.dynbindings,
153 elems->hw_state.num_attribs, elems->hw_state.dynattribs);
154
155 ctx->vertex_buffers_dirty = false;
156 }
157
158 ALWAYS_INLINE static void
update_drawid(struct zink_context * ctx,unsigned draw_id)159 update_drawid(struct zink_context *ctx, unsigned draw_id)
160 {
161 VKCTX(CmdPushConstants)(ctx->bs->cmdbuf, ctx->curr_program->base.layout, VK_SHADER_STAGE_ALL_GRAPHICS,
162 offsetof(struct zink_gfx_push_constant, draw_id), sizeof(unsigned),
163 &draw_id);
164 }
165
166 ALWAYS_INLINE static void
draw_indexed_need_index_buffer_unref(struct zink_context * ctx,const struct pipe_draw_info * dinfo,const struct pipe_draw_start_count_bias * draws,unsigned num_draws,unsigned draw_id,bool needs_drawid)167 draw_indexed_need_index_buffer_unref(struct zink_context *ctx,
168 const struct pipe_draw_info *dinfo,
169 const struct pipe_draw_start_count_bias *draws,
170 unsigned num_draws,
171 unsigned draw_id,
172 bool needs_drawid)
173 {
174 VkCommandBuffer cmdbuf = ctx->bs->cmdbuf;
175 if (dinfo->increment_draw_id && needs_drawid) {
176 for (unsigned i = 0; i < num_draws; i++) {
177 update_drawid(ctx, draw_id);
178 VKCTX(CmdDrawIndexed)(cmdbuf,
179 draws[i].count, dinfo->instance_count,
180 0, draws[i].index_bias, dinfo->start_instance);
181 draw_id++;
182 }
183 } else {
184 if (needs_drawid)
185 update_drawid(ctx, draw_id);
186 for (unsigned i = 0; i < num_draws; i++)
187 VKCTX(CmdDrawIndexed)(cmdbuf,
188 draws[i].count, dinfo->instance_count,
189 0, draws[i].index_bias, dinfo->start_instance);
190
191 }
192 }
193
194 template <zink_multidraw HAS_MULTIDRAW>
195 ALWAYS_INLINE static void
draw_indexed(struct zink_context * ctx,const struct pipe_draw_info * dinfo,const struct pipe_draw_start_count_bias * draws,unsigned num_draws,unsigned draw_id,bool needs_drawid)196 draw_indexed(struct zink_context *ctx,
197 const struct pipe_draw_info *dinfo,
198 const struct pipe_draw_start_count_bias *draws,
199 unsigned num_draws,
200 unsigned draw_id,
201 bool needs_drawid)
202 {
203 VkCommandBuffer cmdbuf = ctx->bs->cmdbuf;
204 if (dinfo->increment_draw_id && needs_drawid) {
205 for (unsigned i = 0; i < num_draws; i++) {
206 update_drawid(ctx, draw_id);
207 VKCTX(CmdDrawIndexed)(cmdbuf,
208 draws[i].count, dinfo->instance_count,
209 draws[i].start, draws[i].index_bias, dinfo->start_instance);
210 draw_id++;
211 }
212 } else {
213 if (needs_drawid)
214 update_drawid(ctx, draw_id);
215 if (HAS_MULTIDRAW) {
216 VKCTX(CmdDrawMultiIndexedEXT)(cmdbuf, num_draws, (const VkMultiDrawIndexedInfoEXT*)draws,
217 dinfo->instance_count,
218 dinfo->start_instance, sizeof(struct pipe_draw_start_count_bias),
219 dinfo->index_bias_varies ? NULL : &draws[0].index_bias);
220 } else {
221 for (unsigned i = 0; i < num_draws; i++)
222 VKCTX(CmdDrawIndexed)(cmdbuf,
223 draws[i].count, dinfo->instance_count,
224 draws[i].start, draws[i].index_bias, dinfo->start_instance);
225 }
226 }
227 }
228
229 template <zink_multidraw HAS_MULTIDRAW>
230 ALWAYS_INLINE static void
draw(struct zink_context * ctx,const struct pipe_draw_info * dinfo,const struct pipe_draw_start_count_bias * draws,unsigned num_draws,unsigned draw_id,bool needs_drawid)231 draw(struct zink_context *ctx,
232 const struct pipe_draw_info *dinfo,
233 const struct pipe_draw_start_count_bias *draws,
234 unsigned num_draws,
235 unsigned draw_id,
236 bool needs_drawid)
237 {
238 VkCommandBuffer cmdbuf = ctx->bs->cmdbuf;
239 if (dinfo->increment_draw_id && needs_drawid) {
240 for (unsigned i = 0; i < num_draws; i++) {
241 update_drawid(ctx, draw_id);
242 VKCTX(CmdDraw)(cmdbuf, draws[i].count, dinfo->instance_count, draws[i].start, dinfo->start_instance);
243 draw_id++;
244 }
245 } else {
246 if (needs_drawid)
247 update_drawid(ctx, draw_id);
248 if (HAS_MULTIDRAW)
249 VKCTX(CmdDrawMultiEXT)(cmdbuf, num_draws, (const VkMultiDrawInfoEXT*)draws,
250 dinfo->instance_count, dinfo->start_instance,
251 sizeof(struct pipe_draw_start_count_bias));
252 else {
253 for (unsigned i = 0; i < num_draws; i++)
254 VKCTX(CmdDraw)(cmdbuf, draws[i].count, dinfo->instance_count, draws[i].start, dinfo->start_instance);
255
256 }
257 }
258 }
259
260 template <zink_dynamic_state DYNAMIC_STATE, bool BATCH_CHANGED>
261 static bool
update_gfx_pipeline(struct zink_context * ctx,struct zink_batch_state * bs,enum mesa_prim mode)262 update_gfx_pipeline(struct zink_context *ctx, struct zink_batch_state *bs, enum mesa_prim mode)
263 {
264 VkPipeline prev_pipeline = ctx->gfx_pipeline_state.pipeline;
265 const struct zink_screen *screen = zink_screen(ctx->base.screen);
266 bool shaders_changed = ctx->gfx_dirty || ctx->dirty_gfx_stages;
267 if (screen->optimal_keys && !ctx->is_generated_gs_bound)
268 zink_gfx_program_update_optimal(ctx);
269 else
270 zink_gfx_program_update(ctx);
271 bool pipeline_changed = false;
272 VkPipeline pipeline = VK_NULL_HANDLE;
273 if (!ctx->curr_program->base.uses_shobj) {
274 if (screen->info.have_EXT_graphics_pipeline_library)
275 pipeline = zink_get_gfx_pipeline<DYNAMIC_STATE, true>(ctx, ctx->curr_program, &ctx->gfx_pipeline_state, mode);
276 else
277 pipeline = zink_get_gfx_pipeline<DYNAMIC_STATE, false>(ctx, ctx->curr_program, &ctx->gfx_pipeline_state, mode);
278 }
279 if (pipeline) {
280 pipeline_changed = prev_pipeline != pipeline;
281 if (BATCH_CHANGED || pipeline_changed || ctx->shobj_draw)
282 VKCTX(CmdBindPipeline)(bs->cmdbuf, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
283 ctx->shobj_draw = false;
284 } else {
285 if (BATCH_CHANGED || shaders_changed || !ctx->shobj_draw) {
286 VkShaderStageFlagBits stages[] = {
287 VK_SHADER_STAGE_VERTEX_BIT,
288 VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT,
289 VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT,
290 VK_SHADER_STAGE_GEOMETRY_BIT,
291 VK_SHADER_STAGE_FRAGMENT_BIT,
292 };
293 /* always rebind all stages */
294 VKCTX(CmdBindShadersEXT)(bs->cmdbuf, ZINK_GFX_SHADER_COUNT, stages, ctx->curr_program->objects);
295 VKCTX(CmdSetDepthBiasEnable)(bs->cmdbuf, VK_TRUE);
296 VKCTX(CmdSetTessellationDomainOriginEXT)(bs->cmdbuf, VK_TESSELLATION_DOMAIN_ORIGIN_LOWER_LEFT);
297 VKCTX(CmdSetSampleLocationsEnableEXT)(bs->cmdbuf, ctx->gfx_pipeline_state.sample_locations_enabled);
298 VKCTX(CmdSetRasterizationStreamEXT)(bs->cmdbuf, 0);
299 }
300 ctx->shobj_draw = true;
301 }
302 return pipeline_changed;
303 }
304
305 static enum mesa_prim
zink_prim_type(const struct zink_context * ctx,const struct pipe_draw_info * dinfo)306 zink_prim_type(const struct zink_context *ctx,
307 const struct pipe_draw_info *dinfo)
308 {
309 if (ctx->gfx_pipeline_state.shader_rast_prim != MESA_PRIM_COUNT)
310 return ctx->gfx_pipeline_state.shader_rast_prim;
311
312 return u_reduced_prim((enum mesa_prim)dinfo->mode);
313 }
314
315 static enum mesa_prim
zink_rast_prim(const struct zink_context * ctx,const struct pipe_draw_info * dinfo)316 zink_rast_prim(const struct zink_context *ctx,
317 const struct pipe_draw_info *dinfo)
318 {
319 enum mesa_prim prim_type = zink_prim_type(ctx, dinfo);
320 assert(prim_type != MESA_PRIM_COUNT);
321
322 if (prim_type == MESA_PRIM_TRIANGLES &&
323 ctx->rast_state->base.fill_front != PIPE_POLYGON_MODE_FILL) {
324 switch(ctx->rast_state->base.fill_front) {
325 case PIPE_POLYGON_MODE_POINT:
326 return MESA_PRIM_POINTS;
327 case PIPE_POLYGON_MODE_LINE:
328 return MESA_PRIM_LINES;
329 default:
330 unreachable("unexpected polygon mode");
331 }
332 }
333
334 return prim_type;
335 }
336
337 template <zink_multidraw HAS_MULTIDRAW, zink_dynamic_state DYNAMIC_STATE, bool BATCH_CHANGED, bool DRAW_STATE>
338 void
zink_draw(struct pipe_context * pctx,const struct pipe_draw_info * dinfo,unsigned drawid_offset,const struct pipe_draw_indirect_info * dindirect,const struct pipe_draw_start_count_bias * draws,unsigned num_draws,struct pipe_vertex_state * vstate,uint32_t partial_velem_mask)339 zink_draw(struct pipe_context *pctx,
340 const struct pipe_draw_info *dinfo,
341 unsigned drawid_offset,
342 const struct pipe_draw_indirect_info *dindirect,
343 const struct pipe_draw_start_count_bias *draws,
344 unsigned num_draws,
345 struct pipe_vertex_state *vstate,
346 uint32_t partial_velem_mask)
347 {
348 if (!dindirect && (!draws[0].count || !dinfo->instance_count))
349 return;
350
351 struct zink_context *ctx = zink_context(pctx);
352 struct zink_screen *screen = zink_screen(pctx->screen);
353 struct zink_rasterizer_state *rast_state = ctx->rast_state;
354 struct zink_depth_stencil_alpha_state *dsa_state = ctx->dsa_state;
355 struct zink_batch_state *bs = ctx->bs;
356 struct zink_so_target *so_target =
357 dindirect && dindirect->count_from_stream_output ?
358 zink_so_target(dindirect->count_from_stream_output) : NULL;
359 VkBuffer counter_buffers[PIPE_MAX_SO_BUFFERS];
360 VkDeviceSize counter_buffer_offsets[PIPE_MAX_SO_BUFFERS];
361 bool need_index_buffer_unref = false;
362 bool mode_changed = ctx->gfx_pipeline_state.gfx_prim_mode != dinfo->mode;
363 bool reads_drawid = ctx->shader_reads_drawid;
364 bool reads_basevertex = ctx->shader_reads_basevertex;
365 unsigned work_count = ctx->work_count;
366 enum mesa_prim mode = (enum mesa_prim)dinfo->mode;
367
368 if (ctx->memory_barrier && !ctx->blitting)
369 zink_flush_memory_barrier(ctx, false);
370
371 if (unlikely(ctx->buffer_rebind_counter < screen->buffer_rebind_counter && !ctx->blitting)) {
372 ctx->buffer_rebind_counter = screen->buffer_rebind_counter;
373 zink_rebind_all_buffers(ctx);
374 }
375
376 if (unlikely(ctx->image_rebind_counter < screen->image_rebind_counter && !ctx->blitting)) {
377 ctx->image_rebind_counter = screen->image_rebind_counter;
378 zink_rebind_all_images(ctx);
379 }
380
381 unsigned index_offset = 0;
382 unsigned index_size = dinfo->index_size;
383 struct pipe_resource *index_buffer = NULL;
384 if (index_size > 0) {
385 if (dinfo->has_user_indices) {
386 if (!util_upload_index_buffer(pctx, dinfo, &draws[0], &index_buffer, &index_offset, 4)) {
387 debug_printf("util_upload_index_buffer() failed\n");
388 return;
389 }
390 /* this will have extra refs from tc */
391 if (screen->threaded)
392 zink_batch_reference_resource_move(ctx, zink_resource(index_buffer));
393 else
394 zink_batch_reference_resource(ctx, zink_resource(index_buffer));
395 } else {
396 index_buffer = dinfo->index.resource;
397 zink_batch_reference_resource_rw(ctx, zink_resource(index_buffer), false);
398 }
399 assert(index_size <= 4 && index_size != 3);
400 assert(index_size != 1 || screen->info.have_EXT_index_type_uint8);
401 }
402
403 ctx->was_line_loop = dinfo->was_line_loop;
404
405 bool have_streamout = !!ctx->num_so_targets;
406 if (have_streamout) {
407 zink_emit_xfb_counter_barrier(ctx);
408 if (ctx->dirty_so_targets) {
409 /* have to loop here and below because barriers must be emitted out of renderpass,
410 * but xfb buffers can't be bound before the renderpass is active to avoid
411 * breaking from recursion
412 */
413 for (unsigned i = 0; i < ctx->num_so_targets; i++) {
414 struct zink_so_target *t = (struct zink_so_target *)ctx->so_targets[i];
415 if (t) {
416 struct zink_resource *res = zink_resource(t->base.buffer);
417 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res,
418 VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT, VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT);
419 if (!ctx->unordered_blitting)
420 res->obj->unordered_read = res->obj->unordered_write = false;
421 }
422 }
423 }
424 }
425
426 barrier_draw_buffers(ctx, dinfo, dindirect, index_buffer);
427 /* this may re-emit draw buffer barriers, but such synchronization is harmless */
428 if (!ctx->blitting)
429 zink_update_barriers(ctx, false, index_buffer, dindirect ? dindirect->buffer : NULL, dindirect ? dindirect->indirect_draw_count : NULL);
430
431 /* ensure synchronization between doing streamout with counter buffer
432 * and using counter buffer for indirect draw
433 */
434 if (so_target && so_target->counter_buffer_valid) {
435 struct zink_resource *res = zink_resource(so_target->counter_buffer);
436 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res,
437 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT,
438 VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT);
439 if (!ctx->unordered_blitting)
440 res->obj->unordered_read = false;
441 }
442
443 zink_query_update_gs_states(ctx);
444
445 if (unlikely(zink_debug & ZINK_DEBUG_SYNC)) {
446 zink_batch_no_rp(ctx);
447 VkMemoryBarrier mb;
448 mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
449 mb.pNext = NULL;
450 mb.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
451 mb.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
452 VKSCR(CmdPipelineBarrier)(ctx->bs->cmdbuf,
453 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
454 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
455 0, 1, &mb, 0, NULL, 0, NULL);
456 }
457
458 zink_batch_rp(ctx);
459 /* check dead swapchain */
460 if (unlikely(!ctx->in_rp))
461 return;
462
463 if (BATCH_CHANGED)
464 zink_update_descriptor_refs(ctx, false);
465
466 /* these must be after renderpass start to avoid issues with recursion */
467 bool drawid_broken = false;
468 if (reads_drawid && (!dindirect || !dindirect->buffer))
469 drawid_broken = (drawid_offset != 0 ||
470 (!HAS_MULTIDRAW && num_draws > 1) ||
471 (HAS_MULTIDRAW && num_draws > 1 && !dinfo->increment_draw_id));
472 if (drawid_broken != zink_get_last_vertex_key(ctx)->push_drawid)
473 zink_set_last_vertex_key(ctx)->push_drawid = drawid_broken;
474
475 bool rast_prim_changed = false;
476 bool prim_changed = false;
477 bool rast_state_changed = ctx->rast_state_changed;
478 if (mode_changed || ctx->gfx_pipeline_state.modules_changed ||
479 rast_state_changed) {
480 enum mesa_prim rast_prim = zink_rast_prim(ctx, dinfo);
481 if (rast_prim != ctx->gfx_pipeline_state.rast_prim) {
482 bool points_changed =
483 (ctx->gfx_pipeline_state.rast_prim == MESA_PRIM_POINTS) !=
484 (rast_prim == MESA_PRIM_POINTS);
485
486 prim_changed = ctx->gfx_pipeline_state.rast_prim != rast_prim;
487
488 static bool rect_warned = false;
489 if (DYNAMIC_STATE >= ZINK_DYNAMIC_STATE3 && rast_prim == MESA_PRIM_LINES && !rect_warned &&
490 (VkLineRasterizationModeEXT)rast_state->hw_state.line_mode == VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT) {
491 if (screen->info.line_rast_feats.rectangularLines)
492 rect_warned = true;
493 else
494 warn_missing_feature(rect_warned, "rectangularLines");
495 }
496
497 ctx->gfx_pipeline_state.rast_prim = rast_prim;
498 rast_prim_changed = true;
499
500 if (points_changed && ctx->rast_state->base.point_quad_rasterization)
501 zink_set_fs_point_coord_key(ctx);
502 }
503 }
504 ctx->gfx_pipeline_state.gfx_prim_mode = mode;
505
506 if ((mode_changed || prim_changed || rast_state_changed || ctx->gfx_pipeline_state.modules_changed)) {
507 zink_set_primitive_emulation_keys(ctx);
508 }
509
510 if (index_size) {
511 const VkIndexType index_type[3] = {
512 VK_INDEX_TYPE_UINT8_EXT,
513 VK_INDEX_TYPE_UINT16,
514 VK_INDEX_TYPE_UINT32,
515 };
516 struct zink_resource *res = zink_resource(index_buffer);
517 VKCTX(CmdBindIndexBuffer)(bs->cmdbuf, res->obj->buffer, index_offset, index_type[index_size >> 1]);
518 }
519 if (DYNAMIC_STATE < ZINK_DYNAMIC_STATE2) {
520 if (ctx->gfx_pipeline_state.dyn_state2.primitive_restart != dinfo->primitive_restart)
521 ctx->gfx_pipeline_state.dirty = true;
522 ctx->gfx_pipeline_state.dyn_state2.primitive_restart = dinfo->primitive_restart;
523 }
524
525 if (have_streamout && ctx->dirty_so_targets)
526 zink_emit_stream_output_targets(pctx);
527
528 bool pipeline_changed = update_gfx_pipeline<DYNAMIC_STATE, BATCH_CHANGED>(ctx, bs, mode);
529
530 if (BATCH_CHANGED || ctx->vp_state_changed || (DYNAMIC_STATE == ZINK_NO_DYNAMIC_STATE && pipeline_changed)) {
531 VkViewport viewports[PIPE_MAX_VIEWPORTS];
532 for (unsigned i = 0; i < ctx->vp_state.num_viewports; i++) {
533 VkViewport viewport = {
534 ctx->vp_state.viewport_states[i].translate[0] - ctx->vp_state.viewport_states[i].scale[0],
535 ctx->vp_state.viewport_states[i].translate[1] - ctx->vp_state.viewport_states[i].scale[1],
536 MAX2(ctx->vp_state.viewport_states[i].scale[0] * 2, 1),
537 ctx->vp_state.viewport_states[i].scale[1] * 2,
538 CLAMP(ctx->rast_state->base.clip_halfz ?
539 ctx->vp_state.viewport_states[i].translate[2] :
540 ctx->vp_state.viewport_states[i].translate[2] - ctx->vp_state.viewport_states[i].scale[2],
541 0, 1),
542 CLAMP(ctx->vp_state.viewport_states[i].translate[2] + ctx->vp_state.viewport_states[i].scale[2],
543 0, 1)
544 };
545 if (!ctx->rast_state->base.half_pixel_center) {
546 /* magic constant value from dxvk */
547 float cf = 0.5f - (1.0f / 128.0f);
548 viewport.x += cf;
549 if (viewport.height < 0)
550 viewport.y += cf;
551 else
552 viewport.y -= cf;
553 }
554 viewports[i] = viewport;
555 }
556 if (DYNAMIC_STATE != ZINK_NO_DYNAMIC_STATE)
557 VKCTX(CmdSetViewportWithCount)(bs->cmdbuf, ctx->vp_state.num_viewports, viewports);
558 else
559 VKCTX(CmdSetViewport)(bs->cmdbuf, 0, ctx->vp_state.num_viewports, viewports);
560 }
561 if (BATCH_CHANGED || ctx->scissor_changed || ctx->vp_state_changed || (DYNAMIC_STATE == ZINK_NO_DYNAMIC_STATE && pipeline_changed)) {
562 VkRect2D scissors[PIPE_MAX_VIEWPORTS];
563 if (ctx->rast_state->base.scissor) {
564 for (unsigned i = 0; i < ctx->vp_state.num_viewports; i++) {
565 scissors[i].offset.x = ctx->vp_state.scissor_states[i].minx;
566 scissors[i].offset.y = ctx->vp_state.scissor_states[i].miny;
567 scissors[i].extent.width = ctx->vp_state.scissor_states[i].maxx - ctx->vp_state.scissor_states[i].minx;
568 scissors[i].extent.height = ctx->vp_state.scissor_states[i].maxy - ctx->vp_state.scissor_states[i].miny;
569 }
570 } else {
571 for (unsigned i = 0; i < ctx->vp_state.num_viewports; i++) {
572 scissors[i].offset.x = 0;
573 scissors[i].offset.y = 0;
574 scissors[i].extent.width = ctx->fb_state.width;
575 scissors[i].extent.height = ctx->fb_state.height;
576 }
577 }
578 if (DYNAMIC_STATE != ZINK_NO_DYNAMIC_STATE)
579 VKCTX(CmdSetScissorWithCount)(bs->cmdbuf, ctx->vp_state.num_viewports, scissors);
580 else
581 VKCTX(CmdSetScissor)(bs->cmdbuf, 0, ctx->vp_state.num_viewports, scissors);
582 }
583 ctx->vp_state_changed = false;
584 ctx->scissor_changed = false;
585
586 if (BATCH_CHANGED || ctx->stencil_ref_changed) {
587 VKCTX(CmdSetStencilReference)(bs->cmdbuf, VK_STENCIL_FACE_FRONT_BIT,
588 ctx->stencil_ref.ref_value[0]);
589 VKCTX(CmdSetStencilReference)(bs->cmdbuf, VK_STENCIL_FACE_BACK_BIT,
590 ctx->stencil_ref.ref_value[1]);
591 ctx->stencil_ref_changed = false;
592 }
593
594 if (DYNAMIC_STATE != ZINK_NO_DYNAMIC_STATE && (BATCH_CHANGED || ctx->dsa_state_changed)) {
595 VKCTX(CmdSetDepthBoundsTestEnable)(bs->cmdbuf, dsa_state->hw_state.depth_bounds_test);
596 if (dsa_state->hw_state.depth_bounds_test)
597 VKCTX(CmdSetDepthBounds)(bs->cmdbuf,
598 dsa_state->hw_state.min_depth_bounds,
599 dsa_state->hw_state.max_depth_bounds);
600 VKCTX(CmdSetDepthTestEnable)(bs->cmdbuf, dsa_state->hw_state.depth_test);
601 VKCTX(CmdSetDepthCompareOp)(bs->cmdbuf, dsa_state->hw_state.depth_compare_op);
602 VKCTX(CmdSetDepthWriteEnable)(bs->cmdbuf, dsa_state->hw_state.depth_write);
603 VKCTX(CmdSetStencilTestEnable)(bs->cmdbuf, dsa_state->hw_state.stencil_test);
604 if (dsa_state->hw_state.stencil_test) {
605 VKCTX(CmdSetStencilOp)(bs->cmdbuf, VK_STENCIL_FACE_FRONT_BIT,
606 dsa_state->hw_state.stencil_front.failOp,
607 dsa_state->hw_state.stencil_front.passOp,
608 dsa_state->hw_state.stencil_front.depthFailOp,
609 dsa_state->hw_state.stencil_front.compareOp);
610 VKCTX(CmdSetStencilOp)(bs->cmdbuf, VK_STENCIL_FACE_BACK_BIT,
611 dsa_state->hw_state.stencil_back.failOp,
612 dsa_state->hw_state.stencil_back.passOp,
613 dsa_state->hw_state.stencil_back.depthFailOp,
614 dsa_state->hw_state.stencil_back.compareOp);
615 if (dsa_state->base.stencil[1].enabled) {
616 VKCTX(CmdSetStencilWriteMask)(bs->cmdbuf, VK_STENCIL_FACE_FRONT_BIT, dsa_state->hw_state.stencil_front.writeMask);
617 VKCTX(CmdSetStencilWriteMask)(bs->cmdbuf, VK_STENCIL_FACE_BACK_BIT, dsa_state->hw_state.stencil_back.writeMask);
618 VKCTX(CmdSetStencilCompareMask)(bs->cmdbuf, VK_STENCIL_FACE_FRONT_BIT, dsa_state->hw_state.stencil_front.compareMask);
619 VKCTX(CmdSetStencilCompareMask)(bs->cmdbuf, VK_STENCIL_FACE_BACK_BIT, dsa_state->hw_state.stencil_back.compareMask);
620 } else {
621 VKCTX(CmdSetStencilWriteMask)(bs->cmdbuf, VK_STENCIL_FACE_FRONT_AND_BACK, dsa_state->hw_state.stencil_front.writeMask);
622 VKCTX(CmdSetStencilCompareMask)(bs->cmdbuf, VK_STENCIL_FACE_FRONT_AND_BACK, dsa_state->hw_state.stencil_front.compareMask);
623 }
624 } else {
625 VKCTX(CmdSetStencilWriteMask)(bs->cmdbuf, VK_STENCIL_FACE_FRONT_AND_BACK, dsa_state->hw_state.stencil_front.writeMask);
626 VKCTX(CmdSetStencilCompareMask)(bs->cmdbuf, VK_STENCIL_FACE_FRONT_AND_BACK, dsa_state->hw_state.stencil_front.compareMask);
627 VKCTX(CmdSetStencilOp)(bs->cmdbuf, VK_STENCIL_FACE_FRONT_AND_BACK, VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_COMPARE_OP_ALWAYS);
628 }
629 }
630 ctx->dsa_state_changed = false;
631
632 if (BATCH_CHANGED || rast_state_changed) {
633 if (DYNAMIC_STATE != ZINK_NO_DYNAMIC_STATE) {
634 VKCTX(CmdSetFrontFace)(bs->cmdbuf, (VkFrontFace)ctx->gfx_pipeline_state.dyn_state1.front_face);
635 VKCTX(CmdSetCullMode)(bs->cmdbuf, ctx->gfx_pipeline_state.dyn_state1.cull_mode);
636 }
637
638 if (DYNAMIC_STATE >= ZINK_DYNAMIC_STATE3) {
639 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_RAST_STIPPLE))
640 VKCTX(CmdSetLineStippleEXT)(bs->cmdbuf, rast_state->base.line_stipple_factor, rast_state->base.line_stipple_pattern);
641 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_RAST_CLIP))
642 VKCTX(CmdSetDepthClipEnableEXT)(bs->cmdbuf, rast_state->hw_state.depth_clip);
643 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_RAST_CLAMP))
644 VKCTX(CmdSetDepthClampEnableEXT)(bs->cmdbuf, rast_state->hw_state.depth_clamp);
645 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_RAST_POLYGON))
646 VKCTX(CmdSetPolygonModeEXT)(bs->cmdbuf, (VkPolygonMode)rast_state->hw_state.polygon_mode);
647 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_RAST_HALFZ))
648 VKCTX(CmdSetDepthClipNegativeOneToOneEXT)(bs->cmdbuf, !rast_state->hw_state.clip_halfz);
649 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_RAST_PV))
650 VKCTX(CmdSetProvokingVertexModeEXT)(bs->cmdbuf,
651 rast_state->hw_state.pv_last ?
652 VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT :
653 VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT);
654 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_RAST_CLIP))
655 VKCTX(CmdSetLineRasterizationModeEXT)(bs->cmdbuf, rast_state->dynamic_line_mode);
656 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_RAST_STIPPLE_ON))
657 VKCTX(CmdSetLineStippleEnableEXT)(bs->cmdbuf, rast_state->hw_state.line_stipple_enable);
658 }
659 }
660 if ((BATCH_CHANGED || ctx->sample_mask_changed) && screen->have_full_ds3) {
661 VKCTX(CmdSetRasterizationSamplesEXT)(bs->cmdbuf, (VkSampleCountFlagBits)(ctx->gfx_pipeline_state.rast_samples + 1));
662 VKCTX(CmdSetSampleMaskEXT)(bs->cmdbuf, (VkSampleCountFlagBits)(ctx->gfx_pipeline_state.rast_samples + 1), &ctx->gfx_pipeline_state.sample_mask);
663 ctx->sample_mask_changed = false;
664 }
665 if ((BATCH_CHANGED || ctx->blend_state_changed)) {
666 if (ctx->gfx_pipeline_state.blend_state) {
667 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_BLEND_A2C))
668 VKCTX(CmdSetAlphaToCoverageEnableEXT)(bs->cmdbuf, ctx->gfx_pipeline_state.blend_state->alpha_to_coverage &&
669 ctx->gfx_stages[MESA_SHADER_FRAGMENT]->info.outputs_written & BITFIELD_BIT(FRAG_RESULT_DATA0));
670 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_BLEND_A21))
671 VKCTX(CmdSetAlphaToOneEnableEXT)(bs->cmdbuf, ctx->gfx_pipeline_state.blend_state->alpha_to_one);
672 if (ctx->fb_state.nr_cbufs) {
673 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_BLEND_ON))
674 VKCTX(CmdSetColorBlendEnableEXT)(bs->cmdbuf, 0, ctx->fb_state.nr_cbufs, ctx->gfx_pipeline_state.blend_state->ds3.enables);
675 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_BLEND_WRITE))
676 VKCTX(CmdSetColorWriteMaskEXT)(bs->cmdbuf, 0, ctx->fb_state.nr_cbufs, ctx->gfx_pipeline_state.blend_state->ds3.wrmask);
677 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_BLEND_EQ))
678 VKCTX(CmdSetColorBlendEquationEXT)(bs->cmdbuf, 0, ctx->fb_state.nr_cbufs, ctx->gfx_pipeline_state.blend_state->ds3.eq);
679 }
680 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_BLEND_LOGIC_ON))
681 VKCTX(CmdSetLogicOpEnableEXT)(bs->cmdbuf, ctx->gfx_pipeline_state.blend_state->logicop_enable);
682 if (ctx->ds3_states & BITFIELD_BIT(ZINK_DS3_BLEND_LOGIC))
683 VKCTX(CmdSetLogicOpEXT)(bs->cmdbuf, ctx->gfx_pipeline_state.blend_state->logicop_func);
684 }
685 }
686 ctx->ds3_states = 0;
687
688 if (BATCH_CHANGED ||
689 /* only re-emit on non-batch change when actually drawing lines */
690 ((ctx->line_width_changed || rast_prim_changed) && ctx->gfx_pipeline_state.rast_prim == MESA_PRIM_LINES)) {
691 VKCTX(CmdSetLineWidth)(bs->cmdbuf, rast_state->line_width);
692 ctx->line_width_changed = false;
693 }
694
695 if (BATCH_CHANGED || mode_changed ||
696 ctx->gfx_pipeline_state.modules_changed ||
697 rast_state_changed) {
698 bool depth_bias =
699 zink_prim_type(ctx, dinfo) == MESA_PRIM_TRIANGLES &&
700 rast_state->offset_fill;
701
702 if (depth_bias) {
703 if (rast_state->base.offset_units_unscaled) {
704 VKCTX(CmdSetDepthBias)(bs->cmdbuf, rast_state->offset_units * ctx->depth_bias_scale_factor, rast_state->offset_clamp, rast_state->offset_scale);
705 } else {
706 VKCTX(CmdSetDepthBias)(bs->cmdbuf, rast_state->offset_units, rast_state->offset_clamp, rast_state->offset_scale);
707 }
708 } else {
709 VKCTX(CmdSetDepthBias)(bs->cmdbuf, 0.0f, 0.0f, 0.0f);
710 }
711 }
712 ctx->rast_state_changed = false;
713
714 if (DYNAMIC_STATE != ZINK_NO_DYNAMIC_STATE) {
715 if (ctx->sample_locations_changed) {
716 VkSampleLocationsInfoEXT loc;
717 zink_init_vk_sample_locations(ctx, &loc);
718 VKCTX(CmdSetSampleLocationsEXT)(bs->cmdbuf, &loc);
719 }
720 ctx->sample_locations_changed = false;
721 }
722
723 if (BATCH_CHANGED || ctx->blend_color_changed) {
724 VKCTX(CmdSetBlendConstants)(bs->cmdbuf, ctx->blend_constants);
725 }
726 ctx->blend_state_changed = false;
727 ctx->blend_color_changed = false;
728
729 if (!DRAW_STATE) {
730 if (BATCH_CHANGED || ctx->vertex_buffers_dirty) {
731 if (DYNAMIC_STATE == ZINK_DYNAMIC_VERTEX_INPUT || ctx->gfx_pipeline_state.uses_dynamic_stride)
732 zink_bind_vertex_buffers<DYNAMIC_STATE>(ctx);
733 else
734 zink_bind_vertex_buffers<ZINK_NO_DYNAMIC_STATE>(ctx);
735 }
736 }
737
738 if (BATCH_CHANGED) {
739 ctx->pipeline_changed[0] = false;
740 zink_select_draw_vbo(ctx);
741 }
742
743 if (DYNAMIC_STATE != ZINK_NO_DYNAMIC_STATE && (BATCH_CHANGED || mode_changed))
744 VKCTX(CmdSetPrimitiveTopology)(bs->cmdbuf, zink_primitive_topology(mode));
745
746 if (DYNAMIC_STATE >= ZINK_DYNAMIC_STATE2 && (BATCH_CHANGED || ctx->primitive_restart != dinfo->primitive_restart)) {
747 VKCTX(CmdSetPrimitiveRestartEnable)(bs->cmdbuf, dinfo->primitive_restart);
748 ctx->primitive_restart = dinfo->primitive_restart;
749 }
750
751 if (DYNAMIC_STATE >= ZINK_DYNAMIC_STATE2 && (BATCH_CHANGED || ctx->rasterizer_discard_changed)) {
752 VKCTX(CmdSetRasterizerDiscardEnable)(bs->cmdbuf, ctx->gfx_pipeline_state.dyn_state2.rasterizer_discard);
753 ctx->rasterizer_discard_changed = false;
754 }
755
756 if (zink_program_has_descriptors(&ctx->curr_program->base))
757 zink_descriptors_update(ctx, false);
758
759 if (ctx->di.any_bindless_dirty &&
760 /* some apps (d3dretrace) call MakeTextureHandleResidentARB randomly */
761 zink_program_has_descriptors(&ctx->curr_program->base) &&
762 ctx->curr_program->base.dd.bindless)
763 zink_descriptors_update_bindless(ctx);
764
765 if (reads_basevertex) {
766 unsigned draw_mode_is_indexed = index_size > 0;
767 VKCTX(CmdPushConstants)(bs->cmdbuf, ctx->curr_program->base.layout, VK_SHADER_STAGE_ALL_GRAPHICS,
768 offsetof(struct zink_gfx_push_constant, draw_mode_is_indexed), sizeof(unsigned),
769 &draw_mode_is_indexed);
770 }
771 if (ctx->curr_program->shaders[MESA_SHADER_TESS_CTRL] &&
772 ctx->curr_program->shaders[MESA_SHADER_TESS_CTRL]->non_fs.is_generated) {
773 VKCTX(CmdPushConstants)(bs->cmdbuf, ctx->curr_program->base.layout, VK_SHADER_STAGE_ALL_GRAPHICS,
774 offsetof(struct zink_gfx_push_constant, default_inner_level), sizeof(float) * 6,
775 &ctx->tess_levels[0]);
776 }
777
778 if (!screen->optimal_keys) {
779 if (zink_get_fs_key(ctx)->lower_line_stipple ||
780 zink_get_gs_key(ctx)->lower_gl_point ||
781 zink_get_fs_key(ctx)->lower_line_smooth) {
782
783 assert(zink_get_gs_key(ctx)->lower_line_stipple ==
784 zink_get_fs_key(ctx)->lower_line_stipple);
785
786 assert(zink_get_gs_key(ctx)->lower_line_smooth ==
787 zink_get_fs_key(ctx)->lower_line_smooth);
788
789 float viewport_scale[2] = {
790 ctx->vp_state.viewport_states[0].scale[0],
791 ctx->vp_state.viewport_states[0].scale[1]
792 };
793 VKCTX(CmdPushConstants)(bs->cmdbuf,
794 ctx->curr_program->base.layout,
795 VK_SHADER_STAGE_ALL_GRAPHICS,
796 offsetof(struct zink_gfx_push_constant, viewport_scale),
797 sizeof(float) * 2, &viewport_scale);
798
799 uint32_t stipple = ctx->rast_state->base.line_stipple_pattern;
800 stipple |= ctx->rast_state->base.line_stipple_factor << 16;
801 VKCTX(CmdPushConstants)(bs->cmdbuf,
802 ctx->curr_program->base.layout,
803 VK_SHADER_STAGE_ALL_GRAPHICS,
804 offsetof(struct zink_gfx_push_constant, line_stipple_pattern),
805 sizeof(uint32_t), &stipple);
806
807 if (ctx->gfx_pipeline_state.shader_keys.key[MESA_SHADER_FRAGMENT].key.fs.lower_line_smooth) {
808 float line_width = ctx->rast_state->base.line_width;
809 VKCTX(CmdPushConstants)(bs->cmdbuf,
810 ctx->curr_program->base.layout,
811 VK_SHADER_STAGE_ALL_GRAPHICS,
812 offsetof(struct zink_gfx_push_constant, line_width),
813 sizeof(uint32_t), &line_width);
814 }
815 }
816 }
817
818 if (have_streamout) {
819 for (unsigned i = 0; i < ctx->num_so_targets; i++) {
820 struct zink_so_target *t = zink_so_target(ctx->so_targets[i]);
821 counter_buffers[i] = VK_NULL_HANDLE;
822 if (t) {
823 struct zink_resource *res = zink_resource(t->counter_buffer);
824 t->stride = ctx->last_vertex_stage->sinfo.stride[i];
825 zink_batch_reference_resource_rw(ctx, res, true);
826 if (!ctx->unordered_blitting)
827 res->obj->unordered_read = res->obj->unordered_write = false;
828 if (t->counter_buffer_valid) {
829 counter_buffers[i] = res->obj->buffer;
830 counter_buffer_offsets[i] = t->counter_buffer_offset;
831 }
832 }
833 }
834 VKCTX(CmdBeginTransformFeedbackEXT)(bs->cmdbuf, 0, ctx->num_so_targets, counter_buffers, counter_buffer_offsets);
835 }
836
837 bool marker = false;
838 if (unlikely(zink_tracing)) {
839 VkViewport viewport = {
840 ctx->vp_state.viewport_states[0].translate[0] - ctx->vp_state.viewport_states[0].scale[0],
841 ctx->vp_state.viewport_states[0].translate[1] - ctx->vp_state.viewport_states[0].scale[1],
842 MAX2(ctx->vp_state.viewport_states[0].scale[0] * 2, 1),
843 ctx->vp_state.viewport_states[0].scale[1] * 2,
844 CLAMP(ctx->rast_state->base.clip_halfz ?
845 ctx->vp_state.viewport_states[0].translate[2] :
846 ctx->vp_state.viewport_states[0].translate[2] - ctx->vp_state.viewport_states[0].scale[2],
847 0, 1),
848 CLAMP(ctx->vp_state.viewport_states[0].translate[2] + ctx->vp_state.viewport_states[0].scale[2],
849 0, 1)
850 };
851 if (ctx->blitting) {
852 bool is_zs = util_format_is_depth_or_stencil(ctx->sampler_views[MESA_SHADER_FRAGMENT][0]->format);
853 marker = zink_cmd_debug_marker_begin(ctx, VK_NULL_HANDLE, "u_blitter(%s->%s, %dx%d)",
854 util_format_short_name(ctx->sampler_views[MESA_SHADER_FRAGMENT][0]->format),
855 util_format_short_name((is_zs ? ctx->fb_state.zsbuf : ctx->fb_state.cbufs[0])->format),
856 lround(viewport.width), lround(viewport.height));
857 } else {
858 marker = zink_cmd_debug_marker_begin(ctx, VK_NULL_HANDLE, "draw(%u cbufs|%s, %dx%d)",
859 ctx->fb_state.nr_cbufs,
860 ctx->fb_state.zsbuf ? "zsbuf" : "",
861 lround(viewport.width), lround(viewport.height));
862 }
863 }
864
865 bool needs_drawid = reads_drawid && zink_get_last_vertex_key(ctx)->push_drawid;
866 work_count += num_draws;
867 if (index_size > 0) {
868 if (dindirect && dindirect->buffer) {
869 assert(num_draws == 1);
870 if (needs_drawid)
871 update_drawid(ctx, drawid_offset);
872 struct zink_resource *indirect = zink_resource(dindirect->buffer);
873 zink_batch_reference_resource_rw(ctx, indirect, false);
874 if (dindirect->indirect_draw_count) {
875 struct zink_resource *indirect_draw_count = zink_resource(dindirect->indirect_draw_count);
876 zink_batch_reference_resource_rw(ctx, indirect_draw_count, false);
877 VKCTX(CmdDrawIndexedIndirectCount)(bs->cmdbuf, indirect->obj->buffer, dindirect->offset,
878 indirect_draw_count->obj->buffer, dindirect->indirect_draw_count_offset,
879 dindirect->draw_count, dindirect->stride);
880 } else
881 VKCTX(CmdDrawIndexedIndirect)(bs->cmdbuf, indirect->obj->buffer, dindirect->offset, dindirect->draw_count, dindirect->stride);
882 } else {
883 if (need_index_buffer_unref)
884 draw_indexed_need_index_buffer_unref(ctx, dinfo, draws, num_draws, drawid_offset, needs_drawid);
885 else
886 draw_indexed<HAS_MULTIDRAW>(ctx, dinfo, draws, num_draws, drawid_offset, needs_drawid);
887 }
888 } else {
889 if (so_target && screen->info.tf_props.transformFeedbackDraw) {
890 /* GTF-GL46.gtf40.GL3Tests.transform_feedback2.transform_feedback2_api attempts a bogus xfb
891 * draw using a streamout target that has no data
892 * to avoid hanging the gpu, reject any such draws
893 */
894 if (so_target->counter_buffer_valid) {
895 if (needs_drawid)
896 update_drawid(ctx, drawid_offset);
897 zink_batch_reference_resource_rw(ctx, zink_resource(so_target->base.buffer), false);
898 zink_batch_reference_resource_rw(ctx, zink_resource(so_target->counter_buffer), true);
899 VKCTX(CmdDrawIndirectByteCountEXT)(bs->cmdbuf, dinfo->instance_count, dinfo->start_instance,
900 zink_resource(so_target->counter_buffer)->obj->buffer, so_target->counter_buffer_offset, 0,
901 MIN2(so_target->stride, screen->info.tf_props.maxTransformFeedbackBufferDataStride));
902 }
903 } else if (dindirect && dindirect->buffer) {
904 assert(num_draws == 1);
905 if (needs_drawid)
906 update_drawid(ctx, drawid_offset);
907 struct zink_resource *indirect = zink_resource(dindirect->buffer);
908 zink_batch_reference_resource_rw(ctx, indirect, false);
909 if (dindirect->indirect_draw_count) {
910 struct zink_resource *indirect_draw_count = zink_resource(dindirect->indirect_draw_count);
911 zink_batch_reference_resource_rw(ctx, indirect_draw_count, false);
912 VKCTX(CmdDrawIndirectCount)(bs->cmdbuf, indirect->obj->buffer, dindirect->offset,
913 indirect_draw_count->obj->buffer, dindirect->indirect_draw_count_offset,
914 dindirect->draw_count, dindirect->stride);
915 } else
916 VKCTX(CmdDrawIndirect)(bs->cmdbuf, indirect->obj->buffer, dindirect->offset, dindirect->draw_count, dindirect->stride);
917 } else {
918 draw<HAS_MULTIDRAW>(ctx, dinfo, draws, num_draws, drawid_offset, needs_drawid);
919 }
920 }
921
922 if (unlikely(zink_tracing))
923 zink_cmd_debug_marker_end(ctx, bs->cmdbuf, marker);
924
925 if (have_streamout) {
926 for (unsigned i = 0; i < ctx->num_so_targets; i++) {
927 struct zink_so_target *t = zink_so_target(ctx->so_targets[i]);
928 if (t) {
929 counter_buffers[i] = zink_resource(t->counter_buffer)->obj->buffer;
930 counter_buffer_offsets[i] = t->counter_buffer_offset;
931 t->counter_buffer_valid = true;
932 }
933 }
934 VKCTX(CmdEndTransformFeedbackEXT)(bs->cmdbuf, 0, ctx->num_so_targets, counter_buffers, counter_buffer_offsets);
935 }
936
937 ctx->last_work_was_compute = false;
938 ctx->work_count = work_count;
939 /* flush if there's >100k draws */
940 if (!ctx->unordered_blitting && (unlikely(work_count >= 30000) || ctx->oom_flush))
941 pctx->flush(pctx, NULL, 0);
942 }
943
944 template <zink_multidraw HAS_MULTIDRAW, zink_dynamic_state DYNAMIC_STATE, bool BATCH_CHANGED>
945 static void
zink_draw_vbo(struct pipe_context * pctx,const struct pipe_draw_info * info,unsigned drawid_offset,const struct pipe_draw_indirect_info * indirect,const struct pipe_draw_start_count_bias * draws,unsigned num_draws)946 zink_draw_vbo(struct pipe_context *pctx,
947 const struct pipe_draw_info *info,
948 unsigned drawid_offset,
949 const struct pipe_draw_indirect_info *indirect,
950 const struct pipe_draw_start_count_bias *draws,
951 unsigned num_draws)
952 {
953 zink_draw<HAS_MULTIDRAW, DYNAMIC_STATE, BATCH_CHANGED, false>(pctx, info, drawid_offset, indirect, draws, num_draws, NULL, 0);
954 }
955
956 template <util_popcnt HAS_POPCNT>
957 static void
zink_vertex_state_mask(struct zink_context * ctx,struct pipe_vertex_state * vstate,uint32_t partial_velem_mask)958 zink_vertex_state_mask(struct zink_context *ctx, struct pipe_vertex_state *vstate, uint32_t partial_velem_mask)
959 {
960 struct zink_vertex_state *zstate = (struct zink_vertex_state *)vstate;
961 VkCommandBuffer cmdbuf = ctx->bs->cmdbuf;
962
963 if (partial_velem_mask == vstate->input.full_velem_mask) {
964 VKCTX(CmdSetVertexInputEXT)(cmdbuf,
965 zstate->velems.hw_state.num_bindings, zstate->velems.hw_state.dynbindings,
966 zstate->velems.hw_state.num_attribs, zstate->velems.hw_state.dynattribs);
967 return;
968 }
969
970 VkVertexInputAttributeDescription2EXT dynattribs[PIPE_MAX_ATTRIBS];
971 unsigned num_attribs = 0;
972 u_foreach_bit(elem, vstate->input.full_velem_mask & partial_velem_mask) {
973 unsigned idx = util_bitcount_fast<HAS_POPCNT>(vstate->input.full_velem_mask & BITFIELD_MASK(elem));
974 dynattribs[num_attribs] = zstate->velems.hw_state.dynattribs[idx];
975 dynattribs[num_attribs].location = num_attribs;
976 num_attribs++;
977 }
978
979 VKCTX(CmdSetVertexInputEXT)(cmdbuf,
980 zstate->velems.hw_state.num_bindings, zstate->velems.hw_state.dynbindings,
981 num_attribs, dynattribs);
982 }
983
984 template <util_popcnt HAS_POPCNT>
985 static void
zink_bind_vertex_state(struct zink_context * ctx,struct pipe_vertex_state * vstate,uint32_t partial_velem_mask)986 zink_bind_vertex_state(struct zink_context *ctx, struct pipe_vertex_state *vstate, uint32_t partial_velem_mask)
987 {
988 struct zink_vertex_state *zstate = (struct zink_vertex_state *)vstate;
989 VkCommandBuffer cmdbuf = ctx->bs->cmdbuf;
990 if (!vstate->input.vbuffer.buffer.resource)
991 return;
992
993 zink_vertex_state_mask<HAS_POPCNT>(ctx, vstate, partial_velem_mask);
994
995 struct zink_resource *res = zink_resource(vstate->input.vbuffer.buffer.resource);
996 zink_batch_resource_usage_set(ctx->bs, res, false, true);
997 VkDeviceSize offset = vstate->input.vbuffer.buffer_offset;
998 VKCTX(CmdBindVertexBuffers)(cmdbuf, 0,
999 zstate->velems.hw_state.num_bindings,
1000 &res->obj->buffer, &offset);
1001 }
1002
1003 template <zink_multidraw HAS_MULTIDRAW, zink_dynamic_state DYNAMIC_STATE, util_popcnt HAS_POPCNT, bool BATCH_CHANGED>
1004 static void
zink_draw_vertex_state(struct pipe_context * pctx,struct pipe_vertex_state * vstate,uint32_t partial_velem_mask,struct pipe_draw_vertex_state_info info,const struct pipe_draw_start_count_bias * draws,unsigned num_draws)1005 zink_draw_vertex_state(struct pipe_context *pctx,
1006 struct pipe_vertex_state *vstate,
1007 uint32_t partial_velem_mask,
1008 struct pipe_draw_vertex_state_info info,
1009 const struct pipe_draw_start_count_bias *draws,
1010 unsigned num_draws)
1011 {
1012 struct pipe_draw_info dinfo = {};
1013
1014 dinfo.mode = info.mode;
1015 dinfo.index_size = 4;
1016 dinfo.instance_count = 1;
1017 dinfo.index.resource = vstate->input.indexbuf;
1018 struct zink_context *ctx = zink_context(pctx);
1019 struct zink_resource *res = zink_resource(vstate->input.vbuffer.buffer.resource);
1020 zink_screen(ctx->base.screen)->buffer_barrier(ctx, res, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT,
1021 VK_PIPELINE_STAGE_VERTEX_INPUT_BIT);
1022 if (!ctx->unordered_blitting)
1023 res->obj->unordered_read = false;
1024 zink_bind_vertex_state<HAS_POPCNT>(ctx, vstate, partial_velem_mask);
1025
1026 zink_draw<HAS_MULTIDRAW, DYNAMIC_STATE, BATCH_CHANGED, true>(pctx, &dinfo, 0, NULL, draws, num_draws, vstate, partial_velem_mask);
1027 /* ensure ctx->vertex_buffers gets rebound on next non-vstate draw */
1028 ctx->vertex_buffers_dirty = true;
1029
1030 if (info.take_vertex_state_ownership)
1031 pipe_vertex_state_reference(&vstate, NULL);
1032 }
1033
1034 template <bool BATCH_CHANGED>
1035 static void
zink_launch_grid(struct pipe_context * pctx,const struct pipe_grid_info * info)1036 zink_launch_grid(struct pipe_context *pctx, const struct pipe_grid_info *info)
1037 {
1038 struct zink_context *ctx = zink_context(pctx);
1039 struct zink_batch_state *bs = ctx->bs;
1040 struct zink_screen *screen = zink_screen(pctx->screen);
1041
1042 if (ctx->render_condition_active)
1043 zink_start_conditional_render(ctx);
1044
1045 if (info->indirect) {
1046 /*
1047 VK_ACCESS_INDIRECT_COMMAND_READ_BIT specifies read access to indirect command data read as
1048 part of an indirect build, trace, drawing or dispatching command. Such access occurs in the
1049 VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT pipeline stage.
1050
1051 - Chapter 7. Synchronization and Cache Control
1052 */
1053 check_buffer_barrier(ctx, info->indirect, VK_ACCESS_INDIRECT_COMMAND_READ_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT);
1054 }
1055
1056 zink_update_barriers(ctx, true, NULL, info->indirect, NULL);
1057 if (ctx->memory_barrier)
1058 zink_flush_memory_barrier(ctx, true);
1059
1060 if (unlikely(zink_debug & ZINK_DEBUG_SYNC)) {
1061 zink_batch_no_rp(ctx);
1062 VkMemoryBarrier mb;
1063 mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
1064 mb.pNext = NULL;
1065 mb.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
1066 mb.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT;
1067 VKSCR(CmdPipelineBarrier)(ctx->bs->cmdbuf,
1068 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
1069 VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
1070 0, 1, &mb, 0, NULL, 0, NULL);
1071 }
1072
1073 zink_program_update_compute_pipeline_state(ctx, ctx->curr_compute, info);
1074 VkPipeline prev_pipeline = ctx->compute_pipeline_state.pipeline;
1075
1076 if (BATCH_CHANGED) {
1077 zink_update_descriptor_refs(ctx, true);
1078 }
1079 if (ctx->compute_dirty) {
1080 /* update inlinable constants */
1081 zink_update_compute_program(ctx);
1082 ctx->compute_dirty = false;
1083 }
1084
1085 VkPipeline pipeline = zink_get_compute_pipeline(screen, ctx->curr_compute,
1086 &ctx->compute_pipeline_state);
1087
1088 if (prev_pipeline != pipeline || BATCH_CHANGED)
1089 VKCTX(CmdBindPipeline)(bs->cmdbuf, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
1090 if (BATCH_CHANGED) {
1091 ctx->pipeline_changed[1] = false;
1092 zink_select_launch_grid(ctx);
1093 }
1094
1095 if (zink_program_has_descriptors(&ctx->curr_compute->base))
1096 zink_descriptors_update(ctx, true);
1097 if (ctx->di.any_bindless_dirty && ctx->curr_compute->base.dd.bindless)
1098 zink_descriptors_update_bindless(ctx);
1099
1100 ctx->work_count++;
1101 zink_batch_no_rp(ctx);
1102 if (!ctx->queries_disabled)
1103 zink_resume_cs_query(ctx);
1104 if (info->indirect) {
1105 VKCTX(CmdDispatchIndirect)(bs->cmdbuf, zink_resource(info->indirect)->obj->buffer, info->indirect_offset);
1106 zink_batch_reference_resource_rw(ctx, zink_resource(info->indirect), false);
1107 } else
1108 VKCTX(CmdDispatch)(bs->cmdbuf, info->grid[0], info->grid[1], info->grid[2]);
1109 bs->has_work = true;
1110 ctx->last_work_was_compute = true;
1111 /* flush if there's >100k computes */
1112 if (!ctx->unordered_blitting && (unlikely(ctx->work_count >= 30000) || ctx->oom_flush))
1113 pctx->flush(pctx, NULL, 0);
1114 }
1115
1116 template <zink_multidraw HAS_MULTIDRAW, zink_dynamic_state DYNAMIC_STATE, bool BATCH_CHANGED>
1117 static void
init_batch_changed_functions(struct zink_context * ctx,pipe_draw_func draw_vbo_array[2][6][2],pipe_draw_vertex_state_func draw_state_array[2][6][2][2])1118 init_batch_changed_functions(struct zink_context *ctx, pipe_draw_func draw_vbo_array[2][6][2], pipe_draw_vertex_state_func draw_state_array[2][6][2][2])
1119 {
1120 draw_vbo_array[HAS_MULTIDRAW][DYNAMIC_STATE][BATCH_CHANGED] = zink_draw_vbo<HAS_MULTIDRAW, DYNAMIC_STATE, BATCH_CHANGED>;
1121 draw_state_array[HAS_MULTIDRAW][DYNAMIC_STATE][0][BATCH_CHANGED] = zink_draw_vertex_state<HAS_MULTIDRAW, DYNAMIC_STATE, POPCNT_NO, BATCH_CHANGED>;
1122 draw_state_array[HAS_MULTIDRAW][DYNAMIC_STATE][1][BATCH_CHANGED] = zink_draw_vertex_state<HAS_MULTIDRAW, DYNAMIC_STATE, POPCNT_YES, BATCH_CHANGED>;
1123 }
1124
1125 template <zink_multidraw HAS_MULTIDRAW, zink_dynamic_state DYNAMIC_STATE>
1126 static void
init_dynamic_state_functions(struct zink_context * ctx,pipe_draw_func draw_vbo_array[2][6][2],pipe_draw_vertex_state_func draw_state_array[2][6][2][2])1127 init_dynamic_state_functions(struct zink_context *ctx, pipe_draw_func draw_vbo_array[2][6][2], pipe_draw_vertex_state_func draw_state_array[2][6][2][2])
1128 {
1129 init_batch_changed_functions<HAS_MULTIDRAW, DYNAMIC_STATE, false>(ctx, draw_vbo_array, draw_state_array);
1130 init_batch_changed_functions<HAS_MULTIDRAW, DYNAMIC_STATE, true>(ctx, draw_vbo_array, draw_state_array);
1131 }
1132
1133 template <zink_multidraw HAS_MULTIDRAW>
1134 static void
init_multidraw_functions(struct zink_context * ctx,pipe_draw_func draw_vbo_array[2][6][2],pipe_draw_vertex_state_func draw_state_array[2][6][2][2])1135 init_multidraw_functions(struct zink_context *ctx, pipe_draw_func draw_vbo_array[2][6][2], pipe_draw_vertex_state_func draw_state_array[2][6][2][2])
1136 {
1137 init_dynamic_state_functions<HAS_MULTIDRAW, ZINK_NO_DYNAMIC_STATE>(ctx, draw_vbo_array, draw_state_array);
1138 init_dynamic_state_functions<HAS_MULTIDRAW, ZINK_DYNAMIC_STATE>(ctx, draw_vbo_array, draw_state_array);
1139 init_dynamic_state_functions<HAS_MULTIDRAW, ZINK_DYNAMIC_STATE2>(ctx, draw_vbo_array, draw_state_array);
1140 init_dynamic_state_functions<HAS_MULTIDRAW, ZINK_DYNAMIC_VERTEX_INPUT2>(ctx, draw_vbo_array, draw_state_array);
1141 init_dynamic_state_functions<HAS_MULTIDRAW, ZINK_DYNAMIC_STATE3>(ctx, draw_vbo_array, draw_state_array);
1142 init_dynamic_state_functions<HAS_MULTIDRAW, ZINK_DYNAMIC_VERTEX_INPUT>(ctx, draw_vbo_array, draw_state_array);
1143 }
1144
1145 static void
init_all_draw_functions(struct zink_context * ctx,pipe_draw_func draw_vbo_array[2][6][2],pipe_draw_vertex_state_func draw_state_array[2][6][2][2])1146 init_all_draw_functions(struct zink_context *ctx, pipe_draw_func draw_vbo_array[2][6][2], pipe_draw_vertex_state_func draw_state_array[2][6][2][2])
1147 {
1148 init_multidraw_functions<ZINK_NO_MULTIDRAW>(ctx, draw_vbo_array, draw_state_array);
1149 init_multidraw_functions<ZINK_MULTIDRAW>(ctx, draw_vbo_array, draw_state_array);
1150 }
1151
1152 template <bool BATCH_CHANGED>
1153 static void
init_grid_batch_changed_functions(struct zink_context * ctx)1154 init_grid_batch_changed_functions(struct zink_context *ctx)
1155 {
1156 ctx->launch_grid[BATCH_CHANGED] = zink_launch_grid<BATCH_CHANGED>;
1157 }
1158
1159 static void
init_all_grid_functions(struct zink_context * ctx)1160 init_all_grid_functions(struct zink_context *ctx)
1161 {
1162 init_grid_batch_changed_functions<false>(ctx);
1163 init_grid_batch_changed_functions<true>(ctx);
1164 }
1165
1166 static void
zink_invalid_draw_vbo(struct pipe_context * pipe,const struct pipe_draw_info * dinfo,unsigned drawid_offset,const struct pipe_draw_indirect_info * dindirect,const struct pipe_draw_start_count_bias * draws,unsigned num_draws)1167 zink_invalid_draw_vbo(struct pipe_context *pipe,
1168 const struct pipe_draw_info *dinfo,
1169 unsigned drawid_offset,
1170 const struct pipe_draw_indirect_info *dindirect,
1171 const struct pipe_draw_start_count_bias *draws,
1172 unsigned num_draws)
1173 {
1174 unreachable("vertex shader not bound");
1175 }
1176
1177 static void
zink_invalid_draw_vertex_state(struct pipe_context * pipe,struct pipe_vertex_state * vstate,uint32_t partial_velem_mask,struct pipe_draw_vertex_state_info info,const struct pipe_draw_start_count_bias * draws,unsigned num_draws)1178 zink_invalid_draw_vertex_state(struct pipe_context *pipe,
1179 struct pipe_vertex_state *vstate,
1180 uint32_t partial_velem_mask,
1181 struct pipe_draw_vertex_state_info info,
1182 const struct pipe_draw_start_count_bias *draws,
1183 unsigned num_draws)
1184 {
1185 unreachable("vertex shader not bound");
1186 }
1187
1188 static void
zink_invalid_launch_grid(struct pipe_context * pctx,const struct pipe_grid_info * info)1189 zink_invalid_launch_grid(struct pipe_context *pctx, const struct pipe_grid_info *info)
1190 {
1191 unreachable("compute shader not bound");
1192 }
1193
1194 #define STAGE_BASE 0
1195 #define STAGE_BASE_GS (BITFIELD_BIT(MESA_SHADER_GEOMETRY) >> 1)
1196 #define STAGE_BASE_TES (BITFIELD_BIT(MESA_SHADER_TESS_EVAL) >> 1)
1197 #define STAGE_BASE_TES_GS ((BITFIELD_BIT(MESA_SHADER_TESS_EVAL) | BITFIELD_BIT(MESA_SHADER_GEOMETRY)) >> 1)
1198 #define STAGE_BASE_TCS_TES ((BITFIELD_BIT(MESA_SHADER_TESS_CTRL) | BITFIELD_BIT(MESA_SHADER_TESS_EVAL)) >> 1)
1199 #define STAGE_BASE_TCS_TES_GS ((BITFIELD_BIT(MESA_SHADER_TESS_CTRL) | BITFIELD_BIT(MESA_SHADER_TESS_EVAL) | BITFIELD_BIT(MESA_SHADER_GEOMETRY)) >> 1)
1200
1201 template <unsigned STAGE_MASK>
1202 static uint32_t
hash_gfx_program(const void * key)1203 hash_gfx_program(const void *key)
1204 {
1205 const struct zink_shader **shaders = (const struct zink_shader**)key;
1206 uint32_t base_hash = shaders[MESA_SHADER_VERTEX]->hash ^ shaders[MESA_SHADER_FRAGMENT]->hash;
1207 if (STAGE_MASK == STAGE_BASE) //VS+FS
1208 return base_hash;
1209 if (STAGE_MASK == STAGE_BASE_GS) //VS+GS+FS
1210 return base_hash ^ shaders[MESA_SHADER_GEOMETRY]->hash;
1211 /*VS+TCS+FS isn't a thing */
1212 /*VS+TCS+GS+FS isn't a thing */
1213 if (STAGE_MASK == STAGE_BASE_TES) //VS+TES+FS
1214 return base_hash ^ shaders[MESA_SHADER_TESS_EVAL]->hash;
1215 if (STAGE_MASK == STAGE_BASE_TES_GS) //VS+TES+GS+FS
1216 return base_hash ^ shaders[MESA_SHADER_GEOMETRY]->hash ^ shaders[MESA_SHADER_TESS_EVAL]->hash;
1217 if (STAGE_MASK == STAGE_BASE_TCS_TES) //VS+TCS+TES+FS
1218 return base_hash ^ shaders[MESA_SHADER_TESS_CTRL]->hash ^ shaders[MESA_SHADER_TESS_EVAL]->hash;
1219
1220 /* all stages */
1221 return base_hash ^ shaders[MESA_SHADER_GEOMETRY]->hash ^ shaders[MESA_SHADER_TESS_CTRL]->hash ^ shaders[MESA_SHADER_TESS_EVAL]->hash;
1222 }
1223
1224 template <unsigned STAGE_MASK>
1225 static bool
equals_gfx_program(const void * a,const void * b)1226 equals_gfx_program(const void *a, const void *b)
1227 {
1228 const void **sa = (const void**)a;
1229 const void **sb = (const void**)b;
1230 STATIC_ASSERT(MESA_SHADER_VERTEX == 0);
1231 STATIC_ASSERT(MESA_SHADER_TESS_CTRL == 1);
1232 STATIC_ASSERT(MESA_SHADER_TESS_EVAL == 2);
1233 STATIC_ASSERT(MESA_SHADER_GEOMETRY == 3);
1234 STATIC_ASSERT(MESA_SHADER_FRAGMENT == 4);
1235 if (STAGE_MASK == STAGE_BASE) //VS+FS
1236 return sa[MESA_SHADER_VERTEX] == sb[MESA_SHADER_VERTEX] &&
1237 sa[MESA_SHADER_FRAGMENT] == sb[MESA_SHADER_FRAGMENT];
1238 if (STAGE_MASK == STAGE_BASE_GS) //VS+GS+FS
1239 return sa[MESA_SHADER_VERTEX] == sb[MESA_SHADER_VERTEX] &&
1240 !memcmp(&sa[MESA_SHADER_GEOMETRY], &sb[MESA_SHADER_GEOMETRY], sizeof(void*) * 2);
1241 /*VS+TCS+FS isn't a thing */
1242 /*VS+TCS+GS+FS isn't a thing */
1243 if (STAGE_MASK == STAGE_BASE_TES) //VS+TES+FS
1244 return sa[MESA_SHADER_VERTEX] == sb[MESA_SHADER_VERTEX] &&
1245 sa[MESA_SHADER_TESS_EVAL] == sb[MESA_SHADER_TESS_EVAL] &&
1246 sa[MESA_SHADER_FRAGMENT] == sb[MESA_SHADER_FRAGMENT];
1247 if (STAGE_MASK == STAGE_BASE_TES_GS) //VS+TES+GS+FS
1248 return sa[MESA_SHADER_VERTEX] == sb[MESA_SHADER_VERTEX] &&
1249 !memcmp(&sa[MESA_SHADER_TESS_EVAL], &sb[MESA_SHADER_TESS_EVAL], sizeof(void*) * 3);
1250 if (STAGE_MASK == STAGE_BASE_TCS_TES) //VS+TCS+TES+FS
1251 return !memcmp(sa, sb, sizeof(void*) * 3) &&
1252 sa[MESA_SHADER_FRAGMENT] == sb[MESA_SHADER_FRAGMENT];
1253
1254 /* all stages */
1255 return !memcmp(a, b, sizeof(void*) * ZINK_GFX_SHADER_COUNT);
1256 }
1257
1258 extern "C"
1259 void
zink_init_draw_functions(struct zink_context * ctx,struct zink_screen * screen)1260 zink_init_draw_functions(struct zink_context *ctx, struct zink_screen *screen)
1261 {
1262 pipe_draw_func draw_vbo_array[2][6] //multidraw, zink_dynamic_state
1263 [2]; //batch changed
1264 pipe_draw_vertex_state_func draw_state_array[2][6] //multidraw, zink_dynamic_state
1265 [2][2]; //has_popcnt, batch changed
1266 zink_dynamic_state dynamic;
1267 if (screen->info.have_EXT_extended_dynamic_state) {
1268 if (screen->info.have_EXT_extended_dynamic_state2) {
1269 if (screen->info.have_EXT_extended_dynamic_state3) {
1270 if (screen->info.have_EXT_vertex_input_dynamic_state)
1271 dynamic = ZINK_DYNAMIC_VERTEX_INPUT;
1272 else
1273 dynamic = ZINK_DYNAMIC_STATE3;
1274 } else {
1275 if (screen->info.have_EXT_vertex_input_dynamic_state)
1276 dynamic = ZINK_DYNAMIC_VERTEX_INPUT2;
1277 else
1278 dynamic = ZINK_DYNAMIC_STATE2;
1279 }
1280 } else {
1281 dynamic = ZINK_DYNAMIC_STATE;
1282 }
1283 } else {
1284 dynamic = ZINK_NO_DYNAMIC_STATE;
1285 }
1286 init_all_draw_functions(ctx, draw_vbo_array, draw_state_array);
1287 memcpy(ctx->draw_vbo, &draw_vbo_array[screen->info.have_EXT_multi_draw]
1288 [dynamic],
1289 sizeof(ctx->draw_vbo));
1290 memcpy(ctx->draw_state, &draw_state_array[screen->info.have_EXT_multi_draw]
1291 [dynamic][util_get_cpu_caps()->has_popcnt],
1292 sizeof(ctx->draw_state));
1293
1294 /* Bind a fake draw_vbo, so that draw_vbo isn't NULL, which would skip
1295 * initialization of callbacks in upper layers (such as u_threaded_context).
1296 */
1297 ctx->base.draw_vbo = zink_invalid_draw_vbo;
1298 ctx->base.draw_vertex_state = zink_invalid_draw_vertex_state;
1299
1300 _mesa_hash_table_init(&ctx->program_cache[0], ctx, hash_gfx_program<0>, equals_gfx_program<0>);
1301 _mesa_hash_table_init(&ctx->program_cache[1], ctx, hash_gfx_program<1>, equals_gfx_program<1>);
1302 _mesa_hash_table_init(&ctx->program_cache[2], ctx, hash_gfx_program<2>, equals_gfx_program<2>);
1303 _mesa_hash_table_init(&ctx->program_cache[3], ctx, hash_gfx_program<3>, equals_gfx_program<3>);
1304 _mesa_hash_table_init(&ctx->program_cache[4], ctx, hash_gfx_program<4>, equals_gfx_program<4>);
1305 _mesa_hash_table_init(&ctx->program_cache[5], ctx, hash_gfx_program<5>, equals_gfx_program<5>);
1306 _mesa_hash_table_init(&ctx->program_cache[6], ctx, hash_gfx_program<6>, equals_gfx_program<6>);
1307 _mesa_hash_table_init(&ctx->program_cache[7], ctx, hash_gfx_program<7>, equals_gfx_program<7>);
1308 for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_lock); i++)
1309 simple_mtx_init(&ctx->program_lock[i], mtx_plain);
1310 }
1311
1312 void
zink_init_grid_functions(struct zink_context * ctx)1313 zink_init_grid_functions(struct zink_context *ctx)
1314 {
1315 init_all_grid_functions(ctx);
1316 /* Bind a fake launch_grid, so that draw_vbo isn't NULL, which would skip
1317 * initialization of callbacks in upper layers (such as u_threaded_context).
1318 */
1319 ctx->base.launch_grid = zink_invalid_launch_grid;
1320 }
1321
1322 void
zink_init_screen_pipeline_libs(struct zink_screen * screen)1323 zink_init_screen_pipeline_libs(struct zink_screen *screen)
1324 {
1325 _mesa_set_init(&screen->pipeline_libs[0], screen, hash_gfx_program<0>, equals_gfx_program<0>);
1326 _mesa_set_init(&screen->pipeline_libs[1], screen, hash_gfx_program<1>, equals_gfx_program<1>);
1327 _mesa_set_init(&screen->pipeline_libs[2], screen, hash_gfx_program<2>, equals_gfx_program<2>);
1328 _mesa_set_init(&screen->pipeline_libs[3], screen, hash_gfx_program<3>, equals_gfx_program<3>);
1329 _mesa_set_init(&screen->pipeline_libs[4], screen, hash_gfx_program<4>, equals_gfx_program<4>);
1330 _mesa_set_init(&screen->pipeline_libs[5], screen, hash_gfx_program<5>, equals_gfx_program<5>);
1331 _mesa_set_init(&screen->pipeline_libs[6], screen, hash_gfx_program<6>, equals_gfx_program<6>);
1332 _mesa_set_init(&screen->pipeline_libs[7], screen, hash_gfx_program<7>, equals_gfx_program<7>);
1333 for (unsigned i = 0; i < ARRAY_SIZE(screen->pipeline_libs_lock); i++)
1334 simple_mtx_init(&screen->pipeline_libs_lock[i], mtx_plain);
1335 }
1336