1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26 #include <string.h>
27 #include <unistd.h>
28 #include <fcntl.h>
29
30 #include "anv_private.h"
31 #include "anv_measure.h"
32
33 #include "vk_util.h"
34
35 /** \file anv_cmd_buffer.c
36 *
37 * This file contains all of the stuff for emitting commands into a command
38 * buffer. This includes implementations of most of the vkCmd*
39 * entrypoints. This file is concerned entirely with state emission and
40 * not with the command buffer data structure itself. As far as this file
41 * is concerned, most of anv_cmd_buffer is magic.
42 */
43
44 static void
anv_cmd_state_init(struct anv_cmd_buffer * cmd_buffer)45 anv_cmd_state_init(struct anv_cmd_buffer *cmd_buffer)
46 {
47 struct anv_cmd_state *state = &cmd_buffer->state;
48
49 memset(state, 0, sizeof(*state));
50
51 state->current_pipeline = UINT32_MAX;
52 state->gfx.restart_index = UINT32_MAX;
53 state->gfx.object_preemption = true;
54 state->gfx.coarse_pixel_active = ANV_COARSE_PIXEL_STATE_UNKNOWN;
55 state->gfx.dirty = 0;
56
57 memcpy(state->gfx.dyn_state.dirty,
58 cmd_buffer->device->gfx_dirty_state,
59 sizeof(state->gfx.dyn_state.dirty));
60 }
61
62 static void
anv_cmd_pipeline_state_finish(struct anv_cmd_buffer * cmd_buffer,struct anv_cmd_pipeline_state * pipe_state)63 anv_cmd_pipeline_state_finish(struct anv_cmd_buffer *cmd_buffer,
64 struct anv_cmd_pipeline_state *pipe_state)
65 {
66 anv_push_descriptor_set_finish(&pipe_state->push_descriptor);
67 }
68
69 static void
anv_cmd_state_finish(struct anv_cmd_buffer * cmd_buffer)70 anv_cmd_state_finish(struct anv_cmd_buffer *cmd_buffer)
71 {
72 struct anv_cmd_state *state = &cmd_buffer->state;
73
74 anv_cmd_pipeline_state_finish(cmd_buffer, &state->gfx.base);
75 anv_cmd_pipeline_state_finish(cmd_buffer, &state->compute.base);
76 }
77
78 static void
anv_cmd_state_reset(struct anv_cmd_buffer * cmd_buffer)79 anv_cmd_state_reset(struct anv_cmd_buffer *cmd_buffer)
80 {
81 anv_cmd_state_finish(cmd_buffer);
82 anv_cmd_state_init(cmd_buffer);
83 }
84
85 VkResult
anv_cmd_buffer_ensure_rcs_companion(struct anv_cmd_buffer * cmd_buffer)86 anv_cmd_buffer_ensure_rcs_companion(struct anv_cmd_buffer *cmd_buffer)
87 {
88 if (cmd_buffer->companion_rcs_cmd_buffer)
89 return VK_SUCCESS;
90
91 VkResult result = VK_SUCCESS;
92 pthread_mutex_lock(&cmd_buffer->device->mutex);
93 VK_FROM_HANDLE(vk_command_pool, pool,
94 cmd_buffer->device->companion_rcs_cmd_pool);
95 assert(pool != NULL);
96
97 struct vk_command_buffer *tmp_cmd_buffer = NULL;
98 result = pool->command_buffer_ops->create(pool, cmd_buffer->vk.level, &tmp_cmd_buffer);
99
100 if (result != VK_SUCCESS)
101 goto unlock_and_return;
102
103 cmd_buffer->companion_rcs_cmd_buffer =
104 container_of(tmp_cmd_buffer, struct anv_cmd_buffer, vk);
105 anv_genX(cmd_buffer->device->info, cmd_buffer_begin_companion)(
106 cmd_buffer->companion_rcs_cmd_buffer, cmd_buffer->vk.level);
107
108 unlock_and_return:
109 pthread_mutex_unlock(&cmd_buffer->device->mutex);
110 return result;
111 }
112
113 static VkResult
anv_create_cmd_buffer(struct vk_command_pool * pool,VkCommandBufferLevel level,struct vk_command_buffer ** cmd_buffer_out)114 anv_create_cmd_buffer(struct vk_command_pool *pool,
115 VkCommandBufferLevel level,
116 struct vk_command_buffer **cmd_buffer_out)
117 {
118 struct anv_device *device =
119 container_of(pool->base.device, struct anv_device, vk);
120 struct anv_cmd_buffer *cmd_buffer;
121 VkResult result;
122
123 cmd_buffer = vk_zalloc(&pool->alloc, sizeof(*cmd_buffer), 8,
124 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
125 if (cmd_buffer == NULL)
126 return vk_error(pool, VK_ERROR_OUT_OF_HOST_MEMORY);
127
128 result = vk_command_buffer_init(pool, &cmd_buffer->vk,
129 &anv_cmd_buffer_ops, level);
130 if (result != VK_SUCCESS)
131 goto fail_alloc;
132
133 cmd_buffer->vk.dynamic_graphics_state.ms.sample_locations =
134 &cmd_buffer->state.gfx.sample_locations;
135 cmd_buffer->vk.dynamic_graphics_state.vi =
136 &cmd_buffer->state.gfx.vertex_input;
137
138 cmd_buffer->batch.status = VK_SUCCESS;
139 cmd_buffer->generation.batch.status = VK_SUCCESS;
140
141 cmd_buffer->device = device;
142
143 assert(pool->queue_family_index < device->physical->queue.family_count);
144 cmd_buffer->queue_family =
145 &device->physical->queue.families[pool->queue_family_index];
146
147 result = anv_cmd_buffer_init_batch_bo_chain(cmd_buffer);
148 if (result != VK_SUCCESS)
149 goto fail_vk;
150
151 anv_state_stream_init(&cmd_buffer->surface_state_stream,
152 &device->internal_surface_state_pool, 4096);
153 anv_state_stream_init(&cmd_buffer->dynamic_state_stream,
154 &device->dynamic_state_pool, 16384);
155 anv_state_stream_init(&cmd_buffer->general_state_stream,
156 &device->general_state_pool, 16384);
157 anv_state_stream_init(&cmd_buffer->indirect_push_descriptor_stream,
158 &device->indirect_push_descriptor_pool, 4096);
159 anv_state_stream_init(&cmd_buffer->push_descriptor_buffer_stream,
160 &device->push_descriptor_buffer_pool, 4096);
161
162 int success = u_vector_init_pow2(&cmd_buffer->dynamic_bos, 8,
163 sizeof(struct anv_bo *));
164 if (!success)
165 goto fail_batch_bo;
166
167 cmd_buffer->self_mod_locations = NULL;
168 cmd_buffer->companion_rcs_cmd_buffer = NULL;
169 cmd_buffer->is_companion_rcs_cmd_buffer = false;
170
171 cmd_buffer->generation.jump_addr = ANV_NULL_ADDRESS;
172 cmd_buffer->generation.return_addr = ANV_NULL_ADDRESS;
173
174 memset(&cmd_buffer->generation.shader_state, 0,
175 sizeof(cmd_buffer->generation.shader_state));
176
177 anv_cmd_state_init(cmd_buffer);
178
179 anv_measure_init(cmd_buffer);
180
181 u_trace_init(&cmd_buffer->trace, &device->ds.trace_context);
182
183 *cmd_buffer_out = &cmd_buffer->vk;
184
185 return VK_SUCCESS;
186
187 fail_batch_bo:
188 anv_cmd_buffer_fini_batch_bo_chain(cmd_buffer);
189 fail_vk:
190 vk_command_buffer_finish(&cmd_buffer->vk);
191 fail_alloc:
192 vk_free2(&device->vk.alloc, &pool->alloc, cmd_buffer);
193
194 return result;
195 }
196
197 static void
destroy_cmd_buffer(struct anv_cmd_buffer * cmd_buffer)198 destroy_cmd_buffer(struct anv_cmd_buffer *cmd_buffer)
199 {
200 u_trace_fini(&cmd_buffer->trace);
201
202 anv_measure_destroy(cmd_buffer);
203
204 anv_cmd_buffer_fini_batch_bo_chain(cmd_buffer);
205
206 anv_state_stream_finish(&cmd_buffer->surface_state_stream);
207 anv_state_stream_finish(&cmd_buffer->dynamic_state_stream);
208 anv_state_stream_finish(&cmd_buffer->general_state_stream);
209 anv_state_stream_finish(&cmd_buffer->indirect_push_descriptor_stream);
210 anv_state_stream_finish(&cmd_buffer->push_descriptor_buffer_stream);
211
212 while (u_vector_length(&cmd_buffer->dynamic_bos) > 0) {
213 struct anv_bo **bo = u_vector_remove(&cmd_buffer->dynamic_bos);
214 anv_bo_pool_free((*bo)->map != NULL ?
215 &cmd_buffer->device->batch_bo_pool :
216 &cmd_buffer->device->bvh_bo_pool, *bo);
217 }
218 u_vector_finish(&cmd_buffer->dynamic_bos);
219
220 anv_cmd_state_finish(cmd_buffer);
221
222 vk_free(&cmd_buffer->vk.pool->alloc, cmd_buffer->self_mod_locations);
223
224 vk_command_buffer_finish(&cmd_buffer->vk);
225 vk_free(&cmd_buffer->vk.pool->alloc, cmd_buffer);
226 }
227
228 static void
anv_cmd_buffer_destroy(struct vk_command_buffer * vk_cmd_buffer)229 anv_cmd_buffer_destroy(struct vk_command_buffer *vk_cmd_buffer)
230 {
231 struct anv_cmd_buffer *cmd_buffer =
232 container_of(vk_cmd_buffer, struct anv_cmd_buffer, vk);
233 struct anv_device *device = cmd_buffer->device;
234
235 pthread_mutex_lock(&device->mutex);
236 if (cmd_buffer->companion_rcs_cmd_buffer) {
237 destroy_cmd_buffer(cmd_buffer->companion_rcs_cmd_buffer);
238 cmd_buffer->companion_rcs_cmd_buffer = NULL;
239 }
240
241 ANV_RMV(cmd_buffer_destroy, cmd_buffer->device, cmd_buffer);
242
243 destroy_cmd_buffer(cmd_buffer);
244 pthread_mutex_unlock(&device->mutex);
245 }
246
247 static void
reset_cmd_buffer(struct anv_cmd_buffer * cmd_buffer,UNUSED VkCommandBufferResetFlags flags)248 reset_cmd_buffer(struct anv_cmd_buffer *cmd_buffer,
249 UNUSED VkCommandBufferResetFlags flags)
250 {
251 vk_command_buffer_reset(&cmd_buffer->vk);
252
253 cmd_buffer->usage_flags = 0;
254 cmd_buffer->perf_query_pool = NULL;
255 cmd_buffer->is_companion_rcs_cmd_buffer = false;
256 anv_cmd_buffer_reset_batch_bo_chain(cmd_buffer);
257 anv_cmd_state_reset(cmd_buffer);
258
259 memset(&cmd_buffer->generation.shader_state, 0,
260 sizeof(cmd_buffer->generation.shader_state));
261
262 cmd_buffer->generation.jump_addr = ANV_NULL_ADDRESS;
263 cmd_buffer->generation.return_addr = ANV_NULL_ADDRESS;
264
265 anv_state_stream_finish(&cmd_buffer->surface_state_stream);
266 anv_state_stream_init(&cmd_buffer->surface_state_stream,
267 &cmd_buffer->device->internal_surface_state_pool, 4096);
268
269 anv_state_stream_finish(&cmd_buffer->dynamic_state_stream);
270 anv_state_stream_init(&cmd_buffer->dynamic_state_stream,
271 &cmd_buffer->device->dynamic_state_pool, 16384);
272
273 anv_state_stream_finish(&cmd_buffer->general_state_stream);
274 anv_state_stream_init(&cmd_buffer->general_state_stream,
275 &cmd_buffer->device->general_state_pool, 16384);
276
277 anv_state_stream_finish(&cmd_buffer->indirect_push_descriptor_stream);
278 anv_state_stream_init(&cmd_buffer->indirect_push_descriptor_stream,
279 &cmd_buffer->device->indirect_push_descriptor_pool,
280 4096);
281
282 anv_state_stream_finish(&cmd_buffer->push_descriptor_buffer_stream);
283 anv_state_stream_init(&cmd_buffer->push_descriptor_buffer_stream,
284 &cmd_buffer->device->push_descriptor_buffer_pool, 4096);
285
286 while (u_vector_length(&cmd_buffer->dynamic_bos) > 0) {
287 struct anv_bo **bo = u_vector_remove(&cmd_buffer->dynamic_bos);
288 anv_device_release_bo(cmd_buffer->device, *bo);
289 }
290
291 anv_measure_reset(cmd_buffer);
292
293 u_trace_fini(&cmd_buffer->trace);
294 u_trace_init(&cmd_buffer->trace, &cmd_buffer->device->ds.trace_context);
295 }
296
297 void
anv_cmd_buffer_reset(struct vk_command_buffer * vk_cmd_buffer,UNUSED VkCommandBufferResetFlags flags)298 anv_cmd_buffer_reset(struct vk_command_buffer *vk_cmd_buffer,
299 UNUSED VkCommandBufferResetFlags flags)
300 {
301 struct anv_cmd_buffer *cmd_buffer =
302 container_of(vk_cmd_buffer, struct anv_cmd_buffer, vk);
303
304 if (cmd_buffer->companion_rcs_cmd_buffer) {
305 reset_cmd_buffer(cmd_buffer->companion_rcs_cmd_buffer, flags);
306 destroy_cmd_buffer(cmd_buffer->companion_rcs_cmd_buffer);
307 cmd_buffer->companion_rcs_cmd_buffer = NULL;
308 }
309
310 ANV_RMV(cmd_buffer_destroy, cmd_buffer->device, cmd_buffer);
311
312 reset_cmd_buffer(cmd_buffer, flags);
313 }
314
315 const struct vk_command_buffer_ops anv_cmd_buffer_ops = {
316 .create = anv_create_cmd_buffer,
317 .reset = anv_cmd_buffer_reset,
318 .destroy = anv_cmd_buffer_destroy,
319 };
320
321 void
anv_cmd_buffer_emit_bt_pool_base_address(struct anv_cmd_buffer * cmd_buffer)322 anv_cmd_buffer_emit_bt_pool_base_address(struct anv_cmd_buffer *cmd_buffer)
323 {
324 const struct intel_device_info *devinfo = cmd_buffer->device->info;
325 anv_genX(devinfo, cmd_buffer_emit_bt_pool_base_address)(cmd_buffer);
326 }
327
328 void
anv_cmd_buffer_mark_image_written(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,VkImageAspectFlagBits aspect,enum isl_aux_usage aux_usage,uint32_t level,uint32_t base_layer,uint32_t layer_count)329 anv_cmd_buffer_mark_image_written(struct anv_cmd_buffer *cmd_buffer,
330 const struct anv_image *image,
331 VkImageAspectFlagBits aspect,
332 enum isl_aux_usage aux_usage,
333 uint32_t level,
334 uint32_t base_layer,
335 uint32_t layer_count)
336 {
337 const struct intel_device_info *devinfo = cmd_buffer->device->info;
338 anv_genX(devinfo, cmd_buffer_mark_image_written)(cmd_buffer, image,
339 aspect, aux_usage,
340 level, base_layer,
341 layer_count);
342 }
343
344 void
anv_cmd_buffer_mark_image_fast_cleared(struct anv_cmd_buffer * cmd_buffer,const struct anv_image * image,const enum isl_format format,union isl_color_value clear_color)345 anv_cmd_buffer_mark_image_fast_cleared(struct anv_cmd_buffer *cmd_buffer,
346 const struct anv_image *image,
347 const enum isl_format format,
348 union isl_color_value clear_color)
349 {
350 const struct intel_device_info *devinfo = cmd_buffer->device->info;
351 anv_genX(devinfo, set_fast_clear_state)(cmd_buffer, image, format,
352 clear_color);
353 }
354
355 void
anv_cmd_buffer_load_clear_color_from_image(struct anv_cmd_buffer * cmd_buffer,struct anv_state state,const struct anv_image * image)356 anv_cmd_buffer_load_clear_color_from_image(struct anv_cmd_buffer *cmd_buffer,
357 struct anv_state state,
358 const struct anv_image *image)
359 {
360 const struct intel_device_info *devinfo = cmd_buffer->device->info;
361 anv_genX(devinfo, load_image_clear_color)(cmd_buffer, state, image);
362 }
363
364 void
anv_cmd_emit_conditional_render_predicate(struct anv_cmd_buffer * cmd_buffer)365 anv_cmd_emit_conditional_render_predicate(struct anv_cmd_buffer *cmd_buffer)
366 {
367 const struct intel_device_info *devinfo = cmd_buffer->device->info;
368 anv_genX(devinfo, cmd_emit_conditional_render_predicate)(cmd_buffer);
369 }
370
371 static void
clear_pending_query_bits(enum anv_query_bits * query_bits,enum anv_pipe_bits flushed_bits)372 clear_pending_query_bits(enum anv_query_bits *query_bits,
373 enum anv_pipe_bits flushed_bits)
374 {
375 if (flushed_bits & ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT)
376 *query_bits &= ~ANV_QUERY_WRITES_RT_FLUSH;
377
378 if (flushed_bits & ANV_PIPE_TILE_CACHE_FLUSH_BIT)
379 *query_bits &= ~ANV_QUERY_WRITES_TILE_FLUSH;
380
381 if ((flushed_bits & ANV_PIPE_DATA_CACHE_FLUSH_BIT) &&
382 (flushed_bits & ANV_PIPE_HDC_PIPELINE_FLUSH_BIT) &&
383 (flushed_bits & ANV_PIPE_UNTYPED_DATAPORT_CACHE_FLUSH_BIT))
384 *query_bits &= ~ANV_QUERY_WRITES_TILE_FLUSH;
385
386 /* Once RT/TILE have been flushed, we can consider the CS_STALL flush */
387 if ((*query_bits & (ANV_QUERY_WRITES_TILE_FLUSH |
388 ANV_QUERY_WRITES_RT_FLUSH |
389 ANV_QUERY_WRITES_DATA_FLUSH)) == 0 &&
390 (flushed_bits & (ANV_PIPE_END_OF_PIPE_SYNC_BIT | ANV_PIPE_CS_STALL_BIT)))
391 *query_bits &= ~ANV_QUERY_WRITES_CS_STALL;
392 }
393
394 void
anv_cmd_buffer_update_pending_query_bits(struct anv_cmd_buffer * cmd_buffer,enum anv_pipe_bits flushed_bits)395 anv_cmd_buffer_update_pending_query_bits(struct anv_cmd_buffer *cmd_buffer,
396 enum anv_pipe_bits flushed_bits)
397 {
398 clear_pending_query_bits(&cmd_buffer->state.queries.clear_bits, flushed_bits);
399 clear_pending_query_bits(&cmd_buffer->state.queries.buffer_write_bits, flushed_bits);
400 }
401
402 static bool
mem_update(void * dst,const void * src,size_t size)403 mem_update(void *dst, const void *src, size_t size)
404 {
405 if (memcmp(dst, src, size) == 0)
406 return false;
407
408 memcpy(dst, src, size);
409 return true;
410 }
411
412 static void
set_dirty_for_bind_map(struct anv_cmd_buffer * cmd_buffer,gl_shader_stage stage,const struct anv_pipeline_bind_map * map)413 set_dirty_for_bind_map(struct anv_cmd_buffer *cmd_buffer,
414 gl_shader_stage stage,
415 const struct anv_pipeline_bind_map *map)
416 {
417 assert(stage < ARRAY_SIZE(cmd_buffer->state.surface_sha1s));
418 if (mem_update(cmd_buffer->state.surface_sha1s[stage],
419 map->surface_sha1, sizeof(map->surface_sha1)))
420 cmd_buffer->state.descriptors_dirty |= mesa_to_vk_shader_stage(stage);
421
422 assert(stage < ARRAY_SIZE(cmd_buffer->state.sampler_sha1s));
423 if (mem_update(cmd_buffer->state.sampler_sha1s[stage],
424 map->sampler_sha1, sizeof(map->sampler_sha1)))
425 cmd_buffer->state.descriptors_dirty |= mesa_to_vk_shader_stage(stage);
426
427 assert(stage < ARRAY_SIZE(cmd_buffer->state.push_sha1s));
428 if (mem_update(cmd_buffer->state.push_sha1s[stage],
429 map->push_sha1, sizeof(map->push_sha1)))
430 cmd_buffer->state.push_constants_dirty |= mesa_to_vk_shader_stage(stage);
431 }
432
433 static void
anv_cmd_buffer_set_ray_query_buffer(struct anv_cmd_buffer * cmd_buffer,struct anv_cmd_pipeline_state * pipeline_state,struct anv_pipeline * pipeline,VkShaderStageFlags stages)434 anv_cmd_buffer_set_ray_query_buffer(struct anv_cmd_buffer *cmd_buffer,
435 struct anv_cmd_pipeline_state *pipeline_state,
436 struct anv_pipeline *pipeline,
437 VkShaderStageFlags stages)
438 {
439 struct anv_device *device = cmd_buffer->device;
440
441 uint64_t ray_shadow_size =
442 align64(brw_rt_ray_queries_shadow_stacks_size(device->info,
443 pipeline->ray_queries),
444 4096);
445 if (ray_shadow_size > 0 &&
446 (!cmd_buffer->state.ray_query_shadow_bo ||
447 cmd_buffer->state.ray_query_shadow_bo->size < ray_shadow_size)) {
448 unsigned shadow_size_log2 = MAX2(util_logbase2_ceil(ray_shadow_size), 16);
449 unsigned bucket = shadow_size_log2 - 16;
450 assert(bucket < ARRAY_SIZE(device->ray_query_shadow_bos));
451
452 struct anv_bo *bo = p_atomic_read(&device->ray_query_shadow_bos[bucket]);
453 if (bo == NULL) {
454 struct anv_bo *new_bo;
455 VkResult result = anv_device_alloc_bo(device, "RT queries shadow",
456 ray_shadow_size,
457 ANV_BO_ALLOC_INTERNAL, /* alloc_flags */
458 0, /* explicit_address */
459 &new_bo);
460 if (result != VK_SUCCESS) {
461 anv_batch_set_error(&cmd_buffer->batch, result);
462 return;
463 }
464
465 bo = p_atomic_cmpxchg(&device->ray_query_shadow_bos[bucket], NULL, new_bo);
466 if (bo != NULL) {
467 anv_device_release_bo(device, new_bo);
468 } else {
469 bo = new_bo;
470 }
471 }
472 cmd_buffer->state.ray_query_shadow_bo = bo;
473
474 /* Add the ray query buffers to the batch list. */
475 anv_reloc_list_add_bo(cmd_buffer->batch.relocs,
476 cmd_buffer->state.ray_query_shadow_bo);
477 }
478
479 /* Add the HW buffer to the list of BO used. */
480 assert(device->ray_query_bo);
481 anv_reloc_list_add_bo(cmd_buffer->batch.relocs,
482 device->ray_query_bo);
483
484 /* Fill the push constants & mark them dirty. */
485 struct anv_address ray_query_globals_addr =
486 anv_genX(device->info, cmd_buffer_ray_query_globals)(cmd_buffer);
487 pipeline_state->push_constants.ray_query_globals =
488 anv_address_physical(ray_query_globals_addr);
489 cmd_buffer->state.push_constants_dirty |= stages;
490 pipeline_state->push_constants_data_dirty = true;
491 }
492
493 /**
494 * This function compute changes between 2 pipelines and flags the dirty HW
495 * state appropriately.
496 */
497 static void
anv_cmd_buffer_flush_pipeline_state(struct anv_cmd_buffer * cmd_buffer,struct anv_graphics_pipeline * old_pipeline,struct anv_graphics_pipeline * new_pipeline)498 anv_cmd_buffer_flush_pipeline_state(struct anv_cmd_buffer *cmd_buffer,
499 struct anv_graphics_pipeline *old_pipeline,
500 struct anv_graphics_pipeline *new_pipeline)
501 {
502 struct anv_cmd_graphics_state *gfx = &cmd_buffer->state.gfx;
503 struct anv_gfx_dynamic_state *hw_state = &gfx->dyn_state;
504
505 #define diff_fix_state(bit, name) \
506 do { \
507 /* Fixed states should always have matching sizes */ \
508 assert(old_pipeline == NULL || \
509 old_pipeline->name.len == new_pipeline->name.len); \
510 /* Don't bother memcmp if the state is already dirty */ \
511 if (!BITSET_TEST(hw_state->dirty, ANV_GFX_STATE_##bit) && \
512 (old_pipeline == NULL || \
513 memcmp(&old_pipeline->batch_data[old_pipeline->name.offset], \
514 &new_pipeline->batch_data[new_pipeline->name.offset], \
515 4 * new_pipeline->name.len) != 0)) \
516 BITSET_SET(hw_state->dirty, ANV_GFX_STATE_##bit); \
517 } while (0)
518 #define diff_var_state(bit, name) \
519 do { \
520 /* Don't bother memcmp if the state is already dirty */ \
521 /* Also if the new state is empty, avoid marking dirty */ \
522 if (!BITSET_TEST(hw_state->dirty, ANV_GFX_STATE_##bit) && \
523 new_pipeline->name.len != 0 && \
524 (old_pipeline == NULL || \
525 old_pipeline->name.len != new_pipeline->name.len || \
526 memcmp(&old_pipeline->batch_data[old_pipeline->name.offset], \
527 &new_pipeline->batch_data[new_pipeline->name.offset], \
528 4 * new_pipeline->name.len) != 0)) \
529 BITSET_SET(hw_state->dirty, ANV_GFX_STATE_##bit); \
530 } while (0)
531 #define assert_identical(bit, name) \
532 do { \
533 /* Fixed states should always have matching sizes */ \
534 assert(old_pipeline == NULL || \
535 old_pipeline->name.len == new_pipeline->name.len); \
536 assert(old_pipeline == NULL || \
537 memcmp(&old_pipeline->batch_data[old_pipeline->name.offset], \
538 &new_pipeline->batch_data[new_pipeline->name.offset], \
539 4 * new_pipeline->name.len) == 0); \
540 } while (0)
541 #define assert_empty(name) assert(new_pipeline->name.len == 0)
542
543 /* Compare all states, including partial packed ones, the dynamic part is
544 * left at 0 but the static part could still change.
545 *
546 * We avoid comparing protected packets as all the fields but the scratch
547 * surface are identical. we just need to select the right one at emission.
548 */
549 diff_fix_state(URB, final.urb);
550 diff_fix_state(VF_SGVS, final.vf_sgvs);
551 if (cmd_buffer->device->info->ver >= 11)
552 diff_fix_state(VF_SGVS_2, final.vf_sgvs_2);
553 if (cmd_buffer->device->info->ver >= 12)
554 diff_fix_state(PRIMITIVE_REPLICATION, final.primitive_replication);
555 diff_fix_state(SBE, final.sbe);
556 diff_fix_state(SBE_SWIZ, final.sbe_swiz);
557 diff_fix_state(VS, final.vs);
558 diff_fix_state(HS, final.hs);
559 diff_fix_state(DS, final.ds);
560
561 diff_fix_state(CLIP, partial.clip);
562 diff_fix_state(SF, partial.sf);
563 diff_fix_state(RASTER, partial.raster);
564 diff_fix_state(MULTISAMPLE, partial.ms);
565 diff_fix_state(WM, partial.wm);
566 diff_fix_state(STREAMOUT, partial.so);
567 diff_fix_state(GS, partial.gs);
568 diff_fix_state(TE, partial.te);
569 diff_fix_state(VFG, partial.vfg);
570 diff_fix_state(PS, partial.ps);
571 diff_fix_state(PS_EXTRA, partial.ps_extra);
572
573 if (cmd_buffer->device->vk.enabled_extensions.EXT_mesh_shader) {
574 diff_fix_state(TASK_CONTROL, final.task_control);
575 diff_fix_state(TASK_SHADER, final.task_shader);
576 diff_fix_state(TASK_REDISTRIB, final.task_redistrib);
577 diff_fix_state(MESH_CONTROL, final.mesh_control);
578 diff_fix_state(MESH_SHADER, final.mesh_shader);
579 diff_fix_state(MESH_DISTRIB, final.mesh_distrib);
580 diff_fix_state(CLIP_MESH, final.clip_mesh);
581 diff_fix_state(SBE_MESH, final.sbe_mesh);
582 } else {
583 assert_empty(final.task_control);
584 assert_empty(final.task_shader);
585 assert_empty(final.task_redistrib);
586 assert_empty(final.mesh_control);
587 assert_empty(final.mesh_shader);
588 assert_empty(final.mesh_distrib);
589 assert_empty(final.clip_mesh);
590 assert_empty(final.sbe_mesh);
591 }
592
593 /* States that should never vary between pipelines, but can be affected by
594 * blorp etc...
595 */
596 assert_identical(VF_STATISTICS, final.vf_statistics);
597
598 /* States that can vary in length */
599 diff_var_state(VF_SGVS_INSTANCING, final.vf_sgvs_instancing);
600 diff_var_state(SO_DECL_LIST, final.so_decl_list);
601
602 #undef diff_fix_state
603 #undef diff_var_state
604 #undef assert_identical
605 #undef assert_empty
606
607 /* We're not diffing the following :
608 * - anv_graphics_pipeline::vertex_input_data
609 * - anv_graphics_pipeline::final::vf_instancing
610 *
611 * since they are tracked by the runtime.
612 */
613 }
614
anv_CmdBindPipeline(VkCommandBuffer commandBuffer,VkPipelineBindPoint pipelineBindPoint,VkPipeline _pipeline)615 void anv_CmdBindPipeline(
616 VkCommandBuffer commandBuffer,
617 VkPipelineBindPoint pipelineBindPoint,
618 VkPipeline _pipeline)
619 {
620 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
621 ANV_FROM_HANDLE(anv_pipeline, pipeline, _pipeline);
622 struct anv_cmd_pipeline_state *state;
623 VkShaderStageFlags stages = 0;
624
625 switch (pipelineBindPoint) {
626 case VK_PIPELINE_BIND_POINT_COMPUTE: {
627 if (cmd_buffer->state.compute.base.pipeline == pipeline)
628 return;
629
630 cmd_buffer->state.compute.base.pipeline = pipeline;
631 cmd_buffer->state.compute.pipeline_dirty = true;
632
633 struct anv_compute_pipeline *compute_pipeline =
634 anv_pipeline_to_compute(pipeline);
635 set_dirty_for_bind_map(cmd_buffer, MESA_SHADER_COMPUTE,
636 &compute_pipeline->cs->bind_map);
637
638 state = &cmd_buffer->state.compute.base;
639 stages = VK_SHADER_STAGE_COMPUTE_BIT;
640 break;
641 }
642
643 case VK_PIPELINE_BIND_POINT_GRAPHICS: {
644 struct anv_graphics_pipeline *new_pipeline =
645 anv_pipeline_to_graphics(pipeline);
646
647 /* Apply the non dynamic state from the pipeline */
648 vk_cmd_set_dynamic_graphics_state(&cmd_buffer->vk,
649 &new_pipeline->dynamic_state);
650
651 if (cmd_buffer->state.gfx.base.pipeline == pipeline)
652 return;
653
654 struct anv_graphics_pipeline *old_pipeline =
655 cmd_buffer->state.gfx.base.pipeline == NULL ? NULL :
656 anv_pipeline_to_graphics(cmd_buffer->state.gfx.base.pipeline);
657
658 cmd_buffer->state.gfx.base.pipeline = pipeline;
659 cmd_buffer->state.gfx.dirty |= ANV_CMD_DIRTY_PIPELINE;
660
661 anv_foreach_stage(stage, new_pipeline->base.base.active_stages) {
662 set_dirty_for_bind_map(cmd_buffer, stage,
663 &new_pipeline->base.shaders[stage]->bind_map);
664 }
665
666 state = &cmd_buffer->state.gfx.base;
667 stages = new_pipeline->base.base.active_stages;
668
669
670 /* When the pipeline is using independent states and dynamic buffers,
671 * this will trigger an update of anv_push_constants::dynamic_base_index
672 * & anv_push_constants::dynamic_offsets.
673 */
674 struct anv_push_constants *push =
675 &cmd_buffer->state.gfx.base.push_constants;
676 struct anv_pipeline_sets_layout *layout = &new_pipeline->base.base.layout;
677 if (layout->independent_sets && layout->num_dynamic_buffers > 0) {
678 bool modified = false;
679 for (uint32_t s = 0; s < layout->num_sets; s++) {
680 if (layout->set[s].layout == NULL)
681 continue;
682
683 assert(layout->set[s].dynamic_offset_start < MAX_DYNAMIC_BUFFERS);
684 if (layout->set[s].layout->dynamic_offset_count > 0 &&
685 (push->desc_surface_offsets[s] & ANV_DESCRIPTOR_SET_DYNAMIC_INDEX_MASK) !=
686 layout->set[s].dynamic_offset_start) {
687 push->desc_surface_offsets[s] &= ~ANV_DESCRIPTOR_SET_DYNAMIC_INDEX_MASK;
688 push->desc_surface_offsets[s] |= (layout->set[s].dynamic_offset_start &
689 ANV_DESCRIPTOR_SET_DYNAMIC_INDEX_MASK);
690 modified = true;
691 }
692 }
693 if (modified) {
694 cmd_buffer->state.push_constants_dirty |= stages;
695 state->push_constants_data_dirty = true;
696 }
697 }
698
699 anv_cmd_buffer_flush_pipeline_state(cmd_buffer, old_pipeline, new_pipeline);
700 break;
701 }
702
703 case VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR: {
704 if (cmd_buffer->state.rt.base.pipeline == pipeline)
705 return;
706
707 cmd_buffer->state.rt.base.pipeline = pipeline;
708 cmd_buffer->state.rt.pipeline_dirty = true;
709
710 struct anv_ray_tracing_pipeline *rt_pipeline =
711 anv_pipeline_to_ray_tracing(pipeline);
712 if (rt_pipeline->stack_size > 0) {
713 anv_CmdSetRayTracingPipelineStackSizeKHR(commandBuffer,
714 rt_pipeline->stack_size);
715 }
716
717 state = &cmd_buffer->state.rt.base;
718 break;
719 }
720
721 default:
722 unreachable("invalid bind point");
723 break;
724 }
725
726 if (pipeline->ray_queries > 0)
727 anv_cmd_buffer_set_ray_query_buffer(cmd_buffer, state, pipeline, stages);
728 }
729
730 static struct anv_cmd_pipeline_state *
anv_cmd_buffer_get_pipeline_layout_state(struct anv_cmd_buffer * cmd_buffer,VkPipelineBindPoint bind_point,const struct anv_descriptor_set_layout * set_layout,VkShaderStageFlags * out_stages)731 anv_cmd_buffer_get_pipeline_layout_state(struct anv_cmd_buffer *cmd_buffer,
732 VkPipelineBindPoint bind_point,
733 const struct anv_descriptor_set_layout *set_layout,
734 VkShaderStageFlags *out_stages)
735 {
736 *out_stages = set_layout->shader_stages;
737
738 switch (bind_point) {
739 case VK_PIPELINE_BIND_POINT_GRAPHICS:
740 *out_stages &= VK_SHADER_STAGE_ALL_GRAPHICS |
741 (cmd_buffer->device->vk.enabled_extensions.EXT_mesh_shader ?
742 (VK_SHADER_STAGE_TASK_BIT_EXT |
743 VK_SHADER_STAGE_MESH_BIT_EXT) : 0);
744 return &cmd_buffer->state.gfx.base;
745
746 case VK_PIPELINE_BIND_POINT_COMPUTE:
747 *out_stages &= VK_SHADER_STAGE_COMPUTE_BIT;
748 return &cmd_buffer->state.compute.base;
749
750 case VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR:
751 *out_stages &= VK_SHADER_STAGE_RAYGEN_BIT_KHR |
752 VK_SHADER_STAGE_ANY_HIT_BIT_KHR |
753 VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR |
754 VK_SHADER_STAGE_MISS_BIT_KHR |
755 VK_SHADER_STAGE_INTERSECTION_BIT_KHR |
756 VK_SHADER_STAGE_CALLABLE_BIT_KHR;
757 return &cmd_buffer->state.rt.base;
758
759 default:
760 unreachable("invalid bind point");
761 }
762 }
763
764 static void
anv_cmd_buffer_maybe_dirty_descriptor_mode(struct anv_cmd_buffer * cmd_buffer,enum anv_cmd_descriptor_buffer_mode new_mode)765 anv_cmd_buffer_maybe_dirty_descriptor_mode(struct anv_cmd_buffer *cmd_buffer,
766 enum anv_cmd_descriptor_buffer_mode new_mode)
767 {
768 if (cmd_buffer->state.current_db_mode == new_mode)
769 return;
770
771 /* Ensure we program the STATE_BASE_ADDRESS properly at least once */
772 cmd_buffer->state.descriptor_buffers.dirty = true;
773 cmd_buffer->state.pending_db_mode = new_mode;
774 }
775
776 static void
anv_cmd_buffer_bind_descriptor_set(struct anv_cmd_buffer * cmd_buffer,VkPipelineBindPoint bind_point,struct anv_pipeline_sets_layout * layout,uint32_t set_index,struct anv_descriptor_set * set,uint32_t * dynamic_offset_count,const uint32_t ** dynamic_offsets)777 anv_cmd_buffer_bind_descriptor_set(struct anv_cmd_buffer *cmd_buffer,
778 VkPipelineBindPoint bind_point,
779 struct anv_pipeline_sets_layout *layout,
780 uint32_t set_index,
781 struct anv_descriptor_set *set,
782 uint32_t *dynamic_offset_count,
783 const uint32_t **dynamic_offsets)
784 {
785 /* Either we have no pool because it's a push descriptor or the pool is not
786 * host only :
787 *
788 * VUID-vkCmdBindDescriptorSets-pDescriptorSets-04616:
789 *
790 * "Each element of pDescriptorSets must not have been allocated from a
791 * VkDescriptorPool with the
792 * VK_DESCRIPTOR_POOL_CREATE_HOST_ONLY_BIT_EXT flag set"
793 */
794 assert(!set->pool || !set->pool->host_only);
795
796 struct anv_descriptor_set_layout *set_layout =
797 layout->set[set_index].layout;
798
799 anv_cmd_buffer_maybe_dirty_descriptor_mode(
800 cmd_buffer,
801 (set->layout->flags &
802 VK_DESCRIPTOR_SET_LAYOUT_CREATE_DESCRIPTOR_BUFFER_BIT_EXT) != 0 ?
803 ANV_CMD_DESCRIPTOR_BUFFER_MODE_BUFFER :
804 ANV_CMD_DESCRIPTOR_BUFFER_MODE_LEGACY);
805
806 VkShaderStageFlags stages;
807 struct anv_cmd_pipeline_state *pipe_state =
808 anv_cmd_buffer_get_pipeline_layout_state(cmd_buffer, bind_point,
809 set_layout, &stages);
810
811 VkShaderStageFlags dirty_stages = 0;
812 /* If it's a push descriptor set, we have to flag things as dirty
813 * regardless of whether or not the CPU-side data structure changed as we
814 * may have edited in-place.
815 */
816 if (pipe_state->descriptors[set_index] != set ||
817 anv_descriptor_set_is_push(set)) {
818 pipe_state->descriptors[set_index] = set;
819
820 if (set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_DESCRIPTOR_BUFFER_BIT_EXT) {
821 assert(set->is_push);
822
823 pipe_state->descriptor_buffers[set_index].buffer_index = -1;
824 pipe_state->descriptor_buffers[set_index].buffer_offset = set->desc_offset;
825 pipe_state->descriptor_buffers[set_index].bound = true;
826 cmd_buffer->state.descriptors_dirty |= stages;
827 cmd_buffer->state.descriptor_buffers.offsets_dirty |= stages;
828 } else {
829 /* When using indirect descriptors, stages that have access to the HW
830 * binding tables, never need to access the
831 * anv_push_constants::desc_offsets fields, because any data they
832 * need from the descriptor buffer is accessible through a binding
833 * table entry. For stages that are "bindless" (Mesh/Task/RT), we
834 * need to provide anv_push_constants::desc_offsets matching the
835 * bound descriptor so that shaders can access the descriptor buffer
836 * through A64 messages.
837 *
838 * With direct descriptors, the shaders can use the
839 * anv_push_constants::desc_offsets to build bindless offsets. So
840 * it's we always need to update the push constant data.
841 */
842 bool update_desc_sets =
843 !cmd_buffer->device->physical->indirect_descriptors ||
844 (stages & (VK_SHADER_STAGE_TASK_BIT_EXT |
845 VK_SHADER_STAGE_MESH_BIT_EXT |
846 VK_SHADER_STAGE_RAYGEN_BIT_KHR |
847 VK_SHADER_STAGE_ANY_HIT_BIT_KHR |
848 VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR |
849 VK_SHADER_STAGE_MISS_BIT_KHR |
850 VK_SHADER_STAGE_INTERSECTION_BIT_KHR |
851 VK_SHADER_STAGE_CALLABLE_BIT_KHR));
852
853 if (update_desc_sets) {
854 struct anv_push_constants *push = &pipe_state->push_constants;
855 uint64_t offset =
856 anv_address_physical(set->desc_surface_addr) -
857 cmd_buffer->device->physical->va.internal_surface_state_pool.addr;
858 assert((offset & ~ANV_DESCRIPTOR_SET_OFFSET_MASK) == 0);
859 push->desc_surface_offsets[set_index] &= ~ANV_DESCRIPTOR_SET_OFFSET_MASK;
860 push->desc_surface_offsets[set_index] |= offset;
861 push->desc_sampler_offsets[set_index] =
862 anv_address_physical(set->desc_sampler_addr) -
863 cmd_buffer->device->physical->va.dynamic_state_pool.addr;
864
865 anv_reloc_list_add_bo(cmd_buffer->batch.relocs,
866 set->desc_surface_addr.bo);
867 anv_reloc_list_add_bo(cmd_buffer->batch.relocs,
868 set->desc_sampler_addr.bo);
869 }
870 }
871
872 dirty_stages |= stages;
873 }
874
875 if (dynamic_offsets) {
876 if (set_layout->dynamic_offset_count > 0) {
877 struct anv_push_constants *push = &pipe_state->push_constants;
878 uint32_t dynamic_offset_start =
879 layout->set[set_index].dynamic_offset_start;
880 uint32_t *push_offsets =
881 &push->dynamic_offsets[dynamic_offset_start];
882
883 memcpy(pipe_state->dynamic_offsets[set_index].offsets,
884 *dynamic_offsets,
885 sizeof(uint32_t) * MIN2(*dynamic_offset_count,
886 set_layout->dynamic_offset_count));
887
888 /* Assert that everything is in range */
889 assert(set_layout->dynamic_offset_count <= *dynamic_offset_count);
890 assert(dynamic_offset_start + set_layout->dynamic_offset_count <=
891 ARRAY_SIZE(push->dynamic_offsets));
892
893 for (uint32_t i = 0; i < set_layout->dynamic_offset_count; i++) {
894 if (push_offsets[i] != (*dynamic_offsets)[i]) {
895 pipe_state->dynamic_offsets[set_index].offsets[i] =
896 push_offsets[i] = (*dynamic_offsets)[i];
897 /* dynamic_offset_stages[] elements could contain blanket
898 * values like VK_SHADER_STAGE_ALL, so limit this to the
899 * binding point's bits.
900 */
901 dirty_stages |= set_layout->dynamic_offset_stages[i] & stages;
902 }
903 }
904
905 *dynamic_offsets += set_layout->dynamic_offset_count;
906 *dynamic_offset_count -= set_layout->dynamic_offset_count;
907 }
908 }
909
910 if (set->is_push)
911 cmd_buffer->state.push_descriptors_dirty |= dirty_stages;
912 else
913 cmd_buffer->state.descriptors_dirty |= dirty_stages;
914 cmd_buffer->state.push_constants_dirty |= dirty_stages;
915 pipe_state->push_constants_data_dirty = true;
916 }
917
918 #define ANV_GRAPHICS_STAGE_BITS \
919 (VK_SHADER_STAGE_ALL_GRAPHICS | \
920 VK_SHADER_STAGE_MESH_BIT_EXT | \
921 VK_SHADER_STAGE_TASK_BIT_EXT)
922
923 #define ANV_RT_STAGE_BITS \
924 (VK_SHADER_STAGE_RAYGEN_BIT_KHR | \
925 VK_SHADER_STAGE_ANY_HIT_BIT_KHR | \
926 VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR | \
927 VK_SHADER_STAGE_MISS_BIT_KHR | \
928 VK_SHADER_STAGE_INTERSECTION_BIT_KHR | \
929 VK_SHADER_STAGE_CALLABLE_BIT_KHR)
930
anv_CmdBindDescriptorSets2KHR(VkCommandBuffer commandBuffer,const VkBindDescriptorSetsInfoKHR * pInfo)931 void anv_CmdBindDescriptorSets2KHR(
932 VkCommandBuffer commandBuffer,
933 const VkBindDescriptorSetsInfoKHR* pInfo)
934 {
935 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
936 ANV_FROM_HANDLE(anv_pipeline_layout, pipeline_layout, pInfo->layout);
937 struct anv_pipeline_sets_layout *layout = &pipeline_layout->sets_layout;
938
939 assert(pInfo->firstSet + pInfo->descriptorSetCount <= MAX_SETS);
940
941 if (pInfo->stageFlags & VK_SHADER_STAGE_COMPUTE_BIT) {
942 uint32_t dynamicOffsetCount = pInfo->dynamicOffsetCount;
943 const uint32_t *pDynamicOffsets = pInfo->pDynamicOffsets;
944
945 for (uint32_t i = 0; i < pInfo->descriptorSetCount; i++) {
946 ANV_FROM_HANDLE(anv_descriptor_set, set, pInfo->pDescriptorSets[i]);
947 if (set == NULL)
948 continue;
949 anv_cmd_buffer_bind_descriptor_set(cmd_buffer,
950 VK_PIPELINE_BIND_POINT_COMPUTE,
951 layout, pInfo->firstSet + i, set,
952 &dynamicOffsetCount,
953 &pDynamicOffsets);
954 }
955 }
956 if (pInfo->stageFlags & ANV_GRAPHICS_STAGE_BITS) {
957 uint32_t dynamicOffsetCount = pInfo->dynamicOffsetCount;
958 const uint32_t *pDynamicOffsets = pInfo->pDynamicOffsets;
959
960 for (uint32_t i = 0; i < pInfo->descriptorSetCount; i++) {
961 ANV_FROM_HANDLE(anv_descriptor_set, set, pInfo->pDescriptorSets[i]);
962 if (set == NULL)
963 continue;
964 anv_cmd_buffer_bind_descriptor_set(cmd_buffer,
965 VK_PIPELINE_BIND_POINT_GRAPHICS,
966 layout, pInfo->firstSet + i, set,
967 &dynamicOffsetCount,
968 &pDynamicOffsets);
969 }
970 }
971 if (pInfo->stageFlags & ANV_RT_STAGE_BITS) {
972 uint32_t dynamicOffsetCount = pInfo->dynamicOffsetCount;
973 const uint32_t *pDynamicOffsets = pInfo->pDynamicOffsets;
974
975 for (uint32_t i = 0; i < pInfo->descriptorSetCount; i++) {
976 ANV_FROM_HANDLE(anv_descriptor_set, set, pInfo->pDescriptorSets[i]);
977 if (set == NULL)
978 continue;
979 anv_cmd_buffer_bind_descriptor_set(cmd_buffer,
980 VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR,
981 layout, pInfo->firstSet + i, set,
982 &dynamicOffsetCount,
983 &pDynamicOffsets);
984 }
985 }
986 }
987
anv_CmdBindDescriptorBuffersEXT(VkCommandBuffer commandBuffer,uint32_t bufferCount,const VkDescriptorBufferBindingInfoEXT * pBindingInfos)988 void anv_CmdBindDescriptorBuffersEXT(
989 VkCommandBuffer commandBuffer,
990 uint32_t bufferCount,
991 const VkDescriptorBufferBindingInfoEXT* pBindingInfos)
992 {
993 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
994 struct anv_cmd_state *state = &cmd_buffer->state;
995
996 for (uint32_t i = 0; i < bufferCount; i++) {
997 assert(pBindingInfos[i].address >= cmd_buffer->device->physical->va.dynamic_visible_pool.addr &&
998 pBindingInfos[i].address < (cmd_buffer->device->physical->va.dynamic_visible_pool.addr +
999 cmd_buffer->device->physical->va.dynamic_visible_pool.size));
1000
1001 if (state->descriptor_buffers.address[i] != pBindingInfos[i].address) {
1002 state->descriptor_buffers.address[i] = pBindingInfos[i].address;
1003 if (pBindingInfos[i].usage & VK_BUFFER_USAGE_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT)
1004 state->descriptor_buffers.surfaces_address = pBindingInfos[i].address;
1005 if (pBindingInfos[i].usage & VK_BUFFER_USAGE_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT)
1006 state->descriptor_buffers.samplers_address = pBindingInfos[i].address;
1007 state->descriptor_buffers.dirty = true;
1008 state->descriptor_buffers.offsets_dirty = ~0;
1009 }
1010 }
1011
1012 anv_cmd_buffer_maybe_dirty_descriptor_mode(cmd_buffer,
1013 ANV_CMD_DESCRIPTOR_BUFFER_MODE_BUFFER);
1014 }
1015
1016 static void
anv_cmd_buffer_set_descriptor_buffer_offsets(struct anv_cmd_buffer * cmd_buffer,VkPipelineBindPoint bind_point,struct anv_pipeline_layout * layout,uint32_t first_set,uint32_t set_count,const VkDeviceSize * buffer_offsets,const uint32_t * buffer_indices)1017 anv_cmd_buffer_set_descriptor_buffer_offsets(struct anv_cmd_buffer *cmd_buffer,
1018 VkPipelineBindPoint bind_point,
1019 struct anv_pipeline_layout *layout,
1020 uint32_t first_set,
1021 uint32_t set_count,
1022 const VkDeviceSize *buffer_offsets,
1023 const uint32_t *buffer_indices)
1024 {
1025 for (uint32_t i = 0; i < set_count; i++) {
1026 const uint32_t set_index = first_set + i;
1027
1028 const struct anv_descriptor_set_layout *set_layout =
1029 layout->sets_layout.set[set_index].layout;
1030 VkShaderStageFlags stages;
1031 struct anv_cmd_pipeline_state *pipe_state =
1032 anv_cmd_buffer_get_pipeline_layout_state(cmd_buffer, bind_point,
1033 set_layout, &stages);
1034
1035 if (buffer_offsets[i] != pipe_state->descriptor_buffers[set_index].buffer_offset ||
1036 buffer_indices[i] != pipe_state->descriptor_buffers[set_index].buffer_index ||
1037 !pipe_state->descriptor_buffers[set_index].bound) {
1038 pipe_state->descriptor_buffers[set_index].buffer_index = buffer_indices[i];
1039 pipe_state->descriptor_buffers[set_index].buffer_offset = buffer_offsets[i];
1040 cmd_buffer->state.descriptors_dirty |= stages;
1041 cmd_buffer->state.descriptor_buffers.offsets_dirty |= stages;
1042 }
1043 pipe_state->descriptor_buffers[set_index].bound = true;
1044 }
1045 }
1046
anv_CmdSetDescriptorBufferOffsets2EXT(VkCommandBuffer commandBuffer,const VkSetDescriptorBufferOffsetsInfoEXT * pSetDescriptorBufferOffsetsInfo)1047 void anv_CmdSetDescriptorBufferOffsets2EXT(
1048 VkCommandBuffer commandBuffer,
1049 const VkSetDescriptorBufferOffsetsInfoEXT* pSetDescriptorBufferOffsetsInfo)
1050 {
1051 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1052 ANV_FROM_HANDLE(anv_pipeline_layout, layout, pSetDescriptorBufferOffsetsInfo->layout);
1053
1054 if (pSetDescriptorBufferOffsetsInfo->stageFlags & VK_SHADER_STAGE_COMPUTE_BIT) {
1055 anv_cmd_buffer_set_descriptor_buffer_offsets(cmd_buffer,
1056 VK_PIPELINE_BIND_POINT_COMPUTE,
1057 layout,
1058 pSetDescriptorBufferOffsetsInfo->firstSet,
1059 pSetDescriptorBufferOffsetsInfo->setCount,
1060 pSetDescriptorBufferOffsetsInfo->pOffsets,
1061 pSetDescriptorBufferOffsetsInfo->pBufferIndices);
1062 }
1063 if (pSetDescriptorBufferOffsetsInfo->stageFlags & ANV_GRAPHICS_STAGE_BITS) {
1064 anv_cmd_buffer_set_descriptor_buffer_offsets(cmd_buffer,
1065 VK_PIPELINE_BIND_POINT_GRAPHICS,
1066 layout,
1067 pSetDescriptorBufferOffsetsInfo->firstSet,
1068 pSetDescriptorBufferOffsetsInfo->setCount,
1069 pSetDescriptorBufferOffsetsInfo->pOffsets,
1070 pSetDescriptorBufferOffsetsInfo->pBufferIndices);
1071 }
1072 if (pSetDescriptorBufferOffsetsInfo->stageFlags & ANV_RT_STAGE_BITS) {
1073 anv_cmd_buffer_set_descriptor_buffer_offsets(cmd_buffer,
1074 VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR,
1075 layout,
1076 pSetDescriptorBufferOffsetsInfo->firstSet,
1077 pSetDescriptorBufferOffsetsInfo->setCount,
1078 pSetDescriptorBufferOffsetsInfo->pOffsets,
1079 pSetDescriptorBufferOffsetsInfo->pBufferIndices);
1080 }
1081 }
1082
anv_CmdBindDescriptorBufferEmbeddedSamplers2EXT(VkCommandBuffer commandBuffer,const VkBindDescriptorBufferEmbeddedSamplersInfoEXT * pBindDescriptorBufferEmbeddedSamplersInfo)1083 void anv_CmdBindDescriptorBufferEmbeddedSamplers2EXT(
1084 VkCommandBuffer commandBuffer,
1085 const VkBindDescriptorBufferEmbeddedSamplersInfoEXT* pBindDescriptorBufferEmbeddedSamplersInfo)
1086 {
1087 /* no-op */
1088 }
1089
anv_CmdBindVertexBuffers2(VkCommandBuffer commandBuffer,uint32_t firstBinding,uint32_t bindingCount,const VkBuffer * pBuffers,const VkDeviceSize * pOffsets,const VkDeviceSize * pSizes,const VkDeviceSize * pStrides)1090 void anv_CmdBindVertexBuffers2(
1091 VkCommandBuffer commandBuffer,
1092 uint32_t firstBinding,
1093 uint32_t bindingCount,
1094 const VkBuffer* pBuffers,
1095 const VkDeviceSize* pOffsets,
1096 const VkDeviceSize* pSizes,
1097 const VkDeviceSize* pStrides)
1098 {
1099 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1100 struct anv_vertex_binding *vb = cmd_buffer->state.vertex_bindings;
1101
1102 /* We have to defer setting up vertex buffer since we need the buffer
1103 * stride from the pipeline. */
1104
1105 assert(firstBinding + bindingCount <= MAX_VBS);
1106 for (uint32_t i = 0; i < bindingCount; i++) {
1107 ANV_FROM_HANDLE(anv_buffer, buffer, pBuffers[i]);
1108
1109 if (buffer == NULL) {
1110 vb[firstBinding + i] = (struct anv_vertex_binding) {
1111 .buffer = NULL,
1112 };
1113 } else {
1114 vb[firstBinding + i] = (struct anv_vertex_binding) {
1115 .buffer = buffer,
1116 .offset = pOffsets[i],
1117 .size = vk_buffer_range(&buffer->vk, pOffsets[i],
1118 pSizes ? pSizes[i] : VK_WHOLE_SIZE),
1119 };
1120 }
1121 cmd_buffer->state.gfx.vb_dirty |= 1 << (firstBinding + i);
1122 }
1123
1124 if (pStrides != NULL) {
1125 vk_cmd_set_vertex_binding_strides(&cmd_buffer->vk, firstBinding,
1126 bindingCount, pStrides);
1127 }
1128 }
1129
anv_CmdBindTransformFeedbackBuffersEXT(VkCommandBuffer commandBuffer,uint32_t firstBinding,uint32_t bindingCount,const VkBuffer * pBuffers,const VkDeviceSize * pOffsets,const VkDeviceSize * pSizes)1130 void anv_CmdBindTransformFeedbackBuffersEXT(
1131 VkCommandBuffer commandBuffer,
1132 uint32_t firstBinding,
1133 uint32_t bindingCount,
1134 const VkBuffer* pBuffers,
1135 const VkDeviceSize* pOffsets,
1136 const VkDeviceSize* pSizes)
1137 {
1138 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1139 struct anv_xfb_binding *xfb = cmd_buffer->state.xfb_bindings;
1140
1141 /* We have to defer setting up vertex buffer since we need the buffer
1142 * stride from the pipeline. */
1143
1144 assert(firstBinding + bindingCount <= MAX_XFB_BUFFERS);
1145 for (uint32_t i = 0; i < bindingCount; i++) {
1146 if (pBuffers[i] == VK_NULL_HANDLE) {
1147 xfb[firstBinding + i].buffer = NULL;
1148 } else {
1149 ANV_FROM_HANDLE(anv_buffer, buffer, pBuffers[i]);
1150 xfb[firstBinding + i].buffer = buffer;
1151 xfb[firstBinding + i].offset = pOffsets[i];
1152 xfb[firstBinding + i].size =
1153 vk_buffer_range(&buffer->vk, pOffsets[i],
1154 pSizes ? pSizes[i] : VK_WHOLE_SIZE);
1155 }
1156 }
1157 }
1158
1159 enum isl_format
anv_isl_format_for_descriptor_type(const struct anv_device * device,VkDescriptorType type)1160 anv_isl_format_for_descriptor_type(const struct anv_device *device,
1161 VkDescriptorType type)
1162 {
1163 switch (type) {
1164 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
1165 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1166 return device->physical->compiler->indirect_ubos_use_sampler ?
1167 ISL_FORMAT_R32G32B32A32_FLOAT : ISL_FORMAT_RAW;
1168
1169 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
1170 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
1171 return ISL_FORMAT_RAW;
1172
1173 default:
1174 unreachable("Invalid descriptor type");
1175 }
1176 }
1177
1178 struct anv_state
anv_cmd_buffer_emit_dynamic(struct anv_cmd_buffer * cmd_buffer,const void * data,uint32_t size,uint32_t alignment)1179 anv_cmd_buffer_emit_dynamic(struct anv_cmd_buffer *cmd_buffer,
1180 const void *data, uint32_t size, uint32_t alignment)
1181 {
1182 struct anv_state state;
1183
1184 state = anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, size, alignment);
1185 memcpy(state.map, data, size);
1186
1187 VG(VALGRIND_CHECK_MEM_IS_DEFINED(state.map, size));
1188
1189 return state;
1190 }
1191
1192 struct anv_state
anv_cmd_buffer_merge_dynamic(struct anv_cmd_buffer * cmd_buffer,uint32_t * a,uint32_t * b,uint32_t dwords,uint32_t alignment)1193 anv_cmd_buffer_merge_dynamic(struct anv_cmd_buffer *cmd_buffer,
1194 uint32_t *a, uint32_t *b,
1195 uint32_t dwords, uint32_t alignment)
1196 {
1197 struct anv_state state;
1198 uint32_t *p;
1199
1200 state = anv_cmd_buffer_alloc_dynamic_state(cmd_buffer,
1201 dwords * 4, alignment);
1202 p = state.map;
1203 for (uint32_t i = 0; i < dwords; i++) {
1204 assert((a[i] & b[i]) == 0);
1205 p[i] = a[i] | b[i];
1206 }
1207
1208 VG(VALGRIND_CHECK_MEM_IS_DEFINED(p, dwords * 4));
1209
1210 return state;
1211 }
1212
1213 struct anv_state
anv_cmd_buffer_gfx_push_constants(struct anv_cmd_buffer * cmd_buffer)1214 anv_cmd_buffer_gfx_push_constants(struct anv_cmd_buffer *cmd_buffer)
1215 {
1216 struct anv_push_constants *data =
1217 &cmd_buffer->state.gfx.base.push_constants;
1218
1219 struct anv_state state =
1220 anv_cmd_buffer_alloc_temporary_state(cmd_buffer,
1221 sizeof(struct anv_push_constants),
1222 32 /* bottom 5 bits MBZ */);
1223 if (state.alloc_size == 0)
1224 return state;
1225
1226 memcpy(state.map, data, sizeof(struct anv_push_constants));
1227
1228 return state;
1229 }
1230
1231 struct anv_state
anv_cmd_buffer_cs_push_constants(struct anv_cmd_buffer * cmd_buffer)1232 anv_cmd_buffer_cs_push_constants(struct anv_cmd_buffer *cmd_buffer)
1233 {
1234 const struct intel_device_info *devinfo = cmd_buffer->device->info;
1235 struct anv_cmd_pipeline_state *pipe_state = &cmd_buffer->state.compute.base;
1236 struct anv_push_constants *data = &pipe_state->push_constants;
1237 struct anv_compute_pipeline *pipeline =
1238 anv_pipeline_to_compute(cmd_buffer->state.compute.base.pipeline);
1239 const struct brw_cs_prog_data *cs_prog_data = get_cs_prog_data(pipeline);
1240 const struct anv_push_range *range = &pipeline->cs->bind_map.push_ranges[0];
1241
1242 const struct intel_cs_dispatch_info dispatch =
1243 brw_cs_get_dispatch_info(devinfo, cs_prog_data, NULL);
1244 const unsigned total_push_constants_size =
1245 brw_cs_push_const_total_size(cs_prog_data, dispatch.threads);
1246 if (total_push_constants_size == 0)
1247 return (struct anv_state) { .offset = 0 };
1248
1249 const unsigned push_constant_alignment = 64;
1250 const unsigned aligned_total_push_constants_size =
1251 ALIGN(total_push_constants_size, push_constant_alignment);
1252 struct anv_state state;
1253 if (devinfo->verx10 >= 125) {
1254 state = anv_state_stream_alloc(&cmd_buffer->general_state_stream,
1255 aligned_total_push_constants_size,
1256 push_constant_alignment);
1257 } else {
1258 state = anv_cmd_buffer_alloc_dynamic_state(cmd_buffer,
1259 aligned_total_push_constants_size,
1260 push_constant_alignment);
1261 }
1262 if (state.map == NULL)
1263 return state;
1264
1265 void *dst = state.map;
1266 const void *src = (char *)data + (range->start * 32);
1267
1268 if (cs_prog_data->push.cross_thread.size > 0) {
1269 memcpy(dst, src, cs_prog_data->push.cross_thread.size);
1270 dst += cs_prog_data->push.cross_thread.size;
1271 src += cs_prog_data->push.cross_thread.size;
1272 }
1273
1274 if (cs_prog_data->push.per_thread.size > 0) {
1275 for (unsigned t = 0; t < dispatch.threads; t++) {
1276 memcpy(dst, src, cs_prog_data->push.per_thread.size);
1277
1278 uint32_t *subgroup_id = dst +
1279 offsetof(struct anv_push_constants, cs.subgroup_id) -
1280 (range->start * 32 + cs_prog_data->push.cross_thread.size);
1281 *subgroup_id = t;
1282
1283 dst += cs_prog_data->push.per_thread.size;
1284 }
1285 }
1286
1287 return state;
1288 }
1289
anv_CmdPushConstants2KHR(VkCommandBuffer commandBuffer,const VkPushConstantsInfoKHR * pInfo)1290 void anv_CmdPushConstants2KHR(
1291 VkCommandBuffer commandBuffer,
1292 const VkPushConstantsInfoKHR* pInfo)
1293 {
1294 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1295
1296 if (pInfo->stageFlags & ANV_GRAPHICS_STAGE_BITS) {
1297 struct anv_cmd_pipeline_state *pipe_state =
1298 &cmd_buffer->state.gfx.base;
1299
1300 memcpy(pipe_state->push_constants.client_data + pInfo->offset,
1301 pInfo->pValues, pInfo->size);
1302 pipe_state->push_constants_data_dirty = true;
1303 }
1304 if (pInfo->stageFlags & VK_SHADER_STAGE_COMPUTE_BIT) {
1305 struct anv_cmd_pipeline_state *pipe_state =
1306 &cmd_buffer->state.compute.base;
1307
1308 memcpy(pipe_state->push_constants.client_data + pInfo->offset,
1309 pInfo->pValues, pInfo->size);
1310 pipe_state->push_constants_data_dirty = true;
1311 }
1312 if (pInfo->stageFlags & ANV_RT_STAGE_BITS) {
1313 struct anv_cmd_pipeline_state *pipe_state =
1314 &cmd_buffer->state.rt.base;
1315
1316 memcpy(pipe_state->push_constants.client_data + pInfo->offset,
1317 pInfo->pValues, pInfo->size);
1318 pipe_state->push_constants_data_dirty = true;
1319 }
1320
1321 cmd_buffer->state.push_constants_dirty |= pInfo->stageFlags;
1322 }
1323
1324 static struct anv_cmd_pipeline_state *
anv_cmd_buffer_get_pipe_state(struct anv_cmd_buffer * cmd_buffer,VkPipelineBindPoint bind_point)1325 anv_cmd_buffer_get_pipe_state(struct anv_cmd_buffer *cmd_buffer,
1326 VkPipelineBindPoint bind_point)
1327 {
1328 switch (bind_point) {
1329 case VK_PIPELINE_BIND_POINT_GRAPHICS:
1330 return &cmd_buffer->state.gfx.base;
1331 case VK_PIPELINE_BIND_POINT_COMPUTE:
1332 return &cmd_buffer->state.compute.base;
1333 case VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR:
1334 return &cmd_buffer->state.rt.base;
1335 break;
1336 default:
1337 unreachable("invalid bind point");
1338 }
1339 }
1340
1341 static void
anv_cmd_buffer_push_descriptor_sets(struct anv_cmd_buffer * cmd_buffer,VkPipelineBindPoint bind_point,const VkPushDescriptorSetInfoKHR * pInfo)1342 anv_cmd_buffer_push_descriptor_sets(struct anv_cmd_buffer *cmd_buffer,
1343 VkPipelineBindPoint bind_point,
1344 const VkPushDescriptorSetInfoKHR *pInfo)
1345 {
1346 ANV_FROM_HANDLE(anv_pipeline_layout, pipeline_layout, pInfo->layout);
1347 struct anv_pipeline_sets_layout *layout = &pipeline_layout->sets_layout;
1348
1349 assert(pInfo->set < MAX_SETS);
1350
1351 struct anv_descriptor_set_layout *set_layout = layout->set[pInfo->set].layout;
1352 struct anv_push_descriptor_set *push_set =
1353 &anv_cmd_buffer_get_pipe_state(cmd_buffer,
1354 bind_point)->push_descriptor;
1355 if (!anv_push_descriptor_set_init(cmd_buffer, push_set, set_layout))
1356 return;
1357
1358 anv_descriptor_set_write(cmd_buffer->device, &push_set->set,
1359 pInfo->descriptorWriteCount,
1360 pInfo->pDescriptorWrites);
1361
1362 anv_cmd_buffer_bind_descriptor_set(cmd_buffer, bind_point,
1363 layout, pInfo->set, &push_set->set,
1364 NULL, NULL);
1365 }
1366
anv_CmdPushDescriptorSet2KHR(VkCommandBuffer commandBuffer,const VkPushDescriptorSetInfoKHR * pInfo)1367 void anv_CmdPushDescriptorSet2KHR(
1368 VkCommandBuffer commandBuffer,
1369 const VkPushDescriptorSetInfoKHR* pInfo)
1370 {
1371 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1372
1373 if (pInfo->stageFlags & VK_SHADER_STAGE_COMPUTE_BIT)
1374 anv_cmd_buffer_push_descriptor_sets(cmd_buffer,
1375 VK_PIPELINE_BIND_POINT_COMPUTE,
1376 pInfo);
1377 if (pInfo->stageFlags & ANV_GRAPHICS_STAGE_BITS)
1378 anv_cmd_buffer_push_descriptor_sets(cmd_buffer,
1379 VK_PIPELINE_BIND_POINT_GRAPHICS,
1380 pInfo);
1381 if (pInfo->stageFlags & ANV_RT_STAGE_BITS)
1382 anv_cmd_buffer_push_descriptor_sets(cmd_buffer,
1383 VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR,
1384 pInfo);
1385 }
1386
anv_CmdPushDescriptorSetWithTemplate2KHR(VkCommandBuffer commandBuffer,const VkPushDescriptorSetWithTemplateInfoKHR * pInfo)1387 void anv_CmdPushDescriptorSetWithTemplate2KHR(
1388 VkCommandBuffer commandBuffer,
1389 const VkPushDescriptorSetWithTemplateInfoKHR* pInfo)
1390 {
1391 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1392 VK_FROM_HANDLE(vk_descriptor_update_template, template,
1393 pInfo->descriptorUpdateTemplate);
1394 ANV_FROM_HANDLE(anv_pipeline_layout, pipeline_layout, pInfo->layout);
1395 struct anv_pipeline_sets_layout *layout = &pipeline_layout->sets_layout;
1396
1397 assert(pInfo->set < MAX_PUSH_DESCRIPTORS);
1398
1399 struct anv_descriptor_set_layout *set_layout = layout->set[pInfo->set].layout;
1400 UNUSED VkShaderStageFlags stages;
1401 struct anv_cmd_pipeline_state *pipe_state =
1402 anv_cmd_buffer_get_pipeline_layout_state(cmd_buffer, template->bind_point,
1403 set_layout, &stages);
1404 struct anv_push_descriptor_set *push_set = &pipe_state->push_descriptor;
1405 if (!anv_push_descriptor_set_init(cmd_buffer, push_set, set_layout))
1406 return;
1407
1408 anv_descriptor_set_write_template(cmd_buffer->device, &push_set->set,
1409 template,
1410 pInfo->pData);
1411
1412 anv_cmd_buffer_bind_descriptor_set(cmd_buffer, template->bind_point,
1413 layout, pInfo->set, &push_set->set,
1414 NULL, NULL);
1415 }
1416
anv_CmdSetRayTracingPipelineStackSizeKHR(VkCommandBuffer commandBuffer,uint32_t pipelineStackSize)1417 void anv_CmdSetRayTracingPipelineStackSizeKHR(
1418 VkCommandBuffer commandBuffer,
1419 uint32_t pipelineStackSize)
1420 {
1421 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1422 struct anv_cmd_ray_tracing_state *rt = &cmd_buffer->state.rt;
1423 struct anv_device *device = cmd_buffer->device;
1424
1425 if (anv_batch_has_error(&cmd_buffer->batch))
1426 return;
1427
1428 uint32_t stack_ids_per_dss = 2048; /* TODO */
1429
1430 unsigned stack_size_log2 = util_logbase2_ceil(pipelineStackSize);
1431 if (stack_size_log2 < 10)
1432 stack_size_log2 = 10;
1433
1434 if (rt->scratch.layout.total_size == 1 << stack_size_log2)
1435 return;
1436
1437 brw_rt_compute_scratch_layout(&rt->scratch.layout, device->info,
1438 stack_ids_per_dss, 1 << stack_size_log2);
1439
1440 unsigned bucket = stack_size_log2 - 10;
1441 assert(bucket < ARRAY_SIZE(device->rt_scratch_bos));
1442
1443 struct anv_bo *bo = p_atomic_read(&device->rt_scratch_bos[bucket]);
1444 if (bo == NULL) {
1445 struct anv_bo *new_bo;
1446 VkResult result = anv_device_alloc_bo(device, "RT scratch",
1447 rt->scratch.layout.total_size,
1448 ANV_BO_ALLOC_INTERNAL, /* alloc_flags */
1449 0, /* explicit_address */
1450 &new_bo);
1451 if (result != VK_SUCCESS) {
1452 rt->scratch.layout.total_size = 0;
1453 anv_batch_set_error(&cmd_buffer->batch, result);
1454 return;
1455 }
1456
1457 bo = p_atomic_cmpxchg(&device->rt_scratch_bos[bucket], NULL, new_bo);
1458 if (bo != NULL) {
1459 anv_device_release_bo(device, bo);
1460 } else {
1461 bo = new_bo;
1462 }
1463 }
1464
1465 rt->scratch.bo = bo;
1466 }
1467
1468 void
anv_cmd_buffer_save_state(struct anv_cmd_buffer * cmd_buffer,uint32_t flags,struct anv_cmd_saved_state * state)1469 anv_cmd_buffer_save_state(struct anv_cmd_buffer *cmd_buffer,
1470 uint32_t flags,
1471 struct anv_cmd_saved_state *state)
1472 {
1473 state->flags = flags;
1474
1475 /* we only support the compute pipeline at the moment */
1476 assert(state->flags & ANV_CMD_SAVED_STATE_COMPUTE_PIPELINE);
1477 const struct anv_cmd_pipeline_state *pipe_state =
1478 &cmd_buffer->state.compute.base;
1479
1480 if (state->flags & ANV_CMD_SAVED_STATE_COMPUTE_PIPELINE)
1481 state->pipeline = pipe_state->pipeline;
1482
1483 if (state->flags & ANV_CMD_SAVED_STATE_DESCRIPTOR_SET_0)
1484 state->descriptor_set = pipe_state->descriptors[0];
1485
1486 if (state->flags & ANV_CMD_SAVED_STATE_PUSH_CONSTANTS) {
1487 memcpy(state->push_constants, pipe_state->push_constants.client_data,
1488 sizeof(state->push_constants));
1489 }
1490 }
1491
1492 void
anv_cmd_buffer_restore_state(struct anv_cmd_buffer * cmd_buffer,struct anv_cmd_saved_state * state)1493 anv_cmd_buffer_restore_state(struct anv_cmd_buffer *cmd_buffer,
1494 struct anv_cmd_saved_state *state)
1495 {
1496 VkCommandBuffer cmd_buffer_ = anv_cmd_buffer_to_handle(cmd_buffer);
1497
1498 assert(state->flags & ANV_CMD_SAVED_STATE_COMPUTE_PIPELINE);
1499 const VkPipelineBindPoint bind_point = VK_PIPELINE_BIND_POINT_COMPUTE;
1500 const VkShaderStageFlags stage_flags = VK_SHADER_STAGE_COMPUTE_BIT;
1501 struct anv_cmd_pipeline_state *pipe_state = &cmd_buffer->state.compute.base;
1502
1503 if (state->flags & ANV_CMD_SAVED_STATE_COMPUTE_PIPELINE) {
1504 if (state->pipeline) {
1505 anv_CmdBindPipeline(cmd_buffer_, bind_point,
1506 anv_pipeline_to_handle(state->pipeline));
1507 } else {
1508 pipe_state->pipeline = NULL;
1509 }
1510 }
1511
1512 if (state->flags & ANV_CMD_SAVED_STATE_DESCRIPTOR_SET_0) {
1513 if (state->descriptor_set) {
1514 anv_cmd_buffer_bind_descriptor_set(cmd_buffer, bind_point, NULL, 0,
1515 state->descriptor_set, NULL, NULL);
1516 } else {
1517 pipe_state->descriptors[0] = NULL;
1518 }
1519 }
1520
1521 if (state->flags & ANV_CMD_SAVED_STATE_PUSH_CONSTANTS) {
1522 VkPushConstantsInfoKHR push_info = {
1523 .sType = VK_STRUCTURE_TYPE_PUSH_CONSTANTS_INFO_KHR,
1524 .layout = VK_NULL_HANDLE,
1525 .stageFlags = stage_flags,
1526 .offset = 0,
1527 .size = sizeof(state->push_constants),
1528 .pValues = state->push_constants,
1529 };
1530 anv_CmdPushConstants2KHR(cmd_buffer_, &push_info);
1531 }
1532 }
1533