xref: /aosp_15_r20/external/mesa3d/src/amd/vulkan/meta/radv_meta.c (revision 6104692788411f58d303aa86923a9ff6ecaded22)
1 /*
2  * Copyright © 2016 Red Hat
3  * based on intel anv code:
4  * Copyright © 2015 Intel Corporation
5  *
6  * SPDX-License-Identifier: MIT
7  */
8 
9 #include "radv_meta.h"
10 #include "radv_printf.h"
11 
12 #include "vk_common_entrypoints.h"
13 #include "vk_pipeline_cache.h"
14 #include "vk_util.h"
15 
16 #include <fcntl.h>
17 #include <limits.h>
18 #ifndef _WIN32
19 #include <pwd.h>
20 #endif
21 #include <sys/stat.h>
22 
23 static void
radv_suspend_queries(struct radv_meta_saved_state * state,struct radv_cmd_buffer * cmd_buffer)24 radv_suspend_queries(struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer)
25 {
26    const uint32_t num_pipeline_stat_queries = radv_get_num_pipeline_stat_queries(cmd_buffer);
27 
28    if (num_pipeline_stat_queries > 0) {
29       cmd_buffer->state.flush_bits &= ~RADV_CMD_FLAG_START_PIPELINE_STATS;
30       cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_STOP_PIPELINE_STATS;
31    }
32 
33    /* Pipeline statistics queries. */
34    if (cmd_buffer->state.active_pipeline_queries > 0) {
35       state->active_pipeline_gds_queries = cmd_buffer->state.active_pipeline_gds_queries;
36       cmd_buffer->state.active_pipeline_gds_queries = 0;
37       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_SHADER_QUERY;
38    }
39 
40    /* Occlusion queries. */
41    if (cmd_buffer->state.active_occlusion_queries) {
42       state->active_occlusion_queries = cmd_buffer->state.active_occlusion_queries;
43       cmd_buffer->state.active_occlusion_queries = 0;
44       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_OCCLUSION_QUERY;
45    }
46 
47    /* Primitives generated queries (legacy). */
48    if (cmd_buffer->state.active_prims_gen_queries) {
49       cmd_buffer->state.suspend_streamout = true;
50       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_STREAMOUT_ENABLE;
51    }
52 
53    /* Primitives generated queries (NGG). */
54    if (cmd_buffer->state.active_prims_gen_gds_queries) {
55       state->active_prims_gen_gds_queries = cmd_buffer->state.active_prims_gen_gds_queries;
56       cmd_buffer->state.active_prims_gen_gds_queries = 0;
57       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_SHADER_QUERY;
58    }
59 
60    /* Transform feedback queries (NGG). */
61    if (cmd_buffer->state.active_prims_xfb_gds_queries) {
62       state->active_prims_xfb_gds_queries = cmd_buffer->state.active_prims_xfb_gds_queries;
63       cmd_buffer->state.active_prims_xfb_gds_queries = 0;
64       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_SHADER_QUERY;
65    }
66 }
67 
68 static void
radv_resume_queries(const struct radv_meta_saved_state * state,struct radv_cmd_buffer * cmd_buffer)69 radv_resume_queries(const struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer)
70 {
71    const uint32_t num_pipeline_stat_queries = radv_get_num_pipeline_stat_queries(cmd_buffer);
72 
73    if (num_pipeline_stat_queries > 0) {
74       cmd_buffer->state.flush_bits &= ~RADV_CMD_FLAG_STOP_PIPELINE_STATS;
75       cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_START_PIPELINE_STATS;
76    }
77 
78    /* Pipeline statistics queries. */
79    if (cmd_buffer->state.active_pipeline_queries > 0) {
80       cmd_buffer->state.active_pipeline_gds_queries = state->active_pipeline_gds_queries;
81       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_SHADER_QUERY;
82    }
83 
84    /* Occlusion queries. */
85    if (state->active_occlusion_queries) {
86       cmd_buffer->state.active_occlusion_queries = state->active_occlusion_queries;
87       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_OCCLUSION_QUERY;
88    }
89 
90    /* Primitives generated queries (legacy). */
91    if (cmd_buffer->state.active_prims_gen_queries) {
92       cmd_buffer->state.suspend_streamout = false;
93       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_STREAMOUT_ENABLE;
94    }
95 
96    /* Primitives generated queries (NGG). */
97    if (state->active_prims_gen_gds_queries) {
98       cmd_buffer->state.active_prims_gen_gds_queries = state->active_prims_gen_gds_queries;
99       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_SHADER_QUERY;
100    }
101 
102    /* Transform feedback queries (NGG). */
103    if (state->active_prims_xfb_gds_queries) {
104       cmd_buffer->state.active_prims_xfb_gds_queries = state->active_prims_xfb_gds_queries;
105       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_SHADER_QUERY;
106    }
107 }
108 
109 void
radv_meta_save(struct radv_meta_saved_state * state,struct radv_cmd_buffer * cmd_buffer,uint32_t flags)110 radv_meta_save(struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer, uint32_t flags)
111 {
112    VkPipelineBindPoint bind_point =
113       flags & RADV_META_SAVE_GRAPHICS_PIPELINE ? VK_PIPELINE_BIND_POINT_GRAPHICS : VK_PIPELINE_BIND_POINT_COMPUTE;
114    struct radv_descriptor_state *descriptors_state = radv_get_descriptors_state(cmd_buffer, bind_point);
115 
116    assert(flags & (RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_COMPUTE_PIPELINE));
117 
118    state->flags = flags;
119    state->active_occlusion_queries = 0;
120    state->active_prims_gen_gds_queries = 0;
121    state->active_prims_xfb_gds_queries = 0;
122 
123    if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) {
124       assert(!(state->flags & RADV_META_SAVE_COMPUTE_PIPELINE));
125 
126       state->old_graphics_pipeline = cmd_buffer->state.graphics_pipeline;
127 
128       /* Save all dynamic states. */
129       state->dynamic = cmd_buffer->state.dynamic;
130    }
131 
132    if (state->flags & RADV_META_SAVE_COMPUTE_PIPELINE) {
133       assert(!(state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE));
134 
135       state->old_compute_pipeline = cmd_buffer->state.compute_pipeline;
136    }
137 
138    for (unsigned i = 0; i <= MESA_SHADER_MESH; i++) {
139       state->old_shader_objs[i] = cmd_buffer->state.shader_objs[i];
140    }
141 
142    if (state->flags & RADV_META_SAVE_DESCRIPTORS) {
143       state->old_descriptor_set0 = descriptors_state->sets[0];
144       if (!(descriptors_state->valid & 1))
145          state->flags &= ~RADV_META_SAVE_DESCRIPTORS;
146    }
147 
148    if (state->flags & RADV_META_SAVE_CONSTANTS) {
149       memcpy(state->push_constants, cmd_buffer->push_constants, MAX_PUSH_CONSTANTS_SIZE);
150    }
151 
152    if (state->flags & RADV_META_SAVE_RENDER) {
153       state->render = cmd_buffer->state.render;
154       radv_cmd_buffer_reset_rendering(cmd_buffer);
155    }
156 
157    if (state->flags & RADV_META_SUSPEND_PREDICATING) {
158       state->predicating = cmd_buffer->state.predicating;
159       cmd_buffer->state.predicating = false;
160    }
161 
162    radv_suspend_queries(state, cmd_buffer);
163 }
164 
165 void
radv_meta_restore(const struct radv_meta_saved_state * state,struct radv_cmd_buffer * cmd_buffer)166 radv_meta_restore(const struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer)
167 {
168    VkPipelineBindPoint bind_point = state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE ? VK_PIPELINE_BIND_POINT_GRAPHICS
169                                                                                     : VK_PIPELINE_BIND_POINT_COMPUTE;
170 
171    if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) {
172       if (state->old_graphics_pipeline) {
173          radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS,
174                               radv_pipeline_to_handle(&state->old_graphics_pipeline->base));
175       }
176 
177       /* Restore all dynamic states. */
178       cmd_buffer->state.dynamic = state->dynamic;
179       cmd_buffer->state.dirty_dynamic |= RADV_DYNAMIC_ALL;
180 
181       /* Re-emit the guardband state because meta operations changed dynamic states. */
182       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_GUARDBAND;
183    }
184 
185    if (state->flags & RADV_META_SAVE_COMPUTE_PIPELINE) {
186       if (state->old_compute_pipeline) {
187          radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE,
188                               radv_pipeline_to_handle(&state->old_compute_pipeline->base));
189       }
190    }
191 
192    VkShaderEXT shaders[MESA_SHADER_MESH + 1];
193    VkShaderStageFlagBits stages[MESA_SHADER_MESH + 1];
194    uint32_t stage_count = 0;
195 
196    for (unsigned i = 0; i <= MESA_SHADER_MESH; i++) {
197       if (state->old_shader_objs[i]) {
198          stages[stage_count] = mesa_to_vk_shader_stage(i);
199          shaders[stage_count] = radv_shader_object_to_handle(state->old_shader_objs[i]);
200          stage_count++;
201       }
202    }
203 
204    if (stage_count > 0) {
205       radv_CmdBindShadersEXT(radv_cmd_buffer_to_handle(cmd_buffer), stage_count, stages, shaders);
206    }
207 
208    if (state->flags & RADV_META_SAVE_DESCRIPTORS) {
209       radv_set_descriptor_set(cmd_buffer, bind_point, state->old_descriptor_set0, 0);
210    }
211 
212    if (state->flags & RADV_META_SAVE_CONSTANTS) {
213       VkShaderStageFlags stage_flags = VK_SHADER_STAGE_COMPUTE_BIT;
214 
215       if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE)
216          stage_flags |= VK_SHADER_STAGE_ALL_GRAPHICS;
217 
218       vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), VK_NULL_HANDLE, stage_flags, 0,
219                                  MAX_PUSH_CONSTANTS_SIZE, state->push_constants);
220    }
221 
222    if (state->flags & RADV_META_SAVE_RENDER) {
223       cmd_buffer->state.render = state->render;
224       cmd_buffer->state.dirty |= RADV_CMD_DIRTY_FRAMEBUFFER;
225    }
226 
227    if (state->flags & RADV_META_SUSPEND_PREDICATING)
228       cmd_buffer->state.predicating = state->predicating;
229 
230    radv_resume_queries(state, cmd_buffer);
231 }
232 
233 VkImageViewType
radv_meta_get_view_type(const struct radv_image * image)234 radv_meta_get_view_type(const struct radv_image *image)
235 {
236    switch (image->vk.image_type) {
237    case VK_IMAGE_TYPE_1D:
238       return VK_IMAGE_VIEW_TYPE_1D;
239    case VK_IMAGE_TYPE_2D:
240       return VK_IMAGE_VIEW_TYPE_2D;
241    case VK_IMAGE_TYPE_3D:
242       return VK_IMAGE_VIEW_TYPE_3D;
243    default:
244       unreachable("bad VkImageViewType");
245    }
246 }
247 
248 /**
249  * When creating a destination VkImageView, this function provides the needed
250  * VkImageViewCreateInfo::subresourceRange::baseArrayLayer.
251  */
252 uint32_t
radv_meta_get_iview_layer(const struct radv_image * dst_image,const VkImageSubresourceLayers * dst_subresource,const VkOffset3D * dst_offset)253 radv_meta_get_iview_layer(const struct radv_image *dst_image, const VkImageSubresourceLayers *dst_subresource,
254                           const VkOffset3D *dst_offset)
255 {
256    switch (dst_image->vk.image_type) {
257    case VK_IMAGE_TYPE_1D:
258    case VK_IMAGE_TYPE_2D:
259       return dst_subresource->baseArrayLayer;
260    case VK_IMAGE_TYPE_3D:
261       /* HACK: Vulkan does not allow attaching a 3D image to a framebuffer,
262        * but meta does it anyway. When doing so, we translate the
263        * destination's z offset into an array offset.
264        */
265       return dst_offset->z;
266    default:
267       assert(!"bad VkImageType");
268       return 0;
269    }
270 }
271 
272 static VKAPI_ATTR void *VKAPI_CALL
meta_alloc(void * _device,size_t size,size_t alignment,VkSystemAllocationScope allocationScope)273 meta_alloc(void *_device, size_t size, size_t alignment, VkSystemAllocationScope allocationScope)
274 {
275    struct radv_device *device = _device;
276    return device->vk.alloc.pfnAllocation(device->vk.alloc.pUserData, size, alignment,
277                                          VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
278 }
279 
280 static VKAPI_ATTR void *VKAPI_CALL
meta_realloc(void * _device,void * original,size_t size,size_t alignment,VkSystemAllocationScope allocationScope)281 meta_realloc(void *_device, void *original, size_t size, size_t alignment, VkSystemAllocationScope allocationScope)
282 {
283    struct radv_device *device = _device;
284    return device->vk.alloc.pfnReallocation(device->vk.alloc.pUserData, original, size, alignment,
285                                            VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
286 }
287 
288 static VKAPI_ATTR void VKAPI_CALL
meta_free(void * _device,void * data)289 meta_free(void *_device, void *data)
290 {
291    struct radv_device *device = _device;
292    device->vk.alloc.pfnFree(device->vk.alloc.pUserData, data);
293 }
294 
295 #ifndef _WIN32
296 static bool
radv_builtin_cache_path(char * path)297 radv_builtin_cache_path(char *path)
298 {
299    char *xdg_cache_home = secure_getenv("XDG_CACHE_HOME");
300    const char *suffix = "/radv_builtin_shaders";
301    const char *suffix2 = "/.cache/radv_builtin_shaders";
302    struct passwd pwd, *result;
303    char path2[PATH_MAX + 1]; /* PATH_MAX is not a real max,but suffices here. */
304    int ret;
305 
306    if (xdg_cache_home) {
307       ret = snprintf(path, PATH_MAX + 1, "%s%s%zd", xdg_cache_home, suffix, sizeof(void *) * 8);
308       return ret > 0 && ret < PATH_MAX + 1;
309    }
310 
311    getpwuid_r(getuid(), &pwd, path2, PATH_MAX - strlen(suffix2), &result);
312    if (!result)
313       return false;
314 
315    strcpy(path, pwd.pw_dir);
316    strcat(path, "/.cache");
317    if (mkdir(path, 0755) && errno != EEXIST)
318       return false;
319 
320    ret = snprintf(path, PATH_MAX + 1, "%s%s%zd", pwd.pw_dir, suffix2, sizeof(void *) * 8);
321    return ret > 0 && ret < PATH_MAX + 1;
322 }
323 #endif
324 
325 static uint32_t
num_cache_entries(VkPipelineCache cache)326 num_cache_entries(VkPipelineCache cache)
327 {
328    struct set *s = vk_pipeline_cache_from_handle(cache)->object_cache;
329    if (!s)
330       return 0;
331    return s->entries;
332 }
333 
334 static bool
radv_load_meta_pipeline(struct radv_device * device)335 radv_load_meta_pipeline(struct radv_device *device)
336 {
337 #ifdef _WIN32
338    return false;
339 #else
340    char path[PATH_MAX + 1];
341    struct stat st;
342    void *data = NULL;
343    bool ret = false;
344    int fd = -1;
345    struct vk_pipeline_cache *cache = NULL;
346 
347    VkPipelineCacheCreateInfo create_info = {
348       .sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO,
349    };
350 
351    struct vk_pipeline_cache_create_info info = {
352       .pCreateInfo = &create_info,
353       .skip_disk_cache = true,
354    };
355 
356    if (!radv_builtin_cache_path(path))
357       goto fail;
358 
359    fd = open(path, O_RDONLY);
360    if (fd < 0)
361       goto fail;
362    if (fstat(fd, &st))
363       goto fail;
364    data = malloc(st.st_size);
365    if (!data)
366       goto fail;
367    if (read(fd, data, st.st_size) == -1)
368       goto fail;
369 
370    create_info.initialDataSize = st.st_size;
371    create_info.pInitialData = data;
372 
373 fail:
374    cache = vk_pipeline_cache_create(&device->vk, &info, NULL);
375 
376    if (cache) {
377       device->meta_state.cache = vk_pipeline_cache_to_handle(cache);
378       device->meta_state.initial_cache_entries = num_cache_entries(device->meta_state.cache);
379       ret = device->meta_state.initial_cache_entries > 0;
380    }
381 
382    free(data);
383    if (fd >= 0)
384       close(fd);
385    return ret;
386 #endif
387 }
388 
389 static void
radv_store_meta_pipeline(struct radv_device * device)390 radv_store_meta_pipeline(struct radv_device *device)
391 {
392 #ifndef _WIN32
393    char path[PATH_MAX + 1], path2[PATH_MAX + 7];
394    size_t size;
395    void *data = NULL;
396 
397    if (device->meta_state.cache == VK_NULL_HANDLE)
398       return;
399 
400    /* Skip serialization if no entries were added. */
401    if (num_cache_entries(device->meta_state.cache) <= device->meta_state.initial_cache_entries)
402       return;
403 
404    if (vk_common_GetPipelineCacheData(radv_device_to_handle(device), device->meta_state.cache, &size, NULL))
405       return;
406 
407    if (!radv_builtin_cache_path(path))
408       return;
409 
410    strcpy(path2, path);
411    strcat(path2, "XXXXXX");
412    int fd = mkstemp(path2); // open(path, O_WRONLY | O_CREAT, 0600);
413    if (fd < 0)
414       return;
415    data = malloc(size);
416    if (!data)
417       goto fail;
418 
419    if (vk_common_GetPipelineCacheData(radv_device_to_handle(device), device->meta_state.cache, &size, data))
420       goto fail;
421    if (write(fd, data, size) == -1)
422       goto fail;
423 
424    rename(path2, path);
425 fail:
426    free(data);
427    close(fd);
428    unlink(path2);
429 #endif
430 }
431 
432 VkResult
radv_device_init_meta(struct radv_device * device)433 radv_device_init_meta(struct radv_device *device)
434 {
435    struct radv_physical_device *pdev = radv_device_physical(device);
436    VkResult result;
437 
438    memset(&device->meta_state, 0, sizeof(device->meta_state));
439 
440    device->meta_state.alloc = (VkAllocationCallbacks){
441       .pUserData = device,
442       .pfnAllocation = meta_alloc,
443       .pfnReallocation = meta_realloc,
444       .pfnFree = meta_free,
445    };
446 
447    bool loaded_cache = radv_load_meta_pipeline(device);
448    bool on_demand = !loaded_cache;
449 
450    mtx_init(&device->meta_state.mtx, mtx_plain);
451 
452    result = radv_device_init_meta_clear_state(device, on_demand);
453    if (result != VK_SUCCESS)
454       goto fail_clear;
455 
456    result = radv_device_init_meta_resolve_state(device, on_demand);
457    if (result != VK_SUCCESS)
458       goto fail_resolve;
459 
460    result = radv_device_init_meta_blit_state(device, on_demand);
461    if (result != VK_SUCCESS)
462       goto fail_blit;
463 
464    result = radv_device_init_meta_blit2d_state(device, on_demand);
465    if (result != VK_SUCCESS)
466       goto fail_blit2d;
467 
468    result = radv_device_init_meta_bufimage_state(device, on_demand);
469    if (result != VK_SUCCESS)
470       goto fail_bufimage;
471 
472    result = radv_device_init_meta_depth_decomp_state(device, on_demand);
473    if (result != VK_SUCCESS)
474       goto fail_depth_decomp;
475 
476    result = radv_device_init_meta_buffer_state(device, on_demand);
477    if (result != VK_SUCCESS)
478       goto fail_buffer;
479 
480    result = radv_device_init_meta_query_state(device, on_demand);
481    if (result != VK_SUCCESS)
482       goto fail_query;
483 
484    result = radv_device_init_meta_fast_clear_flush_state(device, on_demand);
485    if (result != VK_SUCCESS)
486       goto fail_fast_clear;
487 
488    result = radv_device_init_meta_resolve_compute_state(device, on_demand);
489    if (result != VK_SUCCESS)
490       goto fail_resolve_compute;
491 
492    result = radv_device_init_meta_resolve_fragment_state(device, on_demand);
493    if (result != VK_SUCCESS)
494       goto fail_resolve_fragment;
495 
496    if (pdev->use_fmask) {
497       result = radv_device_init_meta_fmask_expand_state(device, on_demand);
498       if (result != VK_SUCCESS)
499          goto fail_fmask_expand;
500 
501       result = radv_device_init_meta_fmask_copy_state(device, on_demand);
502       if (result != VK_SUCCESS)
503          goto fail_fmask_copy;
504    }
505 
506    result = radv_device_init_meta_etc_decode_state(device, on_demand);
507    if (result != VK_SUCCESS)
508       goto fail_etc_decode;
509 
510    result = radv_device_init_meta_astc_decode_state(device, on_demand);
511    if (result != VK_SUCCESS)
512       goto fail_astc_decode;
513 
514    if (radv_uses_device_generated_commands(device)) {
515       result = radv_device_init_dgc_prepare_state(device, on_demand);
516       if (result != VK_SUCCESS)
517          goto fail_dgc;
518    }
519 
520    if (device->vk.enabled_extensions.KHR_acceleration_structure) {
521       if (device->vk.enabled_features.nullDescriptor) {
522          result = radv_device_init_null_accel_struct(device);
523          if (result != VK_SUCCESS)
524             goto fail_accel_struct;
525       }
526 
527       /* FIXME: Acceleration structure builds hang when the build shaders are compiled with LLVM.
528        * Work around it by forcing ACO for now.
529        */
530       bool use_llvm = pdev->use_llvm;
531       if (loaded_cache || use_llvm) {
532          pdev->use_llvm = false;
533          result = radv_device_init_accel_struct_build_state(device);
534          pdev->use_llvm = use_llvm;
535 
536          if (result != VK_SUCCESS)
537             goto fail_accel_struct;
538       }
539    }
540 
541    return VK_SUCCESS;
542 
543 fail_accel_struct:
544    radv_device_finish_accel_struct_build_state(device);
545 fail_dgc:
546    radv_device_finish_dgc_prepare_state(device);
547 fail_astc_decode:
548    radv_device_finish_meta_astc_decode_state(device);
549 fail_etc_decode:
550    radv_device_finish_meta_etc_decode_state(device);
551 fail_fmask_copy:
552    radv_device_finish_meta_fmask_copy_state(device);
553 fail_fmask_expand:
554    radv_device_finish_meta_fmask_expand_state(device);
555 fail_resolve_fragment:
556    radv_device_finish_meta_resolve_fragment_state(device);
557 fail_resolve_compute:
558    radv_device_finish_meta_resolve_compute_state(device);
559 fail_fast_clear:
560    radv_device_finish_meta_fast_clear_flush_state(device);
561 fail_query:
562    radv_device_finish_meta_query_state(device);
563 fail_buffer:
564    radv_device_finish_meta_buffer_state(device);
565 fail_depth_decomp:
566    radv_device_finish_meta_depth_decomp_state(device);
567 fail_bufimage:
568    radv_device_finish_meta_bufimage_state(device);
569 fail_blit2d:
570    radv_device_finish_meta_blit2d_state(device);
571 fail_blit:
572    radv_device_finish_meta_blit_state(device);
573 fail_resolve:
574    radv_device_finish_meta_resolve_state(device);
575 fail_clear:
576    radv_device_finish_meta_clear_state(device);
577 
578    mtx_destroy(&device->meta_state.mtx);
579    vk_common_DestroyPipelineCache(radv_device_to_handle(device), device->meta_state.cache, NULL);
580    return result;
581 }
582 
583 void
radv_device_finish_meta(struct radv_device * device)584 radv_device_finish_meta(struct radv_device *device)
585 {
586    radv_device_finish_dgc_prepare_state(device);
587    radv_device_finish_meta_etc_decode_state(device);
588    radv_device_finish_meta_astc_decode_state(device);
589    radv_device_finish_accel_struct_build_state(device);
590    radv_device_finish_meta_clear_state(device);
591    radv_device_finish_meta_resolve_state(device);
592    radv_device_finish_meta_blit_state(device);
593    radv_device_finish_meta_blit2d_state(device);
594    radv_device_finish_meta_bufimage_state(device);
595    radv_device_finish_meta_depth_decomp_state(device);
596    radv_device_finish_meta_query_state(device);
597    radv_device_finish_meta_buffer_state(device);
598    radv_device_finish_meta_fast_clear_flush_state(device);
599    radv_device_finish_meta_resolve_compute_state(device);
600    radv_device_finish_meta_resolve_fragment_state(device);
601    radv_device_finish_meta_fmask_expand_state(device);
602    radv_device_finish_meta_dcc_retile_state(device);
603    radv_device_finish_meta_copy_vrs_htile_state(device);
604    radv_device_finish_meta_fmask_copy_state(device);
605 
606    radv_store_meta_pipeline(device);
607    vk_common_DestroyPipelineCache(radv_device_to_handle(device), device->meta_state.cache, NULL);
608    mtx_destroy(&device->meta_state.mtx);
609 }
610 
611 nir_builder PRINTFLIKE(3, 4)
radv_meta_init_shader(struct radv_device * dev,gl_shader_stage stage,const char * name,...)612    radv_meta_init_shader(struct radv_device *dev, gl_shader_stage stage, const char *name, ...)
613 {
614    const struct radv_physical_device *pdev = radv_device_physical(dev);
615    nir_builder b = nir_builder_init_simple_shader(stage, NULL, NULL);
616    if (name) {
617       va_list args;
618       va_start(args, name);
619       b.shader->info.name = ralloc_vasprintf(b.shader, name, args);
620       va_end(args);
621    }
622 
623    b.shader->options = &pdev->nir_options[stage];
624 
625    radv_device_associate_nir(dev, b.shader);
626 
627    return b;
628 }
629 
630 /* vertex shader that generates vertices */
631 nir_shader *
radv_meta_build_nir_vs_generate_vertices(struct radv_device * dev)632 radv_meta_build_nir_vs_generate_vertices(struct radv_device *dev)
633 {
634    const struct glsl_type *vec4 = glsl_vec4_type();
635 
636    nir_variable *v_position;
637 
638    nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_VERTEX, "meta_vs_gen_verts");
639 
640    nir_def *outvec = nir_gen_rect_vertices(&b, NULL, NULL);
641 
642    v_position = nir_variable_create(b.shader, nir_var_shader_out, vec4, "gl_Position");
643    v_position->data.location = VARYING_SLOT_POS;
644 
645    nir_store_var(&b, v_position, outvec, 0xf);
646 
647    return b.shader;
648 }
649 
650 nir_shader *
radv_meta_build_nir_fs_noop(struct radv_device * dev)651 radv_meta_build_nir_fs_noop(struct radv_device *dev)
652 {
653    return radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_noop_fs").shader;
654 }
655 
656 void
radv_meta_build_resolve_shader_core(struct radv_device * device,nir_builder * b,bool is_integer,int samples,nir_variable * input_img,nir_variable * color,nir_def * img_coord)657 radv_meta_build_resolve_shader_core(struct radv_device *device, nir_builder *b, bool is_integer, int samples,
658                                     nir_variable *input_img, nir_variable *color, nir_def *img_coord)
659 {
660    const struct radv_physical_device *pdev = radv_device_physical(device);
661    nir_deref_instr *input_img_deref = nir_build_deref_var(b, input_img);
662    nir_def *sample0 = nir_txf_ms_deref(b, input_img_deref, img_coord, nir_imm_int(b, 0));
663 
664    if (is_integer || samples <= 1) {
665       nir_store_var(b, color, sample0, 0xf);
666       return;
667    }
668 
669    if (pdev->use_fmask) {
670       nir_def *all_same = nir_samples_identical_deref(b, input_img_deref, img_coord);
671       nir_push_if(b, nir_inot(b, all_same));
672    }
673 
674    nir_def *accum = sample0;
675    for (int i = 1; i < samples; i++) {
676       nir_def *sample = nir_txf_ms_deref(b, input_img_deref, img_coord, nir_imm_int(b, i));
677       accum = nir_fadd(b, accum, sample);
678    }
679 
680    accum = nir_fdiv_imm(b, accum, samples);
681    nir_store_var(b, color, accum, 0xf);
682 
683    if (pdev->use_fmask) {
684       nir_push_else(b, NULL);
685       nir_store_var(b, color, sample0, 0xf);
686       nir_pop_if(b, NULL);
687    }
688 }
689 
690 nir_def *
radv_meta_load_descriptor(nir_builder * b,unsigned desc_set,unsigned binding)691 radv_meta_load_descriptor(nir_builder *b, unsigned desc_set, unsigned binding)
692 {
693    nir_def *rsrc = nir_vulkan_resource_index(b, 3, 32, nir_imm_int(b, 0), .desc_set = desc_set, .binding = binding);
694    return nir_trim_vector(b, rsrc, 2);
695 }
696 
697 nir_def *
get_global_ids(nir_builder * b,unsigned num_components)698 get_global_ids(nir_builder *b, unsigned num_components)
699 {
700    unsigned mask = BITFIELD_MASK(num_components);
701 
702    nir_def *local_ids = nir_channels(b, nir_load_local_invocation_id(b), mask);
703    nir_def *block_ids = nir_channels(b, nir_load_workgroup_id(b), mask);
704    nir_def *block_size =
705       nir_channels(b,
706                    nir_imm_ivec4(b, b->shader->info.workgroup_size[0], b->shader->info.workgroup_size[1],
707                                  b->shader->info.workgroup_size[2], 0),
708                    mask);
709 
710    return nir_iadd(b, nir_imul(b, block_ids, block_size), local_ids);
711 }
712 
713 void
radv_break_on_count(nir_builder * b,nir_variable * var,nir_def * count)714 radv_break_on_count(nir_builder *b, nir_variable *var, nir_def *count)
715 {
716    nir_def *counter = nir_load_var(b, var);
717 
718    nir_break_if(b, nir_uge(b, counter, count));
719 
720    counter = nir_iadd_imm(b, counter, 1);
721    nir_store_var(b, var, counter, 0x1);
722 }
723 
724 VkResult
radv_meta_create_compute_pipeline(struct radv_device * device,nir_shader * nir,VkPipelineLayout pipeline_layout,VkPipeline * pipeline)725 radv_meta_create_compute_pipeline(struct radv_device *device, nir_shader *nir, VkPipelineLayout pipeline_layout,
726                                   VkPipeline *pipeline)
727 {
728    const VkPipelineShaderStageCreateInfo stage_info = {
729       .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
730       .stage = VK_SHADER_STAGE_COMPUTE_BIT,
731       .module = vk_shader_module_handle_from_nir(nir),
732       .pName = "main",
733       .pSpecializationInfo = NULL,
734    };
735 
736    const VkComputePipelineCreateInfo pipeline_info = {
737       .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
738       .stage = stage_info,
739       .flags = 0,
740       .layout = pipeline_layout,
741    };
742 
743    return radv_compute_pipeline_create(radv_device_to_handle(device), device->meta_state.cache, &pipeline_info, NULL,
744                                        pipeline);
745 }
746 
747 VkResult
radv_meta_create_pipeline_layout(struct radv_device * device,VkDescriptorSetLayout * set_layout,uint32_t num_pc_ranges,const VkPushConstantRange * pc_ranges,VkPipelineLayout * pipeline_layout)748 radv_meta_create_pipeline_layout(struct radv_device *device, VkDescriptorSetLayout *set_layout, uint32_t num_pc_ranges,
749                                  const VkPushConstantRange *pc_ranges, VkPipelineLayout *pipeline_layout)
750 {
751    const VkPipelineLayoutCreateInfo pipeline_layout_info = {
752       .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
753       .setLayoutCount = !!set_layout,
754       .pSetLayouts = set_layout,
755       .pushConstantRangeCount = num_pc_ranges,
756       .pPushConstantRanges = pc_ranges,
757    };
758 
759    return radv_CreatePipelineLayout(radv_device_to_handle(device), &pipeline_layout_info, &device->meta_state.alloc,
760                                     pipeline_layout);
761 }
762 
763 VkResult
radv_meta_create_descriptor_set_layout(struct radv_device * device,uint32_t num_bindings,const VkDescriptorSetLayoutBinding * bindings,VkDescriptorSetLayout * desc_layout)764 radv_meta_create_descriptor_set_layout(struct radv_device *device, uint32_t num_bindings,
765                                        const VkDescriptorSetLayoutBinding *bindings, VkDescriptorSetLayout *desc_layout)
766 {
767    const VkDescriptorSetLayoutCreateInfo desc_layout_info = {
768       .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
769       .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
770       .bindingCount = num_bindings,
771       .pBindings = bindings,
772    };
773 
774    return radv_CreateDescriptorSetLayout(radv_device_to_handle(device), &desc_layout_info, &device->meta_state.alloc,
775                                          desc_layout);
776 }
777