xref: /aosp_15_r20/external/mesa3d/src/amd/vulkan/meta/radv_meta.h (revision 6104692788411f58d303aa86923a9ff6ecaded22) 
1 /*
2  * Copyright © 2016 Red Hat
3  * based on intel anv code:
4  * Copyright © 2015 Intel Corporation
5  *
6  * SPDX-License-Identifier: MIT
7  */
8 
9 #ifndef RADV_META_H
10 #define RADV_META_H
11 
12 #include "radv_buffer.h"
13 #include "radv_buffer_view.h"
14 #include "radv_cmd_buffer.h"
15 #include "radv_device.h"
16 #include "radv_device_memory.h"
17 #include "radv_entrypoints.h"
18 #include "radv_image.h"
19 #include "radv_image_view.h"
20 #include "radv_physical_device.h"
21 #include "radv_pipeline.h"
22 #include "radv_pipeline_compute.h"
23 #include "radv_pipeline_graphics.h"
24 #include "radv_queue.h"
25 #include "radv_shader.h"
26 #include "radv_shader_object.h"
27 #include "radv_sqtt.h"
28 
29 #include "vk_render_pass.h"
30 #include "vk_shader_module.h"
31 
32 #ifdef __cplusplus
33 extern "C" {
34 #endif
35 
36 enum radv_meta_save_flags {
37    RADV_META_SAVE_RENDER = (1 << 0),
38    RADV_META_SAVE_CONSTANTS = (1 << 1),
39    RADV_META_SAVE_DESCRIPTORS = (1 << 2),
40    RADV_META_SAVE_GRAPHICS_PIPELINE = (1 << 3),
41    RADV_META_SAVE_COMPUTE_PIPELINE = (1 << 4),
42    RADV_META_SUSPEND_PREDICATING = (1 << 5),
43 };
44 
45 struct radv_meta_saved_state {
46    uint32_t flags;
47 
48    struct radv_descriptor_set *old_descriptor_set0;
49    struct radv_graphics_pipeline *old_graphics_pipeline;
50    struct radv_compute_pipeline *old_compute_pipeline;
51    struct radv_dynamic_state dynamic;
52 
53    struct radv_shader_object *old_shader_objs[MESA_VULKAN_SHADER_STAGES];
54 
55    char push_constants[MAX_PUSH_CONSTANTS_SIZE];
56 
57    struct radv_rendering_state render;
58 
59    unsigned active_pipeline_gds_queries;
60    unsigned active_prims_gen_gds_queries;
61    unsigned active_prims_xfb_gds_queries;
62    unsigned active_occlusion_queries;
63 
64    bool predicating;
65 };
66 
67 enum radv_blit_ds_layout {
68    RADV_BLIT_DS_LAYOUT_TILE_ENABLE,
69    RADV_BLIT_DS_LAYOUT_TILE_DISABLE,
70    RADV_BLIT_DS_LAYOUT_COUNT,
71 };
72 
73 static inline enum radv_blit_ds_layout
radv_meta_blit_ds_to_type(VkImageLayout layout)74 radv_meta_blit_ds_to_type(VkImageLayout layout)
75 {
76    return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_BLIT_DS_LAYOUT_TILE_DISABLE : RADV_BLIT_DS_LAYOUT_TILE_ENABLE;
77 }
78 
79 static inline VkImageLayout
radv_meta_blit_ds_to_layout(enum radv_blit_ds_layout ds_layout)80 radv_meta_blit_ds_to_layout(enum radv_blit_ds_layout ds_layout)
81 {
82    return ds_layout == RADV_BLIT_DS_LAYOUT_TILE_ENABLE ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : VK_IMAGE_LAYOUT_GENERAL;
83 }
84 
85 enum radv_meta_dst_layout {
86    RADV_META_DST_LAYOUT_GENERAL,
87    RADV_META_DST_LAYOUT_OPTIMAL,
88    RADV_META_DST_LAYOUT_COUNT,
89 };
90 
91 static inline enum radv_meta_dst_layout
radv_meta_dst_layout_from_layout(VkImageLayout layout)92 radv_meta_dst_layout_from_layout(VkImageLayout layout)
93 {
94    return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_META_DST_LAYOUT_GENERAL : RADV_META_DST_LAYOUT_OPTIMAL;
95 }
96 
97 static inline VkImageLayout
radv_meta_dst_layout_to_layout(enum radv_meta_dst_layout layout)98 radv_meta_dst_layout_to_layout(enum radv_meta_dst_layout layout)
99 {
100    return layout == RADV_META_DST_LAYOUT_OPTIMAL ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : VK_IMAGE_LAYOUT_GENERAL;
101 }
102 
103 extern const VkFormat radv_fs_key_format_exemplars[NUM_META_FS_KEYS];
104 
105 VkResult radv_device_init_meta(struct radv_device *device);
106 void radv_device_finish_meta(struct radv_device *device);
107 
108 VkResult radv_device_init_meta_clear_state(struct radv_device *device, bool on_demand);
109 void radv_device_finish_meta_clear_state(struct radv_device *device);
110 
111 VkResult radv_device_init_meta_resolve_state(struct radv_device *device, bool on_demand);
112 void radv_device_finish_meta_resolve_state(struct radv_device *device);
113 
114 VkResult radv_device_init_meta_depth_decomp_state(struct radv_device *device, bool on_demand);
115 void radv_device_finish_meta_depth_decomp_state(struct radv_device *device);
116 
117 VkResult radv_device_init_meta_fast_clear_flush_state(struct radv_device *device, bool on_demand);
118 void radv_device_finish_meta_fast_clear_flush_state(struct radv_device *device);
119 
120 VkResult radv_device_init_meta_blit_state(struct radv_device *device, bool on_demand);
121 void radv_device_finish_meta_blit_state(struct radv_device *device);
122 
123 VkResult radv_device_init_meta_blit2d_state(struct radv_device *device, bool on_demand);
124 void radv_device_finish_meta_blit2d_state(struct radv_device *device);
125 
126 VkResult radv_device_init_meta_buffer_state(struct radv_device *device, bool on_demand);
127 void radv_device_finish_meta_buffer_state(struct radv_device *device);
128 
129 VkResult radv_device_init_meta_query_state(struct radv_device *device, bool on_demand);
130 void radv_device_finish_meta_query_state(struct radv_device *device);
131 
132 VkResult radv_device_init_meta_resolve_compute_state(struct radv_device *device, bool on_demand);
133 void radv_device_finish_meta_resolve_compute_state(struct radv_device *device);
134 
135 VkResult radv_device_init_meta_resolve_fragment_state(struct radv_device *device, bool on_demand);
136 void radv_device_finish_meta_resolve_fragment_state(struct radv_device *device);
137 
138 VkResult radv_device_init_meta_fmask_copy_state(struct radv_device *device, bool on_demand);
139 void radv_device_finish_meta_fmask_copy_state(struct radv_device *device);
140 
141 VkResult radv_device_init_meta_fmask_expand_state(struct radv_device *device, bool on_demand);
142 void radv_device_finish_meta_fmask_expand_state(struct radv_device *device);
143 
144 void radv_device_finish_meta_dcc_retile_state(struct radv_device *device);
145 
146 void radv_device_finish_meta_copy_vrs_htile_state(struct radv_device *device);
147 
148 VkResult radv_device_init_null_accel_struct(struct radv_device *device);
149 VkResult radv_device_init_accel_struct_build_state(struct radv_device *device);
150 void radv_device_finish_accel_struct_build_state(struct radv_device *device);
151 
152 VkResult radv_device_init_meta_etc_decode_state(struct radv_device *device, bool on_demand);
153 void radv_device_finish_meta_etc_decode_state(struct radv_device *device);
154 
155 VkResult radv_device_init_meta_astc_decode_state(struct radv_device *device, bool on_demand);
156 void radv_device_finish_meta_astc_decode_state(struct radv_device *device);
157 
158 VkResult radv_device_init_dgc_prepare_state(struct radv_device *device, bool on_demand);
159 void radv_device_finish_dgc_prepare_state(struct radv_device *device);
160 
161 void radv_meta_save(struct radv_meta_saved_state *saved_state, struct radv_cmd_buffer *cmd_buffer, uint32_t flags);
162 
163 void radv_meta_restore(const struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer);
164 
165 VkImageViewType radv_meta_get_view_type(const struct radv_image *image);
166 
167 uint32_t radv_meta_get_iview_layer(const struct radv_image *dst_image, const VkImageSubresourceLayers *dst_subresource,
168                                    const VkOffset3D *dst_offset);
169 
170 struct radv_meta_blit2d_surf {
171    /** The size of an element in bytes. */
172    uint8_t bs;
173    VkFormat format;
174 
175    struct radv_image *image;
176    unsigned level;
177    unsigned layer;
178    VkImageAspectFlags aspect_mask;
179    VkImageLayout current_layout;
180    bool disable_compression;
181 };
182 
183 struct radv_meta_blit2d_buffer {
184    struct radv_buffer *buffer;
185    uint32_t offset;
186    uint32_t pitch;
187    uint8_t bs;
188    VkFormat format;
189 };
190 
191 struct radv_meta_blit2d_rect {
192    uint32_t src_x, src_y;
193    uint32_t dst_x, dst_y;
194    uint32_t width, height;
195 };
196 
197 void radv_meta_begin_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_saved_state *save);
198 
199 void radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img,
200                       struct radv_meta_blit2d_buffer *src_buf, struct radv_meta_blit2d_surf *dst,
201                       struct radv_meta_blit2d_rect *rect);
202 
203 void radv_meta_end_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_saved_state *save);
204 
205 VkResult radv_device_init_meta_bufimage_state(struct radv_device *device, bool on_demand);
206 void radv_device_finish_meta_bufimage_state(struct radv_device *device);
207 void radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
208                                struct radv_meta_blit2d_buffer *dst, struct radv_meta_blit2d_rect *rect);
209 
210 void radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_buffer *src,
211                                   struct radv_meta_blit2d_surf *dst, struct radv_meta_blit2d_rect *rect);
212 void radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
213                                  struct radv_meta_blit2d_surf *dst, struct radv_meta_blit2d_rect *rect);
214 void radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *dst,
215                               const VkClearColorValue *clear_color);
216 
217 void radv_expand_depth_stencil(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
218                                const VkImageSubresourceRange *subresourceRange,
219                                struct radv_sample_locations_state *sample_locs);
220 void radv_fast_clear_flush_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
221                                          const VkImageSubresourceRange *subresourceRange);
222 void radv_decompress_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
223                          const VkImageSubresourceRange *subresourceRange);
224 void radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image);
225 void radv_expand_fmask_image_inplace(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
226                                      const VkImageSubresourceRange *subresourceRange);
227 void radv_copy_vrs_htile(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *vrs_iview, const VkRect2D *rect,
228                          struct radv_image *dst_image, struct radv_buffer *htile_buffer, bool read_htile_value);
229 
230 bool radv_can_use_fmask_copy(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_image,
231                              const struct radv_image *dst_image, const struct radv_meta_blit2d_rect *rect);
232 void radv_fmask_copy(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src,
233                      struct radv_meta_blit2d_surf *dst);
234 
235 void radv_meta_resolve_compute_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
236                                      VkFormat src_format, VkImageLayout src_image_layout, struct radv_image *dst_image,
237                                      VkFormat dst_format, VkImageLayout dst_image_layout,
238                                      const VkImageResolve2 *region);
239 
240 void radv_meta_resolve_fragment_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
241                                       VkImageLayout src_image_layout, struct radv_image *dst_image,
242                                       VkImageLayout dst_image_layout, const VkImageResolve2 *region);
243 
244 void radv_decompress_resolve_rendering_src(struct radv_cmd_buffer *cmd_buffer);
245 
246 void radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
247                                  VkImageLayout src_image_layout, const VkImageResolve2 *region);
248 
249 uint32_t radv_clear_cmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
250                           const VkImageSubresourceRange *range, uint32_t value);
251 uint32_t radv_clear_fmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
252                           const VkImageSubresourceRange *range, uint32_t value);
253 uint32_t radv_clear_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
254                         const VkImageSubresourceRange *range, uint32_t value);
255 uint32_t radv_clear_htile(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image,
256                           const VkImageSubresourceRange *range, uint32_t value);
257 
258 void radv_update_buffer_cp(struct radv_cmd_buffer *cmd_buffer, uint64_t va, const void *data, uint64_t size);
259 
260 void radv_meta_decode_etc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout layout,
261                           const VkImageSubresourceLayers *subresource, VkOffset3D offset, VkExtent3D extent);
262 void radv_meta_decode_astc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, VkImageLayout layout,
263                            const VkImageSubresourceLayers *subresource, VkOffset3D offset, VkExtent3D extent);
264 
265 /* common nir builder helpers */
266 #include "nir/nir_builder.h"
267 
268 nir_builder PRINTFLIKE(3, 4)
269    radv_meta_init_shader(struct radv_device *dev, gl_shader_stage stage, const char *name, ...);
270 
271 nir_shader *radv_meta_build_nir_vs_generate_vertices(struct radv_device *dev);
272 nir_shader *radv_meta_build_nir_fs_noop(struct radv_device *dev);
273 
274 void radv_meta_build_resolve_shader_core(struct radv_device *device, nir_builder *b, bool is_integer, int samples,
275                                          nir_variable *input_img, nir_variable *color, nir_def *img_coord);
276 
277 nir_def *radv_meta_load_descriptor(nir_builder *b, unsigned desc_set, unsigned binding);
278 
279 nir_def *get_global_ids(nir_builder *b, unsigned num_components);
280 
281 void radv_break_on_count(nir_builder *b, nir_variable *var, nir_def *count);
282 
283 uint32_t radv_fill_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image,
284                           struct radeon_winsys_bo *bo, uint64_t va, uint64_t size, uint32_t value);
285 
286 void radv_copy_buffer(struct radv_cmd_buffer *cmd_buffer, struct radeon_winsys_bo *src_bo,
287                       struct radeon_winsys_bo *dst_bo, uint64_t src_offset, uint64_t dst_offset, uint64_t size);
288 
289 void radv_cmd_buffer_clear_attachment(struct radv_cmd_buffer *cmd_buffer, const VkClearAttachment *attachment);
290 
291 void radv_cmd_buffer_clear_rendering(struct radv_cmd_buffer *cmd_buffer, const VkRenderingInfo *render_info);
292 
293 void radv_cmd_buffer_resolve_rendering(struct radv_cmd_buffer *cmd_buffer);
294 
295 void radv_cmd_buffer_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
296                                           VkImageLayout src_layout, struct radv_image_view *dst_iview,
297                                           VkImageLayout dst_layout, const VkImageResolve2 *region);
298 
299 void radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects,
300                                              VkResolveModeFlagBits resolve_mode);
301 
302 void radv_cmd_buffer_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
303                                           VkImageLayout src_layout, struct radv_image_view *dst_iview,
304                                           VkImageLayout dst_layout);
305 
306 void radv_depth_stencil_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects,
307                                              VkResolveModeFlagBits resolve_mode);
308 
309 VkResult radv_meta_create_compute_pipeline(struct radv_device *device, nir_shader *nir,
310                                            VkPipelineLayout pipeline_layout, VkPipeline *pipeline);
311 
312 VkResult radv_meta_create_pipeline_layout(struct radv_device *device, VkDescriptorSetLayout *set_layout,
313                                           uint32_t num_pc_ranges, const VkPushConstantRange *pc_ranges,
314                                           VkPipelineLayout *pipeline_layout);
315 
316 VkResult radv_meta_create_descriptor_set_layout(struct radv_device *device, uint32_t num_bindings,
317                                                 const VkDescriptorSetLayoutBinding *bindings,
318                                                 VkDescriptorSetLayout *desc_layout);
319 
320 #ifdef __cplusplus
321 }
322 #endif
323 
324 #endif /* RADV_META_H */
325