1 /*
2 * Copyright © 2022 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Mike Blumenkrantz <[email protected]>
25 */
26
27 #ifndef ZINK_TYPES_H
28 #define ZINK_TYPES_H
29
30 #include <vulkan/vulkan_core.h>
31
32 #include "compiler/nir/nir.h"
33
34 #include "pipe/p_context.h"
35 #include "pipe/p_defines.h"
36 #include "pipe/p_state.h"
37
38 #include "pipebuffer/pb_cache.h"
39 #include "pipebuffer/pb_slab.h"
40
41 #include "util/disk_cache.h"
42 #include "util/hash_table.h"
43 #include "util/list.h"
44 #include "util/log.h"
45 #include "util/rwlock.h"
46 #include "util/set.h"
47 #include "util/simple_mtx.h"
48 #include "util/slab.h"
49 #include "util/u_blitter.h"
50 #include "util/u_dynarray.h"
51 #include "util/u_idalloc.h"
52 #include "util/u_live_shader_cache.h"
53 #include "util/u_queue.h"
54 #include "util/u_range.h"
55 #include "util/u_threaded_context.h"
56 #include "util/u_transfer.h"
57 #include "util/u_vertex_state_cache.h"
58
59 #include "vk_util.h"
60
61 #include "zink_device_info.h"
62 #include "zink_instance.h"
63 #include "zink_shader_keys.h"
64 #include "vk_dispatch_table.h"
65
66 #include "renderdoc_app.h"
67
68 /* the descriptor binding id for fbfetch/input attachment */
69 #define ZINK_FBFETCH_BINDING 5
70 #define ZINK_GFX_SHADER_COUNT 5
71
72 /* number of descriptors to allocate in a pool */
73 #define MAX_LAZY_DESCRIPTORS 500
74 /* explicit clamping because descriptor caching used to exist */
75 #define ZINK_MAX_SHADER_IMAGES 32
76 /* total number of bindless ids that can be allocated */
77 #define ZINK_MAX_BINDLESS_HANDLES 1024
78
79 /* enum zink_descriptor_type */
80 #define ZINK_MAX_DESCRIPTOR_SETS 6
81 #define ZINK_MAX_DESCRIPTORS_PER_TYPE (32 * ZINK_GFX_SHADER_COUNT)
82 /* Descriptor size reported by lavapipe. */
83 #define ZINK_FBFETCH_DESCRIPTOR_SIZE 280
84
85 /* suballocator defines */
86 #define NUM_SLAB_ALLOCATORS 3
87 #define MIN_SLAB_ORDER 8
88
89
90 /* this is the spec minimum */
91 #define ZINK_SPARSE_BUFFER_PAGE_SIZE (64 * 1024)
92
93 /* flag to create screen->copy_context */
94 #define ZINK_CONTEXT_COPY_ONLY (1<<30)
95
96 /* convenience macros for accessing dispatch table functions */
97 #define VKCTX(fn) zink_screen(ctx->base.screen)->vk.fn
98 #define VKSCR(fn) screen->vk.fn
99
100 #ifdef __cplusplus
101 extern "C" {
102 #endif
103
104 extern uint32_t zink_debug;
105 extern bool zink_tracing;
106
107 #ifdef __cplusplus
108 }
109 #endif
110
111
112 /** enums */
113
114 /* features for draw/program templates */
115 typedef enum {
116 ZINK_NO_MULTIDRAW,
117 ZINK_MULTIDRAW,
118 } zink_multidraw;
119
120 typedef enum {
121 ZINK_NO_DYNAMIC_STATE,
122 ZINK_DYNAMIC_STATE,
123 ZINK_DYNAMIC_STATE2,
124 ZINK_DYNAMIC_VERTEX_INPUT2,
125 ZINK_DYNAMIC_STATE3,
126 ZINK_DYNAMIC_VERTEX_INPUT,
127 } zink_dynamic_state;
128
129 typedef enum {
130 ZINK_PIPELINE_NO_DYNAMIC_STATE,
131 ZINK_PIPELINE_DYNAMIC_STATE,
132 ZINK_PIPELINE_DYNAMIC_STATE2,
133 ZINK_PIPELINE_DYNAMIC_STATE2_PCP,
134 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT2,
135 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT2_PCP,
136 ZINK_PIPELINE_DYNAMIC_STATE3,
137 ZINK_PIPELINE_DYNAMIC_STATE3_PCP,
138 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT,
139 ZINK_PIPELINE_DYNAMIC_VERTEX_INPUT_PCP,
140 } zink_pipeline_dynamic_state;
141
142 enum zink_blit_flags {
143 ZINK_BLIT_NORMAL = 1 << 0,
144 ZINK_BLIT_SAVE_FS = 1 << 1,
145 ZINK_BLIT_SAVE_FB = 1 << 2,
146 ZINK_BLIT_SAVE_TEXTURES = 1 << 3,
147 ZINK_BLIT_NO_COND_RENDER = 1 << 4,
148 ZINK_BLIT_SAVE_FS_CONST_BUF = 1 << 5,
149 };
150
151 /* descriptor types; also the ordering of the sets
152 * ...except that ZINK_DESCRIPTOR_BASE_TYPES is actually ZINK_DESCRIPTOR_TYPE_UNIFORMS,
153 * and all base type values are thus +1 to get the set id (using screen->desc_set_id[idx])
154 */
155 enum zink_descriptor_type {
156 ZINK_DESCRIPTOR_TYPE_UBO,
157 ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW,
158 ZINK_DESCRIPTOR_TYPE_SSBO,
159 ZINK_DESCRIPTOR_TYPE_IMAGE,
160 ZINK_DESCRIPTOR_BASE_TYPES, /**< the count/iterator for basic descriptor types */
161 ZINK_DESCRIPTOR_BINDLESS,
162 ZINK_DESCRIPTOR_ALL_TYPES,
163 ZINK_DESCRIPTOR_TYPE_UNIFORMS = ZINK_DESCRIPTOR_BASE_TYPES, /**< this is aliased for convenience */
164 ZINK_DESCRIPTOR_NON_BINDLESS_TYPES = ZINK_DESCRIPTOR_BASE_TYPES + 1, /**< for struct sizing */
165 };
166
167 enum zink_descriptor_mode {
168 ZINK_DESCRIPTOR_MODE_AUTO,
169 ZINK_DESCRIPTOR_MODE_LAZY,
170 ZINK_DESCRIPTOR_MODE_DB,
171 };
172
173 /* the current mode */
174 extern enum zink_descriptor_mode zink_descriptor_mode;
175
176 /* indexing for descriptor template management */
177 enum zink_descriptor_size_index {
178 ZDS_INDEX_UBO,
179 ZDS_INDEX_COMBINED_SAMPLER,
180 ZDS_INDEX_UNIFORM_TEXELS,
181 ZDS_INDEX_SAMPLER,
182 ZDS_INDEX_STORAGE_BUFFER,
183 ZDS_INDEX_STORAGE_IMAGE,
184 ZDS_INDEX_STORAGE_TEXELS,
185 ZDS_INDEX_MAX,
186 };
187
188 /* indexing for descriptor template management in COMPACT mode */
189 enum zink_descriptor_size_index_compact {
190 ZDS_INDEX_COMP_UBO,
191 ZDS_INDEX_COMP_STORAGE_BUFFER,
192 ZDS_INDEX_COMP_COMBINED_SAMPLER,
193 ZDS_INDEX_COMP_UNIFORM_TEXELS,
194 ZDS_INDEX_COMP_SAMPLER,
195 ZDS_INDEX_COMP_STORAGE_IMAGE,
196 ZDS_INDEX_COMP_STORAGE_TEXELS,
197 };
198
199 enum zink_resource_access {
200 ZINK_RESOURCE_ACCESS_READ = 1,
201 ZINK_RESOURCE_ACCESS_WRITE = 32,
202 ZINK_RESOURCE_ACCESS_RW = ZINK_RESOURCE_ACCESS_READ | ZINK_RESOURCE_ACCESS_WRITE,
203 };
204
205
206 /* zink heaps are based off of vulkan memory types, but are not a 1-to-1 mapping to vulkan memory type indices and have no direct relation to vulkan memory heaps*/
207 enum zink_heap {
208 ZINK_HEAP_DEVICE_LOCAL,
209 ZINK_HEAP_DEVICE_LOCAL_SPARSE,
210 ZINK_HEAP_DEVICE_LOCAL_LAZY,
211 ZINK_HEAP_DEVICE_LOCAL_VISIBLE,
212 ZINK_HEAP_HOST_VISIBLE_COHERENT,
213 ZINK_HEAP_HOST_VISIBLE_COHERENT_CACHED,
214 ZINK_HEAP_MAX,
215 };
216
217 enum zink_alloc_flag {
218 ZINK_ALLOC_SPARSE = 1<<0,
219 ZINK_ALLOC_NO_SUBALLOC = 1<<1,
220 };
221
222 enum zink_debug {
223 ZINK_DEBUG_NIR = (1<<0),
224 ZINK_DEBUG_SPIRV = (1<<1),
225 ZINK_DEBUG_TGSI = (1<<2),
226 ZINK_DEBUG_VALIDATION = (1<<3),
227 ZINK_DEBUG_SYNC = (1<<4),
228 ZINK_DEBUG_COMPACT = (1<<5),
229 ZINK_DEBUG_NOREORDER = (1<<6),
230 ZINK_DEBUG_GPL = (1<<7),
231 ZINK_DEBUG_SHADERDB = (1<<8),
232 ZINK_DEBUG_RP = (1<<9),
233 ZINK_DEBUG_NORP = (1<<10),
234 ZINK_DEBUG_MAP = (1<<11),
235 ZINK_DEBUG_FLUSHSYNC = (1<<12),
236 ZINK_DEBUG_NOSHOBJ = (1<<13),
237 ZINK_DEBUG_OPTIMAL_KEYS = (1<<14),
238 ZINK_DEBUG_NOOPT = (1<<15),
239 ZINK_DEBUG_NOBGC = (1<<16),
240 ZINK_DEBUG_MEM = (1<<17),
241 ZINK_DEBUG_QUIET = (1<<18),
242 ZINK_DEBUG_IOOPT = (1<<19),
243 ZINK_DEBUG_NOPC = (1<<20),
244 };
245
246 enum zink_pv_emulation_primitive {
247 ZINK_PVE_PRIMITIVE_NONE = 0,
248 ZINK_PVE_PRIMITIVE_SIMPLE = 1,
249 /* when triangle or quad strips are used and the gs outputs triangles */
250 ZINK_PVE_PRIMITIVE_TRISTRIP = 2,
251 ZINK_PVE_PRIMITIVE_FAN = 3,
252 };
253
254 /** fence types */
255 struct tc_unflushed_batch_token;
256
257 /* an async fence created for tc */
258 struct zink_tc_fence {
259 struct pipe_reference reference;
260 /* enables distinction between tc fence submission and vk queue submission */
261 uint32_t submit_count;
262 /* when the tc fence is signaled for use */
263 struct util_queue_fence ready;
264 struct tc_unflushed_batch_token *tc_token;
265 /* for deferred flushes */
266 struct pipe_context *deferred_ctx;
267 /* multiple tc fences may point to a real fence */
268 struct zink_fence *fence;
269 /* for use with semaphore/imported fences */
270 VkSemaphore sem;
271 };
272
273 /* a fence is actually a zink_batch_state, but these are split out for logical consistency */
274 struct zink_fence {
275 uint64_t batch_id;
276 bool submitted;
277 bool completed;
278 struct util_dynarray mfences;
279 };
280
281
282 /** state types */
283
284 struct zink_vertex_elements_hw_state {
285 uint32_t hash;
286 uint32_t num_bindings, num_attribs;
287 /* VK_EXT_vertex_input_dynamic_state uses different types */
288 union {
289 VkVertexInputAttributeDescription attribs[PIPE_MAX_ATTRIBS];
290 VkVertexInputAttributeDescription2EXT dynattribs[PIPE_MAX_ATTRIBS];
291 };
292 union {
293 struct {
294 VkVertexInputBindingDivisorDescriptionEXT divisors[PIPE_MAX_ATTRIBS];
295 VkVertexInputBindingDescription bindings[PIPE_MAX_ATTRIBS]; // combination of element_state and stride
296 VkDeviceSize strides[PIPE_MAX_ATTRIBS];
297 uint8_t divisors_present;
298 } b;
299 VkVertexInputBindingDescription2EXT dynbindings[PIPE_MAX_ATTRIBS];
300 };
301 uint8_t binding_map[PIPE_MAX_ATTRIBS];
302 };
303
304 struct zink_vertex_elements_state {
305 /* decomposed attributes read only a single component for format compatibility */
306 bool has_decomposed_attrs;
307 struct {
308 uint32_t binding;
309 VkVertexInputRate inputRate;
310 } bindings[PIPE_MAX_ATTRIBS];
311 uint32_t divisor[PIPE_MAX_ATTRIBS];
312 uint32_t min_stride[PIPE_MAX_ATTRIBS]; //for dynamic_state1
313 uint32_t decomposed_attrs;
314 unsigned decomposed_attrs_size;
315 uint32_t decomposed_attrs_without_w;
316 unsigned decomposed_attrs_without_w_size;
317 struct zink_vertex_elements_hw_state hw_state;
318 };
319
320 /* for vertex state draws */
321 struct zink_vertex_state {
322 struct pipe_vertex_state b;
323 struct zink_vertex_elements_state velems;
324 };
325
326 struct zink_rasterizer_hw_state {
327 unsigned polygon_mode : 2; //VkPolygonMode
328 unsigned line_mode : 2; //VkLineRasterizationModeEXT
329 unsigned depth_clip:1;
330 unsigned depth_clamp:1;
331 unsigned pv_last:1;
332 unsigned line_stipple_enable:1;
333 unsigned clip_halfz:1;
334 };
335
336 struct zink_rasterizer_state {
337 struct pipe_rasterizer_state base;
338 bool offset_fill;
339 float offset_units, offset_clamp, offset_scale;
340 float line_width;
341 VkFrontFace front_face;
342 VkCullModeFlags cull_mode;
343 VkLineRasterizationModeEXT dynamic_line_mode;
344 struct zink_rasterizer_hw_state hw_state;
345 };
346
347 struct zink_blend_state {
348 uint32_t hash;
349 unsigned num_rts;
350 VkPipelineColorBlendAttachmentState attachments[PIPE_MAX_COLOR_BUFS];
351
352 struct {
353 VkBool32 enables[PIPE_MAX_COLOR_BUFS];
354 VkColorBlendEquationEXT eq[PIPE_MAX_COLOR_BUFS];
355 VkColorComponentFlags wrmask[PIPE_MAX_COLOR_BUFS];
356 } ds3;
357
358 VkBool32 logicop_enable;
359 VkLogicOp logicop_func;
360
361 VkBool32 alpha_to_coverage;
362 VkBool32 alpha_to_one;
363
364 uint32_t wrmask;
365 uint8_t enables;
366
367 bool dual_src_blend;
368 };
369
370 struct zink_depth_stencil_alpha_hw_state {
371 VkBool32 depth_test;
372 VkCompareOp depth_compare_op;
373
374 VkBool32 depth_bounds_test;
375 float min_depth_bounds, max_depth_bounds;
376
377 VkBool32 stencil_test;
378 VkStencilOpState stencil_front;
379 VkStencilOpState stencil_back;
380
381 VkBool32 depth_write;
382 };
383
384 struct zink_depth_stencil_alpha_state {
385 struct pipe_depth_stencil_alpha_state base;
386 struct zink_depth_stencil_alpha_hw_state hw_state;
387 };
388
389
390 /** descriptor types */
391
392 /* zink_descriptor_layout objects are cached: this is the key for one */
393 struct zink_descriptor_layout_key {
394 unsigned num_bindings;
395 VkDescriptorSetLayoutBinding *bindings;
396 };
397
398 struct zink_descriptor_layout {
399 VkDescriptorSetLayout layout;
400 };
401
402 /* descriptor pools are cached: zink_descriptor_pool_key::id is the id for a type of pool */
403 struct zink_descriptor_pool_key {
404 unsigned use_count;
405 unsigned num_type_sizes;
406 unsigned id;
407 VkDescriptorPoolSize sizes[4];
408 struct zink_descriptor_layout_key *layout;
409 };
410
411 /* a template used for updating descriptor buffers */
412 struct zink_descriptor_template {
413 uint16_t stride; //the stride between mem pointers
414 uint16_t db_size; //the size of the entry in the buffer
415 unsigned count; //the number of descriptors
416 size_t offset; //the offset of the base host pointer to update from
417 };
418
419 /* ctx->dd; created at context creation */
420 struct zink_descriptor_data {
421 bool bindless_bound;
422 bool bindless_init;
423 bool has_fbfetch;
424 bool push_state_changed[2]; //gfx, compute
425 uint8_t state_changed[2]; //gfx, compute
426 struct zink_descriptor_layout_key *push_layout_keys[2]; //gfx, compute
427 struct zink_descriptor_layout *push_dsl[2]; //gfx, compute
428 VkDescriptorUpdateTemplate push_template[2]; //gfx, compute
429
430 struct zink_descriptor_layout *dummy_dsl;
431
432 union {
433 struct {
434 VkDescriptorPool bindless_pool;
435 VkDescriptorSet bindless_set;
436 } t;
437 struct {
438 struct zink_resource *bindless_db;
439 uint8_t *bindless_db_map;
440 struct pipe_transfer *bindless_db_xfer;
441 uint32_t bindless_db_offsets[4];
442 unsigned max_db_size;
443 unsigned size_enlarge_scale;
444 } db;
445 };
446
447 struct zink_program *pg[2]; //gfx, compute
448
449 VkDescriptorUpdateTemplateEntry push_entries[MESA_SHADER_STAGES]; //gfx+fbfetch
450 VkDescriptorUpdateTemplateEntry compute_push_entry;
451
452 /* push descriptor layout size and binding offsets */
453 uint32_t db_size[2]; //gfx, compute
454 uint32_t db_offset[ZINK_GFX_SHADER_COUNT + 1]; //gfx + fbfetch
455 /* compute offset is always 0 */
456 };
457
458 /* pg->dd; created at program creation */
459 struct zink_program_descriptor_data {
460 bool bindless;
461 bool fbfetch;
462 /* bitmask of ubo0 usage for stages */
463 uint8_t push_usage;
464 /* bitmask of which sets are used by the program */
465 uint8_t binding_usage;
466 /* all the pool keys for the program */
467 struct zink_descriptor_pool_key *pool_key[ZINK_DESCRIPTOR_BASE_TYPES]; //push set doesn't need one
468 /* all the layouts for the program */
469 struct zink_descriptor_layout *layouts[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES];
470 /* all the templates for the program */
471 union {
472 VkDescriptorUpdateTemplate templates[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES];
473 struct zink_descriptor_template *db_template[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES];
474 };
475 uint32_t db_size[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the total size of the layout
476 uint32_t *db_offset[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //the offset of each binding in the layout
477 };
478
479 struct zink_descriptor_pool {
480 /* the current index of 'sets' */
481 unsigned set_idx;
482 /* number of sets allocated */
483 unsigned sets_alloc;
484 VkDescriptorPool pool;
485 /* sets are lazily allocated */
486 VkDescriptorSet sets[MAX_LAZY_DESCRIPTORS];
487 };
488
489 /* a zink_descriptor_pool_key matches up to this struct */
490 struct zink_descriptor_pool_multi {
491 /* for flagging when overflowed pools must be destroyed instead of reused */
492 bool reinit_overflow;
493 /* this flips to split usable overflow from in-use overflow */
494 unsigned overflow_idx;
495 /* zink_descriptor_pool objects that have exceeded MAX_LAZY_DESCRIPTORS sets */
496 struct util_dynarray overflowed_pools[2];
497 /* the current pool; may be null */
498 struct zink_descriptor_pool *pool;
499 /* pool key for convenience */
500 const struct zink_descriptor_pool_key *pool_key;
501 };
502
503 /* bs->dd; created on batch state creation */
504 struct zink_batch_descriptor_data {
505 /* pools have fbfetch initialized */
506 bool has_fbfetch;
507 /* are descriptor buffers bound */
508 bool db_bound;
509 /* real size of 'pools' */
510 unsigned pool_size[ZINK_DESCRIPTOR_BASE_TYPES];
511 /* this array is sized based on the max zink_descriptor_pool_key::id used by the batch; members may be NULL */
512 struct util_dynarray pools[ZINK_DESCRIPTOR_BASE_TYPES];
513 struct zink_descriptor_pool_multi push_pool[2]; //gfx, compute
514 /* the current program (for descriptor updating) */
515 struct zink_program *pg[2]; //gfx, compute
516 /* the current pipeline compatibility id (for pipeline compatibility rules) */
517 uint32_t compat_id[2]; //gfx, compute
518 /* the current set layout */
519 VkDescriptorSetLayout dsl[2][ZINK_DESCRIPTOR_BASE_TYPES]; //gfx, compute
520 union {
521 /* the current set for a given type; used for rebinding if pipeline compat id changes and current set must be rebound */
522 VkDescriptorSet sets[2][ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //gfx, compute
523 uint64_t cur_db_offset[ZINK_DESCRIPTOR_NON_BINDLESS_TYPES]; //gfx, compute; the current offset of a descriptor buffer for rebinds
524 };
525 /* mask of push descriptor usage */
526 unsigned push_usage[2]; //gfx, compute
527
528 struct zink_resource *db; //the descriptor buffer for a given type
529 uint8_t *db_map; //the host map for the buffer
530 struct pipe_transfer *db_xfer; //the transfer map for the buffer
531 uint64_t db_offset; //the "next" offset that will be used when the buffer is updated
532 };
533
534 /** batch types */
535 /* zink_batch_usage concepts:
536 * - batch "usage" is an indicator of when and how a BO was accessed
537 * - batch "tracking" is the batch state(s) containing an extra ref for a BO
538 *
539 * - usage prevents a BO from being mapped while it has pending+conflicting access
540 * - usage affects pipeline barrier generation for synchronizing reads and writes
541 * - usage MUST be removed before context destruction to avoid crashing during BO
542 * reclaiming in suballocator
543 *
544 * - tracking prevents a BO from being destroyed early
545 * - tracking enables usage to be pruned
546 *
547 *
548 * tracking is added:
549 * - any time a BO is used in a "one-off" operation (e.g., blit, index buffer, indirect buffer)
550 * - any time a descriptor is unbound
551 * - when a buffer is replaced (IFF: resource is bound as a descriptor or usage previously existed)
552 *
553 * tracking is removed:
554 * - in zink_reset_batch_state()
555 *
556 * usage is added:
557 * - any time a BO is used in a "one-off" operation (e.g., blit, index buffer, indirect buffer)
558 * - any time a descriptor is bound
559 * - any time a descriptor is unbound (IFF: usage previously existed)
560 * - for all bound descriptors on the first draw/dispatch after a flush (zink_update_descriptor_refs)
561 *
562 * usage is removed:
563 * - when tracking is removed (IFF: BO usage == tracking, i.e., this is the last batch that a BO was active on)
564 */
565 struct zink_batch_usage {
566 uint32_t usage;
567 /* this is a monotonic int used to disambiguate internal fences from their tc fence references */
568 uint32_t submit_count;
569 cnd_t flush;
570 mtx_t mtx;
571 bool unflushed;
572 };
573
574 struct zink_bo_usage {
575 uint32_t submit_count;
576 struct zink_batch_usage *u;
577 };
578
579 struct zink_batch_obj_list {
580 unsigned max_buffers;
581 unsigned num_buffers;
582 struct zink_resource_object **objs;
583 };
584
585 struct zink_batch_state {
586 struct zink_fence fence;
587 struct zink_batch_state *next;
588
589 struct zink_batch_usage usage;
590 struct zink_context *ctx;
591 VkCommandPool cmdpool;
592 VkCommandBuffer cmdbuf;
593 VkCommandBuffer reordered_cmdbuf;
594 VkCommandPool unsynchronized_cmdpool;
595 VkCommandBuffer unsynchronized_cmdbuf;
596 VkSemaphore signal_semaphore; //external signal semaphore
597 struct util_dynarray signal_semaphores; //external signal semaphores
598 struct util_dynarray wait_semaphores; //external wait semaphores
599 struct util_dynarray wait_semaphore_stages; //external wait semaphores
600 struct util_dynarray fd_wait_semaphores; //dmabuf wait semaphores
601 struct util_dynarray fd_wait_semaphore_stages; //dmabuf wait semaphores
602 struct util_dynarray tracked_semaphores; //semaphores which are just tracked
603 VkSemaphore sparse_semaphore; //current sparse wait semaphore
604 struct util_dynarray fences; //zink_tc_fence refs
605 simple_mtx_t ref_lock;
606
607 VkSemaphore present;
608 struct zink_resource *swapchain;
609 struct util_dynarray acquires;
610 struct util_dynarray acquire_flags;
611
612 VkAccessFlags unordered_write_access;
613 VkPipelineStageFlags unordered_write_stages;
614
615 simple_mtx_t exportable_lock;
616
617 struct util_queue_fence flush_completed;
618
619 struct set programs;
620 struct set dmabuf_exports;
621
622 #define BUFFER_HASHLIST_SIZE 32768
623 /* buffer_indices_hashlist[hash(bo)] returns -1 if the bo
624 * isn't part of any buffer lists or the index where the bo could be found.
625 * Since 1) hash collisions of 2 different bo can happen and 2) we use a
626 * single hashlist for the 3 buffer list, this is only a hint.
627 * batch_find_resource uses this hint to speed up buffers look up.
628 */
629 int16_t buffer_indices_hashlist[BUFFER_HASHLIST_SIZE];
630 uint16_t hashlist_min;
631 uint16_t hashlist_max;
632 struct zink_batch_obj_list real_objs;
633 struct zink_batch_obj_list slab_objs;
634 struct zink_batch_obj_list sparse_objs;
635 struct zink_resource_object *last_added_obj;
636 struct util_dynarray swapchain_obj; //this doesn't have a zink_bo and must be handled differently
637
638 struct util_dynarray unref_resources;
639 struct util_dynarray bindless_releases[2];
640
641 struct util_dynarray zombie_samplers;
642
643 struct set active_queries; /* zink_query objects which were active at some point in this batch */
644 struct util_dynarray dead_querypools;
645
646 struct util_dynarray freed_sparse_backing_bos;
647
648 struct zink_batch_descriptor_data dd;
649
650 VkDeviceSize resource_size;
651
652 bool is_device_lost;
653 /* these flags correspond to the matching cmdbufs */
654 bool has_work;
655 bool has_reordered_work;
656 bool has_unsync;
657 };
658
659 static inline struct zink_batch_state *
zink_batch_state(struct zink_fence * fence)660 zink_batch_state(struct zink_fence *fence)
661 {
662 return (struct zink_batch_state *)fence;
663 }
664
665 /** bo types */
666 struct bo_export {
667 /** File descriptor associated with a handle export. */
668 int drm_fd;
669
670 /** GEM handle in drm_fd */
671 uint32_t gem_handle;
672
673 struct list_head link;
674 };
675
676 struct zink_bo {
677 struct pb_buffer base;
678
679 union {
680 struct {
681 void *cpu_ptr; /* for user_ptr and permanent maps */
682 int map_count;
683 struct list_head exports;
684 simple_mtx_t export_lock;
685
686 bool is_user_ptr;
687 bool use_reusable_pool;
688
689 /* Whether buffer_get_handle or buffer_from_handle has been called,
690 * it can only transition from false to true. Protected by lock.
691 */
692 bool is_shared;
693 } real;
694 struct {
695 struct pb_slab_entry entry;
696 struct zink_bo *real;
697 } slab;
698 struct {
699 uint32_t num_va_pages;
700 uint32_t num_backing_pages;
701
702 struct list_head backing;
703
704 /* Commitment information for each page of the virtual memory area. */
705 struct zink_sparse_commitment *commitments;
706 } sparse;
707 } u;
708
709 VkDeviceMemory mem;
710 uint64_t offset;
711
712 uint32_t unique_id;
713 const char *name;
714
715 simple_mtx_t lock;
716
717 struct zink_bo_usage reads;
718 struct zink_bo_usage writes;
719
720 struct pb_cache_entry cache_entry[];
721 };
722
723 static inline struct zink_bo *
zink_bo(struct pb_buffer * pbuf)724 zink_bo(struct pb_buffer *pbuf)
725 {
726 return (struct zink_bo*)pbuf;
727 }
728
729 /** clear types */
730 struct zink_framebuffer_clear_data {
731 union {
732 union pipe_color_union color;
733 struct {
734 float depth;
735 unsigned stencil;
736 uint8_t bits : 2; // PIPE_CLEAR_DEPTH, PIPE_CLEAR_STENCIL
737 } zs;
738 };
739 struct pipe_scissor_state scissor;
740 bool has_scissor;
741 bool conditional;
742 };
743
744 struct zink_framebuffer_clear {
745 struct util_dynarray clears;
746 };
747
748
749 /** compiler types */
750 struct zink_shader_info {
751 uint16_t stride[PIPE_MAX_SO_BUFFERS];
752 uint32_t sampler_mask;
753 bool have_sparse;
754 bool have_vulkan_memory_model;
755 bool have_workgroup_memory_explicit_layout;
756 struct {
757 uint8_t flush_denorms:3; // 16, 32, 64
758 uint8_t preserve_denorms:3; // 16, 32, 64
759 bool denorms_32_bit_independence:1;
760 bool denorms_all_independence:1;
761 } float_controls;
762 unsigned bindless_set_idx;
763 };
764
765 enum zink_rast_prim {
766 ZINK_PRIM_POINTS,
767 ZINK_PRIM_LINES,
768 ZINK_PRIM_TRIANGLES,
769 ZINK_PRIM_MAX,
770 };
771
772 struct zink_shader_object {
773 union {
774 VkShaderEXT obj;
775 VkShaderModule mod;
776 };
777 struct spirv_shader *spirv;
778 };
779
780 struct zink_shader {
781 struct util_live_shader base;
782 uint32_t hash;
783 struct blob blob;
784 struct shader_info info;
785 /* this is deleted in zink_shader_init */
786 nir_shader *nir;
787
788 struct zink_shader_info sinfo;
789
790 struct {
791 int index;
792 int binding;
793 VkDescriptorType type;
794 unsigned char size;
795 } bindings[ZINK_DESCRIPTOR_BASE_TYPES][ZINK_MAX_DESCRIPTORS_PER_TYPE];
796 size_t num_bindings[ZINK_DESCRIPTOR_BASE_TYPES];
797 uint32_t ubos_used; // bitfield of which ubo indices are used
798 uint32_t ssbos_used; // bitfield of which ssbo indices are used
799 uint64_t arrayed_inputs; //mask of locations using arrayed io
800 uint64_t arrayed_outputs; //mask of locations using arrayed io
801 uint64_t flat_flags;
802 bool bindless;
803 bool can_inline;
804 bool has_uniforms;
805 bool has_edgeflags;
806 bool needs_inlining;
807 struct spirv_shader *spirv;
808
809 struct {
810 struct util_queue_fence fence;
811 struct zink_shader_object obj;
812 VkDescriptorSetLayout dsl;
813 VkPipelineLayout layout;
814 VkPipeline gpl;
815 VkDescriptorSetLayoutBinding *bindings;
816 unsigned num_bindings;
817 struct zink_descriptor_template *db_template;
818 unsigned db_size;
819 unsigned *db_offset;
820 } precompile;
821
822 simple_mtx_t lock;
823 struct set *programs;
824 struct util_dynarray pipeline_libs;
825
826 union {
827 struct {
828 struct zink_shader *generated_tcs; // a generated shader that this shader "owns"; only valid in the tes stage
829 struct zink_shader *generated_gs[MESA_PRIM_COUNT][ZINK_PRIM_MAX]; // generated shaders that this shader "owns"
830 struct zink_shader *parent; // for a generated gs this points to the shader that "owns" it
831
832 bool is_generated; // if this is a driver-created shader (e.g., tcs)
833 } non_fs;
834
835 struct {
836 /* Bitmask of textures that have shadow sampling result components
837 * other than RED accessed. This is a subset of !is_new_style_shadow
838 * (GLSL <1.30, ARB_fp) shadow sampling usage.
839 */
840 uint32_t legacy_shadow_mask;
841 nir_variable *fbfetch; //for fs output
842 } fs;
843 };
844 };
845
846
847 /** pipeline types */
848 struct zink_pipeline_dynamic_state1 {
849 uint8_t front_face; //VkFrontFace:1
850 uint8_t cull_mode; //VkCullModeFlags:2
851 uint16_t num_viewports;
852 struct zink_depth_stencil_alpha_hw_state *depth_stencil_alpha_state; //must be last
853 };
854
855 struct zink_pipeline_dynamic_state2 {
856 bool primitive_restart;
857 bool rasterizer_discard;
858 uint16_t vertices_per_patch; //5 bits
859 };
860
861 #define zink_pipeline_dynamic_state3 zink_rasterizer_hw_state
862
863 struct zink_gfx_pipeline_state {
864 /* order matches zink_gfx_output_key */
865 unsigned force_persample_interp:1;
866 uint32_t rast_samples:6;
867 uint32_t min_samples:6;
868 uint32_t feedback_loop : 1;
869 uint32_t feedback_loop_zs : 1;
870 uint32_t rast_attachment_order : 1;
871 uint32_t rp_state : 16;
872 VkSampleMask sample_mask;
873 uint32_t blend_id;
874
875 /* Pre-hashed value for table lookup, invalid when zero.
876 * Members after this point are not included in pipeline state hash key */
877 uint32_t hash;
878 bool dirty;
879
880 struct zink_pipeline_dynamic_state1 dyn_state1;
881
882 struct zink_pipeline_dynamic_state2 dyn_state2;
883 struct zink_pipeline_dynamic_state3 dyn_state3;
884
885 union {
886 VkShaderModule modules[MESA_SHADER_STAGES - 1];
887 uint32_t optimal_key;
888 };
889 bool modules_changed;
890
891 uint32_t vertex_hash;
892
893 uint32_t final_hash;
894
895 uint32_t _pad2;
896 /* order matches zink_gfx_input_key */
897 union {
898 struct {
899 unsigned idx:8;
900 bool uses_dynamic_stride;
901 };
902 uint32_t input;
903 };
904 uint32_t vertex_buffers_enabled_mask;
905 uint32_t vertex_strides[PIPE_MAX_ATTRIBS];
906 struct zink_vertex_elements_hw_state *element_state;
907 struct zink_zs_swizzle_key *shadow;
908 bool sample_locations_enabled;
909 enum mesa_prim shader_rast_prim, rast_prim; /* reduced type or max for unknown */
910 union {
911 struct {
912 struct zink_shader_key key[5];
913 struct zink_shader_key last_vertex;
914 } shader_keys;
915 struct {
916 union zink_shader_key_optimal key;
917 } shader_keys_optimal;
918 };
919 struct zink_blend_state *blend_state;
920 struct zink_render_pass *render_pass;
921 struct zink_render_pass *next_render_pass; //will be used next time rp is begun
922 VkFormat rendering_formats[PIPE_MAX_COLOR_BUFS];
923 VkPipelineRenderingCreateInfo rendering_info;
924 VkPipeline pipeline;
925 enum mesa_prim gfx_prim_mode; //pending mode
926 };
927
928 struct zink_compute_pipeline_state {
929 /* Pre-hashed value for table lookup, invalid when zero.
930 * Members after this point are not included in pipeline state hash key */
931 uint32_t hash;
932 uint32_t final_hash;
933 bool dirty;
934 uint32_t local_size[3];
935 uint32_t variable_shared_mem;
936
937 uint32_t module_hash;
938 VkShaderModule module;
939 bool module_changed;
940
941 struct zink_shader_key key;
942
943 VkPipeline pipeline;
944 };
945
946
947 /** program types */
948
949 /* create_gfx_pushconst must be kept in sync with this struct */
950 struct zink_gfx_push_constant {
951 unsigned draw_mode_is_indexed;
952 unsigned draw_id;
953 unsigned framebuffer_is_layered;
954 float default_inner_level[2];
955 float default_outer_level[4];
956 uint32_t line_stipple_pattern;
957 float viewport_scale[2];
958 float line_width;
959 };
960
961 /* The order of the enums MUST match the order of the zink_gfx_push_constant
962 * members.
963 */
964 enum zink_gfx_push_constant_member {
965 ZINK_GFX_PUSHCONST_DRAW_MODE_IS_INDEXED,
966 ZINK_GFX_PUSHCONST_DRAW_ID,
967 ZINK_GFX_PUSHCONST_FRAMEBUFFER_IS_LAYERED,
968 ZINK_GFX_PUSHCONST_DEFAULT_INNER_LEVEL,
969 ZINK_GFX_PUSHCONST_DEFAULT_OUTER_LEVEL,
970 ZINK_GFX_PUSHCONST_LINE_STIPPLE_PATTERN,
971 ZINK_GFX_PUSHCONST_VIEWPORT_SCALE,
972 ZINK_GFX_PUSHCONST_LINE_WIDTH,
973 ZINK_GFX_PUSHCONST_MAX
974 };
975
976 /* a shader module is used for directly reusing a shader module between programs,
977 * e.g., in the case where we're swapping out only one shader,
978 * allowing us to skip going through shader keys
979 */
980 struct zink_shader_module {
981 struct zink_shader_object obj;
982 uint32_t hash;
983 bool shobj;
984 bool default_variant;
985 bool has_nonseamless;
986 bool needs_zs_shader_swizzle;
987 uint8_t num_uniforms;
988 uint8_t key_size;
989 uint8_t key[0]; /* | key | uniforms | zs shader swizzle | */
990 };
991
992 struct zink_program {
993 struct pipe_reference reference;
994 struct zink_context *ctx;
995 blake3_hash blake3;
996 struct util_queue_fence cache_fence;
997 struct u_rwlock pipeline_cache_lock;
998 VkPipelineCache pipeline_cache;
999 size_t pipeline_cache_size;
1000 struct zink_batch_usage *batch_uses;
1001 bool is_compute;
1002 bool can_precompile;
1003 bool uses_shobj; //whether shader objects are used; programs CANNOT mix shader objects and shader modules
1004
1005 struct zink_program_descriptor_data dd;
1006
1007 uint32_t compat_id;
1008 VkPipelineLayout layout;
1009 VkDescriptorSetLayout dsl[ZINK_DESCRIPTOR_ALL_TYPES]; // one for each type + push + bindless
1010 unsigned num_dsl;
1011
1012 bool removed;
1013 };
1014
1015 #define STAGE_MASK_OPTIMAL (1<<16)
1016 #define STAGE_MASK_OPTIMAL_SHADOW (1<<17)
1017 typedef bool (*equals_gfx_pipeline_state_func)(const void *a, const void *b);
1018
1019 struct zink_gfx_library_key {
1020 uint32_t optimal_key; //equals_pipeline_lib_optimal
1021 VkShaderModule modules[ZINK_GFX_SHADER_COUNT];
1022 VkPipeline pipeline;
1023 };
1024
1025 struct zink_gfx_input_key {
1026 union {
1027 struct {
1028 unsigned idx:8;
1029 bool uses_dynamic_stride;
1030 };
1031 uint32_t input;
1032 };
1033 uint32_t vertex_buffers_enabled_mask;
1034 uint32_t vertex_strides[PIPE_MAX_ATTRIBS];
1035 struct zink_vertex_elements_hw_state *element_state;
1036 VkPipeline pipeline;
1037 };
1038
1039 struct zink_gfx_output_key {
1040 /* order matches zink_gfx_output_key */
1041 union {
1042 struct {
1043 unsigned force_persample_interp:1;
1044 uint32_t rast_samples:6;
1045 uint32_t min_samples:6;
1046 uint32_t feedback_loop : 1;
1047 uint32_t feedback_loop_zs : 1;
1048 uint32_t rast_attachment_order : 1;
1049 uint32_t rp_state : 16;
1050 };
1051 uint32_t key;
1052 };
1053
1054 /* TODO: compress these */
1055 VkSampleMask sample_mask;
1056 uint32_t blend_id;
1057 VkPipeline pipeline;
1058 };
1059
1060 struct zink_gfx_pipeline_cache_entry {
1061 struct zink_gfx_pipeline_state state;
1062 VkPipeline pipeline;
1063 struct zink_gfx_program *prog;
1064 /* GPL only */
1065 struct util_queue_fence fence;
1066 union {
1067 struct {
1068 struct zink_gfx_input_key *ikey;
1069 struct zink_gfx_library_key *gkey;
1070 struct zink_gfx_output_key *okey;
1071 VkPipeline unoptimized_pipeline;
1072 } gpl;
1073 struct zink_shader_object shobjs[ZINK_GFX_SHADER_COUNT];
1074 };
1075 };
1076
1077 struct zink_gfx_lib_cache {
1078 /* for hashing */
1079 struct zink_shader *shaders[ZINK_GFX_SHADER_COUNT];
1080 unsigned refcount;
1081 bool removed; //once removed from cache
1082 uint8_t stages_present;
1083
1084 simple_mtx_t lock;
1085 struct set libs; //zink_gfx_library_key -> VkPipeline
1086 };
1087
1088 struct zink_gfx_program {
1089 struct zink_program base;
1090
1091 bool is_separable; //not a full program
1092
1093 uint32_t stages_present; //mask of stages present in this program
1094 uint32_t stages_remaining; //mask of zink_shader remaining in this program
1095 uint32_t gfx_hash; //from ctx->gfx_hash
1096
1097 struct zink_shader *shaders[ZINK_GFX_SHADER_COUNT];
1098 struct zink_shader *last_vertex_stage;
1099 struct zink_shader_object objs[ZINK_GFX_SHADER_COUNT];
1100
1101 /* full */
1102 VkShaderEXT objects[ZINK_GFX_SHADER_COUNT];
1103 uint32_t module_hash[ZINK_GFX_SHADER_COUNT];
1104 struct blob blobs[ZINK_GFX_SHADER_COUNT];
1105 struct util_dynarray shader_cache[ZINK_GFX_SHADER_COUNT][2][2]; //normal, nonseamless cubes, inline uniforms
1106 unsigned inlined_variant_count[ZINK_GFX_SHADER_COUNT];
1107 uint32_t default_variant_hash;
1108 uint8_t inline_variants; //which stages are using inlined uniforms
1109 bool needs_inlining; // whether this program requires some uniforms to be inlined
1110 bool has_edgeflags;
1111 bool optimal_keys;
1112
1113 /* separable */
1114 struct zink_gfx_program *full_prog;
1115
1116 struct hash_table pipelines[2][11]; // [dynamic, renderpass][number of draw modes we support]
1117 uint32_t last_variant_hash;
1118
1119 uint32_t last_finalized_hash[2][4]; //[dynamic, renderpass][primtype idx]
1120 struct zink_gfx_pipeline_cache_entry *last_pipeline[2][4]; //[dynamic, renderpass][primtype idx]
1121
1122 struct zink_gfx_lib_cache *libs;
1123 };
1124
1125 struct zink_compute_program {
1126 struct zink_program base;
1127
1128 bool use_local_size;
1129 bool has_variable_shared_mem;
1130
1131 unsigned scratch_size;
1132
1133 unsigned num_inlinable_uniforms;
1134 nir_shader *nir; //only until precompile finishes
1135
1136 struct zink_shader_module *curr;
1137
1138 struct zink_shader_module *module; //base
1139 struct util_dynarray shader_cache[2]; //nonseamless cubes, inline uniforms
1140 unsigned inlined_variant_count;
1141
1142 struct zink_shader *shader;
1143 struct hash_table pipelines;
1144
1145 simple_mtx_t cache_lock; //extra lock because threads are insane and sand was not meant to think
1146
1147 VkPipeline base_pipeline;
1148 };
1149
1150
1151 /** renderpass types */
1152
1153 struct zink_rt_attrib {
1154 VkFormat format;
1155 VkSampleCountFlagBits samples;
1156 bool clear_color;
1157 union {
1158 bool clear_stencil;
1159 bool fbfetch;
1160 };
1161 bool invalid;
1162 bool needs_write;
1163 bool resolve;
1164 bool feedback_loop;
1165 };
1166
1167 struct zink_render_pass_state {
1168 union {
1169 struct {
1170 uint8_t num_cbufs : 5; /* PIPE_MAX_COLOR_BUFS = 8 */
1171 uint8_t have_zsbuf : 1;
1172 uint8_t samples:1; //for fs samplemask
1173 uint32_t num_zsresolves : 1;
1174 uint32_t num_cresolves : 24; /* PIPE_MAX_COLOR_BUFS, but this is a struct hole */
1175 };
1176 uint32_t val; //for comparison
1177 };
1178 struct zink_rt_attrib rts[PIPE_MAX_COLOR_BUFS + 1];
1179 unsigned num_rts;
1180 uint32_t clears; //for extra verification and update flagging
1181 uint16_t msaa_expand_mask;
1182 uint16_t msaa_samples; //used with VK_EXT_multisampled_render_to_single_sampled
1183 };
1184
1185 struct zink_pipeline_rt {
1186 VkFormat format;
1187 VkSampleCountFlagBits samples;
1188 };
1189
1190 struct zink_render_pass_pipeline_state {
1191 uint32_t num_attachments:14;
1192 uint32_t msaa_samples : 8;
1193 uint32_t fbfetch:1;
1194 uint32_t color_read:1;
1195 uint32_t depth_read:1;
1196 uint32_t depth_write:1;
1197 uint32_t num_cresolves:4;
1198 uint32_t num_zsresolves:1;
1199 bool samples:1; //for fs samplemask
1200 struct zink_pipeline_rt attachments[PIPE_MAX_COLOR_BUFS + 1];
1201 unsigned id;
1202 };
1203
1204 struct zink_render_pass {
1205 VkRenderPass render_pass;
1206 struct zink_render_pass_state state;
1207 unsigned pipeline_state;
1208 };
1209
1210
1211 /** resource types */
1212 struct zink_resource_object {
1213 struct pipe_reference reference;
1214
1215 VkPipelineStageFlags access_stage;
1216 VkAccessFlags access;
1217 VkPipelineStageFlags unordered_access_stage;
1218 VkAccessFlags unordered_access;
1219 VkAccessFlags last_write;
1220
1221 /* 'access' is propagated from unordered_access to handle ops occurring
1222 * in the ordered cmdbuf which can promote barriers to unordered
1223 */
1224 bool ordered_access_is_copied;
1225 bool unordered_read;
1226 bool unordered_write;
1227 bool unsync_access;
1228 bool copies_valid;
1229 bool copies_need_reset; //for use with batch state resets
1230
1231 struct u_rwlock copy_lock;
1232 struct util_dynarray copies[16]; //regions being copied to; for barrier omission
1233
1234 VkBuffer storage_buffer;
1235 simple_mtx_t view_lock;
1236 uint32_t view_prune_count; //how many views to prune
1237 uint32_t view_prune_timeline; //when to prune
1238 struct util_dynarray views;
1239
1240 union {
1241 VkBuffer buffer;
1242 VkImage image;
1243 };
1244 VkDeviceAddress bda;
1245
1246 VkSampleLocationsInfoEXT zs_evaluate;
1247 bool needs_zs_evaluate;
1248
1249 bool storage_init; //layout was set for image
1250 bool transfer_dst;
1251 bool render_target;
1252 bool is_buffer;
1253 bool exportable;
1254
1255 /* TODO: this should be a union */
1256 int handle;
1257 struct zink_bo *bo;
1258 // struct {
1259 struct kopper_displaytarget *dt;
1260 uint32_t dt_idx;
1261 uint32_t last_dt_idx;
1262 VkSemaphore present;
1263 bool new_dt;
1264 bool indefinite_acquire;
1265 // }
1266
1267
1268 unsigned miptail_commits;
1269 VkDeviceSize offset, size, alignment;
1270 uint64_t vkflags;
1271 uint64_t vkusage;
1272 VkFormatFeatureFlags vkfeats;
1273 uint64_t modifier;
1274 VkImageAspectFlags modifier_aspect;
1275 VkSamplerYcbcrConversion sampler_conversion;
1276 unsigned plane_offsets[3];
1277 unsigned plane_strides[3];
1278 unsigned plane_count;
1279
1280 bool host_visible;
1281 bool coherent;
1282 bool is_aux;
1283 };
1284
1285 struct zink_resource {
1286 struct threaded_resource base;
1287
1288 enum pipe_format internal_format:16;
1289
1290 struct zink_resource_object *obj;
1291 struct pipe_surface *surface; //for swapchain images
1292 uint32_t queue;
1293 union {
1294 struct {
1295 struct util_range valid_buffer_range;
1296 struct util_range *real_buffer_range; //only set on tc replace_buffer src
1297 uint32_t vbo_bind_mask : PIPE_MAX_ATTRIBS;
1298 uint8_t ubo_bind_count[2];
1299 uint8_t ssbo_bind_count[2];
1300 uint8_t vbo_bind_count;
1301 uint8_t so_bind_count; //not counted in all_binds
1302 bool so_valid;
1303 uint32_t ubo_bind_mask[MESA_SHADER_STAGES];
1304 uint32_t ssbo_bind_mask[MESA_SHADER_STAGES];
1305 };
1306 struct {
1307 bool linear;
1308 bool need_2D;
1309 bool valid;
1310 uint8_t fb_bind_count; //not counted in all_binds
1311 uint16_t fb_binds; /* mask of attachment idx; zs is PIPE_MAX_COLOR_BUFS */
1312 VkSparseImageMemoryRequirements sparse;
1313 VkFormat format;
1314 VkImageLayout layout;
1315 VkImageAspectFlags aspect;
1316 };
1317 };
1318 uint32_t sampler_binds[MESA_SHADER_STAGES];
1319 uint32_t image_binds[MESA_SHADER_STAGES];
1320 uint16_t sampler_bind_count[2]; //gfx, compute
1321 uint16_t image_bind_count[2]; //gfx, compute
1322 uint16_t write_bind_count[2]; //gfx, compute
1323 union {
1324 uint16_t bindless[2]; //tex, img
1325 uint32_t all_bindless;
1326 };
1327 union {
1328 uint16_t bind_count[2]; //gfx, compute
1329 uint32_t all_binds;
1330 };
1331
1332 VkPipelineStageFlagBits gfx_barrier;
1333 VkAccessFlagBits barrier_access[2]; //gfx, compute
1334
1335 union {
1336 struct {
1337 struct hash_table bufferview_cache;
1338 simple_mtx_t bufferview_mtx;
1339 };
1340 struct {
1341 struct hash_table surface_cache;
1342 simple_mtx_t surface_mtx;
1343 };
1344 };
1345
1346 VkRect2D damage;
1347 bool use_damage;
1348
1349 bool copies_warned;
1350 bool swapchain;
1351 bool dmabuf;
1352 unsigned dt_stride;
1353
1354 uint8_t modifiers_count;
1355 uint64_t *modifiers;
1356 };
1357
1358 static inline struct zink_resource *
zink_resource(struct pipe_resource * r)1359 zink_resource(struct pipe_resource *r)
1360 {
1361 return (struct zink_resource *)r;
1362 }
1363
1364
1365 struct zink_transfer {
1366 struct threaded_transfer base;
1367 struct pipe_resource *staging_res;
1368 unsigned offset;
1369 unsigned depthPitch;
1370 };
1371
1372
1373 /** screen types */
1374 struct zink_modifier_prop {
1375 uint32_t drmFormatModifierCount;
1376 VkDrmFormatModifierPropertiesEXT* pDrmFormatModifierProperties;
1377 };
1378
1379 struct zink_format_props {
1380 VkFormatFeatureFlags2 linearTilingFeatures;
1381 VkFormatFeatureFlags2 optimalTilingFeatures;
1382 VkFormatFeatureFlags2 bufferFeatures;
1383 };
1384
1385 struct zink_screen {
1386 struct pipe_screen base;
1387
1388 const char *vendor_name;
1389 const char *device_name;
1390
1391 struct util_dl_library *loader_lib;
1392 PFN_vkGetInstanceProcAddr vk_GetInstanceProcAddr;
1393 PFN_vkGetDeviceProcAddr vk_GetDeviceProcAddr;
1394
1395 bool threaded;
1396 bool threaded_submit;
1397 bool is_cpu;
1398 bool abort_on_hang;
1399 bool frame_marker_emitted;
1400 bool driver_name_is_inferred;
1401 uint64_t curr_batch; //the current batch id
1402 uint32_t last_finished;
1403 VkSemaphore sem;
1404 VkFence fence;
1405 struct util_queue flush_queue;
1406 simple_mtx_t copy_context_lock;
1407 struct zink_context *copy_context;
1408
1409 struct zink_batch_state *free_batch_states; //unused batch states
1410 struct zink_batch_state *last_free_batch_state; //for appending
1411 simple_mtx_t free_batch_states_lock;
1412
1413 simple_mtx_t semaphores_lock;
1414 struct util_dynarray semaphores;
1415 struct util_dynarray fd_semaphores;
1416
1417 unsigned buffer_rebind_counter;
1418 unsigned image_rebind_counter;
1419 unsigned robust_ctx_count;
1420
1421 struct hash_table dts;
1422 simple_mtx_t dt_lock;
1423
1424 bool device_lost;
1425 int drm_fd;
1426
1427 struct slab_parent_pool transfer_pool;
1428 struct disk_cache *disk_cache;
1429 struct util_queue cache_put_thread;
1430 struct util_queue cache_get_thread;
1431
1432 /* there are 5 gfx stages, but VS and FS are assumed to be always present,
1433 * thus only 3 stages need to be considered, giving 2^3 = 8 program caches.
1434 */
1435 struct set pipeline_libs[8];
1436 simple_mtx_t pipeline_libs_lock[8];
1437
1438 simple_mtx_t desc_set_layouts_lock;
1439 struct hash_table desc_set_layouts[ZINK_DESCRIPTOR_BASE_TYPES];
1440 simple_mtx_t desc_pool_keys_lock;
1441 struct set desc_pool_keys[ZINK_DESCRIPTOR_BASE_TYPES];
1442 struct util_live_shader_cache shaders;
1443
1444 uint64_t db_size[ZINK_DESCRIPTOR_ALL_TYPES];
1445 unsigned base_descriptor_size;
1446 VkDescriptorSetLayout bindless_layout;
1447
1448 struct {
1449 struct pb_cache bo_cache;
1450 struct pb_slabs bo_slabs[NUM_SLAB_ALLOCATORS];
1451 unsigned min_alloc_size;
1452 uint32_t next_bo_unique_id;
1453 } pb;
1454 uint8_t heap_map[ZINK_HEAP_MAX][VK_MAX_MEMORY_TYPES]; // mapping from zink heaps to memory type indices
1455 uint8_t heap_count[ZINK_HEAP_MAX]; // number of memory types per zink heap
1456 bool resizable_bar;
1457
1458 uint64_t total_video_mem;
1459 uint64_t clamp_video_mem;
1460 uint64_t total_mem;
1461 uint64_t mapped_vram;
1462
1463 VkInstance instance;
1464 struct zink_instance_info instance_info;
1465
1466 struct hash_table *debug_mem_sizes;
1467 simple_mtx_t debug_mem_lock;
1468
1469 VkPhysicalDevice pdev;
1470 uint32_t vk_version, spirv_version;
1471 struct util_idalloc_mt buffer_ids;
1472 struct util_vertex_state_cache vertex_state_cache;
1473
1474 struct zink_device_info info;
1475 struct nir_shader_compiler_options nir_options;
1476
1477 bool optimal_keys;
1478 bool have_full_ds3;
1479 bool have_X8_D24_UNORM_PACK32;
1480 bool have_D24_UNORM_S8_UINT;
1481 bool have_D32_SFLOAT_S8_UINT;
1482 bool have_triangle_fans;
1483 bool need_decompose_attrs;
1484 bool need_2D_zs;
1485 bool need_2D_sparse;
1486 bool can_hic_shader_read;
1487
1488 uint32_t gfx_queue;
1489 uint32_t sparse_queue;
1490 uint32_t max_queues;
1491 uint32_t timestamp_valid_bits;
1492 VkDevice dev;
1493 VkQueue queue; //gfx+compute
1494 VkQueue queue_sparse;
1495 simple_mtx_t queue_lock;
1496 VkDebugUtilsMessengerEXT debugUtilsCallbackHandle;
1497
1498 uint32_t cur_custom_border_color_samplers;
1499
1500 unsigned screen_id;
1501
1502 RENDERDOC_API_1_0_0 *renderdoc_api;
1503 unsigned renderdoc_capture_start;
1504 unsigned renderdoc_capture_end;
1505 unsigned renderdoc_frame;
1506 bool renderdoc_capturing;
1507 bool renderdoc_capture_all;
1508
1509 struct vk_uncompacted_dispatch_table vk;
1510
1511 void (*buffer_barrier)(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline);
1512 void (*image_barrier)(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
1513 void (*image_barrier_unsync)(struct zink_context *ctx, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
1514
1515 bool compact_descriptors; /**< toggled if descriptor set ids are compacted */
1516 uint8_t desc_set_id[ZINK_MAX_DESCRIPTOR_SETS]; /**< converts enum zink_descriptor_type -> the actual set id */
1517
1518 struct {
1519 bool dual_color_blend_by_location;
1520 bool inline_uniforms;
1521 bool emulate_point_smooth;
1522 bool zink_shader_object_enable;
1523 } driconf;
1524
1525 struct zink_format_props format_props[PIPE_FORMAT_COUNT];
1526 struct zink_modifier_prop modifier_props[PIPE_FORMAT_COUNT];
1527
1528 VkExtent2D maxSampleLocationGridSize[5];
1529 VkPipelineLayout gfx_push_constant_layout;
1530
1531 struct {
1532 /* these affect shader cache */
1533 bool lower_robustImageAccess2;
1534 bool needs_zs_shader_swizzle;
1535 bool needs_sanitised_layer;
1536 bool io_opt;
1537 } driver_compiler_workarounds;
1538 struct {
1539 bool broken_l4a4;
1540 /* https://gitlab.khronos.org/vulkan/vulkan/-/issues/3306
1541 * HI TURNIP
1542 */
1543 bool broken_cache_semantics;
1544 bool missing_a8_unorm;
1545 bool implicit_sync;
1546 bool disable_optimized_compile;
1547 bool always_feedback_loop;
1548 bool always_feedback_loop_zs;
1549 bool track_renderpasses;
1550 bool no_linestipple;
1551 bool no_linesmooth;
1552 bool no_hw_gl_point;
1553 bool can_do_invalid_linear_modifier;
1554 bool inconsistent_interpolation;
1555 bool can_2d_view_sparse;
1556 bool general_depth_layout;
1557 unsigned z16_unscaled_bias;
1558 unsigned z24_unscaled_bias;
1559 } driver_workarounds;
1560 };
1561
1562 static inline struct zink_screen *
zink_screen(struct pipe_screen * pipe)1563 zink_screen(struct pipe_screen *pipe)
1564 {
1565 return (struct zink_screen *)pipe;
1566 }
1567
1568 /** surface types */
1569
1570 /* info for validating/creating imageless framebuffers */
1571 struct zink_surface_info {
1572 VkImageCreateFlags flags;
1573 VkImageUsageFlags usage;
1574 uint32_t width;
1575 uint32_t height;
1576 uint32_t layerCount;
1577 VkFormat format[2]; //base format, srgb format (for srgb framebuffer)
1578 };
1579
1580 /* an imageview for a zink_resource:
1581 - may be a fb attachment, samplerview, or shader image
1582 - cached on the parent zink_resource_object
1583 - also handles swapchains
1584 */
1585 struct zink_surface {
1586 struct pipe_surface base;
1587 /* all the info for creating a new imageview */
1588 VkImageViewCreateInfo ivci;
1589 VkImageViewUsageCreateInfo usage_info;
1590 /* for framebuffer use */
1591 struct zink_surface_info info;
1592 bool is_swapchain;
1593 /* the current imageview */
1594 VkImageView image_view;
1595 /* array of imageviews for swapchains, one for each image */
1596 VkImageView *swapchain;
1597 unsigned swapchain_size;
1598 void *obj; //backing resource object; used to determine rebinds
1599 void *dt_swapchain; //current swapchain object; used to determine swapchain rebinds
1600 uint32_t hash; //for surface caching
1601 };
1602
1603 /* wrapper object that preserves the gallium expectation of having
1604 * pipe_surface::context match the context used to create the surface
1605 */
1606 struct zink_ctx_surface {
1607 struct pipe_surface base;
1608 struct zink_surface *surf; //the actual surface
1609 struct zink_ctx_surface *transient; //for use with EXT_multisample_render_to_texture
1610 bool transient_init; //whether the transient surface has data
1611 bool needs_mutable;
1612 };
1613
1614 /* use this cast for framebuffer surfaces */
1615 static inline struct zink_surface *
zink_csurface(struct pipe_surface * psurface)1616 zink_csurface(struct pipe_surface *psurface)
1617 {
1618 return psurface ? ((struct zink_ctx_surface *)psurface)->surf : NULL;
1619 }
1620
1621 /* use this cast for checking transient framebuffer surfaces */
1622 static inline struct zink_surface *
zink_transient_surface(struct pipe_surface * psurface)1623 zink_transient_surface(struct pipe_surface *psurface)
1624 {
1625 return psurface ? ((struct zink_ctx_surface *)psurface)->transient ? ((struct zink_ctx_surface *)psurface)->transient->surf : NULL : NULL;
1626 }
1627
1628 /* use this cast for internal surfaces */
1629 static inline struct zink_surface *
zink_surface(struct pipe_surface * psurface)1630 zink_surface(struct pipe_surface *psurface)
1631 {
1632 return (struct zink_surface *)psurface;
1633 }
1634
1635
1636 /** framebuffer types */
1637 struct zink_framebuffer_state {
1638 uint32_t width;
1639 uint16_t height;
1640 uint32_t layers:6;
1641 uint32_t samples:6;
1642 uint32_t num_attachments:4;
1643 struct zink_surface_info infos[PIPE_MAX_COLOR_BUFS + 1];
1644 };
1645
1646 struct zink_framebuffer {
1647 struct pipe_reference reference;
1648
1649 /* current objects */
1650 VkFramebuffer fb;
1651 struct zink_render_pass *rp;
1652
1653 struct zink_framebuffer_state state;
1654 VkFramebufferAttachmentImageInfo infos[PIPE_MAX_COLOR_BUFS + 1];
1655 struct hash_table objects;
1656 };
1657
1658
1659 /** context types */
1660 struct zink_sampler_state {
1661 VkSampler sampler;
1662 VkSampler sampler_clamped;
1663 bool custom_border_color;
1664 bool emulate_nonseamless;
1665 };
1666
1667 struct zink_buffer_view {
1668 struct pipe_reference reference;
1669 struct pipe_resource *pres;
1670 VkBufferViewCreateInfo bvci;
1671 VkBufferView buffer_view;
1672 uint32_t hash;
1673 };
1674
1675 struct zink_sampler_view {
1676 struct pipe_sampler_view base;
1677 union {
1678 struct zink_surface *image_view;
1679 struct zink_buffer_view *buffer_view;
1680 unsigned tbo_size;
1681 };
1682 struct zink_surface *cube_array;
1683 /* Optional sampler view returning red (depth) in all channels, for shader rewrites. */
1684 struct zink_surface *zs_view;
1685 struct zink_zs_swizzle swizzle;
1686 };
1687
1688 struct zink_image_view {
1689 struct pipe_image_view base;
1690 union {
1691 struct zink_surface *surface;
1692 struct zink_buffer_view *buffer_view;
1693 };
1694 };
1695
1696 static inline struct zink_sampler_view *
zink_sampler_view(struct pipe_sampler_view * pview)1697 zink_sampler_view(struct pipe_sampler_view *pview)
1698 {
1699 return (struct zink_sampler_view *)pview;
1700 }
1701
1702 struct zink_so_target {
1703 struct pipe_stream_output_target base;
1704 struct pipe_resource *counter_buffer;
1705 VkDeviceSize counter_buffer_offset;
1706 uint32_t stride;
1707 bool counter_buffer_valid;
1708 };
1709
1710 static inline struct zink_so_target *
zink_so_target(struct pipe_stream_output_target * so_target)1711 zink_so_target(struct pipe_stream_output_target *so_target)
1712 {
1713 return (struct zink_so_target *)so_target;
1714 }
1715
1716 struct zink_viewport_state {
1717 struct pipe_viewport_state viewport_states[PIPE_MAX_VIEWPORTS];
1718 struct pipe_scissor_state scissor_states[PIPE_MAX_VIEWPORTS];
1719 uint8_t num_viewports;
1720 };
1721
1722 struct zink_descriptor_db_info {
1723 unsigned offset;
1724 unsigned size;
1725 enum pipe_format format;
1726 struct pipe_resource *pres;
1727 };
1728
1729 struct zink_descriptor_surface {
1730 union {
1731 struct zink_surface *surface;
1732 struct zink_buffer_view *bufferview;
1733 struct zink_descriptor_db_info db;
1734 };
1735 bool is_buffer;
1736 };
1737
1738 struct zink_bindless_descriptor {
1739 struct zink_descriptor_surface ds;
1740 struct zink_sampler_state *sampler;
1741 uint32_t handle;
1742 uint32_t access; //PIPE_ACCESS_...
1743 };
1744
1745 struct zink_rendering_info {
1746 VkPipelineRenderingCreateInfo info;
1747 unsigned id;
1748 };
1749
1750
1751 typedef void (*pipe_draw_vertex_state_func)(struct pipe_context *ctx,
1752 struct pipe_vertex_state *vstate,
1753 uint32_t partial_velem_mask,
1754 struct pipe_draw_vertex_state_info info,
1755 const struct pipe_draw_start_count_bias *draws,
1756 unsigned num_draws);
1757 typedef void (*pipe_launch_grid_func)(struct pipe_context *pipe, const struct pipe_grid_info *info);
1758
1759
1760 enum zink_ds3_state {
1761 ZINK_DS3_RAST_STIPPLE,
1762 ZINK_DS3_RAST_CLIP,
1763 ZINK_DS3_RAST_CLAMP,
1764 ZINK_DS3_RAST_POLYGON,
1765 ZINK_DS3_RAST_HALFZ,
1766 ZINK_DS3_RAST_PV,
1767 ZINK_DS3_RAST_LINE,
1768 ZINK_DS3_RAST_STIPPLE_ON,
1769 ZINK_DS3_BLEND_A2C,
1770 ZINK_DS3_BLEND_A21,
1771 ZINK_DS3_BLEND_ON,
1772 ZINK_DS3_BLEND_WRITE,
1773 ZINK_DS3_BLEND_EQ,
1774 ZINK_DS3_BLEND_LOGIC_ON,
1775 ZINK_DS3_BLEND_LOGIC,
1776 };
1777
1778 struct zink_context {
1779 struct pipe_context base;
1780 struct threaded_context *tc;
1781 struct slab_child_pool transfer_pool;
1782 struct slab_child_pool transfer_pool_unsync;
1783 struct blitter_context *blitter;
1784 struct util_debug_callback dbg;
1785
1786 unsigned flags;
1787
1788 pipe_draw_func draw_vbo[2]; //batch changed
1789 pipe_draw_vertex_state_func draw_state[2]; //batch changed
1790 pipe_launch_grid_func launch_grid[2]; //batch changed
1791
1792 struct pipe_device_reset_callback reset;
1793
1794 struct util_queue_fence unsync_fence; //unsigned during unsync recording (blocks flush ops)
1795 struct util_queue_fence flush_fence; //unsigned during flush (blocks unsync ops)
1796
1797 struct zink_fence *deferred_fence;
1798 struct zink_batch_state *last_batch_state; //the last command buffer submitted
1799 struct zink_batch_state *batch_states; //list of submitted batch states: ordered by increasing timeline id
1800 unsigned batch_states_count; //number of states in `batch_states`
1801 struct zink_batch_state *free_batch_states; //unused batch states
1802 struct zink_batch_state *last_free_batch_state; //for appending
1803 bool oom_flush;
1804 bool oom_stall;
1805 bool track_renderpasses;
1806 bool no_reorder;
1807 struct zink_batch_state *bs;
1808
1809 unsigned shader_has_inlinable_uniforms_mask;
1810 unsigned inlinable_uniforms_valid_mask;
1811
1812 struct pipe_constant_buffer ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS];
1813 struct pipe_shader_buffer ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS];
1814 uint32_t writable_ssbos[MESA_SHADER_STAGES];
1815 struct zink_image_view image_views[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1816
1817 uint32_t transient_attachments;
1818 struct pipe_framebuffer_state fb_state;
1819 struct hash_table framebuffer_cache;
1820
1821 struct zink_vertex_elements_state *element_state;
1822 struct zink_rasterizer_state *rast_state;
1823 struct zink_depth_stencil_alpha_state *dsa_state;
1824
1825 bool pipeline_changed[2]; //gfx, compute
1826
1827 struct zink_shader *gfx_stages[ZINK_GFX_SHADER_COUNT];
1828 struct zink_shader *last_vertex_stage;
1829 bool shader_reads_drawid;
1830 bool shader_reads_basevertex;
1831 struct zink_gfx_pipeline_state gfx_pipeline_state;
1832 /* there are 5 gfx stages, but VS and FS are assumed to be always present,
1833 * thus only 3 stages need to be considered, giving 2^3 = 8 program caches.
1834 */
1835 struct hash_table program_cache[8];
1836 simple_mtx_t program_lock[8];
1837 uint32_t gfx_hash;
1838 struct zink_gfx_program *curr_program;
1839 struct set gfx_inputs;
1840 struct set gfx_outputs;
1841
1842 struct zink_descriptor_data dd;
1843
1844 struct zink_compute_pipeline_state compute_pipeline_state;
1845 struct zink_compute_program *curr_compute;
1846
1847 unsigned shader_stages : ZINK_GFX_SHADER_COUNT; /* mask of bound gfx shader stages */
1848 uint8_t dirty_gfx_stages; /* mask of changed gfx shader stages */
1849 bool last_vertex_stage_dirty;
1850 bool compute_dirty;
1851 bool is_generated_gs_bound;
1852
1853 struct {
1854 VkRenderingAttachmentInfo attachments[PIPE_MAX_COLOR_BUFS + 2]; //+depth, +stencil
1855 VkRenderingInfo info;
1856 struct tc_renderpass_info tc_info;
1857 } dynamic_fb;
1858 uint32_t fb_layer_mismatch; //bitmask
1859 unsigned depth_bias_scale_factor;
1860 struct set rendering_state_cache[6]; //[util_logbase2_ceil(msrtss samplecount)]
1861 struct set render_pass_state_cache;
1862 struct hash_table *render_pass_cache;
1863 struct zink_resource *swapchain;
1864 VkExtent2D swapchain_size;
1865 bool fb_changed;
1866 bool in_rp; //renderpass is currently active
1867 bool rp_changed; //force renderpass restart
1868 bool rp_layout_changed; //renderpass changed, maybe restart
1869 bool rp_loadop_changed; //renderpass changed, don't restart
1870 bool zsbuf_unused;
1871 bool zsbuf_readonly;
1872
1873 struct zink_framebuffer *framebuffer;
1874 struct zink_framebuffer_clear fb_clears[PIPE_MAX_COLOR_BUFS + 1];
1875 uint16_t clears_enabled;
1876 uint16_t rp_clears_enabled;
1877 uint16_t void_clears;
1878 uint16_t fbfetch_outputs;
1879 uint16_t feedback_loops;
1880 struct zink_resource *needs_present;
1881
1882 struct pipe_vertex_buffer vertex_buffers[PIPE_MAX_ATTRIBS];
1883 bool vertex_buffers_dirty;
1884
1885 struct zink_sampler_state *sampler_states[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1886 struct pipe_sampler_view *sampler_views[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1887
1888 struct zink_viewport_state vp_state;
1889 bool vp_state_changed;
1890 bool scissor_changed;
1891
1892 float blend_constants[4];
1893
1894 bool sample_locations_changed;
1895 VkSampleLocationEXT vk_sample_locations[PIPE_MAX_SAMPLE_LOCATION_GRID_SIZE * PIPE_MAX_SAMPLE_LOCATION_GRID_SIZE];
1896 uint8_t sample_locations[2 * 4 * 8 * 16];
1897
1898 struct pipe_stencil_ref stencil_ref;
1899
1900 union {
1901 struct {
1902 float default_inner_level[2];
1903 float default_outer_level[4];
1904 };
1905 float tess_levels[6];
1906 };
1907
1908 struct zink_vk_query *curr_xfb_queries[PIPE_MAX_VERTEX_STREAMS];
1909 struct zink_shader *null_fs;
1910 struct zink_shader *saved_fs;
1911
1912 struct list_head query_pools;
1913 struct list_head suspended_queries;
1914 struct list_head primitives_generated_queries;
1915 struct zink_query *vertices_query;
1916 bool disable_fs;
1917 bool disable_color_writes;
1918 bool was_line_loop;
1919 bool fs_query_active;
1920 bool occlusion_query_active;
1921 bool primitives_generated_active;
1922 bool primitives_generated_suspended;
1923 bool queries_disabled, render_condition_active;
1924 bool queries_in_rp;
1925 struct {
1926 struct zink_query *query;
1927 bool inverted;
1928 bool active; //this is the internal vk state
1929 } render_condition;
1930 struct {
1931 uint64_t render_passes;
1932 } hud;
1933
1934 struct pipe_resource *dummy_vertex_buffer;
1935 struct pipe_resource *dummy_xfb_buffer;
1936 struct pipe_surface *dummy_surface[7];
1937 struct zink_buffer_view *dummy_bufferview;
1938
1939 unsigned buffer_rebind_counter;
1940 unsigned image_rebind_counter;
1941
1942 struct {
1943 /* descriptor info */
1944 uint8_t num_ubos[MESA_SHADER_STAGES];
1945
1946 uint8_t num_ssbos[MESA_SHADER_STAGES];
1947 struct util_dynarray global_bindings;
1948
1949 VkDescriptorImageInfo textures[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1950 uint32_t emulate_nonseamless[MESA_SHADER_STAGES];
1951 uint32_t cubes[MESA_SHADER_STAGES];
1952 uint8_t num_samplers[MESA_SHADER_STAGES];
1953 uint8_t num_sampler_views[MESA_SHADER_STAGES];
1954
1955 VkDescriptorImageInfo images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1956 uint8_t num_images[MESA_SHADER_STAGES];
1957
1958 union {
1959 struct {
1960 VkDescriptorBufferInfo ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS];
1961 VkDescriptorBufferInfo ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS];
1962 VkBufferView tbos[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1963 VkBufferView texel_images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1964 } t;
1965 struct {
1966 VkDescriptorAddressInfoEXT ubos[MESA_SHADER_STAGES][PIPE_MAX_CONSTANT_BUFFERS];
1967 VkDescriptorAddressInfoEXT ssbos[MESA_SHADER_STAGES][PIPE_MAX_SHADER_BUFFERS];
1968 VkDescriptorAddressInfoEXT tbos[MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1969 VkDescriptorAddressInfoEXT texel_images[MESA_SHADER_STAGES][ZINK_MAX_SHADER_IMAGES];
1970 } db;
1971 };
1972
1973 VkDescriptorImageInfo fbfetch;
1974 uint8_t fbfetch_db[ZINK_FBFETCH_DESCRIPTOR_SIZE];
1975
1976 /* the current state of the zs swizzle data */
1977 struct zink_zs_swizzle_key zs_swizzle[MESA_SHADER_STAGES];
1978
1979 struct zink_resource *descriptor_res[ZINK_DESCRIPTOR_BASE_TYPES][MESA_SHADER_STAGES][PIPE_MAX_SAMPLERS];
1980
1981 struct {
1982 struct util_idalloc tex_slots; //img, buffer
1983 struct util_idalloc img_slots; //img, buffer
1984 struct hash_table tex_handles; //img, buffer
1985 struct hash_table img_handles; //img, buffer
1986 union {
1987 struct {
1988 VkBufferView *buffer_infos; //tex, img
1989 } t;
1990 struct {
1991 VkDescriptorAddressInfoEXT *buffer_infos;
1992 } db;
1993 };
1994 VkDescriptorImageInfo *img_infos; //tex, img
1995 struct util_dynarray updates; //texture, img
1996 struct util_dynarray resident; //texture, img
1997 } bindless[2];
1998 union {
1999 bool bindless_dirty[2]; //tex, img
2000 uint16_t any_bindless_dirty;
2001 };
2002 bool bindless_refs_dirty;
2003 bool null_fbfetch_init;
2004 } di;
2005 void (*invalidate_descriptor_state)(struct zink_context *ctx, gl_shader_stage shader, enum zink_descriptor_type type, unsigned, unsigned);
2006 struct set *need_barriers[2]; //gfx, compute
2007 struct set update_barriers[2][2]; //[gfx, compute][current, next]
2008 uint8_t barrier_set_idx[2];
2009 unsigned memory_barrier;
2010
2011 uint32_t ds3_states;
2012 unsigned work_count;
2013
2014 uint32_t num_so_targets;
2015 struct pipe_stream_output_target *so_targets[PIPE_MAX_SO_BUFFERS];
2016 bool dirty_so_targets;
2017
2018 bool gfx_dirty;
2019
2020 bool shobj_draw : 1; //using shader objects for draw
2021 bool is_device_lost;
2022 bool primitive_restart;
2023 bool blitting : 1;
2024 bool blit_scissor : 1;
2025 bool blit_nearest : 1;
2026 bool unordered_blitting : 1;
2027 bool vertex_state_changed : 1;
2028 bool blend_state_changed : 1;
2029 bool blend_color_changed : 1;
2030 bool sample_mask_changed : 1;
2031 bool rast_state_changed : 1;
2032 bool line_width_changed : 1;
2033 bool dsa_state_changed : 1;
2034 bool stencil_ref_changed : 1;
2035 bool rasterizer_discard_changed : 1;
2036 bool rp_tc_info_updated : 1;
2037 bool last_work_was_compute : 1;
2038 };
2039
2040 static inline struct zink_context *
zink_context(struct pipe_context * context)2041 zink_context(struct pipe_context *context)
2042 {
2043 return (struct zink_context *)context;
2044 }
2045
2046 #endif
2047