1 /*
2 * Copyright 2024 Valve Corporation
3 * Copyright 2024 Alyssa Rosenzweig
4 * Copyright 2022-2023 Collabora Ltd. and Red Hat Inc.
5 * SPDX-License-Identifier: MIT
6 */
7
8 #pragma once
9
10 #include "asahi/lib/agx_device.h"
11 #include "util/simple_mtx.h"
12 #include "agx_bg_eot.h"
13 #include "agx_pack.h"
14 #include "agx_scratch.h"
15 #include "decode.h"
16 #include "vk_cmd_queue.h"
17 #include "vk_dispatch_table.h"
18
19 #include "hk_private.h"
20
21 #include "hk_descriptor_table.h"
22 #include "hk_queue.h"
23 #include "vk_device.h"
24 #include "vk_meta.h"
25 #include "vk_queue.h"
26
27 struct hk_physical_device;
28 struct vk_pipeline_cache;
29
30 /* Fixed offsets for reserved null image descriptors */
31 #define HK_NULL_TEX_OFFSET (0)
32 #define HK_NULL_PBE_OFFSET (24)
33
34 typedef void (*hk_internal_builder_t)(struct nir_builder *b, const void *key);
35
36 struct hk_internal_key {
37 hk_internal_builder_t builder;
38 size_t key_size;
39 uint8_t key[];
40 };
41
42 struct hk_internal_shaders {
43 simple_mtx_t lock;
44 struct hash_table *ht;
45 };
46
47 struct hk_rc_sampler {
48 struct agx_sampler_packed key;
49
50 /* Reference count for this hardware sampler, protected by the heap mutex */
51 uint16_t refcount;
52
53 /* Index of this hardware sampler in the hardware sampler heap */
54 uint16_t index;
55 };
56
57 struct hk_sampler_heap {
58 simple_mtx_t lock;
59
60 struct hk_descriptor_table table;
61
62 /* Map of agx_sampler_packed to hk_rc_sampler */
63 struct hash_table *ht;
64 };
65
66 struct hk_device {
67 struct vk_device vk;
68 struct agx_device dev;
69 struct agxdecode_ctx *decode_ctx;
70
71 struct hk_descriptor_table images;
72 struct hk_descriptor_table occlusion_queries;
73 struct hk_sampler_heap samplers;
74
75 struct hk_queue queue;
76
77 struct vk_pipeline_cache *mem_cache;
78
79 struct vk_meta_device meta;
80 struct agx_bg_eot_cache bg_eot;
81
82 struct {
83 struct agx_bo *bo;
84 struct agx_usc_sampler_packed txf_sampler;
85 struct agx_usc_uniform_packed image_heap;
86 uint64_t null_sink, zero_sink;
87 uint64_t geometry_state;
88 } rodata;
89
90 struct hk_internal_shaders prolog_epilog;
91 struct hk_internal_shaders kernels;
92 struct hk_api_shader *write_shader;
93
94 /* Indirected for common secondary emulation */
95 struct vk_device_dispatch_table cmd_dispatch;
96
97 /* Heap used for GPU-side memory allocation for geometry/tessellation.
98 *
99 * Control streams accessing the heap must be serialized. This is not
100 * expected to be a legitimate problem. If it is, we can rework later.
101 */
102 struct agx_bo *heap;
103
104 struct {
105 struct agx_scratch vs, fs, cs;
106 simple_mtx_t lock;
107 } scratch;
108 };
109
110 VK_DEFINE_HANDLE_CASTS(hk_device, vk.base, VkDevice, VK_OBJECT_TYPE_DEVICE)
111
112 static inline struct hk_physical_device *
hk_device_physical(struct hk_device * dev)113 hk_device_physical(struct hk_device *dev)
114 {
115 return (struct hk_physical_device *)dev->vk.physical;
116 }
117
118 VkResult hk_device_init_meta(struct hk_device *dev);
119 void hk_device_finish_meta(struct hk_device *dev);
120
121 VkResult hk_sampler_heap_add(struct hk_device *dev,
122 struct agx_sampler_packed desc,
123 struct hk_rc_sampler **out);
124
125 void hk_sampler_heap_remove(struct hk_device *dev, struct hk_rc_sampler *rc);
126
127 static inline struct agx_scratch *
hk_device_scratch_locked(struct hk_device * dev,enum pipe_shader_type stage)128 hk_device_scratch_locked(struct hk_device *dev, enum pipe_shader_type stage)
129 {
130 simple_mtx_assert_locked(&dev->scratch.lock);
131
132 switch (stage) {
133 case PIPE_SHADER_FRAGMENT:
134 return &dev->scratch.fs;
135 case PIPE_SHADER_VERTEX:
136 return &dev->scratch.vs;
137 default:
138 return &dev->scratch.cs;
139 }
140 }
141
142 static inline void
hk_device_alloc_scratch(struct hk_device * dev,enum pipe_shader_type stage,unsigned size)143 hk_device_alloc_scratch(struct hk_device *dev, enum pipe_shader_type stage,
144 unsigned size)
145 {
146 simple_mtx_lock(&dev->scratch.lock);
147 agx_scratch_alloc(hk_device_scratch_locked(dev, stage), size, 0);
148 simple_mtx_unlock(&dev->scratch.lock);
149 }
150