1 /*
2 * Copyright © 2022 Collabora Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir.h"
25 #include "nir_builder.h"
26
27 static bool
is_ubo_intrinsic(nir_intrinsic_instr * intrin)28 is_ubo_intrinsic(nir_intrinsic_instr *intrin)
29 {
30 return intrin->intrinsic == nir_intrinsic_load_ubo;
31 }
32
33 static bool
is_ssbo_intrinsic(nir_intrinsic_instr * intrin)34 is_ssbo_intrinsic(nir_intrinsic_instr *intrin)
35 {
36 switch (intrin->intrinsic) {
37 case nir_intrinsic_load_ssbo:
38 case nir_intrinsic_store_ssbo:
39 case nir_intrinsic_ssbo_atomic:
40 case nir_intrinsic_ssbo_atomic_swap:
41 return true;
42
43 default:
44 return false;
45 }
46 }
47
48 static bool
is_image_intrinsic(nir_intrinsic_instr * intrin)49 is_image_intrinsic(nir_intrinsic_instr *intrin)
50 {
51 switch (intrin->intrinsic) {
52 case nir_intrinsic_image_load:
53 case nir_intrinsic_image_sparse_load:
54 case nir_intrinsic_image_store:
55 case nir_intrinsic_image_atomic:
56 case nir_intrinsic_image_atomic_swap:
57 case nir_intrinsic_image_size:
58 case nir_intrinsic_image_samples:
59 case nir_intrinsic_image_fragment_mask_load_amd:
60 case nir_intrinsic_bindless_image_load:
61 case nir_intrinsic_bindless_image_sparse_load:
62 case nir_intrinsic_bindless_image_store:
63 case nir_intrinsic_bindless_image_atomic:
64 case nir_intrinsic_bindless_image_atomic_swap:
65 case nir_intrinsic_bindless_image_size:
66 case nir_intrinsic_bindless_image_samples:
67 case nir_intrinsic_bindless_image_fragment_mask_load_amd:
68 case nir_intrinsic_image_deref_load:
69 case nir_intrinsic_image_deref_sparse_load:
70 case nir_intrinsic_image_deref_store:
71 case nir_intrinsic_image_deref_atomic:
72 case nir_intrinsic_image_deref_atomic_swap:
73 case nir_intrinsic_image_deref_size:
74 case nir_intrinsic_image_deref_samples:
75 case nir_intrinsic_image_deref_fragment_mask_load_amd:
76 return true;
77
78 default:
79 return false;
80 }
81 }
82
83 static bool
has_non_uniform_tex_access(nir_tex_instr * tex)84 has_non_uniform_tex_access(nir_tex_instr *tex)
85 {
86 return tex->texture_non_uniform || tex->sampler_non_uniform;
87 }
88
89 static bool
has_non_uniform_access_intrin(nir_intrinsic_instr * intrin)90 has_non_uniform_access_intrin(nir_intrinsic_instr *intrin)
91 {
92 return (nir_intrinsic_access(intrin) & ACCESS_NON_UNIFORM) != 0;
93 }
94
95 static bool
nir_has_non_uniform_access_impl(nir_function_impl * impl,enum nir_lower_non_uniform_access_type types)96 nir_has_non_uniform_access_impl(nir_function_impl *impl, enum nir_lower_non_uniform_access_type types)
97 {
98 nir_foreach_block_safe(block, impl) {
99 nir_foreach_instr_safe(instr, block) {
100 switch (instr->type) {
101 case nir_instr_type_tex: {
102 nir_tex_instr *tex = nir_instr_as_tex(instr);
103 if ((types & nir_lower_non_uniform_texture_access) &&
104 has_non_uniform_tex_access(tex))
105 return true;
106 break;
107 }
108
109 case nir_instr_type_intrinsic: {
110 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
111 if (is_ubo_intrinsic(intrin)) {
112 if ((types & nir_lower_non_uniform_ubo_access) &&
113 has_non_uniform_access_intrin(intrin))
114 return true;
115 } else if (is_ssbo_intrinsic(intrin)) {
116 if ((types & nir_lower_non_uniform_ssbo_access) &&
117 has_non_uniform_access_intrin(intrin))
118 return true;
119 } else if (is_image_intrinsic(intrin)) {
120 if ((types & nir_lower_non_uniform_image_access) &&
121 has_non_uniform_access_intrin(intrin))
122 return true;
123 } else if (intrin->intrinsic == nir_intrinsic_get_ssbo_size) {
124 if ((types & nir_lower_non_uniform_get_ssbo_size) &&
125 has_non_uniform_access_intrin(intrin))
126 return true;
127 } else {
128 /* Nothing to do */
129 }
130 break;
131 }
132
133 default:
134 /* Nothing to do */
135 break;
136 }
137 }
138 }
139
140 return false;
141 }
142
143 bool
nir_has_non_uniform_access(nir_shader * shader,enum nir_lower_non_uniform_access_type types)144 nir_has_non_uniform_access(nir_shader *shader, enum nir_lower_non_uniform_access_type types)
145 {
146 nir_foreach_function_impl(impl, shader) {
147 if (nir_has_non_uniform_access_impl(impl, types))
148 return true;
149 }
150
151 return false;
152 }
153
154 static bool
opt_non_uniform_tex_access(nir_tex_instr * tex)155 opt_non_uniform_tex_access(nir_tex_instr *tex)
156 {
157 if (!has_non_uniform_tex_access(tex))
158 return false;
159
160 bool progress = false;
161
162 for (unsigned i = 0; i < tex->num_srcs; i++) {
163 switch (tex->src[i].src_type) {
164 case nir_tex_src_texture_offset:
165 case nir_tex_src_texture_handle:
166 case nir_tex_src_texture_deref:
167 if (tex->texture_non_uniform && !tex->src[i].src.ssa->divergent) {
168 tex->texture_non_uniform = false;
169 progress = true;
170 }
171 break;
172
173 case nir_tex_src_sampler_offset:
174 case nir_tex_src_sampler_handle:
175 case nir_tex_src_sampler_deref:
176 if (tex->sampler_non_uniform && !tex->src[i].src.ssa->divergent) {
177 tex->sampler_non_uniform = false;
178 progress = true;
179 }
180 break;
181
182 default:
183 break;
184 }
185 }
186
187 return progress;
188 }
189
190 static bool
opt_non_uniform_access_intrin(nir_intrinsic_instr * intrin,unsigned handle_src)191 opt_non_uniform_access_intrin(nir_intrinsic_instr *intrin, unsigned handle_src)
192 {
193 if (!has_non_uniform_access_intrin(intrin))
194 return false;
195
196 if (intrin->src[handle_src].ssa->divergent)
197 return false;
198
199 nir_intrinsic_set_access(intrin, nir_intrinsic_access(intrin) & ~ACCESS_NON_UNIFORM);
200
201 return true;
202 }
203
204 static bool
nir_opt_non_uniform_access_instr(nir_builder * b,nir_instr * instr,UNUSED void * user_data)205 nir_opt_non_uniform_access_instr(nir_builder *b, nir_instr *instr, UNUSED void *user_data)
206 {
207 switch (instr->type) {
208 case nir_instr_type_tex:
209 return opt_non_uniform_tex_access(nir_instr_as_tex(instr));
210
211 case nir_instr_type_intrinsic: {
212 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
213 if (is_ubo_intrinsic(intrin) || is_ssbo_intrinsic(intrin) || is_image_intrinsic(intrin)) {
214 unsigned handle_src = 0;
215 /* SSBO Stores put the index in the second source */
216 if (intrin->intrinsic == nir_intrinsic_store_ssbo)
217 handle_src = 1;
218 return opt_non_uniform_access_intrin(intrin, handle_src);
219 }
220 break;
221 }
222
223 default:
224 /* Nothing to do */
225 break;
226 }
227
228 return false;
229 }
230
231 bool
nir_opt_non_uniform_access(nir_shader * shader)232 nir_opt_non_uniform_access(nir_shader *shader)
233 {
234 NIR_PASS(_, shader, nir_convert_to_lcssa, true, true);
235 nir_divergence_analysis(shader);
236
237 bool progress = nir_shader_instructions_pass(shader,
238 nir_opt_non_uniform_access_instr,
239 nir_metadata_all, NULL);
240
241 NIR_PASS(_, shader, nir_opt_remove_phis); /* cleanup LCSSA phis */
242
243 return progress;
244 }
245