1 /*
2 * Copyright © 2020 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 /*
26 * Replaces make availability/visible semantics on barriers with
27 * ACCESS_COHERENT on memory loads/stores
28 */
29
30 #include "nir/nir.h"
31 #include "shader_enums.h"
32
33 static bool
get_intrinsic_info(nir_intrinsic_instr * intrin,nir_variable_mode * modes,bool * reads,bool * writes)34 get_intrinsic_info(nir_intrinsic_instr *intrin, nir_variable_mode *modes,
35 bool *reads, bool *writes)
36 {
37 switch (intrin->intrinsic) {
38 case nir_intrinsic_image_deref_load:
39 case nir_intrinsic_image_deref_sparse_load:
40 *modes = nir_src_as_deref(intrin->src[0])->modes;
41 *reads = true;
42 break;
43 case nir_intrinsic_image_deref_store:
44 *modes = nir_src_as_deref(intrin->src[0])->modes;
45 *writes = true;
46 break;
47 case nir_intrinsic_image_deref_atomic:
48 case nir_intrinsic_image_deref_atomic_swap:
49 *modes = nir_src_as_deref(intrin->src[0])->modes;
50 *reads = true;
51 *writes = true;
52 break;
53 case nir_intrinsic_load_ssbo:
54 *modes = nir_var_mem_ssbo;
55 *reads = true;
56 break;
57 case nir_intrinsic_store_ssbo:
58 *modes = nir_var_mem_ssbo;
59 *writes = true;
60 break;
61 case nir_intrinsic_ssbo_atomic:
62 case nir_intrinsic_ssbo_atomic_swap:
63 *modes = nir_var_mem_ssbo;
64 *reads = true;
65 *writes = true;
66 break;
67 case nir_intrinsic_load_global:
68 *modes = nir_var_mem_global;
69 *reads = true;
70 break;
71 case nir_intrinsic_store_global:
72 *modes = nir_var_mem_global;
73 *writes = true;
74 break;
75 case nir_intrinsic_global_atomic:
76 case nir_intrinsic_global_atomic_swap:
77 *modes = nir_var_mem_global;
78 *reads = true;
79 *writes = true;
80 break;
81 case nir_intrinsic_load_deref:
82 *modes = nir_src_as_deref(intrin->src[0])->modes;
83 *reads = true;
84 break;
85 case nir_intrinsic_store_deref:
86 *modes = nir_src_as_deref(intrin->src[0])->modes;
87 *writes = true;
88 break;
89 case nir_intrinsic_deref_atomic:
90 case nir_intrinsic_deref_atomic_swap:
91 *modes = nir_src_as_deref(intrin->src[0])->modes;
92 *reads = true;
93 *writes = true;
94 break;
95 default:
96 return false;
97 }
98 return true;
99 }
100
101 static bool
visit_instr(nir_instr * instr,uint32_t * cur_modes,unsigned vis_avail_sem)102 visit_instr(nir_instr *instr, uint32_t *cur_modes, unsigned vis_avail_sem)
103 {
104 if (instr->type != nir_instr_type_intrinsic)
105 return false;
106 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
107
108 if (intrin->intrinsic == nir_intrinsic_barrier &&
109 (nir_intrinsic_memory_semantics(intrin) & vis_avail_sem)) {
110 *cur_modes |= nir_intrinsic_memory_modes(intrin);
111
112 unsigned semantics = nir_intrinsic_memory_semantics(intrin);
113 nir_intrinsic_set_memory_semantics(
114 intrin, semantics & ~vis_avail_sem);
115 return true;
116 }
117
118 if (!*cur_modes)
119 return false; /* early exit */
120
121 nir_variable_mode modes;
122 bool reads = false, writes = false;
123 if (!get_intrinsic_info(intrin, &modes, &reads, &writes))
124 return false;
125
126 if (!reads && vis_avail_sem == NIR_MEMORY_MAKE_VISIBLE)
127 return false;
128 if (!writes && vis_avail_sem == NIR_MEMORY_MAKE_AVAILABLE)
129 return false;
130
131 if (!nir_intrinsic_has_access(intrin))
132 return false;
133
134 unsigned access = nir_intrinsic_access(intrin);
135
136 if (access & (ACCESS_NON_READABLE | ACCESS_NON_WRITEABLE | ACCESS_CAN_REORDER | ACCESS_COHERENT))
137 return false;
138
139 if (*cur_modes & modes) {
140 nir_intrinsic_set_access(intrin, access | ACCESS_COHERENT);
141 return true;
142 }
143
144 return false;
145 }
146
147 static bool
lower_make_visible(nir_cf_node * cf_node,uint32_t * cur_modes)148 lower_make_visible(nir_cf_node *cf_node, uint32_t *cur_modes)
149 {
150 bool progress = false;
151 switch (cf_node->type) {
152 case nir_cf_node_block: {
153 nir_block *block = nir_cf_node_as_block(cf_node);
154 nir_foreach_instr(instr, block)
155 progress |= visit_instr(instr, cur_modes, NIR_MEMORY_MAKE_VISIBLE);
156 break;
157 }
158 case nir_cf_node_if: {
159 nir_if *nif = nir_cf_node_as_if(cf_node);
160 uint32_t cur_modes_then = *cur_modes;
161 uint32_t cur_modes_else = *cur_modes;
162 foreach_list_typed(nir_cf_node, if_node, node, &nif->then_list)
163 progress |= lower_make_visible(if_node, &cur_modes_then);
164 foreach_list_typed(nir_cf_node, if_node, node, &nif->else_list)
165 progress |= lower_make_visible(if_node, &cur_modes_else);
166 *cur_modes |= cur_modes_then | cur_modes_else;
167 break;
168 }
169 case nir_cf_node_loop: {
170 nir_loop *loop = nir_cf_node_as_loop(cf_node);
171 assert(!nir_loop_has_continue_construct(loop));
172 bool loop_progress;
173 do {
174 loop_progress = false;
175 foreach_list_typed(nir_cf_node, loop_node, node, &loop->body)
176 loop_progress |= lower_make_visible(loop_node, cur_modes);
177 progress |= loop_progress;
178 } while (loop_progress);
179 break;
180 }
181 case nir_cf_node_function:
182 unreachable("Invalid cf type");
183 }
184 return progress;
185 }
186
187 static bool
lower_make_available(nir_cf_node * cf_node,uint32_t * cur_modes)188 lower_make_available(nir_cf_node *cf_node, uint32_t *cur_modes)
189 {
190 bool progress = false;
191 switch (cf_node->type) {
192 case nir_cf_node_block: {
193 nir_block *block = nir_cf_node_as_block(cf_node);
194 nir_foreach_instr_reverse(instr, block)
195 progress |= visit_instr(instr, cur_modes, NIR_MEMORY_MAKE_AVAILABLE);
196 break;
197 }
198 case nir_cf_node_if: {
199 nir_if *nif = nir_cf_node_as_if(cf_node);
200 uint32_t cur_modes_then = *cur_modes;
201 uint32_t cur_modes_else = *cur_modes;
202 foreach_list_typed_reverse(nir_cf_node, if_node, node, &nif->then_list)
203 progress |= lower_make_available(if_node, &cur_modes_then);
204 foreach_list_typed_reverse(nir_cf_node, if_node, node, &nif->else_list)
205 progress |= lower_make_available(if_node, &cur_modes_else);
206 *cur_modes |= cur_modes_then | cur_modes_else;
207 break;
208 }
209 case nir_cf_node_loop: {
210 nir_loop *loop = nir_cf_node_as_loop(cf_node);
211 assert(!nir_loop_has_continue_construct(loop));
212 bool loop_progress;
213 do {
214 loop_progress = false;
215 foreach_list_typed_reverse(nir_cf_node, loop_node, node, &loop->body)
216 loop_progress |= lower_make_available(loop_node, cur_modes);
217 progress |= loop_progress;
218 } while (loop_progress);
219 break;
220 }
221 case nir_cf_node_function:
222 unreachable("Invalid cf type");
223 }
224 return progress;
225 }
226
227 bool
nir_lower_memory_model(nir_shader * shader)228 nir_lower_memory_model(nir_shader *shader)
229 {
230 bool progress = false;
231
232 nir_function_impl *impl = nir_shader_get_entrypoint(shader);
233 struct exec_list *cf_list = &impl->body;
234
235 uint32_t modes = 0;
236 foreach_list_typed(nir_cf_node, cf_node, node, cf_list)
237 progress |= lower_make_visible(cf_node, &modes);
238
239 modes = 0;
240 foreach_list_typed_reverse(nir_cf_node, cf_node, node, cf_list)
241 progress |= lower_make_available(cf_node, &modes);
242
243 if (progress)
244 nir_metadata_preserve(impl, nir_metadata_control_flow);
245 else
246 nir_metadata_preserve(impl, nir_metadata_all);
247
248 return progress;
249 }
250