1 /*
2 * Copyright © 2023 Valve Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "lvp_private.h"
8
9 #include "nir.h"
10 #include "nir_builder.h"
11
12 #define lvp_load_internal_field(b, bit_size, field) \
13 nir_load_ssbo(b, 1, bit_size, nir_imm_int(b, 0), \
14 nir_imm_int(b, offsetof(struct lvp_exec_graph_internal_data, field)))
15
16 #define lvp_store_internal_field(b, value, field, scope) \
17 nir_store_ssbo(b, value, nir_imm_int(b, 0), \
18 nir_iadd_imm(b, \
19 nir_imul_imm(b, nir_load_local_invocation_index(b), \
20 scope == SCOPE_INVOCATION \
21 ? sizeof(struct lvp_exec_graph_shader_output) \
22 : 0), \
23 offsetof(struct lvp_exec_graph_internal_data, outputs) + \
24 offsetof(struct lvp_exec_graph_shader_output, field)))
25
26 static bool
lvp_lower_node_payload_deref(nir_builder * b,nir_instr * instr,void * data)27 lvp_lower_node_payload_deref(nir_builder *b, nir_instr *instr, void *data)
28 {
29 if (instr->type != nir_instr_type_deref)
30 return false;
31
32 nir_deref_instr *deref = nir_instr_as_deref(instr);
33
34 bool is_payload = nir_deref_mode_is(deref, nir_var_mem_node_payload);
35 bool is_payload_in = nir_deref_mode_is(deref, nir_var_mem_node_payload_in);
36 if (!is_payload && !is_payload_in)
37 return false;
38
39 deref->modes = nir_var_mem_global;
40
41 if (deref->deref_type != nir_deref_type_var)
42 return true;
43
44 if (is_payload_in) {
45 b->cursor = nir_after_instr(instr);
46 nir_def *payload = lvp_load_internal_field(b, 64, payload_in);
47 nir_deref_instr *cast = nir_build_deref_cast(b, payload, nir_var_mem_global, deref->type, 0);
48 nir_def_rewrite_uses(&deref->def, &cast->def);
49 } else {
50 nir_foreach_use_safe(use, &deref->def) {
51 b->cursor = nir_before_instr(nir_src_parent_instr(use));
52 nir_def *payload = nir_load_var(b, deref->var);
53 nir_deref_instr *cast =
54 nir_build_deref_cast(b, payload, nir_var_mem_global, deref->type, 0);
55 nir_src_rewrite(use, &cast->def);
56 }
57 }
58
59 nir_instr_remove(instr);
60
61 return true;
62 }
63
64 static bool
lvp_lower_node_payload_derefs(nir_shader * nir)65 lvp_lower_node_payload_derefs(nir_shader *nir)
66 {
67 return nir_shader_instructions_pass(nir, lvp_lower_node_payload_deref,
68 nir_metadata_control_flow, NULL);
69 }
70
71 static void
lvp_build_initialize_node_payloads(nir_builder * b,nir_intrinsic_instr * intr)72 lvp_build_initialize_node_payloads(nir_builder *b, nir_intrinsic_instr *intr)
73 {
74 mesa_scope scope = nir_intrinsic_execution_scope(intr);
75 assert(scope == SCOPE_INVOCATION || scope == SCOPE_WORKGROUP);
76
77 nir_deref_instr *payloads_deref = nir_src_as_deref(intr->src[0]);
78 assert(payloads_deref->deref_type == nir_deref_type_var);
79 nir_variable *payloads_var = payloads_deref->var;
80
81 nir_def *addr = lvp_load_internal_field(b, 64, payloads);
82 if (scope == SCOPE_INVOCATION) {
83 nir_def *payloads_offset =
84 nir_imul_imm(b, nir_load_local_invocation_index(b), b->shader->info.cs.node_payloads_size);
85 addr = nir_iadd(b, addr, nir_u2u64(b, payloads_offset));
86 }
87 nir_store_var(b, payloads_var, addr, 0x1);
88
89 nir_def *payload_count = intr->src[1].ssa;
90 lvp_store_internal_field(b, payload_count, payload_count, scope);
91
92 nir_def *node_index = intr->src[1].ssa;
93 lvp_store_internal_field(b, node_index, node_index, scope);
94 }
95
96 static bool
lvp_lower_node_payload_intrinsic(nir_builder * b,nir_intrinsic_instr * intr,void * data)97 lvp_lower_node_payload_intrinsic(nir_builder *b, nir_intrinsic_instr *intr,
98 void *data)
99 {
100 if (intr->intrinsic == nir_intrinsic_enqueue_node_payloads) {
101 nir_instr_remove(&intr->instr);
102 return false;
103 }
104
105 b->cursor = nir_after_instr(&intr->instr);
106
107 switch (intr->intrinsic) {
108 case nir_intrinsic_initialize_node_payloads:
109 lvp_build_initialize_node_payloads(b, intr);
110 nir_instr_remove(&intr->instr);
111 return true;
112 case nir_intrinsic_finalize_incoming_node_payload:
113 nir_def_replace(&intr->def, nir_imm_true(b));
114 return true;
115 case nir_intrinsic_load_coalesced_input_count:
116 nir_def_replace(&intr->def, nir_imm_int(b, 1));
117 return true;
118 default:
119 return false;
120 }
121 }
122
123 static bool
lvp_lower_exec_graph_intrinsics(nir_shader * nir)124 lvp_lower_exec_graph_intrinsics(nir_shader *nir)
125 {
126 return nir_shader_intrinsics_pass(nir, lvp_lower_node_payload_intrinsic,
127 nir_metadata_control_flow, NULL);
128 }
129
130 static void
lvp_lower_node_payload_vars(struct lvp_pipeline * pipeline,nir_shader * nir)131 lvp_lower_node_payload_vars(struct lvp_pipeline *pipeline, nir_shader *nir)
132 {
133 nir_foreach_variable_in_shader(var, nir) {
134 if (var->data.mode != nir_var_mem_node_payload &&
135 var->data.mode != nir_var_mem_node_payload_in)
136 continue;
137
138 if (var->data.mode == nir_var_mem_node_payload) {
139 assert(var->data.node_name);
140 assert(!pipeline->exec_graph.next_name);
141 pipeline->exec_graph.next_name = var->data.node_name;
142 }
143
144 var->data.mode = nir_var_shader_temp;
145 var->type = glsl_uint64_t_type();
146 }
147 }
148
149 bool
lvp_lower_exec_graph(struct lvp_pipeline * pipeline,nir_shader * nir)150 lvp_lower_exec_graph(struct lvp_pipeline *pipeline, nir_shader *nir)
151 {
152 bool progress = false;
153 NIR_PASS(progress, nir, nir_lower_vars_to_explicit_types,
154 nir_var_mem_node_payload | nir_var_mem_node_payload_in,
155 glsl_get_natural_size_align_bytes);
156
157 if (!progress)
158 return false;
159
160 /* Lower node payload variables to 64-bit addresses. */
161 lvp_lower_node_payload_vars(pipeline, nir);
162
163 /* Lower exec graph intrinsics to their actual implementation. */
164 lvp_lower_exec_graph_intrinsics(nir);
165
166 /* Lower node payloads to load/store_global intructions. */
167 lvp_lower_node_payload_derefs(nir);
168 NIR_PASS(_, nir, nir_lower_explicit_io, nir_var_mem_global, nir_address_format_64bit_global);
169
170 /* Cleanup passes */
171 NIR_PASS(_, nir, nir_lower_global_vars_to_local);
172 NIR_PASS(_, nir, nir_lower_vars_to_ssa);
173 NIR_PASS(_, nir, nir_opt_constant_folding);
174 NIR_PASS(_, nir, nir_opt_dce);
175
176 return true;
177 }
178