1 /**************************************************************************
2 *
3 * Copyright 2022 Red Hat
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 *
24 **************************************************************************/
25
26 #include "lp_bld_nir.h"
27 #include "lp_bld_init.h"
28 #include "lp_bld_const.h"
29 #include "lp_bld_flow.h"
30 #include "lp_bld_struct.h"
31 #include "lp_bld_swizzle.h"
32 #include "lp_bld_debug.h"
33 #include "util/u_math.h"
34
35
36 struct lp_build_nir_aos_context
37 {
38 struct lp_build_nir_context bld_base;
39
40 /* Builder for integer masks and indices */
41 struct lp_build_context int_bld;
42
43 /*
44 * AoS swizzle used:
45 * - swizzles[0] = red index
46 * - swizzles[1] = green index
47 * - swizzles[2] = blue index
48 * - swizzles[3] = alpha index
49 */
50 unsigned char swizzles[4];
51 unsigned char inv_swizzles[4];
52
53 LLVMValueRef consts_ptr;
54 const LLVMValueRef *inputs;
55 LLVMValueRef *outputs;
56
57 const struct lp_build_sampler_aos *sampler;
58 };
59
60
61 static inline struct lp_build_nir_aos_context *
lp_nir_aos_context(struct lp_build_nir_context * bld_base)62 lp_nir_aos_context(struct lp_build_nir_context *bld_base)
63 {
64 return (struct lp_build_nir_aos_context *) bld_base;
65 }
66
67
68 static LLVMValueRef
swizzle_aos(struct lp_build_nir_context * bld_base,LLVMValueRef a,unsigned swizzle_x,unsigned swizzle_y,unsigned swizzle_z,unsigned swizzle_w)69 swizzle_aos(struct lp_build_nir_context *bld_base,
70 LLVMValueRef a,
71 unsigned swizzle_x,
72 unsigned swizzle_y,
73 unsigned swizzle_z,
74 unsigned swizzle_w)
75 {
76 unsigned char swizzles[4];
77 struct lp_build_nir_aos_context *bld = lp_nir_aos_context(bld_base);
78
79 assert(swizzle_x < 4);
80 assert(swizzle_y < 4);
81 assert(swizzle_z < 4);
82 assert(swizzle_w < 4);
83
84 swizzles[bld->inv_swizzles[0]] = bld->swizzles[swizzle_x];
85 swizzles[bld->inv_swizzles[1]] = bld->swizzles[swizzle_y];
86 swizzles[bld->inv_swizzles[2]] = bld->swizzles[swizzle_z];
87 swizzles[bld->inv_swizzles[3]] = bld->swizzles[swizzle_w];
88
89 return lp_build_swizzle_aos(&bld->bld_base.base, a, swizzles);
90 }
91
92
93 static void
init_var_slots(struct lp_build_nir_context * bld_base,nir_variable * var)94 init_var_slots(struct lp_build_nir_context *bld_base,
95 nir_variable *var)
96 {
97 struct lp_build_nir_aos_context *bld = lp_nir_aos_context(bld_base);
98
99 if (!bld->outputs)
100 return;
101 unsigned this_loc = var->data.driver_location;
102
103 bld->outputs[this_loc] = lp_build_alloca(bld_base->base.gallivm,
104 bld_base->base.vec_type,
105 "output");
106 }
107
108
109 static void
emit_var_decl(struct lp_build_nir_context * bld_base,nir_variable * var)110 emit_var_decl(struct lp_build_nir_context *bld_base,
111 nir_variable *var)
112 {
113 if (var->data.mode == nir_var_shader_out) {
114 init_var_slots(bld_base, var);
115 }
116 }
117
118
119 static void
emit_load_var(struct lp_build_nir_context * bld_base,nir_variable_mode deref_mode,unsigned num_components,unsigned bit_size,nir_variable * var,unsigned vertex_index,LLVMValueRef indir_vertex_index,unsigned const_index,LLVMValueRef indir_index,LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])120 emit_load_var(struct lp_build_nir_context *bld_base,
121 nir_variable_mode deref_mode,
122 unsigned num_components,
123 unsigned bit_size,
124 nir_variable *var,
125 unsigned vertex_index,
126 LLVMValueRef indir_vertex_index,
127 unsigned const_index,
128 LLVMValueRef indir_index,
129 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
130 {
131 struct lp_build_nir_aos_context *bld = lp_nir_aos_context(bld_base);
132 unsigned location = var->data.driver_location;
133
134 if (deref_mode == nir_var_shader_in) {
135 result[0] = bld->inputs[location];
136 }
137 }
138
139
140 static void
emit_store_var(struct lp_build_nir_context * bld_base,nir_variable_mode deref_mode,unsigned num_components,unsigned bit_size,nir_variable * var,unsigned writemask,LLVMValueRef indir_vertex_index,unsigned const_index,LLVMValueRef indir_index,LLVMValueRef vals)141 emit_store_var(struct lp_build_nir_context *bld_base,
142 nir_variable_mode deref_mode,
143 unsigned num_components,
144 unsigned bit_size,
145 nir_variable *var,
146 unsigned writemask,
147 LLVMValueRef indir_vertex_index,
148 unsigned const_index,
149 LLVMValueRef indir_index,
150 LLVMValueRef vals)
151 {
152 struct lp_build_nir_aos_context *bld = lp_nir_aos_context(bld_base);
153 struct gallivm_state *gallivm = bld_base->base.gallivm;
154 unsigned location = var->data.driver_location;
155
156 if (deref_mode == nir_var_shader_out) {
157 LLVMBuildStore(gallivm->builder, vals, bld->outputs[location]);
158 }
159 }
160
161
162 static LLVMValueRef
emit_load_reg(struct lp_build_nir_context * bld_base,struct lp_build_context * reg_bld,const nir_intrinsic_instr * decl,unsigned base,LLVMValueRef indir_src,LLVMValueRef reg_storage)163 emit_load_reg(struct lp_build_nir_context *bld_base,
164 struct lp_build_context *reg_bld,
165 const nir_intrinsic_instr *decl,
166 unsigned base,
167 LLVMValueRef indir_src,
168 LLVMValueRef reg_storage)
169 {
170 struct gallivm_state *gallivm = bld_base->base.gallivm;
171 assert(indir_src == NULL && "no indirects with linear path");
172 return LLVMBuildLoad2(gallivm->builder, reg_bld->vec_type, reg_storage, "");
173 }
174
175
176 unsigned
lp_nir_aos_swizzle(struct lp_build_nir_context * bld_base,unsigned chan)177 lp_nir_aos_swizzle(struct lp_build_nir_context *bld_base, unsigned chan)
178 {
179 struct lp_build_nir_aos_context *bld = lp_nir_aos_context(bld_base);
180 return bld->swizzles[chan];
181 }
182
183
184 /*
185 * If an instruction has a writemask like r0.x = foo and the
186 * AOS/linear context uses swizzle={2,1,0,3} we need to change
187 * the writemask to r0.z
188 */
189 static unsigned
swizzle_writemask(struct lp_build_nir_aos_context * bld,unsigned writemask)190 swizzle_writemask(struct lp_build_nir_aos_context *bld,
191 unsigned writemask)
192 {
193 assert(writemask != 0x0);
194 assert(writemask != 0xf);
195
196 // Ex: swap r/b channels
197 unsigned new_writemask = 0;
198 for (unsigned chan = 0; chan < 4; chan++) {
199 if (writemask & (1 << chan)) {
200 new_writemask |= 1 << bld->swizzles[chan];
201 }
202 }
203 return new_writemask;
204 }
205
206
207 static void
emit_store_reg(struct lp_build_nir_context * bld_base,struct lp_build_context * reg_bld,const nir_intrinsic_instr * decl,unsigned writemask,unsigned base,LLVMValueRef indir_src,LLVMValueRef reg_storage,LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])208 emit_store_reg(struct lp_build_nir_context *bld_base,
209 struct lp_build_context *reg_bld,
210 const nir_intrinsic_instr *decl,
211 unsigned writemask,
212 unsigned base,
213 LLVMValueRef indir_src,
214 LLVMValueRef reg_storage,
215 LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
216 {
217 struct lp_build_nir_aos_context *bld = lp_nir_aos_context(bld_base);
218 struct gallivm_state *gallivm = bld_base->base.gallivm;
219 assert(indir_src == NULL && "no indirects with linear path");
220
221 if (writemask == 0xf) {
222 LLVMBuildStore(gallivm->builder, vals[0], reg_storage);
223 return;
224 }
225
226 writemask = swizzle_writemask(bld, writemask);
227
228 LLVMValueRef cur = LLVMBuildLoad2(gallivm->builder, reg_bld->vec_type,
229 reg_storage, "");
230 LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
231 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
232 for (unsigned j = 0; j < 16; j++) {
233 unsigned comp = j % 4;
234 if (writemask & (1 << comp)) {
235 shuffles[j] = LLVMConstInt(i32t, 16 + j, 0); // new val
236 } else {
237 shuffles[j] = LLVMConstInt(i32t, j, 0); // cur val
238 }
239 }
240 cur = LLVMBuildShuffleVector(gallivm->builder, cur, vals[0],
241 LLVMConstVector(shuffles, 16), "");
242
243 LLVMBuildStore(gallivm->builder, cur, reg_storage);
244 }
245
246
247 static void
emit_load_ubo(struct lp_build_nir_context * bld_base,unsigned nc,unsigned bit_size,bool offset_is_uniform,LLVMValueRef index,LLVMValueRef offset,LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])248 emit_load_ubo(struct lp_build_nir_context *bld_base,
249 unsigned nc,
250 unsigned bit_size,
251 bool offset_is_uniform,
252 LLVMValueRef index,
253 LLVMValueRef offset,
254 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
255 {
256 struct lp_build_nir_aos_context *bld = lp_nir_aos_context(bld_base);
257 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
258 struct gallivm_state *gallivm = bld_base->base.gallivm;
259 struct lp_type type = bld_base->base.type;
260 LLVMValueRef res;
261
262 res = bld->bld_base.base.undef;
263 offset = LLVMBuildExtractElement(builder, offset,
264 lp_build_const_int32(gallivm, 0), "");
265 assert(LLVMIsConstant(offset));
266 unsigned offset_val = LLVMConstIntGetZExtValue(offset) >> 2;
267 for (unsigned chan = 0; chan < nc; ++chan) {
268 LLVMValueRef this_offset = lp_build_const_int32(gallivm,
269 offset_val + chan);
270
271 LLVMTypeRef scalar_type = LLVMInt8TypeInContext(gallivm->context);
272 LLVMValueRef scalar_ptr = LLVMBuildGEP2(builder, scalar_type, bld->consts_ptr, &this_offset, 1, "");
273 LLVMValueRef scalar = LLVMBuildLoad2(builder, scalar_type, scalar_ptr, "");
274
275 lp_build_name(scalar, "const[%u].%c", offset_val, "xyzw"[chan]);
276
277 LLVMValueRef swizzle = lp_build_const_int32(bld->bld_base.base.gallivm,
278 nc == 1 ? 0 : bld->swizzles[chan]);
279
280 res = LLVMBuildInsertElement(builder, res, scalar, swizzle, "");
281 }
282
283 if (type.length > 4) {
284 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
285
286 for (unsigned chan = 0; chan < nc; ++chan) {
287 shuffles[chan] =
288 lp_build_const_int32(bld->bld_base.base.gallivm, chan);
289 }
290
291 for (unsigned i = nc; i < type.length; ++i) {
292 shuffles[i] = shuffles[i % nc];
293 }
294
295 res = LLVMBuildShuffleVector(builder, res, bld->bld_base.base.undef,
296 LLVMConstVector(shuffles, type.length),
297 "");
298 }
299
300 if (nc == 4)
301 swizzle_aos(bld_base, res, 0, 1, 2, 3);
302
303 result[0] = res;
304 }
305
306
307 static void
emit_tex(struct lp_build_nir_context * bld_base,struct lp_sampler_params * params)308 emit_tex(struct lp_build_nir_context *bld_base,
309 struct lp_sampler_params *params)
310 {
311 struct lp_build_nir_aos_context *bld = lp_nir_aos_context(bld_base);
312 static const struct lp_derivatives derivs = { 0 };
313 params->type = bld_base->base.type;
314 params->texel[0] = bld->sampler->emit_fetch_texel(bld->sampler,
315 &bld->bld_base.base,
316 TGSI_TEXTURE_2D,
317 params->texture_index,
318 params->coords[0],
319 params->derivs ? params->derivs[0] : derivs,
320 LP_BLD_TEX_MODIFIER_NONE);
321 }
322
323
324 static void
emit_load_const(struct lp_build_nir_context * bld_base,const nir_load_const_instr * instr,LLVMValueRef outval[NIR_MAX_VEC_COMPONENTS])325 emit_load_const(struct lp_build_nir_context *bld_base,
326 const nir_load_const_instr *instr,
327 LLVMValueRef outval[NIR_MAX_VEC_COMPONENTS])
328 {
329 struct lp_build_nir_aos_context *bld = lp_nir_aos_context(bld_base);
330 LLVMValueRef elems[16];
331 const int nc = instr->def.num_components;
332 bool do_swizzle = false;
333
334 if (nc == 4)
335 do_swizzle = true;
336
337 /* The constant is something like {float, float, float, float}.
338 * We need to convert the float values from [0,1] to ubyte in [0,255].
339 * We previously checked for values outside [0,1] in
340 * llvmpipe_nir_fn_is_linear_compat().
341 * Also, we convert the (typically) 4-element float constant into a
342 * swizzled 16-element ubyte constant (z,y,x,w, z,y,x,w, z,y,x,w, z,y,x,w)
343 * since that's what 'linear' mode operates on.
344 */
345 assert(bld_base->base.type.length <= ARRAY_SIZE(elems));
346 for (unsigned i = 0; i < bld_base->base.type.length; i++) {
347 const unsigned j = do_swizzle ? bld->swizzles[i % nc] : i % nc;
348 assert(instr->value[j].f32 >= 0.0f);
349 assert(instr->value[j].f32 <= 1.0f);
350 const unsigned u8val = float_to_ubyte(instr->value[j].f32);
351 elems[i] = LLVMConstInt(bld_base->uint_bld.int_elem_type, u8val, 0);
352 }
353 outval[0] = LLVMConstVector(elems, bld_base->base.type.length);
354 outval[1] = outval[2] = outval[3] = NULL;
355 }
356
357
358 void
lp_build_nir_aos(struct gallivm_state * gallivm,struct nir_shader * shader,struct lp_type type,const unsigned char swizzles[4],LLVMValueRef consts_ptr,const LLVMValueRef * inputs,LLVMValueRef * outputs,const struct lp_build_sampler_aos * sampler)359 lp_build_nir_aos(struct gallivm_state *gallivm,
360 struct nir_shader *shader,
361 struct lp_type type,
362 const unsigned char swizzles[4],
363 LLVMValueRef consts_ptr,
364 const LLVMValueRef *inputs,
365 LLVMValueRef *outputs,
366 const struct lp_build_sampler_aos *sampler)
367 {
368 struct lp_build_nir_aos_context bld;
369
370 memset(&bld, 0, sizeof bld);
371 lp_build_context_init(&bld.bld_base.base, gallivm, type);
372 lp_build_context_init(&bld.bld_base.uint_bld, gallivm, lp_uint_type(type));
373 lp_build_context_init(&bld.bld_base.int_bld, gallivm, lp_int_type(type));
374
375 for (unsigned chan = 0; chan < 4; ++chan) {
376 bld.swizzles[chan] = swizzles[chan];
377 bld.inv_swizzles[swizzles[chan]] = chan;
378 }
379 bld.sampler = sampler;
380
381 bld.bld_base.shader = shader;
382
383 bld.inputs = inputs;
384 bld.outputs = outputs;
385 bld.consts_ptr = consts_ptr;
386
387 bld.bld_base.load_var = emit_load_var;
388 bld.bld_base.store_var = emit_store_var;
389 bld.bld_base.load_reg = emit_load_reg;
390 bld.bld_base.store_reg = emit_store_reg;
391 bld.bld_base.load_ubo = emit_load_ubo;
392 bld.bld_base.load_const = emit_load_const;
393
394 bld.bld_base.tex = emit_tex;
395 bld.bld_base.emit_var_decl = emit_var_decl;
396
397 lp_build_nir_prepasses(shader);
398 NIR_PASS_V(shader, nir_move_vec_src_uses_to_dest, false);
399 NIR_PASS_V(shader, nir_lower_vec_to_regs, NULL, NULL);
400 lp_build_nir_llvm(&bld.bld_base, shader,
401 nir_shader_get_entrypoint(shader));
402 }
403