1 /**************************************************************************
2 *
3 * Copyright 2010-2021 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include <limits.h>
29
30 #include "pipe/p_defines.h"
31 #include "util/u_inlines.h"
32 #include "util/u_memory.h"
33 #include "util/u_pointer.h"
34 #include "util/format/u_format.h"
35 #include "util/u_dump.h"
36 #include "util/u_string.h"
37 #include "util/os_time.h"
38 #include "pipe/p_shader_tokens.h"
39 #include "draw/draw_context.h"
40 #include "gallivm/lp_bld_type.h"
41 #include "gallivm/lp_bld_const.h"
42 #include "gallivm/lp_bld_conv.h"
43 #include "gallivm/lp_bld_init.h"
44 #include "gallivm/lp_bld_intr.h"
45 #include "gallivm/lp_bld_logic.h"
46 #include "gallivm/lp_bld_tgsi.h"
47 #include "gallivm/lp_bld_swizzle.h"
48 #include "gallivm/lp_bld_flow.h"
49 #include "gallivm/lp_bld_printf.h"
50 #include "gallivm/lp_bld_debug.h"
51 #include "gallivm/lp_bld_nir.h"
52
53 #include "lp_bld_alpha.h"
54 #include "lp_bld_blend.h"
55 #include "lp_bld_depth.h"
56 #include "lp_bld_interp.h"
57 #include "lp_context.h"
58 #include "lp_debug.h"
59 #include "lp_perf.h"
60 #include "lp_screen.h"
61 #include "lp_setup.h"
62 #include "lp_state.h"
63 #include "lp_tex_sample.h"
64 #include "lp_flush.h"
65 #include "lp_state_fs.h"
66
67
68 /**
69 * Sampler.
70 */
71 struct linear_sampler
72 {
73 struct lp_build_sampler_aos base;
74 LLVMValueRef texels_ptrs[LP_MAX_LINEAR_TEXTURES];
75 LLVMValueRef counter;
76 unsigned instance;
77 };
78
79
80 /**
81 * Provide texels to the TGSI translation.
82 *
83 * We don't actually do any texture sampling here, but simply hand the
84 * precomputed row of texels.
85 */
86 static LLVMValueRef
emit_fetch_texel_linear(const struct lp_build_sampler_aos * base,struct lp_build_context * bld,enum tgsi_texture_type target,unsigned unit,LLVMValueRef coords,const struct lp_derivatives derivs,enum lp_build_tex_modifier modifier)87 emit_fetch_texel_linear(const struct lp_build_sampler_aos *base,
88 struct lp_build_context *bld,
89 enum tgsi_texture_type target,
90 unsigned unit,
91 LLVMValueRef coords,
92 const struct lp_derivatives derivs,
93 enum lp_build_tex_modifier modifier)
94 {
95 struct linear_sampler *sampler = (struct linear_sampler *)base;
96
97 if (sampler->instance >= LP_MAX_LINEAR_TEXTURES) {
98 assert(false);
99 return bld->undef;
100 }
101
102 /* Pointer to a row of texels */
103 LLVMValueRef texels_ptr = sampler->texels_ptrs[sampler->instance];
104
105 LLVMValueRef texel = lp_build_pointer_get2(bld->gallivm->builder,
106 bld->vec_type,
107 texels_ptr, sampler->counter);
108 assert(LLVMTypeOf(texel) == bld->vec_type);
109
110 /*
111 * We have a struct lp_linear_sampler instance per TEX instruction,
112 * _not_ per unit, as each TEX instruction will need separate storage
113 * for the texels.
114 */
115 (void)unit;
116 ++sampler->instance;
117
118 return texel;
119 }
120
121
122 /**
123 * Generates the main body of the fragment shader
124 * Supports generating code for 4 pixel blocks and individual pixels
125 */
126 static LLVMValueRef
llvm_fragment_body(struct lp_build_context * bld,struct lp_fragment_shader * shader,struct lp_fragment_shader_variant * variant,struct linear_sampler * sampler,LLVMValueRef * inputs_ptrs,LLVMValueRef consts_ptr,LLVMValueRef blend_color,LLVMValueRef alpha_ref,struct lp_type fs_type,LLVMValueRef dst)127 llvm_fragment_body(struct lp_build_context *bld,
128 struct lp_fragment_shader *shader,
129 struct lp_fragment_shader_variant *variant,
130 struct linear_sampler* sampler,
131 LLVMValueRef *inputs_ptrs,
132 LLVMValueRef consts_ptr,
133 LLVMValueRef blend_color,
134 LLVMValueRef alpha_ref,
135 struct lp_type fs_type,
136 LLVMValueRef dst)
137 {
138 static const unsigned char bgra_swizzles[4] = {2, 1, 0, 3};
139 static const unsigned char rgba_swizzles[4] = {0, 1, 2, 3};
140 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS];
141 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS];
142 LLVMBuilderRef builder = bld->gallivm->builder;
143 struct gallivm_state *gallivm = bld->gallivm;
144 LLVMValueRef result = NULL;
145 bool rgba_order = (variant->key.cbuf_format[0] == PIPE_FORMAT_R8G8B8A8_UNORM ||
146 variant->key.cbuf_format[0] == PIPE_FORMAT_R8G8B8X8_UNORM);
147 struct nir_shader *nir = shader->base.ir.nir;
148 sampler->instance = 0;
149
150 /*
151 * Advance inputs
152 */
153 unsigned i;
154 for (i = 0; i < util_bitcount64(nir->info.inputs_read); ++i) {
155 inputs[i] =
156 lp_build_pointer_get2(builder, bld->vec_type, inputs_ptrs[i], sampler->counter);
157 assert(LLVMTypeOf(inputs[i]) == bld->vec_type);
158 }
159 for ( ; i < PIPE_MAX_SHADER_INPUTS; ++i) {
160 inputs[i] = bld->undef;
161 }
162
163 for (i = 0; i < PIPE_MAX_SHADER_OUTPUTS; ++i) {
164 outputs[i] = bld->undef;
165 }
166
167 nir_shader *clone = nir_shader_clone(NULL, nir);
168 lp_build_nir_aos(gallivm, clone, fs_type,
169 rgba_order ? rgba_swizzles : bgra_swizzles,
170 consts_ptr, inputs, outputs,
171 &sampler->base);
172 ralloc_free(clone);
173
174 /*
175 * Blend output color
176 */
177 nir_foreach_shader_out_variable(var, nir) {
178 unsigned slots = nir_variable_count_slots(var, var->type);
179
180 for (unsigned s = 0; s < slots; s++) {
181 unsigned idx = var->data.driver_location + s;
182 if (!outputs[idx])
183 continue;
184
185 LLVMValueRef output = LLVMBuildLoad2(builder, bld->vec_type, outputs[idx], "");
186 lp_build_name(output, "output%u", i);
187
188 unsigned cbuf = var->data.location - FRAG_RESULT_DATA0 + s;
189 lp_build_name(output, "cbuf%u", cbuf);
190
191 if (var->data.location < FRAG_RESULT_DATA0 || s > 0)
192 continue;
193
194 /* Perform alpha test if necessary */
195 LLVMValueRef mask = NULL;
196 if (variant->key.alpha.enabled) {
197 LLVMTypeRef vec_type = lp_build_vec_type(gallivm, fs_type);
198 LLVMValueRef broadcast_alpha = lp_build_broadcast(gallivm, vec_type,
199 alpha_ref);
200
201 mask = lp_build_cmp(bld, variant->key.alpha.func, output,
202 broadcast_alpha);
203 /* XXX is 4 correct? */
204 mask = lp_build_swizzle_scalar_aos(bld, mask, bgra_swizzles[3], 4);
205
206 lp_build_name(mask, "alpha_test_mask");
207 }
208
209 LLVMValueRef src1 = lp_build_zero(gallivm, fs_type);
210
211 result = lp_build_blend_aos(gallivm,
212 &variant->key.blend,
213 variant->key.cbuf_format[idx],
214 fs_type,
215 cbuf, /* rt */
216 output, /* src */
217 NULL, /* src_alpha */
218 src1, /* src1 */
219 NULL, /* src1_alpha */
220 dst,
221 mask,
222 blend_color, /* const_ */
223 NULL, /* const_alpha */
224 rgba_order ? rgba_swizzles : bgra_swizzles,
225 4);
226 }
227 }
228
229 return result;
230 }
231
232
233 /**
234 * Generate a function that executes the fragment shader in a linear fashion.
235 * The shader operates on unorm8[16] vectors.
236 * See lp_state_fs_analysis for the "linear" conditions.
237 */
238 void
llvmpipe_fs_variant_linear_llvm(struct llvmpipe_context * lp,struct lp_fragment_shader * shader,struct lp_fragment_shader_variant * variant)239 llvmpipe_fs_variant_linear_llvm(struct llvmpipe_context *lp,
240 struct lp_fragment_shader *shader,
241 struct lp_fragment_shader_variant *variant)
242 {
243 assert(shader->kind == LP_FS_KIND_BLIT_RGBA ||
244 shader->kind == LP_FS_KIND_BLIT_RGB1 ||
245 shader->kind == LP_FS_KIND_LLVM_LINEAR);
246
247 struct nir_shader *nir = shader->base.ir.nir;
248 struct gallivm_state *gallivm = variant->gallivm;
249 LLVMTypeRef int8t = LLVMInt8TypeInContext(gallivm->context);
250 LLVMTypeRef int32t = LLVMInt32TypeInContext(gallivm->context);
251 LLVMTypeRef pint8t = LLVMPointerType(int8t, 0);
252 LLVMTypeRef pixelt = LLVMVectorType(int32t, 4);
253
254 // unorm8[16] vector type
255 struct lp_type fs_type;
256 memset(&fs_type, 0, sizeof fs_type);
257 fs_type.floating = false;
258 fs_type.sign = false;
259 fs_type.norm = true;
260 fs_type.width = 8;
261 fs_type.length = 16;
262
263 if (LP_DEBUG & DEBUG_TGSI) {
264 if (shader->base.ir.nir) {
265 nir_print_shader(shader->base.ir.nir, stderr);
266 }
267 }
268
269 /*
270 * Generate the function prototype. Any change here must be reflected in
271 * lp_jit.h's lp_jit_frag_func function pointer type, and vice-versa.
272 */
273
274 char func_name[256];
275 snprintf(func_name, sizeof(func_name), "fs_variant_linear2");
276
277 LLVMTypeRef ret_type = pint8t;
278 LLVMTypeRef arg_types[4];
279 arg_types[0] = variant->jit_linear_context_ptr_type; /* context */
280 arg_types[1] = int32t; /* x */
281 arg_types[2] = int32t; /* y */
282 arg_types[3] = int32t; /* width */
283
284 LLVMTypeRef func_type =
285 LLVMFunctionType(ret_type, arg_types, ARRAY_SIZE(arg_types), 0);
286
287 LLVMValueRef function =
288 LLVMAddFunction(gallivm->module, func_name, func_type);
289 LLVMSetFunctionCallConv(function, LLVMCCallConv);
290
291 variant->linear_function = function;
292 variant->linear_function_name = MALLOC(strlen(func_name)+1);
293 strcpy(variant->linear_function_name, func_name);
294
295 /* XXX: need to propagate noalias down into color param now we are
296 * passing a pointer-to-pointer?
297 */
298 for (unsigned i = 0; i < ARRAY_SIZE(arg_types); ++i) {
299 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) {
300 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
301 }
302 }
303
304 if (variant->gallivm->cache->data_size) {
305 gallivm_stub_func(gallivm, function);
306 return;
307 }
308
309 LLVMValueRef context_ptr = LLVMGetParam(function, 0);
310 LLVMValueRef x = LLVMGetParam(function, 1);
311 LLVMValueRef y = LLVMGetParam(function, 2);
312 LLVMValueRef width = LLVMGetParam(function, 3);
313
314 lp_build_name(context_ptr, "context");
315 lp_build_name(x, "x");
316 lp_build_name(y, "y");
317 lp_build_name(width, "width");
318
319 /*
320 * Function body
321 */
322
323 LLVMBasicBlockRef block =
324 LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
325 LLVMBuilderRef builder = gallivm->builder;
326
327 LLVMPositionBuilderAtEnd(builder, block);
328
329 struct lp_build_context bld;
330 lp_build_context_init(&bld, gallivm, fs_type);
331
332 /*
333 * Get context data
334 */
335 LLVMValueRef consts_ptr =
336 lp_jit_linear_context_constants(gallivm,
337 variant->jit_linear_context_type,
338 context_ptr);
339 LLVMValueRef interpolators_ptr =
340 lp_jit_linear_context_inputs(gallivm,
341 variant->jit_linear_context_type,
342 context_ptr);
343 LLVMValueRef samplers_ptr =
344 lp_jit_linear_context_tex(gallivm,
345 variant->jit_linear_context_type,
346 context_ptr);
347
348 LLVMValueRef color0_ptr =
349 lp_jit_linear_context_color0(gallivm,
350 variant->jit_linear_context_type,
351 context_ptr);
352 color0_ptr = LLVMBuildLoad2(builder, LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
353 color0_ptr, "");
354 color0_ptr = LLVMBuildBitCast(builder, color0_ptr,
355 LLVMPointerType(bld.vec_type, 0), "");
356
357 LLVMValueRef blend_color =
358 lp_jit_linear_context_blend_color(gallivm,
359 variant->jit_linear_context_type,
360 context_ptr);
361 blend_color = LLVMBuildLoad2(builder, LLVMInt32TypeInContext(gallivm->context),
362 blend_color, "");
363 blend_color = lp_build_broadcast(gallivm, LLVMVectorType(int32t, 4),
364 blend_color);
365 blend_color = LLVMBuildBitCast(builder, blend_color,
366 LLVMVectorType(int8t, 16), "");
367
368 LLVMValueRef alpha_ref =
369 lp_jit_linear_context_alpha_ref(gallivm,
370 variant->jit_linear_context_type,
371 context_ptr);
372 alpha_ref = LLVMBuildLoad2(builder, LLVMInt8TypeInContext(gallivm->context),
373 alpha_ref, "");
374
375 /*
376 * Invoke the input interpolators
377 */
378 LLVMValueRef inputs_ptrs[LP_MAX_LINEAR_INPUTS];
379
380 nir_foreach_shader_in_variable(var, nir) {
381 unsigned slots = nir_variable_count_slots(var, var->type);
382
383 for (unsigned s = 0; s < slots; s++) {
384 unsigned attrib = var->data.driver_location + s;
385 assert(attrib < LP_MAX_LINEAR_INPUTS);
386 if (attrib >= LP_MAX_LINEAR_INPUTS) {
387 break;
388 }
389
390 LLVMValueRef index = LLVMConstInt(int32t, attrib, 0);
391
392 LLVMTypeRef input_type = variant->jit_linear_inputs_type;
393 LLVMValueRef elem =
394 lp_build_array_get2(bld.gallivm, input_type, interpolators_ptr, index);
395 assert(LLVMGetTypeKind(LLVMTypeOf(elem)) == LLVMPointerTypeKind);
396
397 LLVMTypeRef fetch_type = LLVMPointerType(variant->jit_linear_func_type, 0);
398 LLVMValueRef fetch_ptr = lp_build_pointer_get2(builder, fetch_type, elem,
399 LLVMConstInt(int32t, 0, 0));
400 assert(LLVMGetTypeKind(LLVMTypeOf(fetch_ptr)) == LLVMPointerTypeKind);
401
402 /* Pointer to a row of interpolated inputs */
403 LLVMTypeRef call_type = variant->jit_linear_func_type;
404 elem = LLVMBuildBitCast(builder, elem, pint8t, "");
405 LLVMValueRef inputs_ptr = LLVMBuildCall2(builder, call_type, fetch_ptr, &elem, 1, "");
406 assert(LLVMGetTypeKind(LLVMTypeOf(inputs_ptr)) == LLVMPointerTypeKind);
407
408 lp_add_function_attr(inputs_ptr, -1, LP_FUNC_ATTR_NOUNWIND);
409
410 lp_build_name(inputs_ptr, "input%u_ptr", attrib);
411
412 inputs_ptrs[attrib] = inputs_ptr;
413 }
414 }
415
416 /*
417 * Invoke and hook up the texture samplers.
418 */
419
420 struct linear_sampler sampler;
421 memset(&sampler, 0, sizeof sampler);
422 sampler.base.emit_fetch_texel = &emit_fetch_texel_linear;
423
424 for (unsigned attrib = 0; attrib < shader->info.num_texs; ++attrib) {
425 assert(attrib < LP_MAX_LINEAR_TEXTURES);
426 if (attrib >= LP_MAX_LINEAR_TEXTURES) {
427 break;
428 }
429
430 LLVMValueRef index = LLVMConstInt(int32t, attrib, 0);
431 LLVMTypeRef samp_type = variant->jit_linear_textures_type;
432 LLVMValueRef elem = lp_build_array_get2(bld.gallivm, samp_type, samplers_ptr, index);
433 assert(LLVMGetTypeKind(LLVMTypeOf(elem)) == LLVMPointerTypeKind);
434
435 LLVMTypeRef fetch_type = LLVMPointerType(variant->jit_linear_func_type, 0);
436 LLVMValueRef fetch_ptr =
437 lp_build_pointer_get2(builder, fetch_type,
438 elem, LLVMConstInt(int32t, 0, 0));
439 assert(LLVMGetTypeKind(LLVMTypeOf(fetch_ptr)) == LLVMPointerTypeKind);
440
441 /* Pointer to a row of texels */
442 LLVMTypeRef call_type = variant->jit_linear_func_type;
443 elem = LLVMBuildBitCast(builder, elem, pint8t, "");
444 LLVMValueRef texels_ptr = LLVMBuildCall2(builder, call_type, fetch_ptr, &elem, 1, "");
445 assert(LLVMGetTypeKind(LLVMTypeOf(texels_ptr)) == LLVMPointerTypeKind);
446
447 lp_add_function_attr(texels_ptr, -1, LP_FUNC_ATTR_NOUNWIND);
448
449 lp_build_name(texels_ptr, "tex%u_ptr", attrib);
450
451 sampler.texels_ptrs[attrib] = texels_ptr;
452 }
453
454 /* excess = width & 0x3 */
455 LLVMValueRef excess =
456 LLVMBuildAnd(builder, width, LLVMConstInt(int32t, 3, 0), "");
457 /* width *= 4 */
458 width = LLVMBuildLShr(builder, width, LLVMConstInt(int32t, 2, 0), "");
459
460 /* Loop over blocks of 4 pixels */
461 /* for loop.counter = 0; loop.counter < width; loop.counter++) { */
462 struct lp_build_for_loop_state loop;
463 lp_build_for_loop_begin(&loop, gallivm, LLVMConstInt(int32t, 0, 0),
464 LLVMIntULT, width, LLVMConstInt(int32t, 1, 0));
465 {
466 LLVMValueRef value;
467 sampler.counter = loop.counter;
468
469 /* Read 4 pixels */
470 value = lp_build_pointer_get_unaligned2(builder,
471 bld.vec_type,
472 color0_ptr,
473 loop.counter, 4);
474
475 /* Perform fragment shader body */
476 value = llvm_fragment_body(&bld, shader, variant, &sampler, inputs_ptrs,
477 consts_ptr, blend_color, alpha_ref, fs_type,
478 value);
479
480 /* Write 4 pixels */
481 lp_build_pointer_set_unaligned(builder, color0_ptr, loop.counter,
482 value, 4);
483 }
484 lp_build_for_loop_end(&loop);
485
486 /* Compute the edge pixels (width % 4) */
487 struct lp_build_if_state ifstate;
488 lp_build_if(&ifstate, gallivm, LLVMBuildICmp(builder, LLVMIntNE, excess,
489 LLVMConstInt(int32t, 0, 0), ""));
490 {
491 struct lp_build_loop_state loop_read, loop_write;
492 LLVMValueRef buf, elem, result, pixel_ptr;
493 LLVMValueRef buf_ptr = lp_build_alloca(gallivm, pixelt, "");
494
495 sampler.counter = width;
496
497 /* Get the i32* pixel pointer from the <i16x8>* element pointer */
498 pixel_ptr = LLVMBuildGEP2(gallivm->builder, bld.vec_type,
499 color0_ptr, &width, 1, "");
500 pixel_ptr = LLVMBuildBitCast(gallivm->builder, pixel_ptr,
501 LLVMPointerType(int32t, 0), "");
502
503 /* Copy individual pixels from memory to local buffer */
504 lp_build_loop_begin(&loop_read, gallivm, LLVMConstInt(int32t, 0, 0));
505 {
506 elem = lp_build_pointer_get2(gallivm->builder,
507 int32t,
508 pixel_ptr, loop_read.counter);
509
510 buf = LLVMBuildLoad2(gallivm->builder, pixelt, buf_ptr, "");
511 buf = LLVMBuildInsertElement(builder, buf, elem,
512 loop_read.counter, "");
513 LLVMBuildStore(builder, buf, buf_ptr);
514 }
515 lp_build_loop_end_cond(&loop_read, excess,
516 LLVMConstInt(int32t, 1, 0), LLVMIntUGE);
517
518 /* Perform fragment shader body */
519 buf = LLVMBuildLoad2(gallivm->builder, pixelt, buf_ptr, "");
520 buf = LLVMBuildBitCast(builder, buf, bld.vec_type, "");
521
522 result = llvm_fragment_body(&bld, shader, variant, &sampler,
523 inputs_ptrs, consts_ptr, blend_color,
524 alpha_ref, fs_type, buf);
525 result = LLVMBuildBitCast(builder, result, pixelt, "");
526
527 /* Write individual pixels from local buffer to the memory */
528 lp_build_loop_begin(&loop_write, gallivm, LLVMConstInt(int32t, 0, 0));
529 {
530 elem = LLVMBuildExtractElement(builder, result,
531 loop_write.counter, "");
532
533 lp_build_pointer_set(gallivm->builder, pixel_ptr,
534 loop_write.counter, elem);
535 }
536 lp_build_loop_end_cond(&loop_write, excess,
537 LLVMConstInt(int32t, 1, 0), LLVMIntUGE);
538 }
539 lp_build_endif(&ifstate);
540
541 color0_ptr = LLVMBuildBitCast(builder, color0_ptr, pint8t, "");
542
543 LLVMBuildRet(builder, color0_ptr);
544
545 gallivm_verify_function(gallivm, function);
546 }
547