1 /**************************************************************************
2 *
3 * Copyright 2010 VMware.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28
29 #include "util/u_math.h"
30 #include "util/u_memory.h"
31 #include "util/os_time.h"
32 #include "gallivm/lp_bld_arit.h"
33 #include "gallivm/lp_bld_bitarit.h"
34 #include "gallivm/lp_bld_const.h"
35 #include "gallivm/lp_bld_debug.h"
36 #include "gallivm/lp_bld_init.h"
37 #include "gallivm/lp_bld_logic.h"
38 #include "gallivm/lp_bld_intr.h"
39 #include "gallivm/lp_bld_flow.h"
40 #include "gallivm/lp_bld_type.h"
41
42 #include "lp_perf.h"
43 #include "lp_debug.h"
44 #include "lp_flush.h"
45 #include "lp_screen.h"
46 #include "lp_context.h"
47 #include "lp_state.h"
48 #include "lp_state_fs.h"
49 #include "lp_state_setup.h"
50
51 #include "nir.h"
52
53 /** Setup shader number (for debugging) */
54 static unsigned setup_no = 0;
55
56
57 /* currently organized to interpolate full float[4] attributes even
58 * when some elements are unused. Later, can pack vertex data more
59 * closely.
60 */
61
62
63 struct lp_setup_args
64 {
65 /* Function arguments:
66 */
67 LLVMValueRef v0;
68 LLVMValueRef v1;
69 LLVMValueRef v2;
70 LLVMValueRef facing; /* boolean */
71 LLVMValueRef a0;
72 LLVMValueRef dadx;
73 LLVMValueRef dady;
74 LLVMValueRef key;
75 LLVMTypeRef vec4f_type;
76
77 /* Derived:
78 */
79 LLVMValueRef x0_center;
80 LLVMValueRef y0_center;
81 LLVMValueRef dy20_ooa;
82 LLVMValueRef dy01_ooa;
83 LLVMValueRef dx20_ooa;
84 LLVMValueRef dx01_ooa;
85 struct lp_build_context bld;
86 };
87
88
89 static void
store_coef(struct gallivm_state * gallivm,const struct lp_setup_args * args,unsigned slot,LLVMValueRef a0,LLVMValueRef dadx,LLVMValueRef dady)90 store_coef(struct gallivm_state *gallivm,
91 const struct lp_setup_args *args,
92 unsigned slot,
93 LLVMValueRef a0,
94 LLVMValueRef dadx,
95 LLVMValueRef dady)
96 {
97 LLVMBuilderRef builder = gallivm->builder;
98 LLVMValueRef idx = lp_build_const_int32(gallivm, slot);
99
100 LLVMBuildStore(builder,
101 a0,
102 LLVMBuildGEP2(builder, args->vec4f_type, args->a0, &idx, 1, ""));
103
104 LLVMBuildStore(builder,
105 dadx,
106 LLVMBuildGEP2(builder, args->vec4f_type, args->dadx, &idx, 1, ""));
107
108 LLVMBuildStore(builder,
109 dady,
110 LLVMBuildGEP2(builder, args->vec4f_type, args->dady, &idx, 1, ""));
111 }
112
113
114 static void
emit_constant_coef4(struct gallivm_state * gallivm,const struct lp_setup_args * args,unsigned slot,LLVMValueRef vert)115 emit_constant_coef4(struct gallivm_state *gallivm,
116 const struct lp_setup_args *args,
117 unsigned slot,
118 LLVMValueRef vert)
119 {
120 store_coef(gallivm, args, slot, vert, args->bld.zero, args->bld.zero);
121 }
122
123
124 /**
125 * Setup the fragment input attribute with the front-facing value.
126 * \param frontface is the triangle front facing?
127 */
128 static void
emit_facing_coef(struct gallivm_state * gallivm,struct lp_setup_args * args,unsigned slot)129 emit_facing_coef(struct gallivm_state *gallivm,
130 struct lp_setup_args *args,
131 unsigned slot)
132 {
133 LLVMBuilderRef builder = gallivm->builder;
134 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context);
135 LLVMValueRef a0_0 = args->facing;
136 LLVMValueRef a0_0f = LLVMBuildSIToFP(builder, a0_0, float_type, "");
137 LLVMValueRef a0, face_val;
138 const unsigned char swizzles[4] = { PIPE_SWIZZLE_X, PIPE_SWIZZLE_0,
139 PIPE_SWIZZLE_0, PIPE_SWIZZLE_0 };
140 /* Our face val is either 1 or 0 so we do
141 * face = (val * 2) - 1
142 * to make it 1 or -1
143 */
144 face_val =
145 LLVMBuildFAdd(builder,
146 LLVMBuildFMul(builder, a0_0f,
147 lp_build_const_float(gallivm, 2.0),
148 ""),
149 lp_build_const_float(gallivm, -1.0),
150 "facing");
151 face_val = lp_build_broadcast_scalar(&args->bld, face_val);
152 a0 = lp_build_swizzle_aos(&args->bld, face_val, swizzles);
153
154 store_coef(gallivm, args, slot, a0, args->bld.zero, args->bld.zero);
155 }
156
157
158 static LLVMValueRef
vert_attrib(struct gallivm_state * gallivm,LLVMTypeRef vert_type,LLVMValueRef vert,int attr,int elem,const char * name)159 vert_attrib(struct gallivm_state *gallivm,
160 LLVMTypeRef vert_type,
161 LLVMValueRef vert,
162 int attr,
163 int elem,
164 const char *name)
165 {
166 LLVMBuilderRef b = gallivm->builder;
167 LLVMValueRef idx[2];
168 LLVMTypeRef v_type = LLVMFloatTypeInContext(gallivm->context);
169
170 idx[0] = lp_build_const_int32(gallivm, attr);
171 idx[1] = lp_build_const_int32(gallivm, elem);
172 return LLVMBuildLoad2(b, v_type, LLVMBuildGEP2(b, vert_type, vert, idx, 2, ""), name);
173 }
174
175
176 static void
lp_twoside(struct gallivm_state * gallivm,struct lp_setup_args * args,const struct lp_setup_variant_key * key,int bcolor_slot,LLVMValueRef attribv[3])177 lp_twoside(struct gallivm_state *gallivm,
178 struct lp_setup_args *args,
179 const struct lp_setup_variant_key *key,
180 int bcolor_slot,
181 LLVMValueRef attribv[3])
182 {
183 LLVMBuilderRef b = gallivm->builder;
184 LLVMValueRef a0_back, a1_back, a2_back;
185 LLVMValueRef idx2 = lp_build_const_int32(gallivm, bcolor_slot);
186
187 LLVMValueRef facing = args->facing;
188 LLVMValueRef front_facing = LLVMBuildICmp(b, LLVMIntEQ, facing,
189 lp_build_const_int32(gallivm, 0), ""); /** need i1 for if condition */
190
191 a0_back = LLVMBuildLoad2(b, args->vec4f_type, LLVMBuildGEP2(b, args->vec4f_type, args->v0, &idx2, 1, ""), "v0a_back");
192 a1_back = LLVMBuildLoad2(b, args->vec4f_type, LLVMBuildGEP2(b, args->vec4f_type, args->v1, &idx2, 1, ""), "v1a_back");
193 a2_back = LLVMBuildLoad2(b, args->vec4f_type, LLVMBuildGEP2(b, args->vec4f_type, args->v2, &idx2, 1, ""), "v2a_back");
194
195 /* Possibly swap the front and back attrib values,
196 *
197 * Prefer select to if so we don't have to worry about phis or
198 * allocas.
199 */
200 attribv[0] = LLVMBuildSelect(b, front_facing, a0_back, attribv[0], "");
201 attribv[1] = LLVMBuildSelect(b, front_facing, a1_back, attribv[1], "");
202 attribv[2] = LLVMBuildSelect(b, front_facing, a2_back, attribv[2], "");
203 }
204
205
206 static LLVMValueRef
lp_do_offset_tri(struct gallivm_state * gallivm,struct lp_setup_args * args,const struct lp_setup_variant_key * key,LLVMValueRef inv_det,LLVMValueRef dxyz01,LLVMValueRef dxyz20,LLVMValueRef attribv[3])207 lp_do_offset_tri(struct gallivm_state *gallivm,
208 struct lp_setup_args *args,
209 const struct lp_setup_variant_key *key,
210 LLVMValueRef inv_det,
211 LLVMValueRef dxyz01,
212 LLVMValueRef dxyz20,
213 LLVMValueRef attribv[3])
214 {
215 LLVMBuilderRef b = gallivm->builder;
216 struct lp_build_context flt_scalar_bld;
217 struct lp_build_context int_scalar_bld;
218 struct lp_build_context *bld = &args->bld;
219 LLVMValueRef zoffset, mult;
220 LLVMValueRef dzdxdzdy, dzdx, dzdy, dzxyz20, dyzzx01, dyzzx01_dzxyz20, dzx01_dyz20;
221 LLVMValueRef max, max_value, res12;
222 LLVMValueRef shuffles[4];
223 LLVMTypeRef shuf_type = LLVMInt32TypeInContext(gallivm->context);
224 LLVMValueRef onei = lp_build_const_int32(gallivm, 1);
225 LLVMValueRef zeroi = lp_build_const_int32(gallivm, 0);
226 LLVMValueRef twoi = lp_build_const_int32(gallivm, 2);
227 LLVMValueRef threei = lp_build_const_int32(gallivm, 3);
228
229 /* (res12) = cross(e,f).xy */
230 shuffles[0] = twoi;
231 shuffles[1] = zeroi;
232 shuffles[2] = onei;
233 shuffles[3] = twoi;
234 dzxyz20 = LLVMBuildShuffleVector(b, dxyz20, dxyz20, LLVMConstVector(shuffles, 4), "");
235
236 shuffles[0] = onei;
237 shuffles[1] = twoi;
238 shuffles[2] = twoi;
239 shuffles[3] = zeroi;
240 dyzzx01 = LLVMBuildShuffleVector(b, dxyz01, dxyz01, LLVMConstVector(shuffles, 4), "");
241
242 dyzzx01_dzxyz20 = LLVMBuildFMul(b, dzxyz20, dyzzx01, "dyzzx01_dzxyz20");
243
244 shuffles[0] = twoi;
245 shuffles[1] = threei;
246 shuffles[2] = LLVMGetUndef(shuf_type);
247 shuffles[3] = LLVMGetUndef(shuf_type);
248 dzx01_dyz20 = LLVMBuildShuffleVector(b, dyzzx01_dzxyz20, dyzzx01_dzxyz20,
249 LLVMConstVector(shuffles, 4), "");
250
251 res12 = LLVMBuildFSub(b, dyzzx01_dzxyz20, dzx01_dyz20, "res12");
252
253 /* dzdx = fabsf(res1 * inv_det), dydx = fabsf(res2 * inv_det)*/
254 dzdxdzdy = LLVMBuildFMul(b, res12, inv_det, "dzdxdzdy");
255 dzdxdzdy = lp_build_abs(bld, dzdxdzdy);
256
257 dzdx = LLVMBuildExtractElement(b, dzdxdzdy, zeroi, "");
258 dzdy = LLVMBuildExtractElement(b, dzdxdzdy, onei, "");
259
260 /* mult = MAX2(dzdx, dzdy) * pgon_offset_scale */
261 max = LLVMBuildFCmp(b, LLVMRealUGT, dzdx, dzdy, "");
262 max_value = LLVMBuildSelect(b, max, dzdx, dzdy, "max");
263
264 mult = LLVMBuildFMul(b, max_value,
265 lp_build_const_float(gallivm,
266 key->pgon_offset_scale), "");
267
268 lp_build_context_init(&flt_scalar_bld, gallivm, lp_type_float_vec(32, 32));
269
270 if (key->floating_point_depth) {
271 /*
272 * bias = pgon_offset_units * 2^(exponent(max(abs(z0), abs(z1), abs(z2))) -
273 * mantissa_bits) + MAX2(dzdx, dzdy) * pgon_offset_scale
274 *
275 * NOTE: Assumes IEEE float32.
276 */
277 LLVMValueRef c23_shifted, exp_mask, bias, exp;
278 LLVMValueRef maxz_value, maxz0z1_value;
279
280 lp_build_context_init(&int_scalar_bld, gallivm, lp_type_int_vec(32, 32));
281
282 c23_shifted = lp_build_const_int32(gallivm, 23 << 23);
283 exp_mask = lp_build_const_int32(gallivm, 0xff << 23);
284
285 maxz0z1_value = lp_build_max(&flt_scalar_bld,
286 lp_build_abs(&flt_scalar_bld,
287 LLVMBuildExtractElement(b, attribv[0], twoi, "")),
288 lp_build_abs(&flt_scalar_bld,
289 LLVMBuildExtractElement(b, attribv[1], twoi, "")));
290
291 maxz_value = lp_build_max(&flt_scalar_bld,
292 lp_build_abs(&flt_scalar_bld,
293 LLVMBuildExtractElement(b, attribv[2], twoi, "")),
294 maxz0z1_value);
295
296 exp = LLVMBuildBitCast(b, maxz_value, int_scalar_bld.vec_type, "");
297 exp = lp_build_and(&int_scalar_bld, exp, exp_mask);
298 exp = lp_build_sub(&int_scalar_bld, exp, c23_shifted);
299 /* Clamping to zero means mrd will be zero for very small numbers,
300 * but specs do not indicate this should be prevented by clamping
301 * mrd to smallest normal number instead. */
302 exp = lp_build_max(&int_scalar_bld, exp, int_scalar_bld.zero);
303 exp = LLVMBuildBitCast(b, exp, flt_scalar_bld.vec_type, "");
304
305 bias = LLVMBuildFMul(b, exp,
306 lp_build_const_float(gallivm, key->pgon_offset_units),
307 "bias");
308
309 zoffset = LLVMBuildFAdd(b, bias, mult, "zoffset");
310 } else {
311 /*
312 * bias = pgon_offset_units + MAX2(dzdx, dzdy) * pgon_offset_scale
313 */
314 zoffset = LLVMBuildFAdd(b,
315 lp_build_const_float(gallivm, key->pgon_offset_units),
316 mult, "zoffset");
317 }
318
319 if (key->pgon_offset_clamp > 0) {
320 zoffset = lp_build_min(&flt_scalar_bld,
321 lp_build_const_float(gallivm, key->pgon_offset_clamp),
322 zoffset);
323 } else if (key->pgon_offset_clamp < 0) {
324 zoffset = lp_build_max(&flt_scalar_bld,
325 lp_build_const_float(gallivm, key->pgon_offset_clamp),
326 zoffset);
327 }
328
329 return zoffset;
330 }
331
332
333 static void
load_attribute(struct gallivm_state * gallivm,struct lp_setup_args * args,const struct lp_setup_variant_key * key,unsigned vert_attr,LLVMValueRef attribv[3])334 load_attribute(struct gallivm_state *gallivm,
335 struct lp_setup_args *args,
336 const struct lp_setup_variant_key *key,
337 unsigned vert_attr,
338 LLVMValueRef attribv[3])
339 {
340 LLVMBuilderRef b = gallivm->builder;
341 LLVMValueRef idx = lp_build_const_int32(gallivm, vert_attr);
342
343 /* Load the vertex data
344 */
345 attribv[0] = LLVMBuildLoad2(b, args->vec4f_type, LLVMBuildGEP2(b, args->vec4f_type, args->v0, &idx, 1, ""), "v0a");
346 attribv[1] = LLVMBuildLoad2(b, args->vec4f_type, LLVMBuildGEP2(b, args->vec4f_type, args->v1, &idx, 1, ""), "v1a");
347 attribv[2] = LLVMBuildLoad2(b, args->vec4f_type, LLVMBuildGEP2(b, args->vec4f_type, args->v2, &idx, 1, ""), "v2a");
348
349 /* Potentially modify it according to twoside, etc:
350 */
351 if (key->twoside) {
352 if (vert_attr == key->color_slot && key->bcolor_slot >= 0)
353 lp_twoside(gallivm, args, key, key->bcolor_slot, attribv);
354 else if (vert_attr == key->spec_slot && key->bspec_slot >= 0)
355 lp_twoside(gallivm, args, key, key->bspec_slot, attribv);
356 }
357 }
358
359
360 /*
361 * FIXME: interpolation is always done wrt fb origin (0/0).
362 * However, if some (small) tri is far away from the origin and gradients
363 * are large, this can lead to HUGE errors, since the a0 value calculated
364 * here can get very large (with the actual values inside the triangle way
365 * smaller), leading to complete loss of accuracy. This could be prevented
366 * by using some point inside (or at corner) of the tri as interpolation
367 * origin, or just use barycentric interpolation (which GL suggests and is
368 * what real hw does - you can get the barycentric coordinates from the
369 * edge functions in rasterization in principle (though we skip these
370 * sometimes completely in case of tris covering a block fully,
371 * which obviously wouldn't work)).
372 */
373 static void
calc_coef4(struct gallivm_state * gallivm,struct lp_setup_args * args,LLVMValueRef a0,LLVMValueRef a1,LLVMValueRef a2,LLVMValueRef out[3])374 calc_coef4(struct gallivm_state *gallivm,
375 struct lp_setup_args *args,
376 LLVMValueRef a0,
377 LLVMValueRef a1,
378 LLVMValueRef a2,
379 LLVMValueRef out[3])
380 {
381 LLVMBuilderRef b = gallivm->builder;
382 LLVMValueRef attr_0;
383 LLVMValueRef dy20_ooa = args->dy20_ooa;
384 LLVMValueRef dy01_ooa = args->dy01_ooa;
385 LLVMValueRef dx20_ooa = args->dx20_ooa;
386 LLVMValueRef dx01_ooa = args->dx01_ooa;
387 LLVMValueRef x0_center = args->x0_center;
388 LLVMValueRef y0_center = args->y0_center;
389 LLVMValueRef da01 = LLVMBuildFSub(b, a0, a1, "da01");
390 LLVMValueRef da20 = LLVMBuildFSub(b, a2, a0, "da20");
391
392 /* Calculate dadx (vec4f)
393 */
394 LLVMValueRef da01_dy20_ooa = LLVMBuildFMul(b, da01, dy20_ooa, "da01_dy20_ooa");
395 LLVMValueRef da20_dy01_ooa = LLVMBuildFMul(b, da20, dy01_ooa, "da20_dy01_ooa");
396 LLVMValueRef dadx = LLVMBuildFSub(b, da01_dy20_ooa, da20_dy01_ooa, "dadx");
397
398 /* Calculate dady (vec4f)
399 */
400 LLVMValueRef da01_dx20_ooa = LLVMBuildFMul(b, da01, dx20_ooa, "da01_dx20_ooa");
401 LLVMValueRef da20_dx01_ooa = LLVMBuildFMul(b, da20, dx01_ooa, "da20_dx01_ooa");
402 LLVMValueRef dady = LLVMBuildFSub(b, da20_dx01_ooa, da01_dx20_ooa, "dady");
403
404 /* Calculate a0 - the attribute value at the origin
405 */
406 LLVMValueRef dadx_x0 = LLVMBuildFMul(b, dadx, x0_center, "dadx_x0");
407 LLVMValueRef dady_y0 = LLVMBuildFMul(b, dady, y0_center, "dady_y0");
408 LLVMValueRef attr_v0 = LLVMBuildFAdd(b, dadx_x0, dady_y0, "attr_v0");
409 attr_0 = LLVMBuildFSub(b, a0, attr_v0, "attr_0");
410
411 out[0] = attr_0;
412 out[1] = dadx;
413 out[2] = dady;
414 }
415
416
417 static void
emit_coef4(struct gallivm_state * gallivm,struct lp_setup_args * args,unsigned slot,LLVMValueRef a0,LLVMValueRef a1,LLVMValueRef a2)418 emit_coef4(struct gallivm_state *gallivm,
419 struct lp_setup_args *args,
420 unsigned slot,
421 LLVMValueRef a0,
422 LLVMValueRef a1,
423 LLVMValueRef a2)
424 {
425 LLVMValueRef coeffs[3];
426 calc_coef4(gallivm, args, a0, a1, a2, coeffs);
427 store_coef(gallivm, args, slot, coeffs[0], coeffs[1], coeffs[2]);
428 }
429
430
431 static void
emit_linear_coef(struct gallivm_state * gallivm,struct lp_setup_args * args,unsigned slot,LLVMValueRef attribv[3])432 emit_linear_coef(struct gallivm_state *gallivm,
433 struct lp_setup_args *args,
434 unsigned slot,
435 LLVMValueRef attribv[3])
436 {
437 /* nothing to do anymore */
438 emit_coef4(gallivm, args, slot, attribv[0], attribv[1], attribv[2]);
439 }
440
441
442 /**
443 * Compute a0, dadx and dady for a perspective-corrected interpolant,
444 * for a triangle.
445 * We basically multiply the vertex value by 1/w before computing
446 * the plane coefficients (a0, dadx, dady).
447 * Later, when we compute the value at a particular fragment position we'll
448 * divide the interpolated value by the interpolated W at that fragment.
449 */
450 static void
apply_perspective_corr(struct gallivm_state * gallivm,struct lp_setup_args * args,unsigned slot,LLVMValueRef attribv[3])451 apply_perspective_corr(struct gallivm_state *gallivm,
452 struct lp_setup_args *args,
453 unsigned slot,
454 LLVMValueRef attribv[3])
455 {
456 LLVMBuilderRef b = gallivm->builder;
457
458 /* premultiply by 1/w (v[0][3] is always 1/w):
459 */
460 LLVMValueRef v0_oow = lp_build_broadcast_scalar(&args->bld,
461 vert_attrib(gallivm, args->vec4f_type,
462 args->v0, 0, 3, "v0_oow"));
463 LLVMValueRef v1_oow = lp_build_broadcast_scalar(&args->bld,
464 vert_attrib(gallivm, args->vec4f_type,
465 args->v1, 0, 3, "v1_oow"));
466 LLVMValueRef v2_oow = lp_build_broadcast_scalar(&args->bld,
467 vert_attrib(gallivm, args->vec4f_type,
468 args->v2, 0, 3, "v2_oow"));
469
470 attribv[0] = LLVMBuildFMul(b, attribv[0], v0_oow, "v0_oow_v0a");
471 attribv[1] = LLVMBuildFMul(b, attribv[1], v1_oow, "v1_oow_v1a");
472 attribv[2] = LLVMBuildFMul(b, attribv[2], v2_oow, "v2_oow_v2a");
473 }
474
475
476 /**
477 * Compute the inputs-> dadx, dady, a0 values.
478 */
479 static void
emit_tri_coef(struct gallivm_state * gallivm,const struct lp_setup_variant_key * key,struct lp_setup_args * args)480 emit_tri_coef(struct gallivm_state *gallivm,
481 const struct lp_setup_variant_key *key,
482 struct lp_setup_args *args)
483 {
484 LLVMValueRef attribs[3];
485
486 /* setup interpolation for all the remaining attributes */
487 for (unsigned slot = 0; slot < key->num_inputs; slot++) {
488 switch (key->inputs[slot].interp) {
489 case LP_INTERP_CONSTANT:
490 load_attribute(gallivm, args, key, key->inputs[slot].src_index, attribs);
491 if (key->flatshade_first) {
492 emit_constant_coef4(gallivm, args, slot+1, attribs[0]);
493 } else {
494 emit_constant_coef4(gallivm, args, slot+1, attribs[2]);
495 }
496 break;
497
498 case LP_INTERP_LINEAR:
499 load_attribute(gallivm, args, key, key->inputs[slot].src_index, attribs);
500 emit_linear_coef(gallivm, args, slot+1, attribs);
501 break;
502
503 case LP_INTERP_PERSPECTIVE:
504 load_attribute(gallivm, args, key, key->inputs[slot].src_index, attribs);
505 apply_perspective_corr(gallivm, args, slot+1, attribs);
506 emit_linear_coef(gallivm, args, slot+1, attribs);
507 break;
508
509 case LP_INTERP_POSITION:
510 /*
511 * The generated pixel interpolators will pick up the coeffs from
512 * slot 0.
513 */
514 break;
515
516 case LP_INTERP_FACING:
517 emit_facing_coef(gallivm, args, slot+1);
518 break;
519
520 default:
521 assert(0);
522 }
523 }
524 }
525
526
527 /* XXX: generic code:
528 */
529 static void
set_noalias(LLVMBuilderRef builder,LLVMValueRef function,const LLVMTypeRef * arg_types,int nr_args)530 set_noalias(LLVMBuilderRef builder,
531 LLVMValueRef function,
532 const LLVMTypeRef *arg_types,
533 int nr_args)
534 {
535 for (int i = 0; i < nr_args; ++i) {
536 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) {
537 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
538 }
539 }
540 }
541
542
543 static void
init_args(struct gallivm_state * gallivm,const struct lp_setup_variant_key * key,struct lp_setup_args * args)544 init_args(struct gallivm_state *gallivm,
545 const struct lp_setup_variant_key *key,
546 struct lp_setup_args *args)
547 {
548 LLVMBuilderRef b = gallivm->builder;
549 LLVMTypeRef shuf_type = LLVMInt32TypeInContext(gallivm->context);
550 LLVMValueRef onef = lp_build_const_float(gallivm, 1.0);
551 LLVMValueRef onei = lp_build_const_int32(gallivm, 1);
552 LLVMValueRef zeroi = lp_build_const_int32(gallivm, 0);
553 LLVMValueRef pixel_center, xy0_center, dxy01, dxy20, dyx20;
554 LLVMValueRef e, f, ef, ooa;
555 LLVMValueRef shuffles[4], shuf10;
556 LLVMValueRef attr_pos[3];
557 LLVMValueRef polygon_offset;
558 struct lp_type typef4 = lp_type_float_vec(32, 128);
559 struct lp_build_context bld;
560
561 lp_build_context_init(&bld, gallivm, typef4);
562 args->bld = bld;
563
564 /* The internal position input is in slot zero:
565 */
566 load_attribute(gallivm, args, key, 0, attr_pos);
567
568 pixel_center = lp_build_const_vec(gallivm, typef4,
569 (!key->multisample && key->pixel_center_half) ? 0.5 : 0.0);
570
571 /*
572 * xy are first two elems in v0a/v1a/v2a but just use vec4 arit
573 * also offset_tri uses actually xyz in them
574 */
575 xy0_center = LLVMBuildFSub(b, attr_pos[0], pixel_center, "xy0_center" );
576
577 dxy01 = LLVMBuildFSub(b, attr_pos[0], attr_pos[1], "dxy01");
578 dxy20 = LLVMBuildFSub(b, attr_pos[2], attr_pos[0], "dxy20");
579
580 shuffles[0] = onei;
581 shuffles[1] = zeroi;
582 shuffles[2] = LLVMGetUndef(shuf_type);
583 shuffles[3] = LLVMGetUndef(shuf_type);
584 shuf10 = LLVMConstVector(shuffles, 4);
585
586 dyx20 = LLVMBuildShuffleVector(b, dxy20, dxy20, shuf10, "");
587
588 ef = LLVMBuildFMul(b, dxy01, dyx20, "ef");
589 e = LLVMBuildExtractElement(b, ef, zeroi, "");
590 f = LLVMBuildExtractElement(b, ef, onei, "");
591
592 ooa = LLVMBuildFDiv(b, onef, LLVMBuildFSub(b, e, f, ""), "ooa");
593
594 ooa = lp_build_broadcast_scalar(&bld, ooa);
595
596 /* tri offset calc shares a lot of arithmetic, do it here */
597 if (key->pgon_offset_scale != 0.0f || key->pgon_offset_units != 0.0f) {
598 polygon_offset = lp_do_offset_tri(gallivm, args, key, ooa, dxy01, dxy20, attr_pos);
599 } else {
600 polygon_offset = lp_build_const_float(gallivm, 0.0f);
601 }
602
603 dxy20 = LLVMBuildFMul(b, dxy20, ooa, "");
604 dxy01 = LLVMBuildFMul(b, dxy01, ooa, "");
605
606 args->dy20_ooa = lp_build_extract_broadcast(gallivm, typef4, typef4, dxy20, onei);
607 args->dy01_ooa = lp_build_extract_broadcast(gallivm, typef4, typef4, dxy01, onei);
608
609 args->dx20_ooa = lp_build_extract_broadcast(gallivm, typef4, typef4, dxy20, zeroi);
610 args->dx01_ooa = lp_build_extract_broadcast(gallivm, typef4, typef4, dxy01, zeroi);
611
612 args->x0_center = lp_build_extract_broadcast(gallivm, typef4, typef4, xy0_center, zeroi);
613 args->y0_center = lp_build_extract_broadcast(gallivm, typef4, typef4, xy0_center, onei);
614
615 LLVMValueRef coeffs[3];
616 calc_coef4(gallivm, args, attr_pos[0], attr_pos[1], attr_pos[2], coeffs);
617
618 /* This is a bit sneaky:
619 * Because we observe that the X component of A0 is otherwise unused,
620 * we can overwrite it with the computed polygon-offset value, to make
621 * sure it's available in the fragment shader without having to change
622 * the interface (which is error-prone).
623 */
624 coeffs[0] = LLVMBuildInsertElement(b, coeffs[0], polygon_offset,
625 lp_build_const_int32(gallivm, 0), "");
626
627 store_coef(gallivm, args, 0, coeffs[0], coeffs[1], coeffs[2]);
628 }
629
630
631 /**
632 * Generate the runtime callable function for the coefficient calculation.
633 *
634 */
635 static struct lp_setup_variant *
generate_setup_variant(struct lp_setup_variant_key * key,struct llvmpipe_context * lp)636 generate_setup_variant(struct lp_setup_variant_key *key,
637 struct llvmpipe_context *lp)
638 {
639 int64_t t0 = 0, t1;
640
641 if (0)
642 goto fail;
643
644 struct lp_setup_variant *variant = CALLOC_STRUCT(lp_setup_variant);
645 if (!variant)
646 goto fail;
647
648 variant->no = setup_no++;
649
650 char func_name[64];
651 snprintf(func_name, sizeof(func_name), "setup_variant_%u",
652 variant->no);
653
654 struct gallivm_state *gallivm;
655 variant->gallivm = gallivm = gallivm_create(func_name, &lp->context, NULL);
656 if (!variant->gallivm) {
657 goto fail;
658 }
659
660 LLVMBuilderRef builder = gallivm->builder;
661
662 if (LP_DEBUG & DEBUG_COUNTERS) {
663 t0 = os_time_get();
664 }
665
666 memcpy(&variant->key, key, key->size);
667 variant->list_item_global.base = variant;
668
669 /* Currently always deal with full 4-wide vertex attributes from
670 * the vertices.
671 */
672
673 LLVMTypeRef vec4f_type =
674 LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4);
675
676 LLVMTypeRef arg_types[8];
677 arg_types[0] = LLVMPointerType(vec4f_type, 0); /* v0 */
678 arg_types[1] = LLVMPointerType(vec4f_type, 0); /* v1 */
679 arg_types[2] = LLVMPointerType(vec4f_type, 0); /* v2 */
680 arg_types[3] = LLVMInt32TypeInContext(gallivm->context); /* facing */
681 arg_types[4] = LLVMPointerType(vec4f_type, 0); /* a0, aligned */
682 arg_types[5] = LLVMPointerType(vec4f_type, 0); /* dadx, aligned */
683 arg_types[6] = LLVMPointerType(vec4f_type, 0); /* dady, aligned */
684 arg_types[7] = LLVMPointerType(vec4f_type, 0); /* key (placeholder) */
685
686 LLVMTypeRef func_type =
687 LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context),
688 arg_types, ARRAY_SIZE(arg_types), 0);
689
690 variant->function = LLVMAddFunction(gallivm->module, func_name, func_type);
691 variant->function_name = MALLOC(strlen(func_name)+1);
692 strcpy(variant->function_name, func_name);
693 if (!variant->function)
694 goto fail;
695
696 LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
697
698 struct lp_setup_args args;
699 args.vec4f_type = vec4f_type;
700 args.v0 = LLVMGetParam(variant->function, 0);
701 args.v1 = LLVMGetParam(variant->function, 1);
702 args.v2 = LLVMGetParam(variant->function, 2);
703 args.facing = LLVMGetParam(variant->function, 3);
704 args.a0 = LLVMGetParam(variant->function, 4);
705 args.dadx = LLVMGetParam(variant->function, 5);
706 args.dady = LLVMGetParam(variant->function, 6);
707 args.key = LLVMGetParam(variant->function, 7);
708
709 lp_build_name(args.v0, "in_v0");
710 lp_build_name(args.v1, "in_v1");
711 lp_build_name(args.v2, "in_v2");
712 lp_build_name(args.facing, "in_facing");
713 lp_build_name(args.a0, "out_a0");
714 lp_build_name(args.dadx, "out_dadx");
715 lp_build_name(args.dady, "out_dady");
716 lp_build_name(args.key, "key");
717
718 /*
719 * Function body
720 */
721 LLVMBasicBlockRef block =
722 LLVMAppendBasicBlockInContext(gallivm->context,
723 variant->function, "entry");
724 LLVMPositionBuilderAtEnd(builder, block);
725
726 set_noalias(builder, variant->function, arg_types, ARRAY_SIZE(arg_types));
727 init_args(gallivm, &variant->key, &args);
728 emit_tri_coef(gallivm, &variant->key, &args);
729
730 LLVMBuildRetVoid(builder);
731
732 gallivm_verify_function(gallivm, variant->function);
733
734 gallivm_compile_module(gallivm);
735
736 variant->jit_function = (lp_jit_setup_triangle)
737 gallivm_jit_function(gallivm, variant->function, variant->function_name);
738 if (!variant->jit_function)
739 goto fail;
740
741 gallivm_free_ir(variant->gallivm);
742
743 /*
744 * Update timing information:
745 */
746 if (LP_DEBUG & DEBUG_COUNTERS) {
747 t1 = os_time_get();
748 LP_COUNT_ADD(llvm_compile_time, t1 - t0);
749 LP_COUNT_ADD(nr_llvm_compiles, 1);
750 }
751
752 return variant;
753
754 fail:
755 if (variant) {
756 if (variant->gallivm) {
757 gallivm_destroy(variant->gallivm);
758 }
759 FREE(variant);
760 }
761
762 return NULL;
763 }
764
765
766 static void
lp_make_setup_variant_key(const struct llvmpipe_context * lp,struct lp_setup_variant_key * key)767 lp_make_setup_variant_key(const struct llvmpipe_context *lp,
768 struct lp_setup_variant_key *key)
769 {
770 const struct lp_fragment_shader *fs = lp->fs;
771 struct nir_shader *nir = fs->base.ir.nir;
772
773 assert(sizeof key->inputs[0] == sizeof(uint));
774
775 key->num_inputs = nir->num_inputs;
776 key->flatshade_first = lp->rasterizer->flatshade_first;
777 key->pixel_center_half = lp->rasterizer->half_pixel_center;
778 key->multisample = lp->rasterizer->multisample;
779 key->twoside = lp->rasterizer->light_twoside;
780 key->size = offsetof(struct lp_setup_variant_key, inputs[key->num_inputs]);
781
782 key->color_slot = lp->color_slot[0];
783 key->bcolor_slot = lp->bcolor_slot[0];
784 key->spec_slot = lp->color_slot[1];
785 key->bspec_slot = lp->bcolor_slot[1];
786
787 /*
788 * If depth is floating point, depth bias is calculated with respect
789 * to the primitive's maximum Z value. Retain the original depth bias
790 * value until that stage.
791 */
792 key->floating_point_depth = lp->floating_point_depth;
793
794 if (key->floating_point_depth) {
795 key->pgon_offset_units = (float) lp->rasterizer->offset_units;
796 } else {
797 key->pgon_offset_units =
798 (float) (lp->rasterizer->offset_units * lp->mrd * 2);
799 }
800
801 key->pgon_offset_scale = lp->rasterizer->offset_scale;
802 key->pgon_offset_clamp = lp->rasterizer->offset_clamp;
803 key->uses_constant_interp = 0;
804 key->pad = 0;
805
806 memcpy(key->inputs, fs->inputs, key->num_inputs * sizeof key->inputs[0]);
807
808 for (unsigned i = 0; i < key->num_inputs; i++) {
809 if (key->inputs[i].interp == LP_INTERP_COLOR) {
810 if (lp->rasterizer->flatshade)
811 key->inputs[i].interp = LP_INTERP_CONSTANT;
812 else
813 key->inputs[i].interp = LP_INTERP_PERSPECTIVE;
814 }
815 if (key->inputs[i].interp == LP_INTERP_CONSTANT) {
816 key->uses_constant_interp = 1;
817 }
818 }
819 }
820
821
822 static void
remove_setup_variant(struct llvmpipe_context * lp,struct lp_setup_variant * variant)823 remove_setup_variant(struct llvmpipe_context *lp,
824 struct lp_setup_variant *variant)
825 {
826 if (gallivm_debug & GALLIVM_DEBUG_IR) {
827 debug_printf("llvmpipe: del setup_variant #%u total %u\n",
828 variant->no, lp->nr_setup_variants);
829 }
830
831 if (variant->gallivm) {
832 gallivm_destroy(variant->gallivm);
833 }
834
835 list_del(&variant->list_item_global.list);
836 lp->nr_setup_variants--;
837 FREE(variant->function_name);
838 FREE(variant);
839 }
840
841
842 /* When the number of setup variants exceeds a threshold, cull a
843 * fraction (currently a quarter) of them.
844 */
845 static void
cull_setup_variants(struct llvmpipe_context * lp)846 cull_setup_variants(struct llvmpipe_context *lp)
847 {
848 struct pipe_context *pipe = &lp->pipe;
849
850 /*
851 * XXX: we need to flush the context until we have some sort of reference
852 * counting in fragment shaders as they may still be binned
853 * Flushing alone might not be sufficient we need to wait on it too.
854 */
855 llvmpipe_finish(pipe, __func__);
856
857 for (int i = 0; i < LP_MAX_SETUP_VARIANTS / 4; i++) {
858 struct lp_setup_variant_list_item *item;
859 if (list_is_empty(&lp->setup_variants_list.list)) {
860 break;
861 }
862 item = list_last_entry(&lp->setup_variants_list.list,
863 struct lp_setup_variant_list_item, list);
864 assert(item);
865 assert(item->base);
866 remove_setup_variant(lp, item->base);
867 }
868 }
869
870
871 /**
872 * Update fragment/vertex shader linkage state. This is called just
873 * prior to drawing something when some fragment-related state has
874 * changed.
875 */
876 void
llvmpipe_update_setup(struct llvmpipe_context * lp)877 llvmpipe_update_setup(struct llvmpipe_context *lp)
878 {
879 struct lp_setup_variant_key *key = &lp->setup_variant.key;
880 struct lp_setup_variant *variant = NULL;
881 struct lp_setup_variant_list_item *li;
882
883 lp_make_setup_variant_key(lp, key);
884
885 LIST_FOR_EACH_ENTRY(li, &lp->setup_variants_list.list, list) {
886 if (li->base->key.size == key->size &&
887 memcmp(&li->base->key, key, key->size) == 0) {
888 variant = li->base;
889 break;
890 }
891 }
892
893 if (variant) {
894 list_move_to(&variant->list_item_global.list, &lp->setup_variants_list.list);
895 } else {
896 if (lp->nr_setup_variants >= LP_MAX_SETUP_VARIANTS) {
897 cull_setup_variants(lp);
898 }
899
900 variant = generate_setup_variant(key, lp);
901 if (variant) {
902 list_add(&variant->list_item_global.list, &lp->setup_variants_list.list);
903 lp->nr_setup_variants++;
904 }
905 }
906
907 lp_setup_set_setup_variant(lp->setup, variant);
908 }
909
910
911 void
lp_delete_setup_variants(struct llvmpipe_context * lp)912 lp_delete_setup_variants(struct llvmpipe_context *lp)
913 {
914 struct lp_setup_variant_list_item *li, *next;
915 LIST_FOR_EACH_ENTRY_SAFE(li, next, &lp->setup_variants_list.list, list) {
916 remove_setup_variant(lp, li->base);
917 }
918 }
919
920
921 void
lp_dump_setup_coef(const struct lp_setup_variant_key * key,const float (* sa0)[4],const float (* sdadx)[4],const float (* sdady)[4])922 lp_dump_setup_coef(const struct lp_setup_variant_key *key,
923 const float (*sa0)[4],
924 const float (*sdadx)[4],
925 const float (*sdady)[4])
926 {
927 for (int i = 0; i < TGSI_NUM_CHANNELS; i++) {
928 float a0 = sa0 [0][i];
929 float dadx = sdadx[0][i];
930 float dady = sdady[0][i];
931
932 debug_printf("POS.%c: a0 = %f, dadx = %f, dady = %f\n",
933 "xyzw"[i], a0, dadx, dady);
934 }
935
936 for (int slot = 0; slot < key->num_inputs; slot++) {
937 unsigned usage_mask = key->inputs[slot].usage_mask;
938 for (int i = 0; i < TGSI_NUM_CHANNELS; i++) {
939 if (usage_mask & (1 << i)) {
940 float a0 = sa0 [1 + slot][i];
941 float dadx = sdadx[1 + slot][i];
942 float dady = sdady[1 + slot][i];
943
944 debug_printf("IN[%u].%c: a0 = %f, dadx = %f, dady = %f\n",
945 slot, "xyzw"[i], a0, dadx, dady);
946 }
947 }
948 }
949 }
950