1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * Copyright 2007-2008 VMware, Inc.
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 **************************************************************************/
28
29 #include "util/u_memory.h"
30 #include "lp_bld_const.h"
31 #include "lp_bld_type.h"
32 #include "lp_bld_init.h"
33 #include "lp_bld_flow.h"
34 #include "lp_bld_ir_common.h"
35 #include "lp_bld_logic.h"
36
37 /*
38 * Return the context for the current function.
39 * (always 'main', if shader doesn't do any function calls)
40 */
41 static inline struct function_ctx *
func_ctx(struct lp_exec_mask * mask)42 func_ctx(struct lp_exec_mask *mask)
43 {
44 assert(mask->function_stack_size > 0);
45 assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
46 return &mask->function_stack[mask->function_stack_size - 1];
47 }
48
49 /*
50 * Returns true if we're in a loop.
51 * It's global, meaning that it returns true even if there's
52 * no loop inside the current function, but we were inside
53 * a loop inside another function, from which this one was called.
54 */
55 static inline bool
mask_has_loop(struct lp_exec_mask * mask)56 mask_has_loop(struct lp_exec_mask *mask)
57 {
58 int i;
59 for (i = mask->function_stack_size - 1; i >= 0; --i) {
60 const struct function_ctx *ctx = &mask->function_stack[i];
61 if (ctx->loop_stack_size > 0)
62 return true;
63 }
64 return false;
65 }
66
67 /*
68 * Returns true if we're inside a switch statement.
69 * It's global, meaning that it returns true even if there's
70 * no switch in the current function, but we were inside
71 * a switch inside another function, from which this one was called.
72 */
73 static inline bool
mask_has_switch(struct lp_exec_mask * mask)74 mask_has_switch(struct lp_exec_mask *mask)
75 {
76 int i;
77 for (i = mask->function_stack_size - 1; i >= 0; --i) {
78 const struct function_ctx *ctx = &mask->function_stack[i];
79 if (ctx->switch_stack_size > 0)
80 return true;
81 }
82 return false;
83 }
84
85 /*
86 * Returns true if we're inside a conditional.
87 * It's global, meaning that it returns true even if there's
88 * no conditional in the current function, but we were inside
89 * a conditional inside another function, from which this one was called.
90 */
91 static inline bool
mask_has_cond(struct lp_exec_mask * mask)92 mask_has_cond(struct lp_exec_mask *mask)
93 {
94 int i;
95 for (i = mask->function_stack_size - 1; i >= 0; --i) {
96 const struct function_ctx *ctx = &mask->function_stack[i];
97 if (ctx->cond_stack_size > 0)
98 return true;
99 }
100 return false;
101 }
102
lp_exec_mask_update(struct lp_exec_mask * mask)103 void lp_exec_mask_update(struct lp_exec_mask *mask)
104 {
105 LLVMBuilderRef builder = mask->bld->gallivm->builder;
106 bool has_loop_mask = mask_has_loop(mask);
107 bool has_cond_mask = mask_has_cond(mask);
108 bool has_switch_mask = mask_has_switch(mask);
109 bool has_ret_mask = mask->function_stack_size > 1 ||
110 mask->ret_in_main;
111
112 if (has_loop_mask) {
113 /*for loops we need to update the entire mask at runtime */
114 LLVMValueRef tmp;
115 assert(mask->break_mask);
116 tmp = LLVMBuildAnd(builder,
117 LLVMBuildLoad2(builder, mask->int_vec_type, mask->cont_mask, ""),
118 LLVMBuildLoad2(builder, mask->int_vec_type, mask->break_mask, ""),
119 "maskcb");
120 mask->exec_mask = LLVMBuildAnd(builder,
121 mask->cond_mask,
122 tmp,
123 "maskfull");
124 } else
125 mask->exec_mask = mask->cond_mask;
126
127 if (has_switch_mask) {
128 mask->exec_mask = LLVMBuildAnd(builder,
129 mask->exec_mask,
130 mask->switch_mask,
131 "switchmask");
132 }
133
134 if (has_ret_mask) {
135 mask->exec_mask = LLVMBuildAnd(builder,
136 mask->exec_mask,
137 mask->ret_mask,
138 "callmask");
139 }
140
141 mask->has_mask = (has_cond_mask ||
142 has_loop_mask ||
143 has_switch_mask ||
144 has_ret_mask);
145 }
146
147 /*
148 * Initialize a function context at the specified index.
149 */
150 void
lp_exec_mask_function_init(struct lp_exec_mask * mask,int function_idx)151 lp_exec_mask_function_init(struct lp_exec_mask *mask, int function_idx)
152 {
153 LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
154 LLVMBuilderRef builder = mask->bld->gallivm->builder;
155 struct function_ctx *ctx = &mask->function_stack[function_idx];
156
157 ctx->cond_stack_size = 0;
158 ctx->loop_stack_size = 0;
159 ctx->bgnloop_stack_size = 0;
160 ctx->switch_stack_size = 0;
161
162 if (function_idx == 0) {
163 ctx->ret_mask = mask->ret_mask;
164 }
165
166 ctx->loop_limiter = lp_build_alloca(mask->bld->gallivm,
167 int_type, "looplimiter");
168 LLVMBuildStore(
169 builder,
170 LLVMConstInt(int_type, LP_MAX_TGSI_LOOP_ITERATIONS, false),
171 ctx->loop_limiter);
172 }
173
lp_exec_mask_init(struct lp_exec_mask * mask,struct lp_build_context * bld)174 void lp_exec_mask_init(struct lp_exec_mask *mask, struct lp_build_context *bld)
175 {
176 mask->bld = bld;
177 mask->has_mask = false;
178 mask->ret_in_main = false;
179 /* For the main function */
180 mask->function_stack_size = 1;
181
182 mask->int_vec_type = lp_build_int_vec_type(bld->gallivm, mask->bld->type);
183 mask->exec_mask = mask->ret_mask =
184 mask->cond_mask = mask->switch_mask =
185 LLVMConstAllOnes(mask->int_vec_type);
186
187 mask->break_mask = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "break_mask");
188 LLVMBuildStore(bld->gallivm->builder, LLVMConstAllOnes(mask->int_vec_type), mask->break_mask);
189
190 mask->cont_mask = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "cont_mask");
191 LLVMBuildStore(bld->gallivm->builder, LLVMConstAllOnes(mask->int_vec_type), mask->cont_mask);
192
193 mask->function_stack = CALLOC(LP_MAX_NUM_FUNCS,
194 sizeof(mask->function_stack[0]));
195 lp_exec_mask_function_init(mask, 0);
196 }
197
198 void
lp_exec_mask_fini(struct lp_exec_mask * mask)199 lp_exec_mask_fini(struct lp_exec_mask *mask)
200 {
201 FREE(mask->function_stack);
202 }
203
204 /* stores val into an address pointed to by dst_ptr.
205 * mask->exec_mask is used to figure out which bits of val
206 * should be stored into the address
207 * (0 means don't store this bit, 1 means do store).
208 */
lp_exec_mask_store(struct lp_exec_mask * mask,struct lp_build_context * bld_store,LLVMValueRef val,LLVMValueRef dst_ptr)209 void lp_exec_mask_store(struct lp_exec_mask *mask,
210 struct lp_build_context *bld_store,
211 LLVMValueRef val,
212 LLVMValueRef dst_ptr)
213 {
214 LLVMBuilderRef builder = mask->bld->gallivm->builder;
215 LLVMValueRef exec_mask = mask->has_mask ? mask->exec_mask : NULL;
216
217 assert(lp_check_value(bld_store->type, val));
218 assert(LLVMGetTypeKind(LLVMTypeOf(dst_ptr)) == LLVMPointerTypeKind);
219 assert(LLVM_VERSION_MAJOR >= 15
220 || (LLVMGetElementType(LLVMTypeOf(dst_ptr)) == LLVMTypeOf(val)
221 || LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(dst_ptr))) == LLVMArrayTypeKind));
222
223 if (exec_mask) {
224 LLVMValueRef res, dst;
225
226 dst = LLVMBuildLoad2(builder, LLVMTypeOf(val), dst_ptr, "");
227 if (bld_store->type.width < 32)
228 exec_mask = LLVMBuildTrunc(builder, exec_mask, bld_store->vec_type, "");
229 res = lp_build_select(bld_store, exec_mask, val, dst);
230 LLVMBuildStore(builder, res, dst_ptr);
231 } else
232 LLVMBuildStore(builder, val, dst_ptr);
233 }
234
lp_exec_bgnloop_post_phi(struct lp_exec_mask * mask)235 void lp_exec_bgnloop_post_phi(struct lp_exec_mask *mask)
236 {
237 LLVMBuilderRef builder = mask->bld->gallivm->builder;
238 struct function_ctx *ctx = func_ctx(mask);
239
240 if (ctx->loop_stack_size != ctx->bgnloop_stack_size) {
241 LLVMBuildStore(builder, LLVMBuildLoad2(builder, mask->int_vec_type, ctx->break_var, ""), mask->break_mask);
242 lp_exec_mask_update(mask);
243 ctx->bgnloop_stack_size = ctx->loop_stack_size;
244 }
245 }
246
lp_exec_bgnloop(struct lp_exec_mask * mask,bool load)247 void lp_exec_bgnloop(struct lp_exec_mask *mask, bool load)
248 {
249 LLVMBuilderRef builder = mask->bld->gallivm->builder;
250 struct function_ctx *ctx = func_ctx(mask);
251
252 if (ctx->loop_stack_size >= LP_MAX_TGSI_NESTING) {
253 ++ctx->loop_stack_size;
254 return;
255 }
256
257 ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
258 ctx->break_type;
259 ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_LOOP;
260
261 ctx->loop_stack[ctx->loop_stack_size].loop_block = ctx->loop_block;
262 ctx->loop_stack[ctx->loop_stack_size].cont_mask = mask->cont_mask;
263 ctx->loop_stack[ctx->loop_stack_size].break_mask = mask->break_mask;
264 ctx->loop_stack[ctx->loop_stack_size].break_var = ctx->break_var;
265 ++ctx->loop_stack_size;
266
267 LLVMValueRef cont_mask = LLVMBuildLoad2(builder, mask->int_vec_type, mask->cont_mask, "");
268 LLVMValueRef break_mask = LLVMBuildLoad2(builder, mask->int_vec_type, mask->break_mask, "");
269
270 mask->break_mask = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "");
271 LLVMBuildStore(builder, break_mask, mask->break_mask);
272
273 ctx->break_var = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "");
274 LLVMBuildStore(builder, break_mask, ctx->break_var);
275
276 ctx->loop_block = lp_build_insert_new_block(mask->bld->gallivm, "bgnloop");
277
278 LLVMBuildBr(builder, ctx->loop_block);
279 LLVMPositionBuilderAtEnd(builder, ctx->loop_block);
280
281 mask->cont_mask = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "");
282 LLVMBuildStore(builder, cont_mask, mask->cont_mask);
283
284 if (load) {
285 lp_exec_bgnloop_post_phi(mask);
286 }
287 }
288
lp_exec_endloop(struct gallivm_state * gallivm,struct lp_exec_mask * exec_mask,struct lp_build_mask_context * mask)289 void lp_exec_endloop(struct gallivm_state *gallivm,
290 struct lp_exec_mask *exec_mask,
291 struct lp_build_mask_context *mask)
292 {
293 LLVMBuilderRef builder = exec_mask->bld->gallivm->builder;
294 struct function_ctx *ctx = func_ctx(exec_mask);
295 LLVMBasicBlockRef endloop;
296 LLVMTypeRef int_type = LLVMInt32TypeInContext(exec_mask->bld->gallivm->context);
297 LLVMTypeRef mask_type = LLVMIntTypeInContext(exec_mask->bld->gallivm->context, exec_mask->bld->type.length);
298 LLVMValueRef i1cond, i2cond, icond, limiter;
299
300 assert(exec_mask->break_mask);
301
302 assert(ctx->loop_stack_size);
303 if (ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
304 --ctx->loop_stack_size;
305 --ctx->bgnloop_stack_size;
306 return;
307 }
308
309 /*
310 * Restore the cont_mask, but don't pop
311 */
312 exec_mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size - 1].cont_mask;
313 lp_exec_mask_update(exec_mask);
314
315 /*
316 * Unlike the continue mask, the break_mask must be preserved across loop
317 * iterations
318 */
319 LLVMBuildStore(builder, LLVMBuildLoad2(builder, exec_mask->int_vec_type, exec_mask->break_mask, ""), ctx->break_var);
320
321 /* Decrement the loop limiter */
322 limiter = LLVMBuildLoad2(builder, int_type, ctx->loop_limiter, "");
323
324 limiter = LLVMBuildSub(
325 builder,
326 limiter,
327 LLVMConstInt(int_type, 1, false),
328 "");
329
330 LLVMBuildStore(builder, limiter, ctx->loop_limiter);
331
332 LLVMValueRef end_mask = exec_mask->exec_mask;
333 if (mask)
334 end_mask = LLVMBuildAnd(builder, exec_mask->exec_mask, lp_build_mask_value(mask), "");
335 end_mask = LLVMBuildICmp(builder, LLVMIntNE, end_mask, lp_build_zero(gallivm, exec_mask->bld->type), "");
336 end_mask = LLVMBuildBitCast(builder, end_mask, mask_type, "");
337
338 /* i1cond = (end_mask != 0) */
339 i1cond = LLVMBuildICmp(
340 builder,
341 LLVMIntNE,
342 end_mask,
343 LLVMConstNull(mask_type), "i1cond");
344
345 /* i2cond = (looplimiter > 0) */
346 i2cond = LLVMBuildICmp(
347 builder,
348 LLVMIntSGT,
349 limiter,
350 LLVMConstNull(int_type), "i2cond");
351
352 /* if( i1cond && i2cond ) */
353 icond = LLVMBuildAnd(builder, i1cond, i2cond, "");
354
355 endloop = lp_build_insert_new_block(exec_mask->bld->gallivm, "endloop");
356
357 LLVMBuildCondBr(builder,
358 icond, ctx->loop_block, endloop);
359
360 LLVMPositionBuilderAtEnd(builder, endloop);
361
362 assert(ctx->loop_stack_size);
363 --ctx->loop_stack_size;
364 --ctx->bgnloop_stack_size;
365 exec_mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size].cont_mask;
366 exec_mask->break_mask = ctx->loop_stack[ctx->loop_stack_size].break_mask;
367 ctx->loop_block = ctx->loop_stack[ctx->loop_stack_size].loop_block;
368 ctx->break_var = ctx->loop_stack[ctx->loop_stack_size].break_var;
369 ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size +
370 ctx->switch_stack_size];
371
372 lp_exec_mask_update(exec_mask);
373 }
374
lp_exec_mask_cond_push(struct lp_exec_mask * mask,LLVMValueRef val)375 void lp_exec_mask_cond_push(struct lp_exec_mask *mask,
376 LLVMValueRef val)
377 {
378 LLVMBuilderRef builder = mask->bld->gallivm->builder;
379 struct function_ctx *ctx = func_ctx(mask);
380
381 if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING) {
382 ctx->cond_stack_size++;
383 return;
384 }
385 if (ctx->cond_stack_size == 0 && mask->function_stack_size == 1) {
386 assert(mask->cond_mask == LLVMConstAllOnes(mask->int_vec_type));
387 }
388 ctx->cond_stack[ctx->cond_stack_size++] = mask->cond_mask;
389 assert(LLVMTypeOf(val) == mask->int_vec_type);
390 mask->cond_mask = LLVMBuildAnd(builder,
391 mask->cond_mask,
392 val,
393 "");
394 lp_exec_mask_update(mask);
395 }
396
lp_exec_mask_cond_invert(struct lp_exec_mask * mask)397 void lp_exec_mask_cond_invert(struct lp_exec_mask *mask)
398 {
399 LLVMBuilderRef builder = mask->bld->gallivm->builder;
400 struct function_ctx *ctx = func_ctx(mask);
401 LLVMValueRef prev_mask;
402 LLVMValueRef inv_mask;
403
404 assert(ctx->cond_stack_size);
405 if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
406 return;
407 prev_mask = ctx->cond_stack[ctx->cond_stack_size - 1];
408 if (ctx->cond_stack_size == 1 && mask->function_stack_size == 1) {
409 assert(prev_mask == LLVMConstAllOnes(mask->int_vec_type));
410 }
411
412 inv_mask = LLVMBuildNot(builder, mask->cond_mask, "");
413
414 mask->cond_mask = LLVMBuildAnd(builder,
415 inv_mask,
416 prev_mask, "");
417 lp_exec_mask_update(mask);
418 }
419
lp_exec_mask_cond_pop(struct lp_exec_mask * mask)420 void lp_exec_mask_cond_pop(struct lp_exec_mask *mask)
421 {
422 struct function_ctx *ctx = func_ctx(mask);
423 assert(ctx->cond_stack_size);
424 --ctx->cond_stack_size;
425 if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
426 return;
427 mask->cond_mask = ctx->cond_stack[ctx->cond_stack_size];
428 lp_exec_mask_update(mask);
429 }
430
431
lp_exec_continue(struct lp_exec_mask * mask)432 void lp_exec_continue(struct lp_exec_mask *mask)
433 {
434 LLVMBuilderRef builder = mask->bld->gallivm->builder;
435 LLVMValueRef exec_mask = LLVMBuildNot(builder,
436 mask->exec_mask,
437 "");
438
439 LLVMBuildStore(builder, LLVMBuildAnd(builder,
440 LLVMBuildLoad2(builder, mask->int_vec_type, mask->cont_mask, ""),
441 exec_mask, ""), mask->cont_mask);
442
443 lp_exec_mask_update(mask);
444 }
445
lp_exec_break(struct lp_exec_mask * mask,int * pc,bool break_always)446 void lp_exec_break(struct lp_exec_mask *mask, int *pc,
447 bool break_always)
448 {
449 LLVMBuilderRef builder = mask->bld->gallivm->builder;
450 struct function_ctx *ctx = func_ctx(mask);
451
452 if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
453 LLVMValueRef exec_mask = LLVMBuildNot(builder,
454 mask->exec_mask,
455 "break");
456
457 LLVMBuildStore(builder, LLVMBuildAnd(builder,
458 LLVMBuildLoad2(builder, mask->int_vec_type, mask->break_mask, ""),
459 exec_mask, "break_full"), mask->break_mask);
460 }
461 else {
462 if (ctx->switch_in_default) {
463 /*
464 * stop default execution but only if this is an unconditional switch.
465 * (The condition here is not perfect since dead code after break is
466 * allowed but should be sufficient since false negatives are just
467 * unoptimized - so we don't have to pre-evaluate that).
468 */
469 if(break_always && ctx->switch_pc) {
470 if (pc)
471 *pc = ctx->switch_pc;
472 return;
473 }
474 }
475
476 if (break_always) {
477 mask->switch_mask = LLVMConstNull(mask->bld->int_vec_type);
478 }
479 else {
480 LLVMValueRef exec_mask = LLVMBuildNot(builder,
481 mask->exec_mask,
482 "break");
483 mask->switch_mask = LLVMBuildAnd(builder,
484 mask->switch_mask,
485 exec_mask, "break_switch");
486 }
487 }
488
489 lp_exec_mask_update(mask);
490 }
491