1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "gallivm/lp_bld_sample.h"
29 #include "gallivm/lp_bld_limits.h"
30 #include "gallivm/lp_bld_tgsi.h"
31 #include "gallivm/lp_bld_type.h"
32 #include "gallivm/lp_bld_init.h"
33 #include "gallivm/lp_bld_const.h"
34 #include "gallivm/lp_bld_sample.h"
35 #include "gallivm/lp_bld_jit_types.h"
36 #include "gallivm/lp_bld_jit_sample.h"
37 #include "gallivm/lp_bld_flow.h"
38
39 struct lp_bld_sampler_dynamic_state
40 {
41 struct lp_sampler_dynamic_state base;
42
43 const struct lp_sampler_static_state *static_state;
44 };
45
46 struct lp_bld_llvm_sampler_soa
47 {
48 struct lp_build_sampler_soa base;
49
50 struct lp_bld_sampler_dynamic_state dynamic_state;
51 unsigned nr_samplers;
52 };
53
54
55 struct lp_bld_image_dynamic_state
56 {
57 struct lp_sampler_dynamic_state base;
58
59 const struct lp_image_static_state *static_state;
60 };
61
62 struct lp_bld_llvm_image_soa
63 {
64 struct lp_build_image_soa base;
65
66 struct lp_bld_image_dynamic_state dynamic_state;
67 unsigned nr_images;
68 };
69
70 static LLVMValueRef
load_texture_functions_ptr(struct gallivm_state * gallivm,LLVMValueRef descriptor,uint32_t offset1,uint32_t offset2)71 load_texture_functions_ptr(struct gallivm_state *gallivm, LLVMValueRef descriptor,
72 uint32_t offset1, uint32_t offset2)
73 {
74 LLVMBuilderRef builder = gallivm->builder;
75
76 LLVMValueRef texture_base_offset = lp_build_const_int64(gallivm, offset1);
77 LLVMValueRef texture_base_ptr = LLVMBuildAdd(builder, descriptor, texture_base_offset, "");
78
79 LLVMTypeRef texture_base_type = LLVMInt64TypeInContext(gallivm->context);
80 LLVMTypeRef texture_base_ptr_type = LLVMPointerType(texture_base_type, 0);
81
82 texture_base_ptr = LLVMBuildIntToPtr(builder, texture_base_ptr, texture_base_ptr_type, "");
83 /* struct lp_texture_functions * */
84 LLVMValueRef texture_base = LLVMBuildLoad2(builder, texture_base_type, texture_base_ptr, "");
85
86 LLVMValueRef functions_offset = lp_build_const_int64(gallivm, offset2);
87 return LLVMBuildAdd(builder, texture_base, functions_offset, "");
88 }
89
90 static LLVMValueRef
widen_to_simd_width(struct gallivm_state * gallivm,LLVMValueRef value)91 widen_to_simd_width(struct gallivm_state *gallivm, LLVMValueRef value)
92 {
93 LLVMBuilderRef builder = gallivm->builder;
94 LLVMTypeRef type = LLVMTypeOf(value);
95
96 if (LLVMGetTypeKind(type) == LLVMVectorTypeKind) {
97 LLVMTypeRef element_type = LLVMGetElementType(type);
98 uint32_t element_count = LLVMGetVectorSize(type);
99 LLVMValueRef elements[8] = { 0 };
100 for (uint32_t i = 0; i < lp_native_vector_width / 32; i++) {
101 if (i < element_count)
102 elements[i] = LLVMBuildExtractElement(builder, value, lp_build_const_int32(gallivm, i), "");
103 else
104 elements[i] = LLVMConstNull(element_type);
105 }
106
107 LLVMTypeRef result_type = LLVMVectorType(element_type, lp_native_vector_width / 32);
108 LLVMValueRef result = LLVMGetUndef(result_type);
109 for (unsigned i = 0; i < lp_native_vector_width / 32; i++)
110 result = LLVMBuildInsertElement(builder, result, elements[i], lp_build_const_int32(gallivm, i), "");
111
112 return result;
113 }
114
115 return value;
116 }
117
118 static LLVMValueRef
truncate_to_type_width(struct gallivm_state * gallivm,LLVMValueRef value,struct lp_type target_type)119 truncate_to_type_width(struct gallivm_state *gallivm, LLVMValueRef value, struct lp_type target_type)
120 {
121 LLVMBuilderRef builder = gallivm->builder;
122 LLVMTypeRef type = LLVMTypeOf(value);
123
124 if (LLVMGetTypeKind(type) == LLVMVectorTypeKind) {
125 LLVMTypeRef element_type = LLVMGetElementType(type);
126
127 LLVMValueRef elements[8];
128 for (uint32_t i = 0; i < target_type.length; i++)
129 elements[i] = LLVMBuildExtractElement(builder, value, lp_build_const_int32(gallivm, i), "");
130
131 LLVMTypeRef result_type = LLVMVectorType(element_type, target_type.length);
132 LLVMValueRef result = LLVMGetUndef(result_type);
133 for (unsigned i = 0; i < target_type.length; i++)
134 result = LLVMBuildInsertElement(builder, result, elements[i], lp_build_const_int32(gallivm, i), "");
135
136 return result;
137 }
138
139 return value;
140 }
141
142 /**
143 * Fetch filtered values from texture.
144 * The 'texel' parameter returns four vectors corresponding to R, G, B, A.
145 */
146 static void
lp_bld_llvm_sampler_soa_emit_fetch_texel(const struct lp_build_sampler_soa * base,struct gallivm_state * gallivm,const struct lp_sampler_params * params)147 lp_bld_llvm_sampler_soa_emit_fetch_texel(const struct lp_build_sampler_soa *base,
148 struct gallivm_state *gallivm,
149 const struct lp_sampler_params *params)
150 {
151 struct lp_bld_llvm_sampler_soa *sampler = (struct lp_bld_llvm_sampler_soa *)base;
152 LLVMBuilderRef builder = gallivm->builder;
153
154 if (params->texture_resource) {
155 LLVMTypeRef out_data_type = lp_build_vec_type(gallivm, params->type);
156 LLVMTypeRef out_residency_type = lp_build_vec_type(gallivm, lp_int_type(params->type));
157
158 LLVMValueRef out_data[5];
159 for (uint32_t i = 0; i < ARRAY_SIZE(out_data) - 1; i++) {
160 out_data[i] = lp_build_alloca(gallivm, out_data_type, "");
161 }
162 out_data[ARRAY_SIZE(out_data) - 1] = lp_build_alloca(gallivm, out_residency_type, "");
163
164 struct lp_type uint_type = lp_uint_type(params->type);
165 LLVMValueRef uint_zero = lp_build_const_int_vec(gallivm, uint_type, 0);
166
167 LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, params->exec_mask, uint_zero, "exec_bitvec");
168
169 LLVMTypeRef bitmask_type = LLVMIntTypeInContext(gallivm->context, uint_type.length);
170 bitmask = LLVMBuildBitCast(builder, bitmask, bitmask_type, "exec_bitmask");
171
172 LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active");
173
174 struct lp_build_if_state if_state;
175 lp_build_if(&if_state, gallivm, any_active);
176
177 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
178
179 LLVMValueRef texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->texture_resource, LP_MAX_TGSI_CONST_BUFFERS);
180
181 enum lp_sampler_op_type op_type = (params->sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
182 uint32_t functions_offset = op_type == LP_SAMPLER_OP_FETCH ? offsetof(struct lp_texture_functions, fetch_functions)
183 : offsetof(struct lp_texture_functions, sample_functions);
184
185 LLVMValueRef texture_base_ptr = load_texture_functions_ptr(
186 gallivm, texture_descriptor, offsetof(struct lp_descriptor, functions), functions_offset);
187
188 LLVMTypeRef texture_function_type = lp_build_sample_function_type(gallivm, params->sample_key);
189 LLVMTypeRef texture_function_ptr_type = LLVMPointerType(texture_function_type, 0);
190 LLVMTypeRef texture_functions_type = LLVMPointerType(texture_function_ptr_type, 0);
191 LLVMTypeRef texture_base_type = LLVMPointerType(texture_functions_type, 0);
192 LLVMTypeRef texture_base_ptr_type = LLVMPointerType(texture_base_type, 0);
193
194 texture_base_ptr = LLVMBuildIntToPtr(builder, texture_base_ptr, texture_base_ptr_type, "");
195 LLVMValueRef texture_base = LLVMBuildLoad2(builder, texture_base_type, texture_base_ptr, "");
196
197 LLVMValueRef texture_functions;
198 LLVMValueRef sampler_desc_ptr;
199 if (op_type == LP_SAMPLER_OP_FETCH) {
200 texture_functions = texture_base;
201 sampler_desc_ptr = LLVMGetUndef(LLVMInt64TypeInContext(gallivm->context));
202 } else {
203 sampler_desc_ptr = lp_llvm_descriptor_base(gallivm, consts, params->sampler_resource, LP_MAX_TGSI_CONST_BUFFERS);
204
205 LLVMValueRef sampler_index_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, texture.sampler_index));
206 LLVMValueRef sampler_index_ptr = LLVMBuildAdd(builder, sampler_desc_ptr, sampler_index_offset, "");
207
208 LLVMTypeRef sampler_index_type = LLVMInt32TypeInContext(gallivm->context);
209 LLVMTypeRef sampler_index_ptr_type = LLVMPointerType(sampler_index_type, 0);
210
211 sampler_index_ptr = LLVMBuildIntToPtr(builder, sampler_index_ptr, sampler_index_ptr_type, "");
212 LLVMValueRef sampler_index = LLVMBuildLoad2(builder, sampler_index_type, sampler_index_ptr, "");
213
214 LLVMValueRef texture_functions_ptr = LLVMBuildGEP2(builder, texture_functions_type, texture_base, &sampler_index, 1, "");
215 texture_functions = LLVMBuildLoad2(builder, texture_functions_type, texture_functions_ptr, "");
216 }
217
218 LLVMValueRef sample_key = lp_build_const_int32(gallivm, params->sample_key);
219 LLVMValueRef texture_function_ptr = LLVMBuildGEP2(builder, texture_function_ptr_type, texture_functions, &sample_key, 1, "");
220 LLVMValueRef texture_function = LLVMBuildLoad2(builder, texture_function_ptr_type, texture_function_ptr, "");
221
222 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS];
223 uint32_t num_args = 0;
224
225 args[num_args++] = texture_descriptor;
226 args[num_args++] = sampler_desc_ptr;
227
228 args[num_args++] = params->aniso_filter_table;
229
230 LLVMTypeRef coord_type;
231 if (op_type == LP_SAMPLER_OP_FETCH)
232 coord_type = lp_build_int_vec_type(gallivm, params->type);
233 else
234 coord_type = lp_build_vec_type(gallivm, params->type);
235
236 for (uint32_t i = 0; i < 4; i++) {
237 if (LLVMIsUndef(params->coords[i]))
238 args[num_args++] = LLVMGetUndef(coord_type);
239 else
240 args[num_args++] = params->coords[i];
241 }
242
243 if (params->sample_key & LP_SAMPLER_SHADOW)
244 args[num_args++] = params->coords[4];
245
246 if (params->sample_key & LP_SAMPLER_FETCH_MS)
247 args[num_args++] = params->ms_index;
248
249 if (params->sample_key & LP_SAMPLER_OFFSETS) {
250 for (uint32_t i = 0; i < 3; i++) {
251 if (params->offsets[i])
252 args[num_args++] = params->offsets[i];
253 else
254 args[num_args++] = LLVMGetUndef(lp_build_int_vec_type(gallivm, params->type));
255 }
256 }
257
258 enum lp_sampler_lod_control lod_control = (params->sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >> LP_SAMPLER_LOD_CONTROL_SHIFT;
259 if (lod_control == LP_SAMPLER_LOD_BIAS || lod_control == LP_SAMPLER_LOD_EXPLICIT)
260 args[num_args++] = params->lod;
261
262 if (params->type.length != lp_native_vector_width / 32)
263 for (uint32_t i = 0; i < num_args; i++)
264 args[i] = widen_to_simd_width(gallivm, args[i]);
265
266 LLVMValueRef result = LLVMBuildCall2(builder, texture_function_type, texture_function, args, num_args, "");
267
268 for (unsigned i = 0; i < ARRAY_SIZE(out_data); i++) {
269 params->texel[i] = LLVMBuildExtractValue(builder, result, i, "");
270
271 if (params->type.length != lp_native_vector_width / 32)
272 params->texel[i] = truncate_to_type_width(gallivm, params->texel[i], params->type);
273
274 LLVMBuildStore(builder, params->texel[i], out_data[i]);
275 }
276
277 lp_build_endif(&if_state);
278
279 for (unsigned i = 0; i < ARRAY_SIZE(out_data) - 1; i++)
280 params->texel[i] = LLVMBuildLoad2(builder, out_data_type, out_data[i], "");
281 params->texel[ARRAY_SIZE(out_data) - 1] =
282 LLVMBuildLoad2(builder, out_residency_type, out_data[ARRAY_SIZE(out_data) - 1], "");
283
284 return;
285 }
286
287 const unsigned texture_index = params->texture_index;
288 const unsigned sampler_index = params->sampler_index;
289
290 assert(sampler_index < PIPE_MAX_SAMPLERS);
291 assert(texture_index < PIPE_MAX_SHADER_SAMPLER_VIEWS);
292 #if 0
293 if (LP_PERF & PERF_NO_TEX) {
294 lp_build_sample_nop(gallivm, params->type, params->coords, params->texel);
295 return;
296 }
297 #endif
298
299 if (params->texture_index_offset) {
300 LLVMValueRef unit =
301 LLVMBuildAdd(builder, params->texture_index_offset,
302 lp_build_const_int32(gallivm, texture_index), "");
303
304 struct lp_build_sample_array_switch switch_info;
305 memset(&switch_info, 0, sizeof(switch_info));
306 lp_build_sample_array_init_soa(&switch_info, gallivm, params, unit,
307 0, sampler->nr_samplers);
308 // build the switch cases
309 for (unsigned i = 0; i < sampler->nr_samplers; i++) {
310 lp_build_sample_array_case_soa(&switch_info, i,
311 &sampler->dynamic_state.static_state[i].texture_state,
312 &sampler->dynamic_state.static_state[i].sampler_state,
313 &sampler->dynamic_state.base);
314 }
315 lp_build_sample_array_fini_soa(&switch_info);
316 } else {
317 lp_build_sample_soa(&sampler->dynamic_state.static_state[texture_index].texture_state,
318 &sampler->dynamic_state.static_state[sampler_index].sampler_state,
319 &sampler->dynamic_state.base,
320 gallivm, params);
321 }
322 }
323
324
325 /**
326 * Fetch the texture size.
327 */
328 static void
lp_bld_llvm_sampler_soa_emit_size_query(const struct lp_build_sampler_soa * base,struct gallivm_state * gallivm,const struct lp_sampler_size_query_params * params)329 lp_bld_llvm_sampler_soa_emit_size_query(const struct lp_build_sampler_soa *base,
330 struct gallivm_state *gallivm,
331 const struct lp_sampler_size_query_params *params)
332 {
333 LLVMBuilderRef builder = gallivm->builder;
334
335 if (params->resource) {
336 LLVMTypeRef out_data_type = lp_build_vec_type(gallivm, params->int_type);
337
338 LLVMValueRef out_data[4];
339 for (uint32_t i = 0; i < 4; i++) {
340 out_data[i] = lp_build_alloca(gallivm, out_data_type, "");
341 }
342
343 struct lp_type uint_type = lp_uint_type(params->int_type);
344 LLVMValueRef uint_zero = lp_build_const_int_vec(gallivm, uint_type, 0);
345
346 LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, params->exec_mask, uint_zero, "exec_bitvec");
347
348 LLVMTypeRef bitmask_type = LLVMIntTypeInContext(gallivm->context, uint_type.length);
349 bitmask = LLVMBuildBitCast(builder, bitmask, bitmask_type, "exec_bitmask");
350
351 LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active");
352
353 struct lp_build_if_state if_state;
354 lp_build_if(&if_state, gallivm, any_active);
355
356 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
357
358 LLVMValueRef texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS);
359
360 uint32_t functions_offset = params->samples_only ? offsetof(struct lp_texture_functions, samples_function)
361 : offsetof(struct lp_texture_functions, size_function);
362
363 LLVMValueRef texture_base_ptr = load_texture_functions_ptr(
364 gallivm, texture_descriptor, offsetof(struct lp_descriptor, functions), functions_offset);
365
366 LLVMTypeRef texture_function_type = lp_build_size_function_type(gallivm, params);
367 LLVMTypeRef texture_function_ptr_type = LLVMPointerType(texture_function_type, 0);
368 LLVMTypeRef texture_function_ptr_ptr_type = LLVMPointerType(texture_function_ptr_type, 0);
369
370 texture_base_ptr = LLVMBuildIntToPtr(builder, texture_base_ptr, texture_function_ptr_ptr_type, "");
371 LLVMValueRef texture_function = LLVMBuildLoad2(builder, texture_function_ptr_type, texture_base_ptr, "");
372
373 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS];
374 uint32_t num_args = 0;
375
376 args[num_args++] = texture_descriptor;
377
378 if (!params->samples_only)
379 args[num_args++] = params->explicit_lod;
380
381 if (params->int_type.length != lp_native_vector_width / 32)
382 for (uint32_t i = 0; i < num_args; i++)
383 args[i] = widen_to_simd_width(gallivm, args[i]);
384
385 LLVMValueRef result = LLVMBuildCall2(builder, texture_function_type, texture_function, args, num_args, "");
386
387 for (unsigned i = 0; i < 4; i++) {
388 params->sizes_out[i] = LLVMBuildExtractValue(gallivm->builder, result, i, "");
389
390 if (params->int_type.length != lp_native_vector_width / 32)
391 params->sizes_out[i] = truncate_to_type_width(gallivm, params->sizes_out[i], params->int_type);
392
393 LLVMBuildStore(builder, params->sizes_out[i], out_data[i]);
394 }
395
396 lp_build_endif(&if_state);
397
398 for (unsigned i = 0; i < 4; i++)
399 params->sizes_out[i] = LLVMBuildLoad2(gallivm->builder, out_data_type, out_data[i], "");
400
401 return;
402 }
403
404 struct lp_bld_llvm_sampler_soa *sampler = (struct lp_bld_llvm_sampler_soa *)base;
405
406 assert(params->texture_unit < PIPE_MAX_SHADER_SAMPLER_VIEWS);
407
408 lp_build_size_query_soa(gallivm,
409 &sampler->dynamic_state.static_state[params->texture_unit].texture_state,
410 &sampler->dynamic_state.base,
411 params);
412 }
413
414
415 struct lp_build_sampler_soa *
lp_bld_llvm_sampler_soa_create(const struct lp_sampler_static_state * static_state,unsigned nr_samplers)416 lp_bld_llvm_sampler_soa_create(const struct lp_sampler_static_state *static_state,
417 unsigned nr_samplers)
418 {
419 assert(static_state);
420
421 struct lp_bld_llvm_sampler_soa *sampler = CALLOC_STRUCT(lp_bld_llvm_sampler_soa);
422 if (!sampler)
423 return NULL;
424
425 sampler->base.emit_tex_sample = lp_bld_llvm_sampler_soa_emit_fetch_texel;
426 sampler->base.emit_size_query = lp_bld_llvm_sampler_soa_emit_size_query;
427
428 lp_build_jit_fill_sampler_dynamic_state(&sampler->dynamic_state.base);
429
430 sampler->dynamic_state.static_state = static_state;
431
432 sampler->nr_samplers = nr_samplers;
433 return &sampler->base;
434 }
435
436
437 static void
lp_bld_llvm_image_soa_emit_op(const struct lp_build_image_soa * base,struct gallivm_state * gallivm,const struct lp_img_params * params)438 lp_bld_llvm_image_soa_emit_op(const struct lp_build_image_soa *base,
439 struct gallivm_state *gallivm,
440 const struct lp_img_params *params)
441 {
442 LLVMBuilderRef builder = gallivm->builder;
443
444 if (params->resource) {
445 const struct util_format_description *desc = util_format_description(params->format);
446 struct lp_type texel_type = lp_build_texel_type(params->type, desc);
447 LLVMTypeRef out_data_type = lp_build_vec_type(gallivm, texel_type);
448 LLVMTypeRef out_residency_type = lp_build_vec_type(gallivm, lp_int_type(texel_type));
449
450 LLVMValueRef out_data[5];
451 for (uint32_t i = 0; i < ARRAY_SIZE(out_data) - 1; i++) {
452 out_data[i] = lp_build_alloca(gallivm, out_data_type, "");
453 }
454 out_data[ARRAY_SIZE(out_data) - 1] = lp_build_alloca(gallivm, out_residency_type, "");
455
456 struct lp_type uint_type = lp_uint_type(params->type);
457 LLVMValueRef uint_zero = lp_build_const_int_vec(gallivm, uint_type, 0);
458
459 LLVMValueRef bitmask = LLVMBuildICmp(builder, LLVMIntNE, params->exec_mask, uint_zero, "exec_bitvec");
460
461 LLVMTypeRef bitmask_type = LLVMIntTypeInContext(gallivm->context, uint_type.length);
462 bitmask = LLVMBuildBitCast(builder, bitmask, bitmask_type, "exec_bitmask");
463
464 LLVMValueRef any_active = LLVMBuildICmp(builder, LLVMIntNE, bitmask, LLVMConstInt(bitmask_type, 0, false), "any_active");
465
466 LLVMValueRef binding_index = LLVMBuildExtractValue(builder, params->resource, 1, "");
467 LLVMValueRef inbounds = LLVMBuildICmp(builder, LLVMIntSGE, binding_index, lp_build_const_int32(gallivm, 0), "inbounds");
468
469 struct lp_build_if_state if_state;
470 lp_build_if(&if_state, gallivm, LLVMBuildAnd(builder, any_active, inbounds, ""));
471
472 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
473
474 LLVMValueRef image_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS);
475
476 LLVMValueRef image_base_ptr = load_texture_functions_ptr(
477 gallivm, image_descriptor, offsetof(struct lp_descriptor, functions),
478 offsetof(struct lp_texture_functions, image_functions));
479
480 LLVMTypeRef image_function_type = lp_build_image_function_type(gallivm, params, params->ms_index);
481 LLVMTypeRef image_function_ptr_type = LLVMPointerType(image_function_type, 0);
482 LLVMTypeRef image_functions_type = LLVMPointerType(image_function_ptr_type, 0);
483 LLVMTypeRef image_base_type = LLVMPointerType(image_functions_type, 0);
484
485 image_base_ptr = LLVMBuildIntToPtr(builder, image_base_ptr, image_base_type, "");
486 LLVMValueRef image_functions = LLVMBuildLoad2(builder, image_functions_type, image_base_ptr, "");
487
488 uint32_t op = params->img_op;
489 if (op == LP_IMG_ATOMIC_CAS)
490 op--;
491 else if (op == LP_IMG_ATOMIC)
492 op = params->op + (LP_IMG_OP_COUNT - 1);
493
494 if (params->ms_index)
495 op += LP_TOTAL_IMAGE_OP_COUNT / 2;
496
497 LLVMValueRef function_index = lp_build_const_int32(gallivm, op);
498
499 LLVMValueRef image_function_ptr = LLVMBuildGEP2(builder, image_function_ptr_type, image_functions, &function_index, 1, "");
500 LLVMValueRef image_function = LLVMBuildLoad2(builder, image_function_ptr_type, image_function_ptr, "");
501
502 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS] = { 0 };
503 uint32_t num_args = 0;
504
505 args[num_args++] = image_descriptor;
506
507 if (params->img_op != LP_IMG_LOAD && params->img_op != LP_IMG_LOAD_SPARSE)
508 args[num_args++] = params->exec_mask;
509
510 for (uint32_t i = 0; i < 3; i++)
511 args[num_args++] = params->coords[i];
512
513 if (params->ms_index)
514 args[num_args++] = params->ms_index;
515
516 if (params->img_op != LP_IMG_LOAD && params->img_op != LP_IMG_LOAD_SPARSE)
517 for (uint32_t i = 0; i < 4; i++)
518 args[num_args++] = params->indata[i];
519
520 if (params->img_op == LP_IMG_ATOMIC_CAS)
521 for (uint32_t i = 0; i < 4; i++)
522 args[num_args++] = params->indata2[i];
523
524 assert(num_args == LLVMCountParamTypes(image_function_type));
525
526 LLVMTypeRef param_types[LP_MAX_TEX_FUNC_ARGS];
527 LLVMGetParamTypes(image_function_type, param_types);
528 for (uint32_t i = 0; i < num_args; i++)
529 if (!args[i])
530 args[i] = LLVMGetUndef(param_types[i]);
531
532 if (params->type.length != lp_native_vector_width / 32)
533 for (uint32_t i = 0; i < num_args; i++)
534 args[i] = widen_to_simd_width(gallivm, args[i]);
535
536 LLVMValueRef result = LLVMBuildCall2(builder, image_function_type, image_function, args, num_args, "");
537
538 if (params->img_op != LP_IMG_STORE) {
539 uint32_t channel_count = params->img_op == LP_IMG_LOAD_SPARSE ? 5 : 4;
540 for (unsigned i = 0; i < channel_count; i++) {
541 LLVMValueRef channel = LLVMBuildExtractValue(builder, result, i, "");
542 if (params->type.length != lp_native_vector_width / 32)
543 channel = truncate_to_type_width(gallivm, channel, params->type);
544
545 LLVMBuildStore(builder, channel, out_data[i]);
546 }
547 }
548
549 lp_build_endif(&if_state);
550
551 if (params->img_op != LP_IMG_STORE) {
552 for (unsigned i = 0; i < ARRAY_SIZE(out_data) - 1; i++) {
553 params->outdata[i] = LLVMBuildLoad2(builder, out_data_type, out_data[i], "");
554 }
555 params->outdata[ARRAY_SIZE(out_data) - 1] =
556 LLVMBuildLoad2(builder, out_residency_type, out_data[ARRAY_SIZE(out_data) - 1], "");
557 }
558
559 return;
560 }
561
562 struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)base;
563 const unsigned image_index = params->image_index;
564 assert(image_index < PIPE_MAX_SHADER_IMAGES);
565
566 if (params->image_index_offset) {
567 struct lp_build_img_op_array_switch switch_info;
568 memset(&switch_info, 0, sizeof(switch_info));
569 LLVMValueRef unit = LLVMBuildAdd(builder,
570 params->image_index_offset,
571 lp_build_const_int32(gallivm,
572 image_index), "");
573
574 lp_build_image_op_switch_soa(&switch_info, gallivm, params,
575 unit, 0, image->nr_images);
576
577 for (unsigned i = 0; i < image->nr_images; i++) {
578 lp_build_image_op_array_case(&switch_info, i,
579 &image->dynamic_state.static_state[i].image_state,
580 &image->dynamic_state.base);
581 }
582 lp_build_image_op_array_fini_soa(&switch_info);
583 } else {
584 lp_build_img_op_soa(&image->dynamic_state.static_state[image_index].image_state,
585 &image->dynamic_state.base,
586 gallivm, params, params->outdata);
587 }
588 }
589
590
591 /**
592 * Fetch the texture size.
593 */
594 static void
lp_bld_llvm_image_soa_emit_size_query(const struct lp_build_image_soa * base,struct gallivm_state * gallivm,const struct lp_sampler_size_query_params * params)595 lp_bld_llvm_image_soa_emit_size_query(const struct lp_build_image_soa *base,
596 struct gallivm_state *gallivm,
597 const struct lp_sampler_size_query_params *params)
598 {
599 struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)base;
600
601 if (params->resource) {
602 LLVMValueRef old_texture = gallivm->texture_descriptor;
603
604 LLVMValueRef consts = lp_jit_resources_constants(gallivm, params->resources_type, params->resources_ptr);
605 gallivm->texture_descriptor = lp_llvm_descriptor_base(gallivm, consts, params->resource, LP_MAX_TGSI_CONST_BUFFERS);
606
607 enum pipe_format format = params->format;
608 if (format == PIPE_FORMAT_NONE)
609 format = PIPE_FORMAT_R8G8B8A8_UNORM;
610
611 struct lp_static_texture_state state = {
612 .format = format,
613 .res_format = format,
614 .target = params->target,
615 .level_zero_only = params->ms,
616 };
617
618 lp_build_size_query_soa(gallivm, &state, &image->dynamic_state.base, params);
619
620 gallivm->texture_descriptor = old_texture;
621
622 return;
623 }
624
625 assert(params->texture_unit < PIPE_MAX_SHADER_IMAGES);
626
627 lp_build_size_query_soa(gallivm,
628 &image->dynamic_state.static_state[params->texture_unit].image_state,
629 &image->dynamic_state.base,
630 params);
631 }
632
633
634 struct lp_build_image_soa *
lp_bld_llvm_image_soa_create(const struct lp_image_static_state * static_state,unsigned nr_images)635 lp_bld_llvm_image_soa_create(const struct lp_image_static_state *static_state,
636 unsigned nr_images)
637 {
638 struct lp_bld_llvm_image_soa *image = CALLOC_STRUCT(lp_bld_llvm_image_soa);
639 if (!image)
640 return NULL;
641
642 image->base.emit_op = lp_bld_llvm_image_soa_emit_op;
643 image->base.emit_size_query = lp_bld_llvm_image_soa_emit_size_query;
644
645 lp_build_jit_fill_image_dynamic_state(&image->dynamic_state.base);
646 image->dynamic_state.static_state = static_state;
647
648 image->nr_images = nr_images;
649 return &image->base;
650 }
651
652 struct lp_sampler_dynamic_state *
lp_build_sampler_soa_dynamic_state(struct lp_build_sampler_soa * _sampler)653 lp_build_sampler_soa_dynamic_state(struct lp_build_sampler_soa *_sampler)
654 {
655 struct lp_bld_llvm_sampler_soa *sampler = (struct lp_bld_llvm_sampler_soa *)_sampler;
656 return &sampler->dynamic_state.base;
657 }
658
659 struct lp_sampler_dynamic_state *
lp_build_image_soa_dynamic_state(struct lp_build_image_soa * _image)660 lp_build_image_soa_dynamic_state(struct lp_build_image_soa *_image)
661 {
662 struct lp_bld_llvm_image_soa *image = (struct lp_bld_llvm_image_soa *)_image;
663 return &image->dynamic_state.base;
664 }
665