1 /*
2 * Copyright © 2023 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "lp_context.h"
25 #include "lp_texture_handle.h"
26 #include "lp_screen.h"
27
28 #include "gallivm/lp_bld_const.h"
29 #include "gallivm/lp_bld_debug.h"
30 #include "gallivm/lp_bld_nir.h"
31
32 #include "nir.h"
33 #include "nir_builder.h"
34
35 #include "pipe/p_context.h"
36 #include "pipe/p_screen.h"
37 #include "util/mesa-sha1.h"
38
39 static const char *image_function_base_hash = "8ca89d7a4ab5830be6a1ba1140844081235b01164a8fce8316ca6a2f81f1a899";
40 static const char *sample_function_base_hash = "0789b032c4a1ddba086e07496fe2a992b1ee08f78c0884a2923564b1ed52b9cc";
41 static const char *size_function_base_hash = "6d249ab9c1106c68b87ec9fdb5ade28368171d27f221c687f32ae1544231d2fe";
42 static const char *jit_sample_function_base_hash = "21de75bb5dbcfea1f90d03b8b688f19bdb0d96f95681cbe8b26853e1723846e4";
43
44 static void
45 llvmpipe_register_texture(struct llvmpipe_context *ctx, struct lp_static_texture_state *state, bool sampled);
46
47 static void
48 llvmpipe_register_sampler(struct llvmpipe_context *ctx, struct lp_static_sampler_state *state);
49
50 static uint64_t
llvmpipe_create_texture_handle(struct pipe_context * pctx,struct pipe_sampler_view * view,const struct pipe_sampler_state * sampler)51 llvmpipe_create_texture_handle(struct pipe_context *pctx, struct pipe_sampler_view *view, const struct pipe_sampler_state *sampler)
52 {
53 struct llvmpipe_context *ctx = llvmpipe_context(pctx);
54 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
55
56 struct lp_texture_handle *handle = calloc(1, sizeof(struct lp_texture_handle));
57
58 if (view) {
59 struct lp_static_texture_state state;
60 lp_sampler_static_texture_state(&state, view);
61
62 /* Trade a bit of performance for potentially less sampler/texture combinations. */
63 state.pot_width = false;
64 state.pot_height = false;
65 state.pot_depth = false;
66
67 llvmpipe_register_texture(ctx, &state, true);
68
69 bool found = false;
70 for (uint32_t i = 0; i < matrix->texture_count; i++) {
71 if (!memcmp(&matrix->textures[i]->state, &state, sizeof(struct lp_static_texture_state))) {
72 handle->functions = matrix->textures[i];
73 found = true;
74 break;
75 }
76 }
77 assert(found);
78 }
79
80 if (sampler) {
81 struct lp_static_sampler_state state;
82 lp_sampler_static_sampler_state(&state, sampler);
83
84 llvmpipe_register_sampler(ctx, &state);
85
86 bool found = false;
87 for (uint32_t i = 0; i < matrix->sampler_count; i++) {
88 if (!memcmp(matrix->samplers + i, &state, sizeof(struct lp_static_sampler_state))) {
89 handle->sampler_index = i;
90 found = true;
91 break;
92 }
93 }
94 assert(found);
95 }
96
97 return (uint64_t)(uintptr_t)handle;
98 }
99
100 static void
llvmpipe_delete_texture_handle(struct pipe_context * pctx,uint64_t handle)101 llvmpipe_delete_texture_handle(struct pipe_context *pctx, uint64_t handle)
102 {
103 free((void *)(uintptr_t)handle);
104 }
105
106 static uint64_t
llvmpipe_create_image_handle(struct pipe_context * pctx,const struct pipe_image_view * view)107 llvmpipe_create_image_handle(struct pipe_context *pctx, const struct pipe_image_view *view)
108 {
109 struct llvmpipe_context *ctx = llvmpipe_context(pctx);
110 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
111
112 struct lp_texture_handle *handle = calloc(1, sizeof(struct lp_texture_handle));
113
114 struct lp_static_texture_state state;
115 lp_sampler_static_texture_state_image(&state, view);
116
117 /* Trade a bit of performance for potentially less sampler/texture combinations. */
118 state.pot_width = false;
119 state.pot_height = false;
120 state.pot_depth = false;
121
122 if (view->u.tex.first_layer == view->u.tex.last_layer) {
123 if (state.target == PIPE_TEXTURE_1D_ARRAY)
124 state.target = PIPE_TEXTURE_1D;
125 else if (state.target == PIPE_TEXTURE_2D_ARRAY || (state.target == PIPE_TEXTURE_3D && !state.tiled))
126 state.target = PIPE_TEXTURE_2D;
127 else if (state.target == PIPE_TEXTURE_CUBE_ARRAY)
128 state.target = PIPE_TEXTURE_CUBE;
129 }
130
131 llvmpipe_register_texture(ctx, &state, false);
132
133 bool found = false;
134 for (uint32_t i = 0; i < matrix->texture_count; i++) {
135 if (!memcmp(&matrix->textures[i]->state, &state, sizeof(struct lp_static_texture_state))) {
136 handle->functions = matrix->textures[i];
137 found = true;
138 break;
139 }
140 }
141 assert(found);
142
143 return (uint64_t)(uintptr_t)handle;
144 }
145
146 static void
llvmpipe_delete_image_handle(struct pipe_context * pctx,uint64_t handle)147 llvmpipe_delete_image_handle(struct pipe_context *pctx, uint64_t handle)
148 {
149 free((void *)(uintptr_t)handle);
150 }
151
152 static uint64_t
153 get_sample_function(uint64_t _matrix, uint64_t _texture_functions, uint64_t _sampler_desc, uint32_t sample_key);
154
155 struct sample_function_cache_key {
156 struct lp_texture_functions *texture_functions;
157 uint32_t sampler_index;
158 uint32_t sample_key;
159 };
160
DERIVE_HASH_TABLE(sample_function_cache_key)161 DERIVE_HASH_TABLE(sample_function_cache_key)
162
163 static struct hash_table *
164 acquire_latest_sample_function_cache(struct lp_sampler_matrix *matrix)
165 {
166 uint64_t value = p_atomic_read(&matrix->latest_cache.value);
167 return (struct hash_table *)(uintptr_t)value;
168 }
169
170 static void
replace_sample_function_cache_locked(struct lp_sampler_matrix * matrix,struct hash_table * new_cache)171 replace_sample_function_cache_locked(struct lp_sampler_matrix *matrix, struct hash_table *new_cache)
172 {
173 uint64_t old_value = p_atomic_xchg(&matrix->latest_cache.value, (uint64_t)(uintptr_t)new_cache);
174 /* Like RCU pointers, defer cleanup of old values until we know no readers are left. */
175 struct hash_table *old_cache = (struct hash_table *)(uintptr_t)old_value;
176 util_dynarray_append(&matrix->trash_caches, struct hash_table *, old_cache);
177 }
178
179 void
llvmpipe_init_sampler_matrix(struct llvmpipe_context * ctx)180 llvmpipe_init_sampler_matrix(struct llvmpipe_context *ctx)
181 {
182 ctx->pipe.create_texture_handle = llvmpipe_create_texture_handle;
183 ctx->pipe.delete_texture_handle = llvmpipe_delete_texture_handle;
184 ctx->pipe.create_image_handle = llvmpipe_create_image_handle;
185 ctx->pipe.delete_image_handle = llvmpipe_delete_image_handle;
186
187 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
188
189 util_dynarray_init(&matrix->gallivms, NULL);
190
191 matrix->ctx = ctx;
192
193 matrix->compile_function = get_sample_function;
194
195 struct hash_table *initial_cache = sample_function_cache_key_table_create(NULL);
196 p_atomic_set(&matrix->latest_cache.value, (uint64_t)(uintptr_t)initial_cache);
197 util_dynarray_init(&matrix->trash_caches, NULL);
198
199 simple_mtx_init(&matrix->lock, mtx_plain);
200 }
201
202 void
llvmpipe_sampler_matrix_destroy(struct llvmpipe_context * ctx)203 llvmpipe_sampler_matrix_destroy(struct llvmpipe_context *ctx)
204 {
205 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
206
207 simple_mtx_destroy(&matrix->lock);
208
209 _mesa_hash_table_destroy(acquire_latest_sample_function_cache(matrix), NULL);
210 util_dynarray_foreach (&matrix->trash_caches, struct hash_table *, trash)
211 _mesa_hash_table_destroy(*trash, NULL);
212 util_dynarray_fini(&matrix->trash_caches);
213
214 free(matrix->samplers);
215
216 for (uint32_t texture_index = 0; texture_index < matrix->texture_count; texture_index++) {
217 struct lp_texture_functions *texture = matrix->textures[texture_index];
218
219 uint32_t sampler_count = texture->sampler_count;
220 if (texture->state.format == PIPE_FORMAT_NONE)
221 sampler_count = MIN2(sampler_count, 1);
222
223 for (uint32_t sampler_index = 0; sampler_index < sampler_count; sampler_index++)
224 free(texture->sample_functions[sampler_index]);
225
226 free(texture->sample_functions);
227 free(texture->fetch_functions);
228 free(texture->image_functions);
229 free(texture);
230 }
231 free(matrix->textures);
232
233 util_dynarray_foreach (&matrix->gallivms, struct gallivm_state *, gallivm)
234 gallivm_destroy(*gallivm);
235
236 util_dynarray_fini(&matrix->gallivms);
237
238 if (matrix->context.ref)
239 lp_context_destroy(&matrix->context);
240 }
241
242 static lp_context_ref *
get_llvm_context(struct llvmpipe_context * ctx)243 get_llvm_context(struct llvmpipe_context *ctx)
244 {
245 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
246
247 if (!matrix->context.ref)
248 lp_context_create(&matrix->context);
249
250 return &matrix->context;
251 }
252
253 static void *
compile_function(struct llvmpipe_context * ctx,struct gallivm_state * gallivm,LLVMValueRef function,const char * func_name,bool needs_caching,uint8_t cache_key[SHA1_DIGEST_LENGTH])254 compile_function(struct llvmpipe_context *ctx, struct gallivm_state *gallivm, LLVMValueRef function,
255 const char *func_name,
256 bool needs_caching,
257 uint8_t cache_key[SHA1_DIGEST_LENGTH])
258 {
259 gallivm_verify_function(gallivm, function);
260 gallivm_compile_module(gallivm);
261
262 void *function_ptr = func_to_pointer(gallivm_jit_function(gallivm, function, func_name));
263
264 if (needs_caching)
265 lp_disk_cache_insert_shader(llvmpipe_screen(ctx->pipe.screen), gallivm->cache, cache_key);
266
267 gallivm_free_ir(gallivm);
268
269 util_dynarray_append(&ctx->sampler_matrix.gallivms, struct gallivm_state *, gallivm);
270
271 return function_ptr;
272 }
273
274 static void *
compile_image_function(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,uint32_t op)275 compile_image_function(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture, uint32_t op)
276 {
277 const struct util_format_description *desc = util_format_description(texture->format);
278 if (desc->colorspace != UTIL_FORMAT_COLORSPACE_ZS && !lp_storage_render_image_format_supported(texture->format))
279 return NULL;
280
281 bool ms = op >= LP_TOTAL_IMAGE_OP_COUNT / 2;
282 if (ms)
283 op -= LP_TOTAL_IMAGE_OP_COUNT / 2;
284
285 struct lp_img_params params = { 0 };
286
287 params.img_op = op;
288 if (op >= LP_IMG_OP_COUNT - 1) {
289 params.img_op = LP_IMG_ATOMIC;
290 params.op = op - (LP_IMG_OP_COUNT - 1);
291 } else if (op != LP_IMG_LOAD && op != LP_IMG_LOAD_SPARSE && op != LP_IMG_STORE) {
292 params.img_op = LP_IMG_ATOMIC_CAS;
293 }
294
295 /* Loads need to support a wider range of formats for input attachments. */
296 if (params.img_op != LP_IMG_LOAD)
297 if (texture->format != PIPE_FORMAT_NONE && !lp_storage_image_format_supported(texture->format))
298 return NULL;
299
300 uint8_t cache_key[SHA1_DIGEST_LENGTH];
301 struct mesa_sha1 hash_ctx;
302 _mesa_sha1_init(&hash_ctx);
303 _mesa_sha1_update(&hash_ctx, image_function_base_hash, strlen(image_function_base_hash));
304 _mesa_sha1_update(&hash_ctx, texture, sizeof(*texture));
305 _mesa_sha1_update(&hash_ctx, &op, sizeof(op));
306 _mesa_sha1_update(&hash_ctx, &ms, sizeof(ms));
307 _mesa_sha1_final(&hash_ctx, cache_key);
308
309 struct lp_cached_code cached = { 0 };
310 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
311 bool needs_caching = !cached.data_size;
312
313 struct gallivm_state *gallivm = gallivm_create("sample_function", get_llvm_context(ctx), &cached);
314
315 struct lp_image_static_state state = {
316 .image_state = *texture,
317 };
318 struct lp_build_image_soa *image_soa = lp_bld_llvm_image_soa_create(&state, 1);
319
320 struct lp_type type;
321 memset(&type, 0, sizeof type);
322 type.floating = true; /* floating point values */
323 type.sign = true; /* values are signed */
324 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
325 type.width = 32; /* 32-bit float */
326 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
327
328 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
329 lp_jit_init_cs_types(&cs);
330
331 params.type = type;
332 params.target = texture->target;
333 params.resources_type = cs.jit_resources_type;
334 params.format = texture->format;
335
336 LLVMTypeRef function_type = lp_build_image_function_type(gallivm, ¶ms, ms);
337 if (!function_type) {
338 free(image_soa);
339 gallivm_destroy(gallivm);
340 return NULL;
341 }
342
343 LLVMValueRef function = LLVMAddFunction(gallivm->module, "image", function_type);
344
345 uint32_t arg_index = 0;
346
347 gallivm->texture_descriptor = LLVMGetParam(function, arg_index++);
348
349 if (params.img_op != LP_IMG_LOAD && params.img_op != LP_IMG_LOAD_SPARSE)
350 params.exec_mask = LLVMGetParam(function, arg_index++);
351
352 LLVMValueRef coords[3];
353 params.coords = coords;
354 for (uint32_t i = 0; i < 3; i++)
355 coords[i] = LLVMGetParam(function, arg_index++);
356
357 if (ms)
358 params.ms_index = LLVMGetParam(function, arg_index++);
359
360 if (params.img_op != LP_IMG_LOAD && params.img_op != LP_IMG_LOAD_SPARSE)
361 for (uint32_t i = 0; i < 4; i++)
362 params.indata[i] = LLVMGetParam(function, arg_index++);
363
364 if (params.img_op == LP_IMG_ATOMIC_CAS)
365 for (uint32_t i = 0; i < 4; i++)
366 params.indata2[i] = LLVMGetParam(function, arg_index++);
367
368 LLVMBuilderRef old_builder = gallivm->builder;
369 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
370 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
371 LLVMPositionBuilderAtEnd(gallivm->builder, block);
372
373 LLVMValueRef outdata[5] = { 0 };
374 lp_build_img_op_soa(texture, lp_build_image_soa_dynamic_state(image_soa), gallivm, ¶ms, outdata);
375
376 for (uint32_t i = 1; i < 4; i++)
377 if (!outdata[i])
378 outdata[i] = outdata[0];
379
380 if (outdata[4])
381 outdata[4] = LLVMBuildZExt(gallivm->builder, outdata[4], lp_build_int_vec_type(gallivm, lp_int_type(type)), "");
382 else
383 outdata[4] = lp_build_one(gallivm, lp_int_type(type));
384
385 if (params.img_op != LP_IMG_STORE)
386 LLVMBuildAggregateRet(gallivm->builder, outdata, params.img_op == LP_IMG_LOAD_SPARSE ? 5 : 4);
387 else
388 LLVMBuildRetVoid(gallivm->builder);
389
390 LLVMDisposeBuilder(gallivm->builder);
391 gallivm->builder = old_builder;
392
393 free(image_soa);
394
395 return compile_function(ctx, gallivm, function, "image", needs_caching, cache_key);
396 }
397
398 static void *
compile_sample_function(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,struct lp_static_sampler_state * sampler,uint32_t sample_key)399 compile_sample_function(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture,
400 struct lp_static_sampler_state *sampler, uint32_t sample_key)
401 {
402 enum lp_sampler_lod_control lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >> LP_SAMPLER_LOD_CONTROL_SHIFT;
403
404 bool supported = true;
405 if (texture->format != PIPE_FORMAT_NONE) {
406 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
407 if (op_type != LP_SAMPLER_OP_LODQ)
408 if ((sampler->compare_mode == PIPE_TEX_COMPARE_NONE) == !!(sample_key & LP_SAMPLER_SHADOW))
409 supported = false;
410
411 /* Skip integer formats which would cause a type mismatch in the compare function. */
412 const struct util_format_description *desc = util_format_description(texture->format);
413 struct lp_type texel_type = {
414 .floating = true,
415 .width = 32,
416 .length = 1,
417 };
418 texel_type = lp_build_texel_type(texel_type, desc);
419 if ((sample_key & LP_SAMPLER_SHADOW) && !texel_type.floating)
420 supported = false;
421
422 if (texture_dims(texture->target) != 2 && op_type == LP_SAMPLER_OP_GATHER)
423 supported = false;
424
425 if (op_type != LP_SAMPLER_OP_FETCH) {
426 if (!sampler->normalized_coords) {
427 if (texture->target != PIPE_TEXTURE_1D && texture->target != PIPE_TEXTURE_2D &&
428 texture->target != PIPE_TEXTURE_1D_ARRAY && texture->target != PIPE_TEXTURE_2D_ARRAY)
429 supported = false;
430
431 if (!texture->level_zero_only)
432 supported = false;
433 }
434 }
435
436 if (util_format_is_pure_integer(texture->format) &&
437 (sampler->min_img_filter == PIPE_TEX_FILTER_LINEAR ||
438 sampler->min_mip_filter == PIPE_TEX_MIPFILTER_LINEAR ||
439 sampler->mag_img_filter == PIPE_TEX_FILTER_LINEAR))
440 supported = false;
441
442 if (sampler->aniso) {
443 if (texture_dims(texture->target) != 2)
444 supported = false;
445
446 if (util_format_is_pure_integer(texture->format))
447 supported = false;
448 }
449
450 if (util_format_get_num_planes(texture->format) > 1)
451 return NULL;
452
453 uint32_t bind = op_type == LP_SAMPLER_OP_FETCH ? PIPE_BIND_CONSTANT_BUFFER : PIPE_BIND_SAMPLER_VIEW;
454 if (!ctx->pipe.screen->is_format_supported(ctx->pipe.screen, texture->format, texture->target, 0, 0, bind))
455 supported = false;
456 }
457
458 uint8_t cache_key[SHA1_DIGEST_LENGTH];
459 struct mesa_sha1 hash_ctx;
460 _mesa_sha1_init(&hash_ctx);
461 _mesa_sha1_update(&hash_ctx, sample_function_base_hash, strlen(sample_function_base_hash));
462 _mesa_sha1_update(&hash_ctx, texture, sizeof(*texture));
463 _mesa_sha1_update(&hash_ctx, sampler, sizeof(*sampler));
464 _mesa_sha1_update(&hash_ctx, &sample_key, sizeof(sample_key));
465 _mesa_sha1_final(&hash_ctx, cache_key);
466
467 struct lp_cached_code cached = { 0 };
468 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
469 bool needs_caching = !cached.data_size;
470
471 struct gallivm_state *gallivm = gallivm_create("sample_function", get_llvm_context(ctx), &cached);
472
473 struct lp_sampler_static_state state = {
474 .texture_state = *texture,
475 .sampler_state = *sampler,
476 };
477 struct lp_build_sampler_soa *sampler_soa = lp_llvm_sampler_soa_create(&state, 1);
478
479 struct lp_type type;
480 memset(&type, 0, sizeof type);
481 type.floating = true; /* floating point values */
482 type.sign = true; /* values are signed */
483 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
484 type.width = 32; /* 32-bit float */
485 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
486
487 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
488 lp_jit_init_cs_types(&cs);
489
490 LLVMTypeRef function_type = lp_build_sample_function_type(gallivm, sample_key);
491 LLVMValueRef function = LLVMAddFunction(gallivm->module, "sample", function_type);
492
493 uint32_t arg_index = 0;
494
495 gallivm->texture_descriptor = LLVMGetParam(function, arg_index++);
496 gallivm->sampler_descriptor = LLVMGetParam(function, arg_index++);
497
498 LLVMValueRef aniso_filter_table = LLVMGetParam(function, arg_index++);
499
500 LLVMValueRef coords[5];
501 for (unsigned i = 0; i < 4; i++)
502 coords[i] = LLVMGetParam(function, arg_index++);
503
504 if (sample_key & LP_SAMPLER_SHADOW)
505 coords[4] = LLVMGetParam(function, arg_index++);
506 else
507 coords[4] = lp_build_undef(gallivm, type);
508
509 LLVMValueRef ms_index = NULL;
510 if (sample_key & LP_SAMPLER_FETCH_MS)
511 ms_index = LLVMGetParam(function, arg_index++);
512
513 LLVMValueRef offsets[3] = { 0 };
514 if (sample_key & LP_SAMPLER_OFFSETS)
515 for (unsigned i = 0; i < 3; i++)
516 offsets[i] = LLVMGetParam(function, arg_index++);
517
518 LLVMValueRef lod = NULL;
519 if (lod_control == LP_SAMPLER_LOD_BIAS || lod_control == LP_SAMPLER_LOD_EXPLICIT)
520 lod = LLVMGetParam(function, arg_index++);
521
522 LLVMBuilderRef old_builder = gallivm->builder;
523 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
524 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
525 LLVMPositionBuilderAtEnd(gallivm->builder, block);
526
527 LLVMValueRef texel_out[5] = { 0 };
528 if (supported) {
529 lp_build_sample_soa_code(gallivm, texture, sampler, lp_build_sampler_soa_dynamic_state(sampler_soa),
530 type, sample_key, 0, 0, cs.jit_resources_type, NULL, cs.jit_cs_thread_data_type,
531 NULL, coords, offsets, NULL, lod, ms_index, aniso_filter_table, texel_out);
532 } else {
533 lp_build_sample_nop(gallivm, lp_build_texel_type(type, util_format_description(texture->format)), coords, texel_out);
534 }
535
536 if (texel_out[4])
537 texel_out[4] = LLVMBuildZExt(gallivm->builder, texel_out[4], lp_build_int_vec_type(gallivm, lp_int_type(type)), "");
538 else
539 texel_out[4] = lp_build_one(gallivm, lp_int_type(type));
540
541 LLVMBuildAggregateRet(gallivm->builder, texel_out, 5);
542
543 LLVMDisposeBuilder(gallivm->builder);
544 gallivm->builder = old_builder;
545
546 free(sampler_soa);
547
548 return compile_function(ctx, gallivm, function, "sample", needs_caching, cache_key);
549 }
550
551 static uint64_t
get_sample_function(uint64_t _matrix,uint64_t _texture_functions,uint64_t _sampler_desc,uint32_t sample_key)552 get_sample_function(uint64_t _matrix, uint64_t _texture_functions, uint64_t _sampler_desc, uint32_t sample_key)
553 {
554 struct lp_sampler_matrix *matrix = (void *)(uintptr_t)_matrix;
555 struct lp_descriptor *sampler_desc = (void *)(uintptr_t)_sampler_desc;
556 uint32_t sampler_index = sampler_desc->texture.sampler_index;
557
558 struct lp_texture_functions *texture_functions = (void *)(uintptr_t)_texture_functions;
559 struct sample_function_cache_key key = {
560 .texture_functions = texture_functions,
561 .sampler_index = sampler_index,
562 .sample_key = sample_key,
563 };
564
565 void *result;
566 {
567 struct hash_entry *entry = _mesa_hash_table_search(acquire_latest_sample_function_cache(matrix), &key);
568 result = entry ? entry->data : NULL;
569 }
570
571 if (!result) {
572 simple_mtx_lock(&matrix->lock);
573 /* Check once more in case the cache got modified between the first check and acquiring the lock. */
574 struct hash_table *current_cache = acquire_latest_sample_function_cache(matrix);
575 struct hash_entry *entry = _mesa_hash_table_search(current_cache, &key);
576 result = entry ? entry->data : NULL;
577 if (!result) {
578 result = compile_sample_function(matrix->ctx, &texture_functions->state, matrix->samplers + sampler_index, sample_key);
579 struct sample_function_cache_key *allocated_key = malloc(sizeof(struct sample_function_cache_key));
580 *allocated_key = key;
581 /* RCU style update: swap in an updated copy of the cache.
582 / Old caches are kept as trash to be safely deleted later. */
583 struct hash_table *new_cache = _mesa_hash_table_clone(current_cache, NULL);
584 _mesa_hash_table_insert(new_cache, allocated_key, result);
585 replace_sample_function_cache_locked(matrix, new_cache);
586 }
587 simple_mtx_unlock(&matrix->lock);
588 }
589
590 return (uint64_t)(uintptr_t)result;
591 }
592
593 static LLVMTypeRef
lp_build_compile_function_type(struct gallivm_state * gallivm)594 lp_build_compile_function_type(struct gallivm_state *gallivm)
595 {
596 LLVMTypeRef param_types[4] = {
597 LLVMInt64TypeInContext(gallivm->context),
598 LLVMInt64TypeInContext(gallivm->context),
599 LLVMInt64TypeInContext(gallivm->context),
600 LLVMInt32TypeInContext(gallivm->context),
601 };
602 LLVMTypeRef ret_type = LLVMInt64TypeInContext(gallivm->context);
603
604 return LLVMFunctionType(ret_type, param_types, ARRAY_SIZE(param_types), false);
605 }
606
607 static void *
compile_jit_sample_function(struct llvmpipe_context * ctx,uint32_t sample_key)608 compile_jit_sample_function(struct llvmpipe_context *ctx, uint32_t sample_key)
609 {
610 uint8_t cache_key[SHA1_DIGEST_LENGTH];
611 struct mesa_sha1 hash_ctx;
612 _mesa_sha1_init(&hash_ctx);
613 _mesa_sha1_update(&hash_ctx, jit_sample_function_base_hash, strlen(jit_sample_function_base_hash));
614 _mesa_sha1_update(&hash_ctx, &sample_key, sizeof(sample_key));
615 _mesa_sha1_final(&hash_ctx, cache_key);
616
617 struct lp_cached_code cached = { 0 };
618 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
619 bool needs_caching = !cached.data_size;
620
621 struct gallivm_state *gallivm = gallivm_create("jit_sample_function", get_llvm_context(ctx), &cached);
622
623 struct lp_type type;
624 memset(&type, 0, sizeof type);
625 type.floating = true; /* floating point values */
626 type.sign = true; /* values are signed */
627 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
628 type.width = 32; /* 32-bit float */
629 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
630
631 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
632 lp_jit_init_cs_types(&cs);
633
634 LLVMTypeRef function_type = lp_build_sample_function_type(gallivm, sample_key);
635 LLVMValueRef function = LLVMAddFunction(gallivm->module, "sample", function_type);
636
637 uint32_t arg_index = 0;
638 LLVMValueRef texture_descriptor = LLVMGetParam(function, arg_index++);
639 LLVMValueRef sampler_descriptor = LLVMGetParam(function, arg_index++);
640
641 LLVMBuilderRef old_builder = gallivm->builder;
642 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
643 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
644 LLVMBuilderRef builder = gallivm->builder;
645 LLVMPositionBuilderAtEnd(gallivm->builder, block);
646
647 LLVMValueRef functions_offset =
648 lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, functions));
649 LLVMValueRef functions_ptr =
650 LLVMBuildAdd(builder, texture_descriptor, functions_offset, "");
651
652 LLVMTypeRef functions_ptr_type = LLVMInt64TypeInContext(gallivm->context);
653 LLVMTypeRef functions_ptr_ptr_type = LLVMPointerType(functions_ptr_type, 0);
654
655 functions_ptr = LLVMBuildIntToPtr(builder, functions_ptr, functions_ptr_ptr_type, "");
656 /* struct lp_texture_functions * */
657 functions_ptr = LLVMBuildLoad2(builder, functions_ptr_type, functions_ptr, "");
658
659 LLVMValueRef matrix_offset =
660 lp_build_const_int64(gallivm, offsetof(struct lp_texture_functions, matrix));
661 LLVMValueRef matrix_ptr = LLVMBuildAdd(builder, functions_ptr, matrix_offset, "");
662
663 matrix_ptr = LLVMBuildIntToPtr(builder, matrix_ptr, functions_ptr_ptr_type, "");
664 /* struct lp_sampler_matrix * */
665 matrix_ptr = LLVMBuildLoad2(builder, functions_ptr_type, matrix_ptr, "");
666
667 LLVMTypeRef compile_function_type = lp_build_compile_function_type(gallivm);
668 LLVMTypeRef compile_function_ptr_type = LLVMPointerType(compile_function_type, 0);
669 LLVMTypeRef compile_function_ptr_ptr_type = LLVMPointerType(compile_function_ptr_type, 0);
670
671 LLVMValueRef compile_function_offset =
672 lp_build_const_int64(gallivm, offsetof(struct lp_sampler_matrix, compile_function));
673 LLVMValueRef compile_function_ptr =
674 LLVMBuildAdd(builder, matrix_ptr, compile_function_offset, "");
675
676 compile_function_ptr =
677 LLVMBuildIntToPtr(builder, compile_function_ptr, compile_function_ptr_ptr_type, "");
678 /* struct lp_texture_functions * */
679 compile_function_ptr =
680 LLVMBuildLoad2(builder, compile_function_ptr_type, compile_function_ptr, "");
681
682 LLVMValueRef compile_args[4] = {
683 matrix_ptr, functions_ptr, sampler_descriptor, lp_build_const_int32(gallivm, sample_key)
684 };
685
686 LLVMValueRef sample_function =
687 LLVMBuildCall2(builder, compile_function_type, compile_function_ptr,
688 compile_args, ARRAY_SIZE(compile_args), "");
689
690 sample_function = LLVMBuildIntToPtr(builder, sample_function, LLVMPointerType(function_type, 0), "");
691
692 LLVMValueRef args[LP_MAX_TEX_FUNC_ARGS];
693 uint32_t num_args = 0;
694
695 LLVMValueRef arg = LLVMGetFirstParam(function);
696 while (true) {
697 args[num_args++] = arg;
698 if (arg == LLVMGetLastParam(function))
699 break;
700
701 arg = LLVMGetNextParam(arg);
702 }
703
704 LLVMValueRef result = LLVMBuildCall2(builder, function_type, sample_function, args, num_args, "");
705 LLVMBuildRet(gallivm->builder, result);
706
707 LLVMDisposeBuilder(gallivm->builder);
708 gallivm->builder = old_builder;
709
710 return compile_function(ctx, gallivm, function, "sample", needs_caching, cache_key);
711 }
712
713 static void *
compile_size_function(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,bool samples)714 compile_size_function(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture, bool samples)
715 {
716 uint8_t cache_key[SHA1_DIGEST_LENGTH];
717 struct mesa_sha1 hash_ctx;
718 _mesa_sha1_init(&hash_ctx);
719 _mesa_sha1_update(&hash_ctx, size_function_base_hash, strlen(size_function_base_hash));
720 _mesa_sha1_update(&hash_ctx, texture, sizeof(*texture));
721 _mesa_sha1_update(&hash_ctx, &samples, sizeof(samples));
722 _mesa_sha1_final(&hash_ctx, cache_key);
723
724 struct lp_cached_code cached = { 0 };
725 lp_disk_cache_find_shader(llvmpipe_screen(ctx->pipe.screen), &cached, cache_key);
726 bool needs_caching = !cached.data_size;
727
728 struct gallivm_state *gallivm = gallivm_create("sample_function", get_llvm_context(ctx), &cached);
729
730 struct lp_sampler_static_state state = {
731 .texture_state = *texture,
732 };
733 struct lp_build_sampler_soa *sampler_soa = lp_llvm_sampler_soa_create(&state, 1);
734
735 struct lp_type type;
736 memset(&type, 0, sizeof type);
737 type.floating = true; /* floating point values */
738 type.sign = true; /* values are signed */
739 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
740 type.width = 32; /* 32-bit float */
741 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
742
743 struct lp_compute_shader_variant cs = { .gallivm = gallivm };
744 lp_jit_init_cs_types(&cs);
745
746 struct lp_sampler_size_query_params params = {
747 .int_type = lp_int_type(type),
748 .target = texture->target,
749 .resources_type = cs.jit_resources_type,
750 .is_sviewinfo = true,
751 .samples_only = samples,
752 .ms = samples,
753 };
754
755 if (params.target == PIPE_TEXTURE_1D)
756 params.target = PIPE_TEXTURE_1D_ARRAY;
757 else if (params.target == PIPE_TEXTURE_2D)
758 params.target = PIPE_TEXTURE_2D_ARRAY;
759 else if (params.target == PIPE_TEXTURE_CUBE)
760 params.target = PIPE_TEXTURE_CUBE_ARRAY;
761
762 LLVMTypeRef function_type = lp_build_size_function_type(gallivm, ¶ms);
763 LLVMValueRef function = LLVMAddFunction(gallivm->module, "size", function_type);
764
765 uint32_t arg_index = 0;
766
767 gallivm->texture_descriptor = LLVMGetParam(function, arg_index++);
768
769 if (!samples)
770 params.explicit_lod = LLVMGetParam(function, arg_index++);
771
772 LLVMBuilderRef old_builder = gallivm->builder;
773 LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
774 gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
775 LLVMPositionBuilderAtEnd(gallivm->builder, block);
776
777 LLVMValueRef out_sizes[4] = { 0 };
778 params.sizes_out = out_sizes;
779 lp_build_size_query_soa(gallivm, texture, lp_build_sampler_soa_dynamic_state(sampler_soa), ¶ms);
780
781 for (uint32_t i = 0; i < 4; i++)
782 if (!out_sizes[i])
783 out_sizes[i] = lp_build_const_int_vec(gallivm, params.int_type, 0);
784
785 LLVMBuildAggregateRet(gallivm->builder, out_sizes, 4);
786
787 LLVMDisposeBuilder(gallivm->builder);
788 gallivm->builder = old_builder;
789
790 free(sampler_soa);
791
792 return compile_function(ctx, gallivm, function, "size", needs_caching, cache_key);
793 }
794
795 static void
compile_sample_functions(struct llvmpipe_context * ctx,struct lp_static_texture_state * texture,struct lp_static_sampler_state * sampler,void *** dst)796 compile_sample_functions(struct llvmpipe_context *ctx, struct lp_static_texture_state *texture,
797 struct lp_static_sampler_state *sampler, void ***dst)
798 {
799 void **functions;
800 if (*dst) {
801 functions = *dst;
802 } else {
803 functions = calloc(LP_SAMPLE_KEY_COUNT, sizeof(void *));
804 *dst = functions;
805 }
806
807 bool has_sampler = !!sampler;
808
809 struct lp_static_sampler_state dummy_sampler = { 0 };
810 if (!sampler)
811 sampler = &dummy_sampler;
812
813 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
814 for (uint32_t sample_key = 0; sample_key < LP_SAMPLE_KEY_COUNT; sample_key++) {
815 if (!BITSET_TEST(matrix->sample_keys, sample_key))
816 continue;
817
818 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
819 if (has_sampler && op_type == LP_SAMPLER_OP_FETCH)
820 continue;
821
822 if (!functions[sample_key]) {
823 if (has_sampler)
824 functions[sample_key] = matrix->jit_sample_functions[sample_key];
825 else
826 functions[sample_key] = compile_sample_function(ctx, texture, sampler, sample_key);
827 }
828 }
829 }
830
831 static void
llvmpipe_register_texture(struct llvmpipe_context * ctx,struct lp_static_texture_state * state,bool sampled)832 llvmpipe_register_texture(struct llvmpipe_context *ctx, struct lp_static_texture_state *state, bool sampled)
833 {
834 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
835
836 bool packed = true;
837 uint32_t dst_index = matrix->texture_count;
838 for (uint32_t i = 0; i < matrix->texture_count; i++) {
839 if (memcmp(&matrix->textures[i]->state, state, sizeof(struct lp_static_texture_state)))
840 continue;
841
842 bool has_functions = sampled ? matrix->textures[i]->sampled : matrix->textures[i]->storage;
843 if (has_functions)
844 return;
845
846 packed = false;
847 dst_index = i;
848 break;
849 }
850
851 struct lp_texture_functions *entry;
852 if (packed) {
853 matrix->texture_count++;
854 matrix->textures = realloc(matrix->textures, matrix->texture_count * sizeof(struct lp_texture_functions *));
855
856 entry = calloc(1, sizeof(struct lp_texture_functions));
857 matrix->textures[dst_index] = entry;
858
859 entry->state = *state;
860 entry->image_functions = calloc(LP_TOTAL_IMAGE_OP_COUNT, sizeof(void **));
861 entry->matrix = matrix;
862 } else {
863 entry = matrix->textures[dst_index];
864 }
865
866 if (sampled)
867 entry->sampled = true;
868 else
869 entry->storage = true;
870
871 simple_mtx_lock(&matrix->lock);
872
873 if (entry->sampled) {
874 if (entry->sample_functions) {
875 entry->sample_functions = realloc(entry->sample_functions, matrix->sampler_count * sizeof(void **));
876 memset(entry->sample_functions + entry->sampler_count, 0, (matrix->sampler_count - entry->sampler_count) * sizeof(void **));
877 } else {
878 entry->sample_functions = calloc(matrix->sampler_count, sizeof(void **));
879 }
880 entry->sampler_count = matrix->sampler_count;
881
882 if (state->format == PIPE_FORMAT_NONE) {
883 if (matrix->sampler_count)
884 compile_sample_functions(ctx, state, NULL, entry->sample_functions);
885 for (uint32_t i = 1; i < matrix->sampler_count; i++)
886 entry->sample_functions[i] = entry->sample_functions[0];
887 } else {
888 for (uint32_t i = 0; i < matrix->sampler_count; i++)
889 compile_sample_functions(ctx, state, matrix->samplers + i, entry->sample_functions + i);
890 }
891
892 compile_sample_functions(ctx, state, NULL, &entry->fetch_functions);
893
894 if (!entry->size_function)
895 entry->size_function = compile_size_function(ctx, state, false);
896
897 if (!entry->samples_function)
898 entry->samples_function = compile_size_function(ctx, state, true);
899 }
900
901 if (entry->storage) {
902 uint32_t image_op;
903 BITSET_FOREACH_SET (image_op, matrix->image_ops, LP_TOTAL_IMAGE_OP_COUNT)
904 if (!entry->image_functions[image_op])
905 entry->image_functions[image_op] = compile_image_function(ctx, state, image_op);
906 }
907
908 simple_mtx_unlock(&matrix->lock);
909 }
910
911 static void
llvmpipe_register_sampler(struct llvmpipe_context * ctx,struct lp_static_sampler_state * state)912 llvmpipe_register_sampler(struct llvmpipe_context *ctx, struct lp_static_sampler_state *state)
913 {
914 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
915 for (uint32_t i = 0; i < matrix->sampler_count; i++)
916 if (!memcmp(matrix->samplers + i, state, sizeof(struct lp_static_sampler_state)))
917 return;
918
919 matrix->sampler_count++;
920 matrix->samplers = realloc(matrix->samplers, matrix->sampler_count * sizeof(struct lp_static_sampler_state));
921
922 matrix->samplers[matrix->sampler_count - 1] = *state;
923
924 simple_mtx_lock(&matrix->lock);
925
926 for (uint32_t i = 0; i < matrix->texture_count; i++) {
927 struct lp_texture_functions *texture = matrix->textures[i];
928 if (!texture->sampled)
929 continue;
930
931 texture->sampler_count = matrix->sampler_count;
932 texture->sample_functions = realloc(texture->sample_functions, matrix->sampler_count * sizeof(void **));
933
934 void ***dst = texture->sample_functions + (matrix->sampler_count - 1);
935
936 if (texture->state.format == PIPE_FORMAT_NONE) {
937 if (matrix->sampler_count == 1) {
938 *dst = NULL;
939 compile_sample_functions(ctx, &texture->state, NULL, dst);
940 } else {
941 *dst = texture->sample_functions[0];
942 }
943
944 continue;
945 }
946
947 *dst = NULL;
948 compile_sample_functions(ctx, &texture->state, state, dst);
949 }
950
951 simple_mtx_unlock(&matrix->lock);
952 }
953
954 static void
register_sample_key(struct llvmpipe_context * ctx,uint32_t sample_key)955 register_sample_key(struct llvmpipe_context *ctx, uint32_t sample_key)
956 {
957 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
958 if (BITSET_TEST(matrix->sample_keys, sample_key))
959 return;
960
961 BITSET_SET(matrix->sample_keys, sample_key);
962
963 simple_mtx_lock(&matrix->lock);
964
965 matrix->jit_sample_functions[sample_key] = compile_jit_sample_function(ctx, sample_key);
966
967 for (uint32_t texture_index = 0; texture_index < matrix->texture_count; texture_index++) {
968 struct lp_texture_functions *texture = matrix->textures[texture_index];
969 if (!texture->sampled)
970 continue;
971
972 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
973 if (op_type == LP_SAMPLER_OP_FETCH) {
974 struct lp_static_sampler_state dummy_sampler = { 0 };
975 texture->fetch_functions[sample_key] = compile_sample_function(ctx, &texture->state, &dummy_sampler, sample_key);
976 continue;
977 }
978
979 if (texture->state.format == PIPE_FORMAT_NONE) {
980 if (matrix->sampler_count) {
981 struct lp_static_sampler_state dummy_sampler = { 0 };
982 texture->sample_functions[0][sample_key] = compile_sample_function(ctx, &texture->state, &dummy_sampler, sample_key);
983 }
984 continue;
985 }
986
987 for (uint32_t sampler_index = 0; sampler_index < matrix->sampler_count; sampler_index++)
988 texture->sample_functions[sampler_index][sample_key] = matrix->jit_sample_functions[sample_key];
989 }
990
991 simple_mtx_unlock(&matrix->lock);
992 }
993
994 static void
register_image_op(struct llvmpipe_context * ctx,uint32_t op)995 register_image_op(struct llvmpipe_context *ctx, uint32_t op)
996 {
997 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
998 if (BITSET_TEST(matrix->image_ops, op))
999 return;
1000
1001 BITSET_SET(matrix->image_ops, op);
1002
1003 simple_mtx_lock(&matrix->lock);
1004
1005 for (uint32_t texture_index = 0; texture_index < matrix->texture_count; texture_index++) {
1006 struct lp_texture_functions *texture = matrix->textures[texture_index];
1007 if (texture->storage)
1008 texture->image_functions[op] = compile_image_function(ctx, &texture->state, op);
1009 }
1010
1011 simple_mtx_unlock(&matrix->lock);
1012 }
1013
1014 static bool
register_instr(nir_builder * b,nir_instr * instr,void * data)1015 register_instr(nir_builder *b, nir_instr *instr, void *data)
1016 {
1017 struct llvmpipe_context *ctx = data;
1018
1019 if (instr->type == nir_instr_type_tex) {
1020 nir_tex_instr *tex = nir_instr_as_tex(instr);
1021 uint32_t sample_key = lp_build_nir_sample_key(b->shader->info.stage, tex);
1022
1023 register_sample_key(ctx, sample_key);
1024 } else if (instr->type == nir_instr_type_intrinsic) {
1025 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
1026
1027 struct lp_img_params params;
1028 lp_img_op_from_intrinsic(¶ms, intrin);
1029
1030 if (params.img_op == -1)
1031 return false;
1032
1033 uint32_t op = params.img_op;
1034 if (op == LP_IMG_ATOMIC_CAS)
1035 op--;
1036 else if (op == LP_IMG_ATOMIC)
1037 op = params.op + (LP_IMG_OP_COUNT - 1);
1038
1039 if (nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_MS ||
1040 nir_intrinsic_image_dim(intrin) == GLSL_SAMPLER_DIM_SUBPASS_MS)
1041 op += LP_TOTAL_IMAGE_OP_COUNT / 2;
1042
1043 register_image_op(ctx, op);
1044 }
1045
1046 return false;
1047 }
1048
1049 void
llvmpipe_register_shader(struct pipe_context * ctx,const struct pipe_shader_state * shader)1050 llvmpipe_register_shader(struct pipe_context *ctx, const struct pipe_shader_state *shader)
1051 {
1052 if (shader->type == PIPE_SHADER_IR_NIR)
1053 nir_shader_instructions_pass(shader->ir.nir, register_instr, nir_metadata_all, ctx);
1054 }
1055
1056 void
llvmpipe_clear_sample_functions_cache(struct llvmpipe_context * ctx,struct pipe_fence_handle ** fence)1057 llvmpipe_clear_sample_functions_cache(struct llvmpipe_context *ctx, struct pipe_fence_handle **fence)
1058 {
1059 if (!fence)
1060 return;
1061
1062 struct lp_sampler_matrix *matrix = &ctx->sampler_matrix;
1063
1064 /* If the cache is empty, there is nothing to do. */
1065 if (!_mesa_hash_table_num_entries(acquire_latest_sample_function_cache(matrix)))
1066 return;
1067
1068 ctx->pipe.screen->fence_finish(ctx->pipe.screen, NULL, *fence, OS_TIMEOUT_INFINITE);
1069
1070 /* All work is finished, it's safe to move cache entries into the table. */
1071 hash_table_foreach_remove(acquire_latest_sample_function_cache(matrix), entry) {
1072 struct sample_function_cache_key *key = (void *)entry->key;
1073 key->texture_functions->sample_functions[key->sampler_index][key->sample_key] = entry->data;
1074 free(key);
1075 }
1076
1077 util_dynarray_foreach (&matrix->trash_caches, struct hash_table *, trash)
1078 _mesa_hash_table_destroy(*trash, NULL);
1079 util_dynarray_clear(&matrix->trash_caches);
1080 }
1081