1 /*
2 * Copyright © 2015 Red Hat
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include "st_nir.h"
25
26 #include "pipe/p_defines.h"
27 #include "pipe/p_screen.h"
28 #include "pipe/p_context.h"
29
30 #include "program/program.h"
31 #include "program/prog_statevars.h"
32 #include "program/prog_parameter.h"
33 #include "main/context.h"
34 #include "main/mtypes.h"
35 #include "main/errors.h"
36 #include "main/glspirv.h"
37 #include "main/shaderapi.h"
38 #include "main/uniforms.h"
39
40 #include "main/shaderobj.h"
41 #include "st_context.h"
42 #include "st_program.h"
43 #include "st_shader_cache.h"
44
45 #include "compiler/nir/nir.h"
46 #include "compiler/nir/nir_builder.h"
47 #include "compiler/glsl_types.h"
48 #include "compiler/glsl/glsl_to_nir.h"
49 #include "compiler/glsl/gl_nir.h"
50 #include "compiler/glsl/gl_nir_linker.h"
51 #include "compiler/glsl/ir.h"
52 #include "compiler/glsl/ir_optimization.h"
53 #include "compiler/glsl/linker_util.h"
54 #include "compiler/glsl/program.h"
55 #include "compiler/glsl/shader_cache.h"
56 #include "compiler/glsl/string_to_uint_map.h"
57
58 #include "util/log.h"
59
60 static int
type_size(const struct glsl_type * type)61 type_size(const struct glsl_type *type)
62 {
63 return glsl_count_attribute_slots(type, false);
64 }
65
66 /* Depending on PIPE_CAP_TGSI_TEXCOORD (st->needs_texcoord_semantic) we
67 * may need to fix up varying slots so the glsl->nir path is aligned
68 * with the anything->tgsi->nir path.
69 */
70 static void
st_nir_fixup_varying_slots(struct st_context * st,nir_shader * shader,nir_variable_mode mode)71 st_nir_fixup_varying_slots(struct st_context *st, nir_shader *shader,
72 nir_variable_mode mode)
73 {
74 if (st->needs_texcoord_semantic)
75 return;
76
77 /* This is called from finalize, but we don't want to do this adjustment twice. */
78 assert(!st->allow_st_finalize_nir_twice);
79
80 nir_foreach_variable_with_modes(var, shader, mode) {
81 if (var->data.location >= VARYING_SLOT_VAR0 && var->data.location < VARYING_SLOT_PATCH0) {
82 var->data.location += 9;
83 } else if (var->data.location == VARYING_SLOT_PNTC) {
84 var->data.location = VARYING_SLOT_VAR8;
85 } else if ((var->data.location >= VARYING_SLOT_TEX0) &&
86 (var->data.location <= VARYING_SLOT_TEX7)) {
87 var->data.location += VARYING_SLOT_VAR0 - VARYING_SLOT_TEX0;
88 }
89 }
90 }
91
92 /* input location assignment for VS inputs must be handled specially, so
93 * that it is aligned w/ st's vbo state.
94 * (This isn't the case with, for ex, FS inputs, which only need to agree
95 * on varying-slot w/ the VS outputs)
96 */
97 void
st_nir_assign_vs_in_locations(struct nir_shader * nir)98 st_nir_assign_vs_in_locations(struct nir_shader *nir)
99 {
100 if (nir->info.stage != MESA_SHADER_VERTEX || nir->info.io_lowered)
101 return;
102
103 nir->num_inputs = util_bitcount64(nir->info.inputs_read);
104
105 bool removed_inputs = false;
106
107 nir_foreach_shader_in_variable_safe(var, nir) {
108 /* NIR already assigns dual-slot inputs to two locations so all we have
109 * to do is compact everything down.
110 */
111 if (nir->info.inputs_read & BITFIELD64_BIT(var->data.location)) {
112 var->data.driver_location =
113 util_bitcount64(nir->info.inputs_read &
114 BITFIELD64_MASK(var->data.location));
115 } else {
116 /* Convert unused input variables to shader_temp (with no
117 * initialization), to avoid confusing drivers looking through the
118 * inputs array and expecting to find inputs with a driver_location
119 * set.
120 */
121 var->data.mode = nir_var_shader_temp;
122 removed_inputs = true;
123 }
124 }
125
126 /* Re-lower global vars, to deal with any dead VS inputs. */
127 if (removed_inputs)
128 NIR_PASS(_, nir, nir_lower_global_vars_to_local);
129 }
130
131 static int
st_nir_lookup_parameter_index(struct gl_program * prog,nir_variable * var)132 st_nir_lookup_parameter_index(struct gl_program *prog, nir_variable *var)
133 {
134 struct gl_program_parameter_list *params = prog->Parameters;
135
136 /* Lookup the first parameter that the uniform storage that match the
137 * variable location.
138 */
139 for (unsigned i = 0; i < params->NumParameters; i++) {
140 int index = params->Parameters[i].MainUniformStorageIndex;
141 if (index == var->data.location)
142 return i;
143 }
144
145 /* TODO: Handle this fallback for SPIR-V. We need this for GLSL e.g. in
146 * dEQP-GLES2.functional.uniform_api.random.3
147 */
148
149 /* is there a better way to do this? If we have something like:
150 *
151 * struct S {
152 * float f;
153 * vec4 v;
154 * };
155 * uniform S color;
156 *
157 * Then what we get in prog->Parameters looks like:
158 *
159 * 0: Name=color.f, Type=6, DataType=1406, Size=1
160 * 1: Name=color.v, Type=6, DataType=8b52, Size=4
161 *
162 * So the name doesn't match up and _mesa_lookup_parameter_index()
163 * fails. In this case just find the first matching "color.*"..
164 *
165 * Note for arrays you could end up w/ color[n].f, for example.
166 */
167 if (!prog->sh.data->spirv) {
168 int namelen = strlen(var->name);
169 for (unsigned i = 0; i < params->NumParameters; i++) {
170 struct gl_program_parameter *p = ¶ms->Parameters[i];
171 if ((strncmp(p->Name, var->name, namelen) == 0) &&
172 ((p->Name[namelen] == '.') || (p->Name[namelen] == '['))) {
173 return i;
174 }
175 }
176 }
177
178 return -1;
179 }
180
181 static void
st_nir_assign_uniform_locations(struct gl_context * ctx,struct gl_program * prog,nir_shader * nir)182 st_nir_assign_uniform_locations(struct gl_context *ctx,
183 struct gl_program *prog,
184 nir_shader *nir)
185 {
186 int shaderidx = 0;
187 int imageidx = 0;
188
189 nir_foreach_variable_with_modes(uniform, nir, nir_var_uniform |
190 nir_var_image) {
191 int loc;
192
193 const struct glsl_type *type = glsl_without_array(uniform->type);
194 if (!uniform->data.bindless && (glsl_type_is_sampler(type) || glsl_type_is_image(type))) {
195 if (glsl_type_is_sampler(type)) {
196 loc = shaderidx;
197 shaderidx += type_size(uniform->type);
198 } else {
199 loc = imageidx;
200 imageidx += type_size(uniform->type);
201 }
202 } else if (uniform->state_slots) {
203 const gl_state_index16 *const stateTokens = uniform->state_slots[0].tokens;
204
205 unsigned comps;
206 if (glsl_type_is_struct_or_ifc(type)) {
207 comps = 4;
208 } else {
209 comps = glsl_get_vector_elements(type);
210 }
211
212 if (ctx->Const.PackedDriverUniformStorage) {
213 loc = _mesa_add_sized_state_reference(prog->Parameters,
214 stateTokens, comps, false);
215 loc = prog->Parameters->Parameters[loc].ValueOffset;
216 } else {
217 loc = _mesa_add_state_reference(prog->Parameters, stateTokens);
218 }
219 } else {
220 loc = st_nir_lookup_parameter_index(prog, uniform);
221
222 /* We need to check that loc is not -1 here before accessing the
223 * array. It can be negative for example when we have a struct that
224 * only contains opaque types.
225 */
226 if (loc >= 0 && ctx->Const.PackedDriverUniformStorage) {
227 loc = prog->Parameters->Parameters[loc].ValueOffset;
228 }
229 }
230
231 uniform->data.driver_location = loc;
232 }
233 }
234
235 static bool
def_is_64bit(nir_def * def,void * state)236 def_is_64bit(nir_def *def, void *state)
237 {
238 bool *lower = (bool *)state;
239 if (def && (def->bit_size == 64)) {
240 *lower = true;
241 return false;
242 }
243 return true;
244 }
245
246 static bool
src_is_64bit(nir_src * src,void * state)247 src_is_64bit(nir_src *src, void *state)
248 {
249 bool *lower = (bool *)state;
250 if (src && (nir_src_bit_size(*src) == 64)) {
251 *lower = true;
252 return false;
253 }
254 return true;
255 }
256
257 static bool
filter_64_bit_instr(const nir_instr * const_instr,UNUSED const void * data)258 filter_64_bit_instr(const nir_instr *const_instr, UNUSED const void *data)
259 {
260 bool lower = false;
261 /* lower_alu_to_scalar required nir_instr to be const, but nir_foreach_*
262 * doesn't have const variants, so do the ugly const_cast here. */
263 nir_instr *instr = const_cast<nir_instr *>(const_instr);
264
265 nir_foreach_def(instr, def_is_64bit, &lower);
266 if (lower)
267 return true;
268 nir_foreach_src(instr, src_is_64bit, &lower);
269 return lower;
270 }
271
272 /* Second third of converting glsl_to_nir. This creates uniforms, gathers
273 * info on varyings, etc after NIR link time opts have been applied.
274 */
275 static char *
st_glsl_to_nir_post_opts(struct st_context * st,struct gl_program * prog,struct gl_shader_program * shader_program)276 st_glsl_to_nir_post_opts(struct st_context *st, struct gl_program *prog,
277 struct gl_shader_program *shader_program)
278 {
279 nir_shader *nir = prog->nir;
280 struct pipe_screen *screen = st->screen;
281
282 /* Make a pass over the IR to add state references for any built-in
283 * uniforms that are used. This has to be done now (during linking).
284 * Code generation doesn't happen until the first time this shader is
285 * used for rendering. Waiting until then to generate the parameters is
286 * too late. At that point, the values for the built-in uniforms won't
287 * get sent to the shader.
288 */
289 nir_foreach_uniform_variable(var, nir) {
290 const nir_state_slot *const slots = var->state_slots;
291 if (slots != NULL) {
292 const struct glsl_type *type = glsl_without_array(var->type);
293 for (unsigned int i = 0; i < var->num_state_slots; i++) {
294 unsigned comps;
295 if (glsl_type_is_struct_or_ifc(type)) {
296 comps = _mesa_program_state_value_size(slots[i].tokens);
297 } else {
298 comps = glsl_get_vector_elements(type);
299 }
300
301 if (st->ctx->Const.PackedDriverUniformStorage) {
302 _mesa_add_sized_state_reference(prog->Parameters,
303 slots[i].tokens,
304 comps, false);
305 } else {
306 _mesa_add_state_reference(prog->Parameters,
307 slots[i].tokens);
308 }
309 }
310 }
311 }
312
313 /* Avoid reallocation of the program parameter list, because the uniform
314 * storage is only associated with the original parameter list.
315 * This should be enough for Bitmap and DrawPixels constants.
316 */
317 _mesa_ensure_and_associate_uniform_storage(st->ctx, shader_program, prog, 28);
318
319 /* None of the builtins being lowered here can be produced by SPIR-V. See
320 * _mesa_builtin_uniform_desc. Also drivers that support packed uniform
321 * storage don't need to lower builtins.
322 */
323 if (!shader_program->data->spirv &&
324 !st->ctx->Const.PackedDriverUniformStorage)
325 NIR_PASS(_, nir, st_nir_lower_builtin);
326
327 if (!screen->get_param(screen, PIPE_CAP_NIR_ATOMICS_AS_DEREF))
328 NIR_PASS(_, nir, gl_nir_lower_atomics, shader_program, true);
329
330 NIR_PASS(_, nir, nir_opt_intrinsics);
331 NIR_PASS(_, nir, nir_opt_fragdepth);
332
333 /* Lower 64-bit ops. */
334 if (nir->options->lower_int64_options ||
335 nir->options->lower_doubles_options) {
336 bool lowered_64bit_ops = false;
337 bool revectorize = false;
338
339 if (nir->options->lower_doubles_options) {
340 /* nir_lower_doubles is not prepared for vector ops, so if the backend doesn't
341 * request lower_alu_to_scalar until now, lower all 64 bit ops, and try to
342 * vectorize them afterwards again */
343 if (!nir->options->lower_to_scalar) {
344 NIR_PASS(revectorize, nir, nir_lower_alu_to_scalar, filter_64_bit_instr, nullptr);
345 NIR_PASS(revectorize, nir, nir_lower_phis_to_scalar, false);
346 }
347 /* doubles lowering requires frexp to be lowered first if it will be,
348 * since the pass generates other 64-bit ops. Most backends lower
349 * frexp, and using doubles is rare, and using frexp is even more rare
350 * (no instances in shader-db), so we're not too worried about
351 * accidentally lowering a 32-bit frexp here.
352 */
353 NIR_PASS(lowered_64bit_ops, nir, nir_lower_frexp);
354
355 NIR_PASS(lowered_64bit_ops, nir, nir_lower_doubles,
356 st->ctx->SoftFP64, nir->options->lower_doubles_options);
357 }
358 if (nir->options->lower_int64_options)
359 NIR_PASS(lowered_64bit_ops, nir, nir_lower_int64);
360
361 if (revectorize && !nir->options->vectorize_vec2_16bit)
362 NIR_PASS(_, nir, nir_opt_vectorize, nullptr, nullptr);
363
364 if (revectorize || lowered_64bit_ops)
365 gl_nir_opts(nir);
366 }
367
368 nir_variable_mode mask =
369 nir_var_shader_in | nir_var_shader_out | nir_var_function_temp;
370 nir_remove_dead_variables(nir, mask, NULL);
371
372 if (!st->has_hw_atomics && !screen->get_param(screen, PIPE_CAP_NIR_ATOMICS_AS_DEREF)) {
373 unsigned align_offset_state = 0;
374 if (st->ctx->Const.ShaderStorageBufferOffsetAlignment > 4) {
375 struct gl_program_parameter_list *params = prog->Parameters;
376 for (unsigned i = 0; i < shader_program->data->NumAtomicBuffers; i++) {
377 gl_state_index16 state[STATE_LENGTH] = { STATE_ATOMIC_COUNTER_OFFSET, (short)shader_program->data->AtomicBuffers[i].Binding };
378 _mesa_add_state_reference(params, state);
379 }
380 align_offset_state = STATE_ATOMIC_COUNTER_OFFSET;
381 }
382 NIR_PASS(_, nir, nir_lower_atomics_to_ssbo, align_offset_state);
383 }
384
385 st_set_prog_affected_state_flags(prog);
386
387 st_finalize_nir_before_variants(nir);
388
389 char *msg = NULL;
390 if (st->allow_st_finalize_nir_twice) {
391 st_serialize_base_nir(prog, nir);
392 msg = st_finalize_nir(st, prog, shader_program, nir, true, true, false);
393 }
394
395 if (st->ctx->_Shader->Flags & GLSL_DUMP) {
396 _mesa_log("\n");
397 _mesa_log("NIR IR for linked %s program %d:\n",
398 _mesa_shader_stage_to_string(prog->info.stage),
399 shader_program->Name);
400 nir_print_shader(nir, mesa_log_get_file());
401 _mesa_log("\n\n");
402 }
403
404 return msg;
405 }
406
407 static void
st_nir_vectorize_io(nir_shader * producer,nir_shader * consumer)408 st_nir_vectorize_io(nir_shader *producer, nir_shader *consumer)
409 {
410 if (consumer)
411 NIR_PASS(_, consumer, nir_lower_io_to_vector, nir_var_shader_in);
412
413 if (!producer)
414 return;
415
416 NIR_PASS(_, producer, nir_lower_io_to_vector, nir_var_shader_out);
417
418 if (producer->info.stage == MESA_SHADER_TESS_CTRL &&
419 producer->options->vectorize_tess_levels)
420 NIR_PASS(_, producer, nir_vectorize_tess_levels);
421
422 NIR_PASS(_, producer, nir_opt_combine_stores, nir_var_shader_out);
423
424 if ((producer)->info.stage != MESA_SHADER_TESS_CTRL) {
425 /* Calling lower_io_to_vector creates output variable writes with
426 * write-masks. We only support these for TCS outputs, so for other
427 * stages, we need to call nir_lower_io_to_temporaries to get rid of
428 * them. This, in turn, creates temporary variables and extra
429 * copy_deref intrinsics that we need to clean up.
430 */
431 NIR_PASS(_, producer, nir_lower_io_to_temporaries,
432 nir_shader_get_entrypoint(producer), true, false);
433 NIR_PASS(_, producer, nir_lower_global_vars_to_local);
434 NIR_PASS(_, producer, nir_split_var_copies);
435 NIR_PASS(_, producer, nir_lower_var_copies);
436 }
437
438 /* Undef scalar store_deref intrinsics are not ignored by nir_lower_io,
439 * so they must be removed before that. These passes remove them.
440 */
441 NIR_PASS(_, producer, nir_lower_vars_to_ssa);
442 NIR_PASS(_, producer, nir_opt_undef);
443 NIR_PASS(_, producer, nir_opt_dce);
444 }
445
446 extern "C" {
447
448 bool
st_nir_lower_wpos_ytransform(struct nir_shader * nir,struct gl_program * prog,struct pipe_screen * pscreen)449 st_nir_lower_wpos_ytransform(struct nir_shader *nir,
450 struct gl_program *prog,
451 struct pipe_screen *pscreen)
452 {
453 bool progress = false;
454
455 if (nir->info.stage != MESA_SHADER_FRAGMENT) {
456 nir_shader_preserve_all_metadata(nir);
457 return progress;
458 }
459
460 static const gl_state_index16 wposTransformState[STATE_LENGTH] = {
461 STATE_FB_WPOS_Y_TRANSFORM
462 };
463 nir_lower_wpos_ytransform_options wpos_options = { { 0 } };
464
465 memcpy(wpos_options.state_tokens, wposTransformState,
466 sizeof(wpos_options.state_tokens));
467 wpos_options.fs_coord_origin_upper_left =
468 pscreen->get_param(pscreen,
469 PIPE_CAP_FS_COORD_ORIGIN_UPPER_LEFT);
470 wpos_options.fs_coord_origin_lower_left =
471 pscreen->get_param(pscreen,
472 PIPE_CAP_FS_COORD_ORIGIN_LOWER_LEFT);
473 wpos_options.fs_coord_pixel_center_integer =
474 pscreen->get_param(pscreen,
475 PIPE_CAP_FS_COORD_PIXEL_CENTER_INTEGER);
476 wpos_options.fs_coord_pixel_center_half_integer =
477 pscreen->get_param(pscreen,
478 PIPE_CAP_FS_COORD_PIXEL_CENTER_HALF_INTEGER);
479
480 if (nir_lower_wpos_ytransform(nir, &wpos_options)) {
481 _mesa_add_state_reference(prog->Parameters, wposTransformState);
482 progress = true;
483 }
484
485 static const gl_state_index16 pntcTransformState[STATE_LENGTH] = {
486 STATE_FB_PNTC_Y_TRANSFORM
487 };
488
489 if (nir_lower_pntc_ytransform(nir, &pntcTransformState)) {
490 _mesa_add_state_reference(prog->Parameters, pntcTransformState);
491 progress = true;
492 }
493
494 return progress;
495 }
496
497 static bool
st_link_glsl_to_nir(struct gl_context * ctx,struct gl_shader_program * shader_program)498 st_link_glsl_to_nir(struct gl_context *ctx,
499 struct gl_shader_program *shader_program)
500 {
501 struct st_context *st = st_context(ctx);
502 struct gl_linked_shader *linked_shader[MESA_SHADER_STAGES];
503 unsigned num_shaders = 0;
504
505 /* Return early if we are loading the shader from on-disk cache */
506 if (st_load_nir_from_disk_cache(ctx, shader_program)) {
507 return GL_TRUE;
508 }
509
510 MESA_TRACE_FUNC();
511
512 assert(shader_program->data->LinkStatus);
513
514 for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
515 if (shader_program->_LinkedShaders[i])
516 linked_shader[num_shaders++] = shader_program->_LinkedShaders[i];
517 }
518
519 for (unsigned i = 0; i < num_shaders; i++) {
520 struct gl_linked_shader *shader = linked_shader[i];
521 const nir_shader_compiler_options *options =
522 st->ctx->Const.ShaderCompilerOptions[shader->Stage].NirOptions;
523 struct gl_program *prog = shader->Program;
524
525 shader->Program->info.separate_shader = shader_program->SeparateShader;
526
527 assert(!prog->nir);
528 prog->shader_program = shader_program;
529 prog->state.type = PIPE_SHADER_IR_NIR;
530
531 /* Parameters will be filled during NIR linking. */
532 prog->Parameters = _mesa_new_parameter_list();
533
534 if (shader_program->data->spirv) {
535 prog->nir = _mesa_spirv_to_nir(ctx, shader_program, shader->Stage, options);
536 } else {
537 if (ctx->_Shader->Flags & GLSL_DUMP) {
538 _mesa_log("\n");
539 _mesa_log("GLSL IR for linked %s program %d:\n",
540 _mesa_shader_stage_to_string(shader->Stage),
541 shader_program->Name);
542 _mesa_print_ir(mesa_log_get_file(), shader->ir, NULL);
543 _mesa_log("\n\n");
544 }
545
546 prog->nir = glsl_to_nir(&st->ctx->Const, &shader->ir,
547 &shader->Program->info, shader->Stage, options);
548
549 prog->nir->info.name =
550 ralloc_asprintf(shader, "GLSL%d", shader_program->Name);
551 if (shader_program->Label)
552 prog->nir->info.label = ralloc_strdup(shader, shader_program->Label);
553
554 if (prog->nir->info.stage == MESA_SHADER_FRAGMENT) {
555 prog->nir->info.fs.pixel_center_integer = prog->info.fs.pixel_center_integer;
556 prog->nir->info.fs.origin_upper_left = prog->info.fs.origin_upper_left;
557 prog->nir->info.fs.advanced_blend_modes = prog->info.fs.advanced_blend_modes;
558 }
559 }
560
561 memcpy(prog->nir->info.source_blake3, shader->linked_source_blake3,
562 BLAKE3_OUT_LEN);
563
564 nir_shader_gather_info(prog->nir, nir_shader_get_entrypoint(prog->nir));
565 if (!st->ctx->SoftFP64 && ((prog->nir->info.bit_sizes_int | prog->nir->info.bit_sizes_float) & 64) &&
566 (options->lower_doubles_options & nir_lower_fp64_full_software) != 0) {
567
568 /* It's not possible to use float64 on GLSL ES, so don't bother trying to
569 * build the support code. The support code depends on higher versions of
570 * desktop GLSL, so it will fail to compile (below) anyway.
571 */
572 if (_mesa_is_desktop_gl(st->ctx) && st->ctx->Const.GLSLVersion >= 400)
573 st->ctx->SoftFP64 = glsl_float64_funcs_to_nir(st->ctx, options);
574 }
575 }
576
577 if (shader_program->data->spirv) {
578 static const gl_nir_linker_options opts = {
579 true /*fill_parameters */
580 };
581 if (!gl_nir_link_spirv(&ctx->Const, &ctx->Extensions, shader_program,
582 &opts))
583 return GL_FALSE;
584 } else {
585 if (!gl_nir_link_glsl(ctx, shader_program))
586 return GL_FALSE;
587 }
588
589 for (unsigned i = 0; i < num_shaders; i++) {
590 struct gl_program *prog = linked_shader[i]->Program;
591 prog->ExternalSamplersUsed = gl_external_samplers(prog);
592 _mesa_update_shader_textures_used(shader_program, prog);
593 }
594
595 nir_build_program_resource_list(&ctx->Const, shader_program,
596 shader_program->data->spirv);
597
598 for (unsigned i = 0; i < num_shaders; i++) {
599 struct gl_linked_shader *shader = linked_shader[i];
600 nir_shader *nir = shader->Program->nir;
601 gl_shader_stage stage = shader->Stage;
602 const struct gl_shader_compiler_options *options =
603 &ctx->Const.ShaderCompilerOptions[stage];
604
605 if (nir->info.io_lowered) {
606 /* Since IO is lowered, we won't need the IO variables from now on.
607 * nir_build_program_resource_list was the last pass that needed them.
608 */
609 NIR_PASS_V(nir, nir_remove_dead_variables,
610 nir_var_shader_in | nir_var_shader_out, NULL);
611 }
612
613 /* If there are forms of indirect addressing that the driver
614 * cannot handle, perform the lowering pass.
615 */
616 if (options->EmitNoIndirectInput || options->EmitNoIndirectOutput ||
617 options->EmitNoIndirectTemp || options->EmitNoIndirectUniform) {
618 nir_variable_mode mode = (nir_variable_mode)0;
619
620 if (!nir->info.io_lowered) {
621 mode |= options->EmitNoIndirectInput ?
622 nir_var_shader_in : (nir_variable_mode)0;
623 mode |= options->EmitNoIndirectOutput ?
624 nir_var_shader_out : (nir_variable_mode)0;
625 }
626 mode |= options->EmitNoIndirectTemp ?
627 nir_var_function_temp : (nir_variable_mode)0;
628 mode |= options->EmitNoIndirectUniform ?
629 nir_var_uniform | nir_var_mem_ubo | nir_var_mem_ssbo :
630 (nir_variable_mode)0;
631
632 if (mode)
633 nir_lower_indirect_derefs(nir, mode, UINT32_MAX);
634 }
635
636 /* This needs to run after the initial pass of nir_lower_vars_to_ssa, so
637 * that the buffer indices are constants in nir where they where
638 * constants in GLSL. */
639 NIR_PASS(_, nir, gl_nir_lower_buffers, shader_program);
640
641 NIR_PASS(_, nir, st_nir_lower_wpos_ytransform, shader->Program,
642 st->screen);
643
644 /* needed to lower base_workgroup_id and base_global_invocation_id */
645 struct nir_lower_compute_system_values_options cs_options = {};
646 NIR_PASS(_, nir, nir_lower_system_values);
647 NIR_PASS(_, nir, nir_lower_compute_system_values, &cs_options);
648
649 if (nir->info.io_lowered)
650 continue; /* the rest is for non-lowered IO only */
651
652 /* Remap the locations to slots so those requiring two slots will occupy
653 * two locations. For instance, if we have in the IR code a dvec3 attr0 in
654 * location 0 and vec4 attr1 in location 1, in NIR attr0 will use
655 * locations/slots 0 and 1, and attr1 will use location/slot 2
656 */
657 if (nir->info.stage == MESA_SHADER_VERTEX && !shader_program->data->spirv)
658 nir_remap_dual_slot_attributes(nir, &shader->Program->DualSlotInputs);
659
660 if (i >= 1) {
661 struct gl_program *prev_shader = linked_shader[i - 1]->Program;
662
663 /* We can't use nir_compact_varyings with transform feedback, since
664 * the pipe_stream_output->output_register field is based on the
665 * pre-compacted driver_locations.
666 */
667 if (!(prev_shader->sh.LinkedTransformFeedback &&
668 prev_shader->sh.LinkedTransformFeedback->NumVarying > 0))
669 nir_compact_varyings(prev_shader->nir,
670 nir, ctx->API != API_OPENGL_COMPAT);
671
672 if (ctx->Const.ShaderCompilerOptions[shader->Stage].NirOptions->vectorize_io)
673 st_nir_vectorize_io(prev_shader->nir, nir);
674 }
675 }
676
677 /* If the program is a separate shader program check if we need to vectorise
678 * the first and last program interfaces too.
679 */
680 if (shader_program->SeparateShader && num_shaders > 0) {
681 struct gl_linked_shader *first_shader = linked_shader[0];
682 struct gl_linked_shader *last_shader = linked_shader[num_shaders - 1];
683 if (first_shader->Stage != MESA_SHADER_COMPUTE) {
684 if (ctx->Const.ShaderCompilerOptions[first_shader->Stage].NirOptions->vectorize_io &&
685 first_shader->Stage > MESA_SHADER_VERTEX)
686 st_nir_vectorize_io(NULL, first_shader->Program->nir);
687
688 if (ctx->Const.ShaderCompilerOptions[last_shader->Stage].NirOptions->vectorize_io &&
689 last_shader->Stage < MESA_SHADER_FRAGMENT)
690 st_nir_vectorize_io(last_shader->Program->nir, NULL);
691 }
692 }
693
694 struct shader_info *prev_info = NULL;
695
696 for (unsigned i = 0; i < num_shaders; i++) {
697 struct gl_linked_shader *shader = linked_shader[i];
698 struct shader_info *info = &shader->Program->nir->info;
699
700 char *msg = st_glsl_to_nir_post_opts(st, shader->Program, shader_program);
701 if (msg) {
702 linker_error(shader_program, msg);
703 return false;
704 }
705
706 if (prev_info &&
707 ctx->Const.ShaderCompilerOptions[shader->Stage].NirOptions->unify_interfaces) {
708 prev_info->outputs_written |= info->inputs_read &
709 ~(VARYING_BIT_TESS_LEVEL_INNER | VARYING_BIT_TESS_LEVEL_OUTER);
710 info->inputs_read |= prev_info->outputs_written &
711 ~(VARYING_BIT_TESS_LEVEL_INNER | VARYING_BIT_TESS_LEVEL_OUTER);
712
713 prev_info->patch_outputs_written |= info->patch_inputs_read;
714 info->patch_inputs_read |= prev_info->patch_outputs_written;
715 }
716 prev_info = info;
717 }
718
719 for (unsigned i = 0; i < num_shaders; i++) {
720 struct gl_linked_shader *shader = linked_shader[i];
721 struct gl_program *prog = shader->Program;
722
723 /* Make sure that prog->info is in sync with nir->info, but st/mesa
724 * expects some of the values to be from before lowering.
725 */
726 shader_info old_info = prog->info;
727 prog->info = prog->nir->info;
728 prog->info.name = old_info.name;
729 prog->info.label = old_info.label;
730 prog->info.num_ssbos = old_info.num_ssbos;
731 prog->info.num_ubos = old_info.num_ubos;
732 prog->info.num_abos = old_info.num_abos;
733
734 if (prog->info.stage == MESA_SHADER_VERTEX) {
735 if (prog->nir->info.io_lowered &&
736 prog->nir->options->io_options & nir_io_glsl_opt_varyings) {
737 prog->info.inputs_read = prog->nir->info.inputs_read;
738 prog->DualSlotInputs = prog->nir->info.dual_slot_inputs;
739 } else {
740 /* NIR expands dual-slot inputs out to two locations. We need to
741 * compact things back down GL-style single-slot inputs to avoid
742 * confusing the state tracker.
743 */
744 prog->info.inputs_read =
745 nir_get_single_slot_attribs_mask(prog->nir->info.inputs_read,
746 prog->DualSlotInputs);
747 }
748
749 /* Initialize st_vertex_program members. */
750 st_prepare_vertex_program(prog);
751 }
752
753 /* Get pipe_stream_output_info. */
754 if (shader->Stage == MESA_SHADER_VERTEX ||
755 shader->Stage == MESA_SHADER_TESS_EVAL ||
756 shader->Stage == MESA_SHADER_GEOMETRY)
757 st_translate_stream_output_info(prog);
758
759 st_store_nir_in_disk_cache(st, prog);
760
761 st_release_variants(st, prog);
762 st_finalize_program(st, prog);
763 }
764
765 struct pipe_context *pctx = st_context(ctx)->pipe;
766 if (pctx->link_shader) {
767 void *driver_handles[PIPE_SHADER_TYPES];
768 memset(driver_handles, 0, sizeof(driver_handles));
769
770 for (uint32_t i = 0; i < MESA_SHADER_STAGES; ++i) {
771 struct gl_linked_shader *shader = shader_program->_LinkedShaders[i];
772 if (shader) {
773 struct gl_program *p = shader->Program;
774 if (p && p->variants) {
775 enum pipe_shader_type type = pipe_shader_type_from_mesa(shader->Stage);
776 driver_handles[type] = p->variants->driver_shader;
777 }
778 }
779 }
780
781 pctx->link_shader(pctx, driver_handles);
782 }
783
784 return true;
785 }
786
787 void
st_nir_assign_varying_locations(struct st_context * st,nir_shader * nir)788 st_nir_assign_varying_locations(struct st_context *st, nir_shader *nir)
789 {
790 /* Lowered IO don't have variables, so exit. */
791 if (nir->info.io_lowered)
792 return;
793
794 if (nir->info.stage == MESA_SHADER_VERTEX) {
795 nir_assign_io_var_locations(nir, nir_var_shader_out,
796 &nir->num_outputs,
797 nir->info.stage);
798 st_nir_fixup_varying_slots(st, nir, nir_var_shader_out);
799 } else if (nir->info.stage == MESA_SHADER_GEOMETRY ||
800 nir->info.stage == MESA_SHADER_TESS_CTRL ||
801 nir->info.stage == MESA_SHADER_TESS_EVAL) {
802 nir_assign_io_var_locations(nir, nir_var_shader_in,
803 &nir->num_inputs,
804 nir->info.stage);
805 st_nir_fixup_varying_slots(st, nir, nir_var_shader_in);
806
807 nir_assign_io_var_locations(nir, nir_var_shader_out,
808 &nir->num_outputs,
809 nir->info.stage);
810 st_nir_fixup_varying_slots(st, nir, nir_var_shader_out);
811 } else if (nir->info.stage == MESA_SHADER_FRAGMENT) {
812 nir_assign_io_var_locations(nir, nir_var_shader_in,
813 &nir->num_inputs,
814 nir->info.stage);
815 st_nir_fixup_varying_slots(st, nir, nir_var_shader_in);
816 nir_assign_io_var_locations(nir, nir_var_shader_out,
817 &nir->num_outputs,
818 nir->info.stage);
819 } else if (nir->info.stage == MESA_SHADER_COMPUTE) {
820 /* TODO? */
821 } else {
822 unreachable("invalid shader type");
823 }
824 }
825
826 void
st_nir_lower_samplers(struct pipe_screen * screen,nir_shader * nir,struct gl_shader_program * shader_program,struct gl_program * prog)827 st_nir_lower_samplers(struct pipe_screen *screen, nir_shader *nir,
828 struct gl_shader_program *shader_program,
829 struct gl_program *prog)
830 {
831 if (screen->get_param(screen, PIPE_CAP_NIR_SAMPLERS_AS_DEREF))
832 NIR_PASS(_, nir, gl_nir_lower_samplers_as_deref, shader_program);
833 else
834 NIR_PASS(_, nir, gl_nir_lower_samplers, shader_program);
835
836 if (prog) {
837 BITSET_COPY(prog->info.textures_used, nir->info.textures_used);
838 BITSET_COPY(prog->info.textures_used_by_txf, nir->info.textures_used_by_txf);
839 BITSET_COPY(prog->info.samplers_used, nir->info.samplers_used);
840 BITSET_COPY(prog->info.images_used, nir->info.images_used);
841 BITSET_COPY(prog->info.image_buffers, nir->info.image_buffers);
842 BITSET_COPY(prog->info.msaa_images, nir->info.msaa_images);
843 }
844 }
845
846 static int
st_packed_uniforms_type_size(const struct glsl_type * type,bool bindless)847 st_packed_uniforms_type_size(const struct glsl_type *type, bool bindless)
848 {
849 return glsl_count_dword_slots(type, bindless);
850 }
851
852 static int
st_unpacked_uniforms_type_size(const struct glsl_type * type,bool bindless)853 st_unpacked_uniforms_type_size(const struct glsl_type *type, bool bindless)
854 {
855 return glsl_count_vec4_slots(type, false, bindless);
856 }
857
858 void
st_nir_lower_uniforms(struct st_context * st,nir_shader * nir)859 st_nir_lower_uniforms(struct st_context *st, nir_shader *nir)
860 {
861 if (st->ctx->Const.PackedDriverUniformStorage) {
862 NIR_PASS(_, nir, nir_lower_io, nir_var_uniform,
863 st_packed_uniforms_type_size,
864 (nir_lower_io_options)0);
865 } else {
866 NIR_PASS(_, nir, nir_lower_io, nir_var_uniform,
867 st_unpacked_uniforms_type_size,
868 (nir_lower_io_options)0);
869 }
870
871 if (nir->options->lower_uniforms_to_ubo)
872 NIR_PASS(_, nir, nir_lower_uniforms_to_ubo,
873 st->ctx->Const.PackedDriverUniformStorage,
874 !st->ctx->Const.NativeIntegers);
875 }
876
877 /* Last third of preparing nir from glsl, which happens after shader
878 * variant lowering.
879 */
880 char *
st_finalize_nir(struct st_context * st,struct gl_program * prog,struct gl_shader_program * shader_program,nir_shader * nir,bool finalize_by_driver,bool is_before_variants,bool is_draw_shader)881 st_finalize_nir(struct st_context *st, struct gl_program *prog,
882 struct gl_shader_program *shader_program,
883 nir_shader *nir, bool finalize_by_driver,
884 bool is_before_variants,
885 bool is_draw_shader)
886 {
887 struct pipe_screen *screen = st->screen;
888
889 MESA_TRACE_FUNC();
890
891 NIR_PASS(_, nir, nir_split_var_copies);
892 NIR_PASS(_, nir, nir_lower_var_copies);
893
894 const bool lower_tg4_offsets =
895 !is_draw_shader && !st->screen->get_param(screen, PIPE_CAP_TEXTURE_GATHER_OFFSETS);
896
897 if (!is_draw_shader && (st->lower_rect_tex || lower_tg4_offsets)) {
898 struct nir_lower_tex_options opts = {0};
899 opts.lower_rect = !!st->lower_rect_tex;
900 opts.lower_tg4_offsets = lower_tg4_offsets;
901
902 NIR_PASS(_, nir, nir_lower_tex, &opts);
903 }
904
905 st_nir_assign_varying_locations(st, nir);
906 st_nir_assign_uniform_locations(st->ctx, prog, nir);
907
908 /* Lower load_deref/store_deref of inputs and outputs.
909 * This depends on st_nir_assign_varying_locations.
910 *
911 * TODO: remove this once nir_io_glsl_opt_varyings is enabled by default.
912 */
913 if (!is_draw_shader && nir->options->io_options & nir_io_glsl_lower_derefs &&
914 !(nir->options->io_options & nir_io_glsl_opt_varyings)) {
915 nir_lower_io_passes(nir, false);
916 NIR_PASS(_, nir, nir_remove_dead_variables,
917 nir_var_shader_in | nir_var_shader_out, NULL);
918 }
919
920 /* Set num_uniforms in number of attribute slots (vec4s) */
921 nir->num_uniforms = DIV_ROUND_UP(prog->Parameters->NumParameterValues, 4);
922
923 st_nir_lower_uniforms(st, nir);
924
925 if (!is_draw_shader && is_before_variants && nir->options->lower_uniforms_to_ubo) {
926 /* This must be done after uniforms are lowered to UBO and all
927 * nir_var_uniform variables are removed from NIR to prevent conflicts
928 * between state parameter merging and shader variant generation.
929 */
930 _mesa_optimize_state_parameters(&st->ctx->Const, prog->Parameters);
931 }
932
933 st_nir_lower_samplers(screen, nir, shader_program, prog);
934 if (!is_draw_shader && !screen->get_param(screen, PIPE_CAP_NIR_IMAGES_AS_DEREF))
935 NIR_PASS(_, nir, gl_nir_lower_images, false);
936
937 char *msg = NULL;
938 if (!is_draw_shader && finalize_by_driver && screen->finalize_nir)
939 msg = screen->finalize_nir(screen, nir);
940
941 return msg;
942 }
943
944 /**
945 * Link a GLSL shader program. Called via glLinkProgram().
946 */
947 void
st_link_shader(struct gl_context * ctx,struct gl_shader_program * prog)948 st_link_shader(struct gl_context *ctx, struct gl_shader_program *prog)
949 {
950 unsigned int i;
951 bool spirv = false;
952
953 MESA_TRACE_FUNC();
954
955 _mesa_clear_shader_program_data(ctx, prog);
956
957 prog->data = _mesa_create_shader_program_data();
958
959 prog->data->LinkStatus = LINKING_SUCCESS;
960
961 for (i = 0; i < prog->NumShaders; i++) {
962 if (!prog->Shaders[i]->CompileStatus) {
963 linker_error(prog, "linking with uncompiled/unspecialized shader");
964 }
965
966 if (!i) {
967 spirv = (prog->Shaders[i]->spirv_data != NULL);
968 } else if (spirv && !prog->Shaders[i]->spirv_data) {
969 /* The GL_ARB_gl_spirv spec adds a new bullet point to the list of
970 * reasons LinkProgram can fail:
971 *
972 * "All the shader objects attached to <program> do not have the
973 * same value for the SPIR_V_BINARY_ARB state."
974 */
975 linker_error(prog,
976 "not all attached shaders have the same "
977 "SPIR_V_BINARY_ARB state");
978 }
979 }
980 prog->data->spirv = spirv;
981
982 if (prog->data->LinkStatus) {
983 if (!spirv)
984 link_shaders(ctx, prog);
985 else
986 _mesa_spirv_link_shaders(ctx, prog);
987 }
988
989 /* If LinkStatus is LINKING_SUCCESS, then reset sampler validated to true.
990 * Validation happens via the LinkShader call below. If LinkStatus is
991 * LINKING_SKIPPED, then SamplersValidated will have been restored from the
992 * shader cache.
993 */
994 if (prog->data->LinkStatus == LINKING_SUCCESS) {
995 prog->SamplersValidated = GL_TRUE;
996 }
997
998 if (prog->data->LinkStatus && !st_link_glsl_to_nir(ctx, prog)) {
999 prog->data->LinkStatus = LINKING_FAILURE;
1000 }
1001
1002 if (prog->data->LinkStatus != LINKING_FAILURE)
1003 _mesa_create_program_resource_hash(prog);
1004
1005 /* Return early if we are loading the shader from on-disk cache */
1006 if (prog->data->LinkStatus == LINKING_SKIPPED)
1007 return;
1008
1009 if (ctx->_Shader->Flags & GLSL_DUMP) {
1010 if (!prog->data->LinkStatus) {
1011 fprintf(stderr, "GLSL shader program %d failed to link\n", prog->Name);
1012 }
1013
1014 if (prog->data->InfoLog && prog->data->InfoLog[0] != 0) {
1015 fprintf(stderr, "GLSL shader program %d info log:\n", prog->Name);
1016 fprintf(stderr, "%s\n", prog->data->InfoLog);
1017 }
1018 }
1019
1020 #ifdef ENABLE_SHADER_CACHE
1021 if (prog->data->LinkStatus)
1022 shader_cache_write_program_metadata(ctx, prog);
1023 #endif
1024 }
1025
1026 } /* extern "C" */
1027