1 /*
2 * Copyright 2023 Alyssa Rosenzweig
3 * Copyright 2021 Intel Corporation
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "agx_compile.h"
8 #include "agx_tilebuffer.h"
9 #include "nir.h"
10 #include "nir_builder.h"
11 #include "nir_builder_opcodes.h"
12 #include "nir_intrinsics.h"
13
14 static bool
lower_to_per_sample(nir_builder * b,nir_intrinsic_instr * intr,void * data)15 lower_to_per_sample(nir_builder *b, nir_intrinsic_instr *intr, void *data)
16 {
17 b->cursor = nir_before_instr(&intr->instr);
18
19 switch (intr->intrinsic) {
20 case nir_intrinsic_load_sample_id: {
21 nir_def *mask = nir_u2u32(b, nir_load_active_samples_agx(b));
22 nir_def *bit = nir_ufind_msb(b, mask);
23 nir_def_replace(&intr->def, nir_u2uN(b, bit, intr->def.bit_size));
24 return true;
25 }
26
27 case nir_intrinsic_load_local_pixel_agx:
28 case nir_intrinsic_store_local_pixel_agx:
29 case nir_intrinsic_store_zs_agx:
30 case nir_intrinsic_discard_agx:
31 case nir_intrinsic_sample_mask_agx: {
32 /* Fragment I/O inside the loop should only affect active samples. */
33 unsigned mask_index =
34 (intr->intrinsic == nir_intrinsic_store_local_pixel_agx) ? 1 : 0;
35
36 nir_def *mask = intr->src[mask_index].ssa;
37 nir_def *id_mask = nir_load_active_samples_agx(b);
38 nir_def *converted = nir_u2uN(b, id_mask, mask->bit_size);
39
40 nir_src_rewrite(&intr->src[mask_index], nir_iand(b, mask, converted));
41 return true;
42 }
43
44 default:
45 return false;
46 }
47 }
48
49 bool
agx_nir_lower_to_per_sample(nir_shader * shader)50 agx_nir_lower_to_per_sample(nir_shader *shader)
51 {
52 return nir_shader_intrinsics_pass(shader, lower_to_per_sample,
53 nir_metadata_control_flow, NULL);
54 }
55
56 static bool
lower_active_samples(nir_builder * b,nir_intrinsic_instr * intr,void * data)57 lower_active_samples(nir_builder *b, nir_intrinsic_instr *intr, void *data)
58 {
59 if (intr->intrinsic != nir_intrinsic_load_active_samples_agx)
60 return false;
61
62 b->cursor = nir_instr_remove(&intr->instr);
63 nir_def_rewrite_uses(&intr->def, data);
64 return true;
65 }
66
67 /*
68 * In a monolithic pixel shader, we wrap the fragment shader in a loop over
69 * each sample, and then let optimizations (like loop unrolling) go to town.
70 * This lowering is not compatible with fragment epilogues, which require
71 * something similar at the binary level since the NIR is long gone by then.
72 */
73 static bool
agx_nir_wrap_per_sample_loop(nir_shader * shader,uint8_t nr_samples)74 agx_nir_wrap_per_sample_loop(nir_shader *shader, uint8_t nr_samples)
75 {
76 assert(nr_samples > 1);
77
78 /* Get the original function */
79 nir_function_impl *impl = nir_shader_get_entrypoint(shader);
80
81 nir_cf_list list;
82 nir_cf_extract(&list, nir_before_impl(impl), nir_after_impl(impl));
83
84 /* Create a builder for the wrapped function */
85 nir_builder b = nir_builder_at(nir_after_block(nir_start_block(impl)));
86
87 nir_variable *i =
88 nir_local_variable_create(impl, glsl_uintN_t_type(16), NULL);
89 nir_store_var(&b, i, nir_imm_intN_t(&b, 1, 16), ~0);
90 nir_def *bit = NULL;
91 nir_def *end_bit = nir_imm_intN_t(&b, 1 << nr_samples, 16);
92
93 /* Create a loop in the wrapped function */
94 nir_loop *loop = nir_push_loop(&b);
95 {
96 bit = nir_load_var(&b, i);
97 nir_push_if(&b, nir_uge(&b, bit, end_bit));
98 {
99 nir_jump(&b, nir_jump_break);
100 }
101 nir_pop_if(&b, NULL);
102
103 b.cursor = nir_cf_reinsert(&list, b.cursor);
104 nir_store_var(&b, i, nir_ishl_imm(&b, bit, 1), ~0);
105 }
106 nir_pop_loop(&b, loop);
107
108 /* We've mucked about with control flow */
109 nir_metadata_preserve(impl, nir_metadata_none);
110
111 /* Use the loop variable for the active sampple mask each iteration */
112 nir_shader_intrinsics_pass(shader, lower_active_samples,
113 nir_metadata_control_flow, bit);
114 return true;
115 }
116
117 /*
118 * Lower a fragment shader into a monolithic pixel shader, with static sample
119 * count, blend state, and tilebuffer formats in the shader key. For dynamic,
120 * epilogs must be used, which have separate lowerings.
121 */
122 bool
agx_nir_lower_monolithic_msaa(nir_shader * shader,uint8_t nr_samples)123 agx_nir_lower_monolithic_msaa(nir_shader *shader, uint8_t nr_samples)
124 {
125 assert(shader->info.stage == MESA_SHADER_FRAGMENT);
126 assert(nr_samples == 1 || nr_samples == 2 || nr_samples == 4);
127
128 agx_nir_lower_sample_mask(shader);
129
130 /* In single sampled programs, interpolateAtSample needs to return the
131 * center pixel.
132 */
133 if (nr_samples == 1)
134 nir_lower_single_sampled(shader);
135 else if (shader->info.fs.uses_sample_shading) {
136 agx_nir_lower_to_per_sample(shader);
137 agx_nir_wrap_per_sample_loop(shader, nr_samples);
138 }
139
140 return true;
141 }
142