1 /*
2 * Copyright 2022 Alyssa Rosenzweig
3 * SPDX-License-Identifier: MIT
4 */
5
6 #include "compiler/nir/nir.h"
7 #include "compiler/nir/nir_builder.h"
8 #include "agx_compiler.h"
9 #include "nir_builder_opcodes.h"
10
11 #define ALL_SAMPLES 0xFF
12 #define BASE_Z 1
13 #define BASE_S 2
14
15 static bool
lower_zs_emit(nir_block * block,bool force_early_z)16 lower_zs_emit(nir_block *block, bool force_early_z)
17 {
18 nir_intrinsic_instr *zs_emit = NULL;
19 bool progress = false;
20
21 nir_foreach_instr_reverse_safe(instr, block) {
22 if (instr->type != nir_instr_type_intrinsic)
23 continue;
24
25 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
26 if (intr->intrinsic != nir_intrinsic_store_output)
27 continue;
28
29 nir_io_semantics sem = nir_intrinsic_io_semantics(intr);
30 if (sem.location != FRAG_RESULT_DEPTH &&
31 sem.location != FRAG_RESULT_STENCIL)
32 continue;
33
34 /* If early-Z is forced, z/s writes are a no-op (and will cause problems
35 * later in the compile). Piglit early-z tests this. Just remove the
36 * offending writes.
37 */
38 if (force_early_z) {
39 nir_instr_remove(instr);
40 progress = true;
41 continue;
42 }
43
44 nir_builder b = nir_builder_at(nir_before_instr(instr));
45
46 nir_def *value = intr->src[0].ssa;
47 bool z = (sem.location == FRAG_RESULT_DEPTH);
48
49 unsigned src_idx = z ? 1 : 2;
50 unsigned base = z ? BASE_Z : BASE_S;
51
52 /* In the hw, depth is 32-bit but stencil is 16-bit. Instruction
53 * selection checks this, so emit the conversion now.
54 */
55 if (z)
56 value = nir_f2f32(&b, value);
57 else
58 value = nir_u2u16(&b, value);
59
60 if (zs_emit == NULL) {
61 /* Multisampling will get lowered later if needed, default to
62 * broadcast
63 */
64 nir_def *sample_mask = nir_imm_intN_t(&b, ALL_SAMPLES, 16);
65 zs_emit =
66 nir_store_zs_agx(&b, sample_mask, nir_undef(&b, 1, 32) /* depth */,
67 nir_undef(&b, 1, 16) /* stencil */);
68 }
69
70 assert((nir_intrinsic_base(zs_emit) & base) == 0 &&
71 "each of depth/stencil may only be written once");
72
73 nir_src_rewrite(&zs_emit->src[src_idx], value);
74 nir_intrinsic_set_base(zs_emit, nir_intrinsic_base(zs_emit) | base);
75
76 nir_instr_remove(instr);
77 progress = true;
78 }
79
80 return progress;
81 }
82
83 static bool
lower_discard(nir_builder * b,nir_intrinsic_instr * intr,UNUSED void * data)84 lower_discard(nir_builder *b, nir_intrinsic_instr *intr, UNUSED void *data)
85 {
86 if (intr->intrinsic != nir_intrinsic_demote &&
87 intr->intrinsic != nir_intrinsic_demote_if)
88 return false;
89
90 b->cursor = nir_before_instr(&intr->instr);
91
92 nir_def *all_samples = nir_imm_intN_t(b, ALL_SAMPLES, 16);
93 nir_def *no_samples = nir_imm_intN_t(b, 0, 16);
94 nir_def *killed_samples = all_samples;
95
96 if (intr->intrinsic == nir_intrinsic_demote_if)
97 killed_samples = nir_bcsel(b, intr->src[0].ssa, all_samples, no_samples);
98
99 /* This will get lowered later as needed */
100 nir_discard_agx(b, killed_samples);
101 nir_instr_remove(&intr->instr);
102 return true;
103 }
104
105 static bool
agx_nir_lower_discard(nir_shader * s)106 agx_nir_lower_discard(nir_shader *s)
107 {
108 if (!s->info.fs.uses_discard)
109 return false;
110
111 return nir_shader_intrinsics_pass(s, lower_discard,
112 nir_metadata_control_flow, NULL);
113 }
114
115 static bool
agx_nir_lower_zs_emit(nir_shader * s)116 agx_nir_lower_zs_emit(nir_shader *s)
117 {
118 /* If depth/stencil isn't written, there's nothing to lower */
119 if (!(s->info.outputs_written & (BITFIELD64_BIT(FRAG_RESULT_STENCIL) |
120 BITFIELD64_BIT(FRAG_RESULT_DEPTH))))
121 return false;
122
123 bool any_progress = false;
124
125 nir_foreach_function_impl(impl, s) {
126 bool progress = false;
127
128 nir_foreach_block(block, impl) {
129 progress |= lower_zs_emit(block, s->info.fs.early_fragment_tests);
130 }
131
132 if (progress) {
133 nir_metadata_preserve(impl, nir_metadata_control_flow);
134 } else {
135 nir_metadata_preserve(impl, nir_metadata_all);
136 }
137
138 any_progress |= progress;
139 }
140
141 return any_progress;
142 }
143
144 bool
agx_nir_lower_discard_zs_emit(nir_shader * s)145 agx_nir_lower_discard_zs_emit(nir_shader *s)
146 {
147 bool progress = false;
148
149 /* Lower depth/stencil writes before discard so the interaction works */
150 progress |= agx_nir_lower_zs_emit(s);
151 progress |= agx_nir_lower_discard(s);
152
153 return progress;
154 }
155