1 /*
2 * Copyright © 2016 Rob Clark <[email protected]>
3 * SPDX-License-Identifier: MIT
4 *
5 * Authors:
6 * Rob Clark <[email protected]>
7 */
8
9 #include "pipe/p_state.h"
10 #include "util/bitset.h"
11 #include "util/format/u_format.h"
12 #include "util/u_inlines.h"
13 #include "util/u_memory.h"
14 #include "util/u_string.h"
15
16 #include "freedreno_program.h"
17
18 #include "fd5_emit.h"
19 #include "fd5_format.h"
20 #include "fd5_program.h"
21 #include "fd5_texture.h"
22
23 #include "ir3_cache.h"
24
25 void
fd5_emit_shader(struct fd_ringbuffer * ring,const struct ir3_shader_variant * so)26 fd5_emit_shader(struct fd_ringbuffer *ring, const struct ir3_shader_variant *so)
27 {
28 const struct ir3_info *si = &so->info;
29 enum a4xx_state_block sb = fd4_stage2shadersb(so->type);
30 enum a4xx_state_src src;
31 uint32_t i, sz, *bin;
32
33 if (FD_DBG(DIRECT)) {
34 sz = si->sizedwords;
35 src = SS4_DIRECT;
36 bin = fd_bo_map(so->bo);
37 } else {
38 sz = 0;
39 src = SS4_INDIRECT;
40 bin = NULL;
41 }
42
43 OUT_PKT7(ring, CP_LOAD_STATE4, 3 + sz);
44 OUT_RING(ring, CP_LOAD_STATE4_0_DST_OFF(0) |
45 CP_LOAD_STATE4_0_STATE_SRC(src) |
46 CP_LOAD_STATE4_0_STATE_BLOCK(sb) |
47 CP_LOAD_STATE4_0_NUM_UNIT(so->instrlen));
48 if (bin) {
49 OUT_RING(ring, CP_LOAD_STATE4_1_EXT_SRC_ADDR(0) |
50 CP_LOAD_STATE4_1_STATE_TYPE(ST4_SHADER));
51 OUT_RING(ring, CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI(0));
52 } else {
53 OUT_RELOC(ring, so->bo, 0, CP_LOAD_STATE4_1_STATE_TYPE(ST4_SHADER), 0);
54 }
55
56 /* for how clever coverity is, it is sometimes rather dull, and
57 * doesn't realize that the only case where bin==NULL, sz==0:
58 */
59 assume(bin || (sz == 0));
60
61 for (i = 0; i < sz; i++) {
62 OUT_RING(ring, bin[i]);
63 }
64 }
65
66 void
fd5_emit_shader_obj(struct fd_context * ctx,struct fd_ringbuffer * ring,const struct ir3_shader_variant * so,uint32_t shader_obj_reg)67 fd5_emit_shader_obj(struct fd_context *ctx, struct fd_ringbuffer *ring,
68 const struct ir3_shader_variant *so,
69 uint32_t shader_obj_reg)
70 {
71 ir3_get_private_mem(ctx, so);
72
73 OUT_PKT4(ring, shader_obj_reg, 6);
74 OUT_RELOC(ring, so->bo, 0, 0, 0); /* SP_VS_OBJ_START_LO/HI */
75
76 uint32_t per_sp_size = ctx->pvtmem[so->pvtmem_per_wave].per_sp_size;
77 OUT_RING(ring, A5XX_SP_VS_PVT_MEM_PARAM_MEMSIZEPERITEM(
78 ctx->pvtmem[so->pvtmem_per_wave].per_fiber_size) |
79 A5XX_SP_VS_PVT_MEM_PARAM_HWSTACKOFFSET(per_sp_size));
80 if (so->pvtmem_size > 0) { /* SP_xS_PVT_MEM_ADDR */
81 OUT_RELOC(ring, ctx->pvtmem[so->pvtmem_per_wave].bo, 0, 0, 0);
82 fd_ringbuffer_attach_bo(ring, ctx->pvtmem[so->pvtmem_per_wave].bo);
83 } else {
84 OUT_RING(ring, 0);
85 OUT_RING(ring, 0);
86 }
87 OUT_RING(ring, A5XX_SP_VS_PVT_MEM_SIZE_TOTALPVTMEMSIZE(per_sp_size));
88 }
89
90 /* TODO maybe some of this we could pre-compute once rather than having
91 * so much draw-time logic?
92 */
93 static void
emit_stream_out(struct fd_ringbuffer * ring,const struct ir3_shader_variant * v,struct ir3_shader_linkage * l)94 emit_stream_out(struct fd_ringbuffer *ring, const struct ir3_shader_variant *v,
95 struct ir3_shader_linkage *l)
96 {
97 const struct ir3_stream_output_info *strmout = &v->stream_output;
98 unsigned ncomp[PIPE_MAX_SO_BUFFERS] = {0};
99 unsigned prog[align(l->max_loc, 2) / 2];
100
101 memset(prog, 0, sizeof(prog));
102
103 for (unsigned i = 0; i < strmout->num_outputs; i++) {
104 const struct ir3_stream_output *out = &strmout->output[i];
105 unsigned k = out->register_index;
106 unsigned idx;
107
108 ncomp[out->output_buffer] += out->num_components;
109
110 /* linkage map sorted by order frag shader wants things, so
111 * a bit less ideal here..
112 */
113 for (idx = 0; idx < l->cnt; idx++)
114 if (l->var[idx].slot == v->outputs[k].slot)
115 break;
116
117 assert(idx < l->cnt);
118
119 for (unsigned j = 0; j < out->num_components; j++) {
120 unsigned c = j + out->start_component;
121 unsigned loc = l->var[idx].loc + c;
122 unsigned off = j + out->dst_offset; /* in dwords */
123
124 if (loc & 1) {
125 prog[loc / 2] |= A5XX_VPC_SO_PROG_B_EN |
126 A5XX_VPC_SO_PROG_B_BUF(out->output_buffer) |
127 A5XX_VPC_SO_PROG_B_OFF(off * 4);
128 } else {
129 prog[loc / 2] |= A5XX_VPC_SO_PROG_A_EN |
130 A5XX_VPC_SO_PROG_A_BUF(out->output_buffer) |
131 A5XX_VPC_SO_PROG_A_OFF(off * 4);
132 }
133 }
134 }
135
136 OUT_PKT7(ring, CP_CONTEXT_REG_BUNCH, 12 + (2 * ARRAY_SIZE(prog)));
137 OUT_RING(ring, REG_A5XX_VPC_SO_BUF_CNTL);
138 OUT_RING(ring, A5XX_VPC_SO_BUF_CNTL_ENABLE |
139 COND(ncomp[0] > 0, A5XX_VPC_SO_BUF_CNTL_BUF0) |
140 COND(ncomp[1] > 0, A5XX_VPC_SO_BUF_CNTL_BUF1) |
141 COND(ncomp[2] > 0, A5XX_VPC_SO_BUF_CNTL_BUF2) |
142 COND(ncomp[3] > 0, A5XX_VPC_SO_BUF_CNTL_BUF3));
143 OUT_RING(ring, REG_A5XX_VPC_SO_NCOMP(0));
144 OUT_RING(ring, ncomp[0]);
145 OUT_RING(ring, REG_A5XX_VPC_SO_NCOMP(1));
146 OUT_RING(ring, ncomp[1]);
147 OUT_RING(ring, REG_A5XX_VPC_SO_NCOMP(2));
148 OUT_RING(ring, ncomp[2]);
149 OUT_RING(ring, REG_A5XX_VPC_SO_NCOMP(3));
150 OUT_RING(ring, ncomp[3]);
151 OUT_RING(ring, REG_A5XX_VPC_SO_CNTL);
152 OUT_RING(ring, A5XX_VPC_SO_CNTL_ENABLE);
153 for (unsigned i = 0; i < ARRAY_SIZE(prog); i++) {
154 OUT_RING(ring, REG_A5XX_VPC_SO_PROG);
155 OUT_RING(ring, prog[i]);
156 }
157 }
158
159 struct stage {
160 const struct ir3_shader_variant *v;
161 const struct ir3_info *i;
162 /* const sizes are in units of 4 * vec4 */
163 uint8_t constoff;
164 uint8_t constlen;
165 /* instr sizes are in units of 16 instructions */
166 uint8_t instroff;
167 uint8_t instrlen;
168 };
169
170 enum { VS = 0, FS = 1, HS = 2, DS = 3, GS = 4, MAX_STAGES };
171
172 static void
setup_stages(struct fd5_emit * emit,struct stage * s)173 setup_stages(struct fd5_emit *emit, struct stage *s)
174 {
175 unsigned i;
176
177 s[VS].v = fd5_emit_get_vp(emit);
178 s[FS].v = fd5_emit_get_fp(emit);
179
180 s[HS].v = s[DS].v = s[GS].v = NULL; /* for now */
181
182 for (i = 0; i < MAX_STAGES; i++) {
183 if (s[i].v) {
184 s[i].i = &s[i].v->info;
185 /* constlen is in units of 4 * vec4: */
186 assert(s[i].v->constlen % 4 == 0);
187 s[i].constlen = s[i].v->constlen / 4;
188 /* instrlen is already in units of 16 instr.. although
189 * probably we should ditch that and not make the compiler
190 * care about instruction group size of a3xx vs a5xx
191 */
192 s[i].instrlen = s[i].v->instrlen;
193 } else {
194 s[i].i = NULL;
195 s[i].constlen = 0;
196 s[i].instrlen = 0;
197 }
198 }
199
200 /* NOTE: at least for gles2, blob partitions VS at bottom of const
201 * space and FS taking entire remaining space. We probably don't
202 * need to do that the same way, but for now mimic what the blob
203 * does to make it easier to diff against register values from blob
204 *
205 * NOTE: if VS.instrlen + FS.instrlen > 64, then one or both shaders
206 * is run from external memory.
207 */
208 if ((s[VS].instrlen + s[FS].instrlen) > 64) {
209 /* prioritize FS for internal memory: */
210 if (s[FS].instrlen < 64) {
211 /* if FS can fit, kick VS out to external memory: */
212 s[VS].instrlen = 0;
213 } else if (s[VS].instrlen < 64) {
214 /* otherwise if VS can fit, kick out FS: */
215 s[FS].instrlen = 0;
216 } else {
217 /* neither can fit, run both from external memory: */
218 s[VS].instrlen = 0;
219 s[FS].instrlen = 0;
220 }
221 }
222
223 unsigned constoff = 0;
224 for (i = 0; i < MAX_STAGES; i++) {
225 s[i].constoff = constoff;
226 constoff += s[i].constlen;
227 }
228
229 s[VS].instroff = 0;
230 s[FS].instroff = 64 - s[FS].instrlen;
231 s[HS].instroff = s[DS].instroff = s[GS].instroff = s[FS].instroff;
232 }
233
234 static inline uint32_t
next_regid(uint32_t reg,uint32_t increment)235 next_regid(uint32_t reg, uint32_t increment)
236 {
237 if (VALIDREG(reg))
238 return reg + increment;
239 else
240 return regid(63, 0);
241 }
242 void
fd5_program_emit(struct fd_context * ctx,struct fd_ringbuffer * ring,struct fd5_emit * emit)243 fd5_program_emit(struct fd_context *ctx, struct fd_ringbuffer *ring,
244 struct fd5_emit *emit)
245 {
246 struct stage s[MAX_STAGES];
247 uint32_t pos_regid, psize_regid, color_regid[8];
248 uint32_t face_regid, coord_regid, zwcoord_regid, samp_id_regid,
249 samp_mask_regid;
250 uint32_t ij_regid[IJ_COUNT], vertex_regid, instance_regid, clip0_regid,
251 clip1_regid;
252 enum a3xx_threadsize fssz;
253 uint8_t psize_loc = ~0;
254 int i, j;
255
256 setup_stages(emit, s);
257
258 bool do_streamout = (s[VS].v->stream_output.num_outputs > 0);
259 uint8_t clip_mask = s[VS].v->clip_mask,
260 cull_mask = s[VS].v->cull_mask;
261 uint8_t clip_cull_mask = clip_mask | cull_mask;
262
263 /* Unlike a6xx, we don't factor the rasterizer's clip enables in here. It's
264 * already handled by the frontend by storing 0.0 to the clipdist in the
265 * shader variant (using either nir_lower_clip_disable for clip distances
266 * from the source shader, or nir_lower_clip_vs for user clip planes).
267 * Masking the disabled clipdists off causes GPU hangs in tests like
268 * [email protected]@execution@clipping@vs-clip-vertex-enables.
269 */
270
271 fssz = (s[FS].i->double_threadsize) ? FOUR_QUADS : TWO_QUADS;
272
273 pos_regid = ir3_find_output_regid(s[VS].v, VARYING_SLOT_POS);
274 psize_regid = ir3_find_output_regid(s[VS].v, VARYING_SLOT_PSIZ);
275 clip0_regid = ir3_find_output_regid(s[VS].v, VARYING_SLOT_CLIP_DIST0);
276 clip1_regid = ir3_find_output_regid(s[VS].v, VARYING_SLOT_CLIP_DIST1);
277 vertex_regid =
278 ir3_find_sysval_regid(s[VS].v, SYSTEM_VALUE_VERTEX_ID_ZERO_BASE);
279 instance_regid = ir3_find_sysval_regid(s[VS].v, SYSTEM_VALUE_INSTANCE_ID);
280
281 if (s[FS].v->color0_mrt) {
282 color_regid[0] = color_regid[1] = color_regid[2] = color_regid[3] =
283 color_regid[4] = color_regid[5] = color_regid[6] = color_regid[7] =
284 ir3_find_output_regid(s[FS].v, FRAG_RESULT_COLOR);
285 } else {
286 color_regid[0] = ir3_find_output_regid(s[FS].v, FRAG_RESULT_DATA0);
287 color_regid[1] = ir3_find_output_regid(s[FS].v, FRAG_RESULT_DATA1);
288 color_regid[2] = ir3_find_output_regid(s[FS].v, FRAG_RESULT_DATA2);
289 color_regid[3] = ir3_find_output_regid(s[FS].v, FRAG_RESULT_DATA3);
290 color_regid[4] = ir3_find_output_regid(s[FS].v, FRAG_RESULT_DATA4);
291 color_regid[5] = ir3_find_output_regid(s[FS].v, FRAG_RESULT_DATA5);
292 color_regid[6] = ir3_find_output_regid(s[FS].v, FRAG_RESULT_DATA6);
293 color_regid[7] = ir3_find_output_regid(s[FS].v, FRAG_RESULT_DATA7);
294 }
295
296 samp_id_regid = ir3_find_sysval_regid(s[FS].v, SYSTEM_VALUE_SAMPLE_ID);
297 samp_mask_regid =
298 ir3_find_sysval_regid(s[FS].v, SYSTEM_VALUE_SAMPLE_MASK_IN);
299 face_regid = ir3_find_sysval_regid(s[FS].v, SYSTEM_VALUE_FRONT_FACE);
300 coord_regid = ir3_find_sysval_regid(s[FS].v, SYSTEM_VALUE_FRAG_COORD);
301 zwcoord_regid = next_regid(coord_regid, 2);
302 for (unsigned i = 0; i < ARRAY_SIZE(ij_regid); i++)
303 ij_regid[i] = ir3_find_sysval_regid(
304 s[FS].v, SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL + i);
305
306 /* we could probably divide this up into things that need to be
307 * emitted if frag-prog is dirty vs if vert-prog is dirty..
308 */
309
310 OUT_PKT4(ring, REG_A5XX_HLSQ_VS_CONFIG, 5);
311 OUT_RING(ring, A5XX_HLSQ_VS_CONFIG_CONSTOBJECTOFFSET(s[VS].constoff) |
312 A5XX_HLSQ_VS_CONFIG_SHADEROBJOFFSET(s[VS].instroff) |
313 COND(s[VS].v, A5XX_HLSQ_VS_CONFIG_ENABLED));
314 OUT_RING(ring, A5XX_HLSQ_FS_CONFIG_CONSTOBJECTOFFSET(s[FS].constoff) |
315 A5XX_HLSQ_FS_CONFIG_SHADEROBJOFFSET(s[FS].instroff) |
316 COND(s[FS].v, A5XX_HLSQ_FS_CONFIG_ENABLED));
317 OUT_RING(ring, A5XX_HLSQ_HS_CONFIG_CONSTOBJECTOFFSET(s[HS].constoff) |
318 A5XX_HLSQ_HS_CONFIG_SHADEROBJOFFSET(s[HS].instroff) |
319 COND(s[HS].v, A5XX_HLSQ_HS_CONFIG_ENABLED));
320 OUT_RING(ring, A5XX_HLSQ_DS_CONFIG_CONSTOBJECTOFFSET(s[DS].constoff) |
321 A5XX_HLSQ_DS_CONFIG_SHADEROBJOFFSET(s[DS].instroff) |
322 COND(s[DS].v, A5XX_HLSQ_DS_CONFIG_ENABLED));
323 OUT_RING(ring, A5XX_HLSQ_GS_CONFIG_CONSTOBJECTOFFSET(s[GS].constoff) |
324 A5XX_HLSQ_GS_CONFIG_SHADEROBJOFFSET(s[GS].instroff) |
325 COND(s[GS].v, A5XX_HLSQ_GS_CONFIG_ENABLED));
326
327 OUT_PKT4(ring, REG_A5XX_HLSQ_CS_CONFIG, 1);
328 OUT_RING(ring, 0x00000000);
329
330 OUT_PKT4(ring, REG_A5XX_HLSQ_VS_CNTL, 5);
331 OUT_RING(ring, A5XX_HLSQ_VS_CNTL_INSTRLEN(s[VS].instrlen) |
332 COND(s[VS].v && s[VS].v->has_ssbo,
333 A5XX_HLSQ_VS_CNTL_SSBO_ENABLE));
334 OUT_RING(ring, A5XX_HLSQ_FS_CNTL_INSTRLEN(s[FS].instrlen) |
335 COND(s[FS].v && s[FS].v->has_ssbo,
336 A5XX_HLSQ_FS_CNTL_SSBO_ENABLE));
337 OUT_RING(ring, A5XX_HLSQ_HS_CNTL_INSTRLEN(s[HS].instrlen) |
338 COND(s[HS].v && s[HS].v->has_ssbo,
339 A5XX_HLSQ_HS_CNTL_SSBO_ENABLE));
340 OUT_RING(ring, A5XX_HLSQ_DS_CNTL_INSTRLEN(s[DS].instrlen) |
341 COND(s[DS].v && s[DS].v->has_ssbo,
342 A5XX_HLSQ_DS_CNTL_SSBO_ENABLE));
343 OUT_RING(ring, A5XX_HLSQ_GS_CNTL_INSTRLEN(s[GS].instrlen) |
344 COND(s[GS].v && s[GS].v->has_ssbo,
345 A5XX_HLSQ_GS_CNTL_SSBO_ENABLE));
346
347 OUT_PKT4(ring, REG_A5XX_SP_VS_CONFIG, 5);
348 OUT_RING(ring, A5XX_SP_VS_CONFIG_CONSTOBJECTOFFSET(s[VS].constoff) |
349 A5XX_SP_VS_CONFIG_SHADEROBJOFFSET(s[VS].instroff) |
350 COND(s[VS].v, A5XX_SP_VS_CONFIG_ENABLED));
351 OUT_RING(ring, A5XX_SP_FS_CONFIG_CONSTOBJECTOFFSET(s[FS].constoff) |
352 A5XX_SP_FS_CONFIG_SHADEROBJOFFSET(s[FS].instroff) |
353 COND(s[FS].v, A5XX_SP_FS_CONFIG_ENABLED));
354 OUT_RING(ring, A5XX_SP_HS_CONFIG_CONSTOBJECTOFFSET(s[HS].constoff) |
355 A5XX_SP_HS_CONFIG_SHADEROBJOFFSET(s[HS].instroff) |
356 COND(s[HS].v, A5XX_SP_HS_CONFIG_ENABLED));
357 OUT_RING(ring, A5XX_SP_DS_CONFIG_CONSTOBJECTOFFSET(s[DS].constoff) |
358 A5XX_SP_DS_CONFIG_SHADEROBJOFFSET(s[DS].instroff) |
359 COND(s[DS].v, A5XX_SP_DS_CONFIG_ENABLED));
360 OUT_RING(ring, A5XX_SP_GS_CONFIG_CONSTOBJECTOFFSET(s[GS].constoff) |
361 A5XX_SP_GS_CONFIG_SHADEROBJOFFSET(s[GS].instroff) |
362 COND(s[GS].v, A5XX_SP_GS_CONFIG_ENABLED));
363
364 OUT_PKT4(ring, REG_A5XX_SP_CS_CONFIG, 1);
365 OUT_RING(ring, 0x00000000);
366
367 OUT_PKT4(ring, REG_A5XX_HLSQ_VS_CONSTLEN, 2);
368 OUT_RING(ring, s[VS].constlen); /* HLSQ_VS_CONSTLEN */
369 OUT_RING(ring, s[VS].instrlen); /* HLSQ_VS_INSTRLEN */
370
371 OUT_PKT4(ring, REG_A5XX_HLSQ_FS_CONSTLEN, 2);
372 OUT_RING(ring, s[FS].constlen); /* HLSQ_FS_CONSTLEN */
373 OUT_RING(ring, s[FS].instrlen); /* HLSQ_FS_INSTRLEN */
374
375 OUT_PKT4(ring, REG_A5XX_HLSQ_HS_CONSTLEN, 2);
376 OUT_RING(ring, s[HS].constlen); /* HLSQ_HS_CONSTLEN */
377 OUT_RING(ring, s[HS].instrlen); /* HLSQ_HS_INSTRLEN */
378
379 OUT_PKT4(ring, REG_A5XX_HLSQ_DS_CONSTLEN, 2);
380 OUT_RING(ring, s[DS].constlen); /* HLSQ_DS_CONSTLEN */
381 OUT_RING(ring, s[DS].instrlen); /* HLSQ_DS_INSTRLEN */
382
383 OUT_PKT4(ring, REG_A5XX_HLSQ_GS_CONSTLEN, 2);
384 OUT_RING(ring, s[GS].constlen); /* HLSQ_GS_CONSTLEN */
385 OUT_RING(ring, s[GS].instrlen); /* HLSQ_GS_INSTRLEN */
386
387 OUT_PKT4(ring, REG_A5XX_HLSQ_CS_CONSTLEN, 2);
388 OUT_RING(ring, 0x00000000); /* HLSQ_CS_CONSTLEN */
389 OUT_RING(ring, 0x00000000); /* HLSQ_CS_INSTRLEN */
390
391 OUT_PKT4(ring, REG_A5XX_SP_VS_CTRL_REG0, 1);
392 OUT_RING(
393 ring,
394 A5XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT(s[VS].i->max_half_reg + 1) |
395 A5XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(s[VS].i->max_reg + 1) |
396 COND(s[VS].instrlen != 0, A5XX_SP_VS_CTRL_REG0_BUFFER) |
397 /* XXX: 0x2 is only unset in
398 * dEQP-GLES3.functional.ubo.single_nested_struct_array.single_buffer.packed_instance_array_vertex
399 * on a collection of blob traces. That shader is 1091 instrs, 0
400 * half, 3 full, 108 constlen. Other >1091 instr non-VS shaders don't
401 * unset it, so that's not the trick.
402 */
403 0x2 |
404 A5XX_SP_VS_CTRL_REG0_BRANCHSTACK(ir3_shader_branchstack_hw(s[VS].v)) |
405 COND(s[VS].v->need_pixlod, A5XX_SP_VS_CTRL_REG0_PIXLODENABLE));
406
407 /* If we have streamout, link against the real FS in the binning program,
408 * rather than the dummy FS used for binning pass state, to ensure the
409 * OUTLOC's match. Depending on whether we end up doing sysmem or gmem, the
410 * actual streamout could happen with either the binning pass or draw pass
411 * program, but the same streamout stateobj is used in either case:
412 */
413 const struct ir3_shader_variant *link_fs = s[FS].v;
414 if (do_streamout && emit->binning_pass)
415 link_fs = emit->prog->fs;
416 struct ir3_shader_linkage l = {0};
417 ir3_link_shaders(&l, s[VS].v, link_fs, true);
418
419 uint8_t clip0_loc = l.clip0_loc;
420 uint8_t clip1_loc = l.clip1_loc;
421
422 OUT_PKT4(ring, REG_A5XX_VPC_VAR_DISABLE(0), 4);
423 OUT_RING(ring, ~l.varmask[0]); /* VPC_VAR[0].DISABLE */
424 OUT_RING(ring, ~l.varmask[1]); /* VPC_VAR[1].DISABLE */
425 OUT_RING(ring, ~l.varmask[2]); /* VPC_VAR[2].DISABLE */
426 OUT_RING(ring, ~l.varmask[3]); /* VPC_VAR[3].DISABLE */
427
428 /* Add stream out outputs after computing the VPC_VAR_DISABLE bitmask. */
429 ir3_link_stream_out(&l, s[VS].v);
430
431 /* a5xx appends pos/psize to end of the linkage map: */
432 if (VALIDREG(pos_regid))
433 ir3_link_add(&l, VARYING_SLOT_POS, pos_regid, 0xf, l.max_loc);
434
435 if (VALIDREG(psize_regid)) {
436 psize_loc = l.max_loc;
437 ir3_link_add(&l, VARYING_SLOT_PSIZ, psize_regid, 0x1, l.max_loc);
438 }
439
440 /* Handle the case where clip/cull distances aren't read by the FS. Make
441 * sure to avoid adding an output with an empty writemask if the user
442 * disables all the clip distances in the API so that the slot is unused.
443 */
444 if (clip0_loc == 0xff && VALIDREG(clip0_regid) &&
445 (clip_cull_mask & 0xf) != 0) {
446 clip0_loc = l.max_loc;
447 ir3_link_add(&l, VARYING_SLOT_CLIP_DIST0, clip0_regid,
448 clip_cull_mask & 0xf, l.max_loc);
449 }
450
451 if (clip1_loc == 0xff && VALIDREG(clip1_regid) &&
452 (clip_cull_mask >> 4) != 0) {
453 clip1_loc = l.max_loc;
454 ir3_link_add(&l, VARYING_SLOT_CLIP_DIST1, clip1_regid,
455 clip_cull_mask >> 4, l.max_loc);
456 }
457
458 /* If we have stream-out, we use the full shader for binning
459 * pass, rather than the optimized binning pass one, so that we
460 * have all the varying outputs available for xfb. So streamout
461 * state should always be derived from the non-binning pass
462 * program:
463 */
464 if (do_streamout && !emit->binning_pass)
465 emit_stream_out(ring, s[VS].v, &l);
466
467 for (i = 0, j = 0; (i < 16) && (j < l.cnt); i++) {
468 uint32_t reg = 0;
469
470 OUT_PKT4(ring, REG_A5XX_SP_VS_OUT_REG(i), 1);
471
472 reg |= A5XX_SP_VS_OUT_REG_A_REGID(l.var[j].regid);
473 reg |= A5XX_SP_VS_OUT_REG_A_COMPMASK(l.var[j].compmask);
474 j++;
475
476 reg |= A5XX_SP_VS_OUT_REG_B_REGID(l.var[j].regid);
477 reg |= A5XX_SP_VS_OUT_REG_B_COMPMASK(l.var[j].compmask);
478 j++;
479
480 OUT_RING(ring, reg);
481 }
482
483 for (i = 0, j = 0; (i < 8) && (j < l.cnt); i++) {
484 uint32_t reg = 0;
485
486 OUT_PKT4(ring, REG_A5XX_SP_VS_VPC_DST_REG(i), 1);
487
488 reg |= A5XX_SP_VS_VPC_DST_REG_OUTLOC0(l.var[j++].loc);
489 reg |= A5XX_SP_VS_VPC_DST_REG_OUTLOC1(l.var[j++].loc);
490 reg |= A5XX_SP_VS_VPC_DST_REG_OUTLOC2(l.var[j++].loc);
491 reg |= A5XX_SP_VS_VPC_DST_REG_OUTLOC3(l.var[j++].loc);
492
493 OUT_RING(ring, reg);
494 }
495
496 fd5_emit_shader_obj(ctx, ring, s[VS].v, REG_A5XX_SP_VS_OBJ_START_LO);
497
498 if (s[VS].instrlen)
499 fd5_emit_shader(ring, s[VS].v);
500
501 // TODO depending on other bits in this reg (if any) set somewhere else?
502 OUT_PKT4(ring, REG_A5XX_PC_PRIM_VTX_CNTL, 1);
503 OUT_RING(ring, COND(s[VS].v->writes_psize, A5XX_PC_PRIM_VTX_CNTL_PSIZE));
504
505 OUT_PKT4(ring, REG_A5XX_SP_PRIMITIVE_CNTL, 1);
506 OUT_RING(ring, A5XX_SP_PRIMITIVE_CNTL_VSOUT(l.cnt));
507
508 OUT_PKT4(ring, REG_A5XX_VPC_CNTL_0, 1);
509 OUT_RING(ring, A5XX_VPC_CNTL_0_STRIDE_IN_VPC(l.max_loc) |
510 COND(s[FS].v->total_in > 0, A5XX_VPC_CNTL_0_VARYING) |
511 0x10000); // XXX
512
513 fd5_context(ctx)->max_loc = l.max_loc;
514
515 if (emit->binning_pass) {
516 OUT_PKT4(ring, REG_A5XX_SP_FS_OBJ_START_LO, 2);
517 OUT_RING(ring, 0x00000000); /* SP_FS_OBJ_START_LO */
518 OUT_RING(ring, 0x00000000); /* SP_FS_OBJ_START_HI */
519 } else {
520 fd5_emit_shader_obj(ctx, ring, s[FS].v, REG_A5XX_SP_FS_OBJ_START_LO);
521 }
522
523 OUT_PKT4(ring, REG_A5XX_HLSQ_CONTROL_0_REG, 5);
524 OUT_RING(ring, A5XX_HLSQ_CONTROL_0_REG_FSTHREADSIZE(fssz) |
525 A5XX_HLSQ_CONTROL_0_REG_CSTHREADSIZE(TWO_QUADS) |
526 0x00000880); /* XXX HLSQ_CONTROL_0 */
527 OUT_RING(ring, A5XX_HLSQ_CONTROL_1_REG_PRIMALLOCTHRESHOLD(63));
528 OUT_RING(ring, A5XX_HLSQ_CONTROL_2_REG_FACEREGID(face_regid) |
529 A5XX_HLSQ_CONTROL_2_REG_SAMPLEID(samp_id_regid) |
530 A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(samp_mask_regid) |
531 A5XX_HLSQ_CONTROL_2_REG_CENTERRHW(ij_regid[IJ_PERSP_CENTER_RHW]));
532 OUT_RING(
533 ring,
534 A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_PIXEL(ij_regid[IJ_PERSP_PIXEL]) |
535 A5XX_HLSQ_CONTROL_3_REG_IJ_LINEAR_PIXEL(ij_regid[IJ_LINEAR_PIXEL]) |
536 A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID(
537 ij_regid[IJ_PERSP_CENTROID]) |
538 A5XX_HLSQ_CONTROL_3_REG_IJ_PERSP_CENTROID(
539 ij_regid[IJ_LINEAR_CENTROID]));
540 OUT_RING(
541 ring,
542 A5XX_HLSQ_CONTROL_4_REG_XYCOORDREGID(coord_regid) |
543 A5XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID(zwcoord_regid) |
544 A5XX_HLSQ_CONTROL_4_REG_IJ_PERSP_SAMPLE(ij_regid[IJ_PERSP_SAMPLE]) |
545 A5XX_HLSQ_CONTROL_4_REG_IJ_LINEAR_SAMPLE(ij_regid[IJ_LINEAR_SAMPLE]));
546
547 OUT_PKT4(ring, REG_A5XX_SP_FS_CTRL_REG0, 1);
548 OUT_RING(
549 ring,
550 COND(s[FS].v->total_in > 0, A5XX_SP_FS_CTRL_REG0_VARYING) |
551 0x40002 | /* XXX set pretty much everywhere */
552 COND(s[FS].instrlen != 0, A5XX_SP_FS_CTRL_REG0_BUFFER) |
553 A5XX_SP_FS_CTRL_REG0_THREADSIZE(fssz) |
554 A5XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT(s[FS].i->max_half_reg + 1) |
555 A5XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(s[FS].i->max_reg + 1) |
556 A5XX_SP_FS_CTRL_REG0_BRANCHSTACK(ir3_shader_branchstack_hw(s[FS].v)) |
557 COND(s[FS].v->need_pixlod, A5XX_SP_FS_CTRL_REG0_PIXLODENABLE));
558
559 OUT_PKT4(ring, REG_A5XX_HLSQ_UPDATE_CNTL, 1);
560 OUT_RING(ring, 0x020fffff); /* XXX */
561
562 OUT_PKT4(ring, REG_A5XX_VPC_GS_SIV_CNTL, 1);
563 OUT_RING(ring, 0x0000ffff); /* XXX */
564
565 OUT_PKT4(ring, REG_A5XX_SP_SP_CNTL, 1);
566 OUT_RING(ring, 0x00000010); /* XXX */
567
568 OUT_PKT4(ring, REG_A5XX_GRAS_CNTL, 1);
569 OUT_RING(ring,
570 CONDREG(ij_regid[IJ_PERSP_PIXEL], A5XX_GRAS_CNTL_IJ_PERSP_PIXEL) |
571 CONDREG(ij_regid[IJ_PERSP_CENTROID],
572 A5XX_GRAS_CNTL_IJ_PERSP_CENTROID) |
573 CONDREG(ij_regid[IJ_PERSP_SAMPLE],
574 A5XX_GRAS_CNTL_IJ_PERSP_SAMPLE) |
575 CONDREG(ij_regid[IJ_LINEAR_PIXEL], A5XX_GRAS_CNTL_IJ_LINEAR_PIXEL) |
576 CONDREG(ij_regid[IJ_LINEAR_CENTROID],
577 A5XX_GRAS_CNTL_IJ_LINEAR_CENTROID) |
578 CONDREG(ij_regid[IJ_LINEAR_SAMPLE],
579 A5XX_GRAS_CNTL_IJ_LINEAR_SAMPLE) |
580 COND(s[FS].v->fragcoord_compmask != 0,
581 A5XX_GRAS_CNTL_COORD_MASK(s[FS].v->fragcoord_compmask) |
582 A5XX_GRAS_CNTL_IJ_LINEAR_PIXEL) |
583 COND(s[FS].v->frag_face, A5XX_GRAS_CNTL_IJ_LINEAR_PIXEL) |
584 CONDREG(ij_regid[IJ_LINEAR_PIXEL], A5XX_GRAS_CNTL_IJ_LINEAR_PIXEL));
585
586 OUT_PKT4(ring, REG_A5XX_RB_RENDER_CONTROL0, 2);
587 OUT_RING(
588 ring,
589 CONDREG(ij_regid[IJ_PERSP_PIXEL],
590 A5XX_RB_RENDER_CONTROL0_IJ_PERSP_PIXEL) |
591 CONDREG(ij_regid[IJ_PERSP_CENTROID],
592 A5XX_RB_RENDER_CONTROL0_IJ_PERSP_CENTROID) |
593 CONDREG(ij_regid[IJ_PERSP_SAMPLE],
594 A5XX_RB_RENDER_CONTROL0_IJ_PERSP_SAMPLE) |
595 CONDREG(ij_regid[IJ_LINEAR_PIXEL],
596 A5XX_RB_RENDER_CONTROL0_IJ_LINEAR_PIXEL) |
597 CONDREG(ij_regid[IJ_LINEAR_CENTROID],
598 A5XX_RB_RENDER_CONTROL0_IJ_LINEAR_CENTROID) |
599 CONDREG(ij_regid[IJ_LINEAR_SAMPLE],
600 A5XX_RB_RENDER_CONTROL0_IJ_LINEAR_SAMPLE) |
601 COND(s[FS].v->fragcoord_compmask != 0,
602 A5XX_RB_RENDER_CONTROL0_COORD_MASK(s[FS].v->fragcoord_compmask) |
603 A5XX_RB_RENDER_CONTROL0_IJ_LINEAR_PIXEL) |
604 COND(s[FS].v->frag_face, A5XX_RB_RENDER_CONTROL0_IJ_LINEAR_PIXEL) |
605 CONDREG(ij_regid[IJ_LINEAR_PIXEL], A5XX_RB_RENDER_CONTROL0_IJ_LINEAR_PIXEL));
606 OUT_RING(ring,
607 CONDREG(samp_mask_regid, A5XX_RB_RENDER_CONTROL1_SAMPLEMASK) |
608 COND(s[FS].v->frag_face, A5XX_RB_RENDER_CONTROL1_FACENESS) |
609 CONDREG(samp_id_regid, A5XX_RB_RENDER_CONTROL1_SAMPLEID));
610
611 OUT_PKT4(ring, REG_A5XX_SP_FS_OUTPUT_REG(0), 8);
612 for (i = 0; i < 8; i++) {
613 OUT_RING(ring, A5XX_SP_FS_OUTPUT_REG_REGID(color_regid[i]) |
614 COND(color_regid[i] & HALF_REG_ID,
615 A5XX_SP_FS_OUTPUT_REG_HALF_PRECISION));
616 }
617
618 OUT_PKT4(ring, REG_A5XX_VPC_PACK, 1);
619 OUT_RING(ring, A5XX_VPC_PACK_NUMNONPOSVAR(s[FS].v->total_in) |
620 A5XX_VPC_PACK_PSIZELOC(psize_loc));
621
622 if (!emit->binning_pass) {
623 uint32_t vinterp[8], vpsrepl[8];
624
625 memset(vinterp, 0, sizeof(vinterp));
626 memset(vpsrepl, 0, sizeof(vpsrepl));
627
628 /* looks like we need to do int varyings in the frag
629 * shader on a5xx (no flatshad reg? or a420.0 bug?):
630 *
631 * (sy)(ss)nop
632 * (sy)ldlv.u32 r0.x,l[r0.x], 1
633 * ldlv.u32 r0.y,l[r0.x+1], 1
634 * (ss)bary.f (ei)r63.x, 0, r0.x
635 * (ss)(rpt1)cov.s32f16 hr0.x, (r)r0.x
636 * (rpt5)nop
637 * sam (f16)(xyzw)hr0.x, hr0.x, s#0, t#0
638 *
639 * Possibly on later a5xx variants we'll be able to use
640 * something like the code below instead of workaround
641 * in the shader:
642 */
643 /* figure out VARYING_INTERP / VARYING_PS_REPL register values: */
644 for (j = -1;
645 (j = ir3_next_varying(s[FS].v, j)) < (int)s[FS].v->inputs_count;) {
646 /* NOTE: varyings are packed, so if compmask is 0xb
647 * then first, third, and fourth component occupy
648 * three consecutive varying slots:
649 */
650 unsigned compmask = s[FS].v->inputs[j].compmask;
651
652 uint32_t inloc = s[FS].v->inputs[j].inloc;
653
654 if (s[FS].v->inputs[j].flat ||
655 (s[FS].v->inputs[j].rasterflat && emit->rasterflat)) {
656 uint32_t loc = inloc;
657
658 for (i = 0; i < 4; i++) {
659 if (compmask & (1 << i)) {
660 vinterp[loc / 16] |= 1 << ((loc % 16) * 2);
661 // flatshade[loc / 32] |= 1 << (loc % 32);
662 loc++;
663 }
664 }
665 }
666
667 bool coord_mode = emit->sprite_coord_mode;
668 if (ir3_point_sprite(s[FS].v, j, emit->sprite_coord_enable,
669 &coord_mode)) {
670 /* mask is two 2-bit fields, where:
671 * '01' -> S
672 * '10' -> T
673 * '11' -> 1 - T (flip mode)
674 */
675 unsigned mask = coord_mode ? 0b1101 : 0b1001;
676 uint32_t loc = inloc;
677 if (compmask & 0x1) {
678 vpsrepl[loc / 16] |= ((mask >> 0) & 0x3) << ((loc % 16) * 2);
679 loc++;
680 }
681 if (compmask & 0x2) {
682 vpsrepl[loc / 16] |= ((mask >> 2) & 0x3) << ((loc % 16) * 2);
683 loc++;
684 }
685 if (compmask & 0x4) {
686 /* .z <- 0.0f */
687 vinterp[loc / 16] |= 0b10 << ((loc % 16) * 2);
688 loc++;
689 }
690 if (compmask & 0x8) {
691 /* .w <- 1.0f */
692 vinterp[loc / 16] |= 0b11 << ((loc % 16) * 2);
693 loc++;
694 }
695 }
696 }
697
698 OUT_PKT4(ring, REG_A5XX_VPC_VARYING_INTERP_MODE(0), 8);
699 for (i = 0; i < 8; i++)
700 OUT_RING(ring, vinterp[i]); /* VPC_VARYING_INTERP[i].MODE */
701
702 OUT_PKT4(ring, REG_A5XX_VPC_VARYING_PS_REPL_MODE(0), 8);
703 for (i = 0; i < 8; i++)
704 OUT_RING(ring, vpsrepl[i]); /* VPC_VARYING_PS_REPL[i] */
705 }
706
707 OUT_PKT4(ring, REG_A5XX_GRAS_VS_CL_CNTL, 1);
708 OUT_RING(ring, A5XX_GRAS_VS_CL_CNTL_CLIP_MASK(clip_mask) |
709 A5XX_GRAS_VS_CL_CNTL_CULL_MASK(cull_mask));
710
711 OUT_PKT4(ring, REG_A5XX_VPC_CLIP_CNTL, 1);
712 OUT_RING(ring, A5XX_VPC_CLIP_CNTL_CLIP_MASK(clip_cull_mask) |
713 A5XX_VPC_CLIP_CNTL_CLIP_DIST_03_LOC(clip0_loc) |
714 A5XX_VPC_CLIP_CNTL_CLIP_DIST_47_LOC(clip1_loc));
715
716 OUT_PKT4(ring, REG_A5XX_PC_CLIP_CNTL, 1);
717 OUT_RING(ring, A5XX_PC_CLIP_CNTL_CLIP_MASK(clip_mask));
718
719 if (!emit->binning_pass)
720 if (s[FS].instrlen)
721 fd5_emit_shader(ring, s[FS].v);
722
723 OUT_PKT4(ring, REG_A5XX_VFD_CONTROL_1, 5);
724 OUT_RING(ring, A5XX_VFD_CONTROL_1_REGID4VTX(vertex_regid) |
725 A5XX_VFD_CONTROL_1_REGID4INST(instance_regid) | 0xfc0000);
726 OUT_RING(ring, 0x0000fcfc); /* VFD_CONTROL_2 */
727 OUT_RING(ring, 0x0000fcfc); /* VFD_CONTROL_3 */
728 OUT_RING(ring, 0x000000fc); /* VFD_CONTROL_4 */
729 OUT_RING(ring, 0x00000000); /* VFD_CONTROL_5 */
730 }
731
732 static struct ir3_program_state *
fd5_program_create(void * data,const struct ir3_shader_variant * bs,const struct ir3_shader_variant * vs,const struct ir3_shader_variant * hs,const struct ir3_shader_variant * ds,const struct ir3_shader_variant * gs,const struct ir3_shader_variant * fs,const struct ir3_cache_key * key)733 fd5_program_create(void *data, const struct ir3_shader_variant *bs,
734 const struct ir3_shader_variant *vs,
735 const struct ir3_shader_variant *hs,
736 const struct ir3_shader_variant *ds,
737 const struct ir3_shader_variant *gs,
738 const struct ir3_shader_variant *fs,
739 const struct ir3_cache_key *key) in_dt
740 {
741 struct fd_context *ctx = fd_context(data);
742 struct fd5_program_state *state = CALLOC_STRUCT(fd5_program_state);
743
744 tc_assert_driver_thread(ctx->tc);
745
746 state->bs = bs;
747 state->vs = vs;
748 state->fs = fs;
749
750 return &state->base;
751 }
752
753 static void
fd5_program_destroy(void * data,struct ir3_program_state * state)754 fd5_program_destroy(void *data, struct ir3_program_state *state)
755 {
756 struct fd5_program_state *so = fd5_program_state(state);
757 free(so);
758 }
759
760 static const struct ir3_cache_funcs cache_funcs = {
761 .create_state = fd5_program_create,
762 .destroy_state = fd5_program_destroy,
763 };
764
765 void
fd5_prog_init(struct pipe_context * pctx)766 fd5_prog_init(struct pipe_context *pctx)
767 {
768 struct fd_context *ctx = fd_context(pctx);
769
770 ctx->shader_cache = ir3_cache_create(&cache_funcs, ctx);
771 ir3_prog_init(pctx);
772 fd_prog_init(pctx);
773 }
774