1 /*
2 * Copyright (C) 2018-2019 Alyssa Rosenzweig <[email protected]>
3 * Copyright (C) 2019-2020 Collabora, Ltd.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include <err.h>
26 #include <fcntl.h>
27 #include <stdint.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <sys/mman.h>
31 #include <sys/stat.h>
32 #include <sys/types.h>
33
34 #include "compiler/glsl/glsl_to_nir.h"
35 #include "compiler/glsl_types.h"
36 #include "compiler/nir/nir_builder.h"
37 #include "util/half_float.h"
38 #include "util/list.h"
39 #include "util/u_debug.h"
40 #include "util/u_dynarray.h"
41 #include "util/u_math.h"
42
43 #include "compiler.h"
44 #include "helpers.h"
45 #include "midgard.h"
46 #include "midgard_compile.h"
47 #include "midgard_nir.h"
48 #include "midgard_ops.h"
49 #include "midgard_quirks.h"
50
51 #include "disassemble.h"
52
53 static const struct debug_named_value midgard_debug_options[] = {
54 {"shaders", MIDGARD_DBG_SHADERS, "Dump shaders in NIR and MIR"},
55 {"shaderdb", MIDGARD_DBG_SHADERDB, "Prints shader-db statistics"},
56 {"inorder", MIDGARD_DBG_INORDER, "Disables out-of-order scheduling"},
57 {"verbose", MIDGARD_DBG_VERBOSE, "Dump shaders verbosely"},
58 {"internal", MIDGARD_DBG_INTERNAL, "Dump internal shaders"},
59 DEBUG_NAMED_VALUE_END};
60
61 DEBUG_GET_ONCE_FLAGS_OPTION(midgard_debug, "MIDGARD_MESA_DEBUG",
62 midgard_debug_options, 0)
63
64 int midgard_debug = 0;
65
66 static midgard_block *
create_empty_block(compiler_context * ctx)67 create_empty_block(compiler_context *ctx)
68 {
69 midgard_block *blk = rzalloc(ctx, midgard_block);
70
71 blk->base.predecessors =
72 _mesa_set_create(blk, _mesa_hash_pointer, _mesa_key_pointer_equal);
73
74 blk->base.name = ctx->block_source_count++;
75
76 return blk;
77 }
78
79 static void
schedule_barrier(compiler_context * ctx)80 schedule_barrier(compiler_context *ctx)
81 {
82 midgard_block *temp = ctx->after_block;
83 ctx->after_block = create_empty_block(ctx);
84 ctx->block_count++;
85 list_addtail(&ctx->after_block->base.link, &ctx->blocks);
86 list_inithead(&ctx->after_block->base.instructions);
87 pan_block_add_successor(&ctx->current_block->base, &ctx->after_block->base);
88 ctx->current_block = ctx->after_block;
89 ctx->after_block = temp;
90 }
91
92 /* Helpers to generate midgard_instruction's using macro magic, since every
93 * driver seems to do it that way */
94
95 #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__));
96
97 #define M_LOAD_STORE(name, store, T) \
98 static midgard_instruction m_##name(unsigned ssa, unsigned address) \
99 { \
100 midgard_instruction i = { \
101 .type = TAG_LOAD_STORE_4, \
102 .mask = 0xF, \
103 .dest = ~0, \
104 .src = {~0, ~0, ~0, ~0}, \
105 .swizzle = SWIZZLE_IDENTITY_4, \
106 .op = midgard_op_##name, \
107 .load_store = \
108 { \
109 .signed_offset = address, \
110 }, \
111 }; \
112 \
113 if (store) { \
114 i.src[0] = ssa; \
115 i.src_types[0] = T; \
116 i.dest_type = T; \
117 } else { \
118 i.dest = ssa; \
119 i.dest_type = T; \
120 } \
121 return i; \
122 }
123
124 #define M_LOAD(name, T) M_LOAD_STORE(name, false, T)
125 #define M_STORE(name, T) M_LOAD_STORE(name, true, T)
126
127 M_LOAD(ld_attr_32, nir_type_uint32);
128 M_LOAD(ld_vary_32, nir_type_uint32);
129 M_LOAD(ld_ubo_u8, nir_type_uint32); /* mandatory extension to 32-bit */
130 M_LOAD(ld_ubo_u16, nir_type_uint32);
131 M_LOAD(ld_ubo_32, nir_type_uint32);
132 M_LOAD(ld_ubo_64, nir_type_uint32);
133 M_LOAD(ld_ubo_128, nir_type_uint32);
134 M_LOAD(ld_u8, nir_type_uint8);
135 M_LOAD(ld_u16, nir_type_uint16);
136 M_LOAD(ld_32, nir_type_uint32);
137 M_LOAD(ld_64, nir_type_uint32);
138 M_LOAD(ld_128, nir_type_uint32);
139 M_STORE(st_u8, nir_type_uint8);
140 M_STORE(st_u16, nir_type_uint16);
141 M_STORE(st_32, nir_type_uint32);
142 M_STORE(st_64, nir_type_uint32);
143 M_STORE(st_128, nir_type_uint32);
144 M_LOAD(ld_tilebuffer_raw, nir_type_uint32);
145 M_LOAD(ld_tilebuffer_16f, nir_type_float16);
146 M_LOAD(ld_tilebuffer_32f, nir_type_float32);
147 M_STORE(st_vary_32, nir_type_uint32);
148 M_LOAD(ld_cubemap_coords, nir_type_uint32);
149 M_LOAD(ldst_mov, nir_type_uint32);
150 M_LOAD(ld_image_32f, nir_type_float32);
151 M_LOAD(ld_image_16f, nir_type_float16);
152 M_LOAD(ld_image_32u, nir_type_uint32);
153 M_LOAD(ld_image_32i, nir_type_int32);
154 M_STORE(st_image_32f, nir_type_float32);
155 M_STORE(st_image_16f, nir_type_float16);
156 M_STORE(st_image_32u, nir_type_uint32);
157 M_STORE(st_image_32i, nir_type_int32);
158 M_LOAD(lea_image, nir_type_uint64);
159
160 #define M_IMAGE(op) \
161 static midgard_instruction op##_image(nir_alu_type type, unsigned val, \
162 unsigned address) \
163 { \
164 switch (type) { \
165 case nir_type_float32: \
166 return m_##op##_image_32f(val, address); \
167 case nir_type_float16: \
168 return m_##op##_image_16f(val, address); \
169 case nir_type_uint32: \
170 return m_##op##_image_32u(val, address); \
171 case nir_type_int32: \
172 return m_##op##_image_32i(val, address); \
173 default: \
174 unreachable("Invalid image type"); \
175 } \
176 }
177
178 M_IMAGE(ld);
179 M_IMAGE(st);
180
181 static midgard_instruction
v_branch(bool conditional,bool invert)182 v_branch(bool conditional, bool invert)
183 {
184 midgard_instruction ins = {
185 .type = TAG_ALU_4,
186 .unit = ALU_ENAB_BRANCH,
187 .compact_branch = true,
188 .branch =
189 {
190 .conditional = conditional,
191 .invert_conditional = invert,
192 },
193 .dest = ~0,
194 .src = {~0, ~0, ~0, ~0},
195 };
196
197 return ins;
198 }
199
200 static void
attach_constants(compiler_context * ctx,midgard_instruction * ins,void * constants,int name)201 attach_constants(compiler_context *ctx, midgard_instruction *ins,
202 void *constants, int name)
203 {
204 ins->has_constants = true;
205 memcpy(&ins->constants, constants, 16);
206 }
207
208 static int
glsl_type_size(const struct glsl_type * type,bool bindless)209 glsl_type_size(const struct glsl_type *type, bool bindless)
210 {
211 return glsl_count_attribute_slots(type, false);
212 }
213
214 static bool
midgard_nir_lower_global_load_instr(nir_builder * b,nir_intrinsic_instr * intr,void * data)215 midgard_nir_lower_global_load_instr(nir_builder *b, nir_intrinsic_instr *intr,
216 void *data)
217 {
218 if (intr->intrinsic != nir_intrinsic_load_global &&
219 intr->intrinsic != nir_intrinsic_load_shared)
220 return false;
221
222 unsigned compsz = intr->def.bit_size;
223 unsigned totalsz = compsz * intr->def.num_components;
224 /* 8, 16, 32, 64 and 128 bit loads don't need to be lowered */
225 if (util_bitcount(totalsz) < 2 && totalsz <= 128)
226 return false;
227
228 b->cursor = nir_before_instr(&intr->instr);
229
230 nir_def *addr = intr->src[0].ssa;
231
232 nir_def *comps[MIR_VEC_COMPONENTS];
233 unsigned ncomps = 0;
234
235 while (totalsz) {
236 unsigned loadsz = MIN2(1 << (util_last_bit(totalsz) - 1), 128);
237 unsigned loadncomps = loadsz / compsz;
238
239 nir_def *load;
240 if (intr->intrinsic == nir_intrinsic_load_global) {
241 load = nir_load_global(b, addr, compsz / 8, loadncomps, compsz);
242 } else {
243 assert(intr->intrinsic == nir_intrinsic_load_shared);
244 nir_intrinsic_instr *shared_load =
245 nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_shared);
246 shared_load->num_components = loadncomps;
247 shared_load->src[0] = nir_src_for_ssa(addr);
248 nir_intrinsic_set_align(shared_load, compsz / 8, 0);
249 nir_intrinsic_set_base(shared_load, nir_intrinsic_base(intr));
250 nir_def_init(&shared_load->instr, &shared_load->def,
251 shared_load->num_components, compsz);
252 nir_builder_instr_insert(b, &shared_load->instr);
253 load = &shared_load->def;
254 }
255
256 for (unsigned i = 0; i < loadncomps; i++)
257 comps[ncomps++] = nir_channel(b, load, i);
258
259 totalsz -= loadsz;
260 addr = nir_iadd_imm(b, addr, loadsz / 8);
261 }
262
263 assert(ncomps == intr->def.num_components);
264 nir_def_rewrite_uses(&intr->def, nir_vec(b, comps, ncomps));
265
266 return true;
267 }
268
269 static bool
midgard_nir_lower_global_load(nir_shader * shader)270 midgard_nir_lower_global_load(nir_shader *shader)
271 {
272 return nir_shader_intrinsics_pass(
273 shader, midgard_nir_lower_global_load_instr,
274 nir_metadata_control_flow, NULL);
275 }
276
277 static bool
mdg_should_scalarize(const nir_instr * instr,const void * _unused)278 mdg_should_scalarize(const nir_instr *instr, const void *_unused)
279 {
280 const nir_alu_instr *alu = nir_instr_as_alu(instr);
281
282 if (nir_src_bit_size(alu->src[0].src) == 64)
283 return true;
284
285 if (alu->def.bit_size == 64)
286 return true;
287
288 switch (alu->op) {
289 case nir_op_fdot2:
290 case nir_op_umul_high:
291 case nir_op_imul_high:
292 case nir_op_pack_half_2x16:
293 case nir_op_unpack_half_2x16:
294
295 /* The LUT unit is scalar */
296 case nir_op_fsqrt:
297 case nir_op_frcp:
298 case nir_op_frsq:
299 case nir_op_fsin_mdg:
300 case nir_op_fcos_mdg:
301 case nir_op_fexp2:
302 case nir_op_flog2:
303 return true;
304 default:
305 return false;
306 }
307 }
308
309 /* Only vectorize int64 up to vec2 */
310 static uint8_t
midgard_vectorize_filter(const nir_instr * instr,const void * data)311 midgard_vectorize_filter(const nir_instr *instr, const void *data)
312 {
313 if (instr->type != nir_instr_type_alu)
314 return 0;
315
316 const nir_alu_instr *alu = nir_instr_as_alu(instr);
317 int src_bit_size = nir_src_bit_size(alu->src[0].src);
318 int dst_bit_size = alu->def.bit_size;
319
320 if (src_bit_size == 64 || dst_bit_size == 64)
321 return 2;
322
323 return 4;
324 }
325
326 static nir_mem_access_size_align
mem_access_size_align_cb(nir_intrinsic_op intrin,uint8_t bytes,uint8_t bit_size,uint32_t align_mul,uint32_t align_offset,bool offset_is_const,const void * cb_data)327 mem_access_size_align_cb(nir_intrinsic_op intrin, uint8_t bytes,
328 uint8_t bit_size, uint32_t align_mul,
329 uint32_t align_offset, bool offset_is_const,
330 const void *cb_data)
331 {
332 uint32_t align = nir_combined_align(align_mul, align_offset);
333 assert(util_is_power_of_two_nonzero(align));
334
335 /* No more than 16 bytes at a time. */
336 bytes = MIN2(bytes, 16);
337
338 /* If the number of bytes is a multiple of 4, use 32-bit loads. Else if it's
339 * a multiple of 2, use 16-bit loads. Else use 8-bit loads.
340 *
341 * But if we're only aligned to 1 byte, use 8-bit loads. If we're only
342 * aligned to 2 bytes, use 16-bit loads, unless we needed 8-bit loads due to
343 * the size.
344 */
345 if ((bytes & 1) || (align == 1))
346 bit_size = 8;
347 else if ((bytes & 2) || (align == 2))
348 bit_size = 16;
349 else if (bit_size >= 32)
350 bit_size = 32;
351
352 unsigned num_comps = MIN2(bytes / (bit_size / 8), 4);
353
354 /* Push constants require 32-bit loads. */
355 if (intrin == nir_intrinsic_load_push_constant) {
356 if (align_mul >= 4) {
357 /* If align_mul is bigger than 4 we can use align_offset to find
358 * the exact number of words we need to read.
359 */
360 num_comps = DIV_ROUND_UP((align_offset % 4) + bytes, 4);
361 } else {
362 /* If bytes is aligned on 32-bit, the access might still cross one
363 * word at the beginning, and one word at the end. If bytes is not
364 * aligned on 32-bit, the extra two words should cover for both the
365 * size and offset mis-alignment.
366 */
367 num_comps = (bytes / 4) + 2;
368 }
369
370 bit_size = MIN2(bit_size, 32);
371 }
372
373 return (nir_mem_access_size_align){
374 .num_components = num_comps,
375 .bit_size = bit_size,
376 .align = bit_size / 8,
377 };
378 }
379
380 static uint8_t
lower_vec816_alu(const nir_instr * instr,const void * cb_data)381 lower_vec816_alu(const nir_instr *instr, const void *cb_data)
382 {
383 return 4;
384 }
385
386 void
midgard_preprocess_nir(nir_shader * nir,unsigned gpu_id)387 midgard_preprocess_nir(nir_shader *nir, unsigned gpu_id)
388 {
389 unsigned quirks = midgard_get_quirks(gpu_id);
390
391 /* Lower gl_Position pre-optimisation, but after lowering vars to ssa
392 * (so we don't accidentally duplicate the epilogue since mesa/st has
393 * messed with our I/O quite a bit already).
394 */
395 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
396
397 if (nir->info.stage == MESA_SHADER_VERTEX) {
398 NIR_PASS_V(nir, nir_lower_viewport_transform);
399 NIR_PASS_V(nir, nir_lower_point_size, 1.0, 0.0);
400 }
401
402 NIR_PASS_V(nir, nir_lower_var_copies);
403 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
404 NIR_PASS_V(nir, nir_split_var_copies);
405 NIR_PASS_V(nir, nir_lower_var_copies);
406 NIR_PASS_V(nir, nir_lower_global_vars_to_local);
407 NIR_PASS_V(nir, nir_lower_var_copies);
408 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
409
410 NIR_PASS_V(nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out,
411 glsl_type_size, 0);
412
413 if (nir->info.stage == MESA_SHADER_VERTEX) {
414 /* nir_lower[_explicit]_io is lazy and emits mul+add chains even
415 * for offsets it could figure out are constant. Do some
416 * constant folding before pan_nir_lower_store_component below.
417 */
418 NIR_PASS_V(nir, nir_opt_constant_folding);
419 NIR_PASS_V(nir, pan_nir_lower_store_component);
420 }
421
422 /* Could be eventually useful for Vulkan, but we don't expect it to have
423 * the support, so limit it to compute */
424 if (gl_shader_stage_is_compute(nir->info.stage)) {
425 nir_lower_mem_access_bit_sizes_options mem_size_options = {
426 .modes = nir_var_mem_ubo | nir_var_mem_push_const | nir_var_mem_ssbo |
427 nir_var_mem_constant | nir_var_mem_task_payload |
428 nir_var_shader_temp | nir_var_function_temp |
429 nir_var_mem_global | nir_var_mem_shared,
430 .callback = mem_access_size_align_cb,
431 };
432
433 NIR_PASS_V(nir, nir_lower_mem_access_bit_sizes, &mem_size_options);
434 NIR_PASS_V(nir, nir_lower_alu_width, lower_vec816_alu, NULL);
435 NIR_PASS_V(nir, nir_lower_alu_vec8_16_srcs);
436 }
437
438 NIR_PASS_V(nir, nir_lower_ssbo, NULL);
439 NIR_PASS_V(nir, pan_nir_lower_zs_store);
440
441 NIR_PASS_V(nir, nir_lower_frexp);
442 NIR_PASS_V(nir, midgard_nir_lower_global_load);
443
444 nir_lower_idiv_options idiv_options = {
445 .allow_fp16 = true,
446 };
447
448 NIR_PASS_V(nir, nir_lower_idiv, &idiv_options);
449
450 nir_lower_tex_options lower_tex_options = {
451 .lower_txs_lod = true,
452 .lower_txp = ~0,
453 .lower_tg4_broadcom_swizzle = true,
454 .lower_txd = true,
455 .lower_invalid_implicit_lod = true,
456 };
457
458 NIR_PASS_V(nir, nir_lower_tex, &lower_tex_options);
459 NIR_PASS_V(nir, nir_lower_image_atomics_to_global);
460
461 /* TEX_GRAD fails to apply sampler descriptor settings on some
462 * implementations, requiring a lowering.
463 */
464 if (quirks & MIDGARD_BROKEN_LOD)
465 NIR_PASS_V(nir, midgard_nir_lod_errata);
466
467 /* lower MSAA image operations to 3D load before coordinate lowering */
468 NIR_PASS_V(nir, pan_nir_lower_image_ms);
469
470 /* Midgard image ops coordinates are 16-bit instead of 32-bit */
471 NIR_PASS_V(nir, midgard_nir_lower_image_bitsize);
472
473 if (nir->info.stage == MESA_SHADER_FRAGMENT)
474 NIR_PASS_V(nir, nir_lower_helper_writes, true);
475
476 NIR_PASS_V(nir, pan_lower_helper_invocation);
477 NIR_PASS_V(nir, pan_lower_sample_pos);
478 NIR_PASS_V(nir, midgard_nir_lower_algebraic_early);
479 NIR_PASS_V(nir, nir_lower_alu_to_scalar, mdg_should_scalarize, NULL);
480 NIR_PASS_V(nir, nir_lower_flrp, 16 | 32 | 64, false /* always_precise */);
481 NIR_PASS_V(nir, nir_lower_var_copies);
482 }
483
484 static void
optimise_nir(nir_shader * nir,unsigned quirks,bool is_blend)485 optimise_nir(nir_shader *nir, unsigned quirks, bool is_blend)
486 {
487 bool progress;
488
489 do {
490 progress = false;
491
492 NIR_PASS(progress, nir, nir_lower_vars_to_ssa);
493
494 NIR_PASS(progress, nir, nir_copy_prop);
495 NIR_PASS(progress, nir, nir_opt_remove_phis);
496 NIR_PASS(progress, nir, nir_opt_dce);
497 NIR_PASS(progress, nir, nir_opt_dead_cf);
498 NIR_PASS(progress, nir, nir_opt_cse);
499 NIR_PASS(progress, nir, nir_opt_peephole_select, 64, false, true);
500 NIR_PASS(progress, nir, nir_opt_algebraic);
501 NIR_PASS(progress, nir, nir_opt_constant_folding);
502 NIR_PASS(progress, nir, nir_opt_undef);
503 NIR_PASS(progress, nir, nir_lower_undef_to_zero);
504
505 NIR_PASS(progress, nir, nir_opt_loop_unroll);
506
507 NIR_PASS(progress, nir, nir_opt_vectorize, midgard_vectorize_filter,
508 NULL);
509 } while (progress);
510
511 NIR_PASS_V(nir, nir_lower_alu_to_scalar, mdg_should_scalarize, NULL);
512
513 /* Run after opts so it can hit more */
514 if (!is_blend)
515 NIR_PASS(progress, nir, nir_fuse_io_16);
516
517 do {
518 progress = false;
519
520 NIR_PASS(progress, nir, nir_opt_dce);
521 NIR_PASS(progress, nir, nir_opt_algebraic);
522 NIR_PASS(progress, nir, nir_opt_constant_folding);
523 NIR_PASS(progress, nir, nir_copy_prop);
524 } while (progress);
525
526 NIR_PASS(progress, nir, nir_opt_algebraic_late);
527 NIR_PASS(progress, nir, nir_opt_algebraic_distribute_src_mods);
528
529 /* We implement booleans as 32-bit 0/~0 */
530 NIR_PASS(progress, nir, nir_lower_bool_to_int32);
531
532 /* Now that booleans are lowered, we can run out late opts */
533 NIR_PASS(progress, nir, midgard_nir_lower_algebraic_late);
534 NIR_PASS(progress, nir, midgard_nir_cancel_inot);
535 NIR_PASS_V(nir, midgard_nir_type_csel);
536
537 /* Clean up after late opts */
538 do {
539 progress = false;
540
541 NIR_PASS(progress, nir, nir_opt_dce);
542 NIR_PASS(progress, nir, nir_opt_constant_folding);
543 NIR_PASS(progress, nir, nir_copy_prop);
544 } while (progress);
545
546 /* Backend scheduler is purely local, so do some global optimizations
547 * to reduce register pressure. */
548 nir_move_options move_all = nir_move_const_undef | nir_move_load_ubo |
549 nir_move_load_input | nir_move_comparisons |
550 nir_move_copies | nir_move_load_ssbo;
551
552 NIR_PASS_V(nir, nir_opt_sink, move_all);
553 NIR_PASS_V(nir, nir_opt_move, move_all);
554
555 /* Take us out of SSA */
556 NIR_PASS(progress, nir, nir_convert_from_ssa, true);
557
558 /* We are a vector architecture; write combine where possible */
559 NIR_PASS(progress, nir, nir_move_vec_src_uses_to_dest, false);
560 NIR_PASS(progress, nir, nir_lower_vec_to_regs, NULL, NULL);
561
562 NIR_PASS(progress, nir, nir_opt_dce);
563 NIR_PASS_V(nir, nir_trivialize_registers);
564 }
565
566 /* Do not actually emit a load; instead, cache the constant for inlining */
567
568 static void
emit_load_const(compiler_context * ctx,nir_load_const_instr * instr)569 emit_load_const(compiler_context *ctx, nir_load_const_instr *instr)
570 {
571 nir_def def = instr->def;
572
573 midgard_constants *consts = rzalloc(ctx, midgard_constants);
574
575 assert(instr->def.num_components * instr->def.bit_size <=
576 sizeof(*consts) * 8);
577
578 #define RAW_CONST_COPY(bits) \
579 nir_const_value_to_array(consts->u##bits, instr->value, \
580 instr->def.num_components, u##bits)
581
582 switch (instr->def.bit_size) {
583 case 64:
584 RAW_CONST_COPY(64);
585 break;
586 case 32:
587 RAW_CONST_COPY(32);
588 break;
589 case 16:
590 RAW_CONST_COPY(16);
591 break;
592 case 8:
593 RAW_CONST_COPY(8);
594 break;
595 default:
596 unreachable("Invalid bit_size for load_const instruction\n");
597 }
598
599 /* Shifted for SSA, +1 for off-by-one */
600 _mesa_hash_table_u64_insert(ctx->ssa_constants, (def.index << 1) + 1,
601 consts);
602 }
603
604 /* Normally constants are embedded implicitly, but for I/O and such we have to
605 * explicitly emit a move with the constant source */
606
607 static void
emit_explicit_constant(compiler_context * ctx,unsigned node)608 emit_explicit_constant(compiler_context *ctx, unsigned node)
609 {
610 void *constant_value =
611 _mesa_hash_table_u64_search(ctx->ssa_constants, node + 1);
612
613 if (constant_value) {
614 midgard_instruction ins =
615 v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), node);
616 attach_constants(ctx, &ins, constant_value, node + 1);
617 emit_mir_instruction(ctx, ins);
618 }
619 }
620
621 static bool
nir_is_non_scalar_swizzle(nir_alu_src * src,unsigned nr_components)622 nir_is_non_scalar_swizzle(nir_alu_src *src, unsigned nr_components)
623 {
624 unsigned comp = src->swizzle[0];
625
626 for (unsigned c = 1; c < nr_components; ++c) {
627 if (src->swizzle[c] != comp)
628 return true;
629 }
630
631 return false;
632 }
633
634 #define ALU_CASE(nir, _op) \
635 case nir_op_##nir: \
636 op = midgard_alu_op_##_op; \
637 assert(src_bitsize == dst_bitsize); \
638 break;
639
640 #define ALU_CASE_RTZ(nir, _op) \
641 case nir_op_##nir: \
642 op = midgard_alu_op_##_op; \
643 roundmode = MIDGARD_RTZ; \
644 break;
645
646 #define ALU_CHECK_CMP() \
647 assert(src_bitsize == 16 || src_bitsize == 32 || src_bitsize == 64); \
648 assert(dst_bitsize == 16 || dst_bitsize == 32);
649
650 #define ALU_CASE_BCAST(nir, _op, count) \
651 case nir_op_##nir: \
652 op = midgard_alu_op_##_op; \
653 broadcast_swizzle = count; \
654 ALU_CHECK_CMP(); \
655 break;
656
657 #define ALU_CASE_CMP(nir, _op) \
658 case nir_op_##nir: \
659 op = midgard_alu_op_##_op; \
660 ALU_CHECK_CMP(); \
661 break;
662
663 static void
mir_copy_src(midgard_instruction * ins,nir_alu_instr * instr,unsigned i,unsigned to,bool is_int,unsigned bcast_count)664 mir_copy_src(midgard_instruction *ins, nir_alu_instr *instr, unsigned i,
665 unsigned to, bool is_int, unsigned bcast_count)
666 {
667 nir_alu_src src = instr->src[i];
668 unsigned bits = nir_src_bit_size(src.src);
669
670 ins->src[to] = nir_src_index(NULL, &src.src);
671 ins->src_types[to] = nir_op_infos[instr->op].input_types[i] | bits;
672
673 /* Figure out which component we should fill unused channels with. This
674 * doesn't matter too much in the non-broadcast case, but it makes
675 * should that scalar sources are packed with replicated swizzles,
676 * which works around issues seen with the combination of source
677 * expansion and destination shrinking.
678 */
679 unsigned replicate_c = 0;
680 if (bcast_count) {
681 replicate_c = bcast_count - 1;
682 } else {
683 for (unsigned c = 0; c < NIR_MAX_VEC_COMPONENTS; ++c) {
684 if (nir_alu_instr_channel_used(instr, i, c))
685 replicate_c = c;
686 }
687 }
688
689 for (unsigned c = 0; c < NIR_MAX_VEC_COMPONENTS; ++c) {
690 ins->swizzle[to][c] =
691 src.swizzle[((!bcast_count || c < bcast_count) &&
692 nir_alu_instr_channel_used(instr, i, c))
693 ? c
694 : replicate_c];
695 }
696 }
697
698 static void
emit_alu(compiler_context * ctx,nir_alu_instr * instr)699 emit_alu(compiler_context *ctx, nir_alu_instr *instr)
700 {
701 unsigned nr_components = instr->def.num_components;
702 unsigned nr_inputs = nir_op_infos[instr->op].num_inputs;
703 unsigned op = 0;
704
705 /* Number of components valid to check for the instruction (the rest
706 * will be forced to the last), or 0 to use as-is. Relevant as
707 * ball-type instructions have a channel count in NIR but are all vec4
708 * in Midgard */
709
710 unsigned broadcast_swizzle = 0;
711
712 /* Should we swap arguments? */
713 bool flip_src12 = false;
714
715 ASSERTED unsigned src_bitsize = nir_src_bit_size(instr->src[0].src);
716 unsigned dst_bitsize = instr->def.bit_size;
717
718 enum midgard_roundmode roundmode = MIDGARD_RTE;
719
720 switch (instr->op) {
721 ALU_CASE(fadd, fadd);
722 ALU_CASE(fmul, fmul);
723 ALU_CASE(fmin, fmin);
724 ALU_CASE(fmax, fmax);
725 ALU_CASE(imin, imin);
726 ALU_CASE(imax, imax);
727 ALU_CASE(umin, umin);
728 ALU_CASE(umax, umax);
729 ALU_CASE(ffloor, ffloor);
730 ALU_CASE(fround_even, froundeven);
731 ALU_CASE(ftrunc, ftrunc);
732 ALU_CASE(fceil, fceil);
733 ALU_CASE(fdot3, fdot3);
734 ALU_CASE(fdot4, fdot4);
735 ALU_CASE(iadd, iadd);
736 ALU_CASE(isub, isub);
737 ALU_CASE(iadd_sat, iaddsat);
738 ALU_CASE(isub_sat, isubsat);
739 ALU_CASE(uadd_sat, uaddsat);
740 ALU_CASE(usub_sat, usubsat);
741 ALU_CASE(imul, imul);
742 ALU_CASE(imul_high, imul);
743 ALU_CASE(umul_high, imul);
744 ALU_CASE(uclz, iclz);
745
746 /* Zero shoved as second-arg */
747 ALU_CASE(iabs, iabsdiff);
748
749 ALU_CASE(uabs_isub, iabsdiff);
750 ALU_CASE(uabs_usub, uabsdiff);
751
752 ALU_CASE(mov, imov);
753
754 ALU_CASE_CMP(feq32, feq);
755 ALU_CASE_CMP(fneu32, fne);
756 ALU_CASE_CMP(flt32, flt);
757 ALU_CASE_CMP(ieq32, ieq);
758 ALU_CASE_CMP(ine32, ine);
759 ALU_CASE_CMP(ilt32, ilt);
760 ALU_CASE_CMP(ult32, ult);
761
762 /* We don't have a native b2f32 instruction. Instead, like many
763 * GPUs, we exploit booleans as 0/~0 for false/true, and
764 * correspondingly AND
765 * by 1.0 to do the type conversion. For the moment, prime us
766 * to emit:
767 *
768 * iand [whatever], #0
769 *
770 * At the end of emit_alu (as MIR), we'll fix-up the constant
771 */
772
773 ALU_CASE_CMP(b2f32, iand);
774 ALU_CASE_CMP(b2f16, iand);
775 ALU_CASE_CMP(b2i32, iand);
776 ALU_CASE_CMP(b2i16, iand);
777
778 ALU_CASE(frcp, frcp);
779 ALU_CASE(frsq, frsqrt);
780 ALU_CASE(fsqrt, fsqrt);
781 ALU_CASE(fexp2, fexp2);
782 ALU_CASE(flog2, flog2);
783
784 ALU_CASE_RTZ(f2i64, f2i_rte);
785 ALU_CASE_RTZ(f2u64, f2u_rte);
786 ALU_CASE_RTZ(i2f64, i2f_rte);
787 ALU_CASE_RTZ(u2f64, u2f_rte);
788
789 ALU_CASE_RTZ(f2i32, f2i_rte);
790 ALU_CASE_RTZ(f2u32, f2u_rte);
791 ALU_CASE_RTZ(i2f32, i2f_rte);
792 ALU_CASE_RTZ(u2f32, u2f_rte);
793
794 ALU_CASE_RTZ(f2i8, f2i_rte);
795 ALU_CASE_RTZ(f2u8, f2u_rte);
796
797 ALU_CASE_RTZ(f2i16, f2i_rte);
798 ALU_CASE_RTZ(f2u16, f2u_rte);
799 ALU_CASE_RTZ(i2f16, i2f_rte);
800 ALU_CASE_RTZ(u2f16, u2f_rte);
801
802 ALU_CASE(fsin_mdg, fsinpi);
803 ALU_CASE(fcos_mdg, fcospi);
804
805 /* We'll get 0 in the second arg, so:
806 * ~a = ~(a | 0) = nor(a, 0) */
807 ALU_CASE(inot, inor);
808 ALU_CASE(iand, iand);
809 ALU_CASE(ior, ior);
810 ALU_CASE(ixor, ixor);
811 ALU_CASE(ishl, ishl);
812 ALU_CASE(ishr, iasr);
813 ALU_CASE(ushr, ilsr);
814
815 ALU_CASE_BCAST(b32all_fequal2, fball_eq, 2);
816 ALU_CASE_BCAST(b32all_fequal3, fball_eq, 3);
817 ALU_CASE_CMP(b32all_fequal4, fball_eq);
818
819 ALU_CASE_BCAST(b32any_fnequal2, fbany_neq, 2);
820 ALU_CASE_BCAST(b32any_fnequal3, fbany_neq, 3);
821 ALU_CASE_CMP(b32any_fnequal4, fbany_neq);
822
823 ALU_CASE_BCAST(b32all_iequal2, iball_eq, 2);
824 ALU_CASE_BCAST(b32all_iequal3, iball_eq, 3);
825 ALU_CASE_CMP(b32all_iequal4, iball_eq);
826
827 ALU_CASE_BCAST(b32any_inequal2, ibany_neq, 2);
828 ALU_CASE_BCAST(b32any_inequal3, ibany_neq, 3);
829 ALU_CASE_CMP(b32any_inequal4, ibany_neq);
830
831 /* Source mods will be shoved in later */
832 ALU_CASE(fabs, fmov);
833 ALU_CASE(fneg, fmov);
834 ALU_CASE(fsat, fmov);
835 ALU_CASE(fsat_signed_mali, fmov);
836 ALU_CASE(fclamp_pos_mali, fmov);
837
838 /* For size conversion, we use a move. Ideally though we would squash
839 * these ops together; maybe that has to happen after in NIR as part of
840 * propagation...? An earlier algebraic pass ensured we step down by
841 * only / exactly one size. If stepping down, we use a dest override to
842 * reduce the size; if stepping up, we use a larger-sized move with a
843 * half source and a sign/zero-extension modifier */
844
845 case nir_op_i2i8:
846 case nir_op_i2i16:
847 case nir_op_i2i32:
848 case nir_op_i2i64:
849 case nir_op_u2u8:
850 case nir_op_u2u16:
851 case nir_op_u2u32:
852 case nir_op_u2u64:
853 case nir_op_f2f16:
854 case nir_op_f2f32:
855 case nir_op_f2f64: {
856 if (instr->op == nir_op_f2f16 || instr->op == nir_op_f2f32 ||
857 instr->op == nir_op_f2f64)
858 op = midgard_alu_op_fmov;
859 else
860 op = midgard_alu_op_imov;
861
862 break;
863 }
864
865 /* For greater-or-equal, we lower to less-or-equal and flip the
866 * arguments */
867
868 case nir_op_fge:
869 case nir_op_fge32:
870 case nir_op_ige32:
871 case nir_op_uge32: {
872 op = instr->op == nir_op_fge ? midgard_alu_op_fle
873 : instr->op == nir_op_fge32 ? midgard_alu_op_fle
874 : instr->op == nir_op_ige32 ? midgard_alu_op_ile
875 : instr->op == nir_op_uge32 ? midgard_alu_op_ule
876 : 0;
877
878 flip_src12 = true;
879 ALU_CHECK_CMP();
880 break;
881 }
882
883 case nir_op_b32csel:
884 case nir_op_b32fcsel_mdg: {
885 bool mixed = nir_is_non_scalar_swizzle(&instr->src[0], nr_components);
886 bool is_float = instr->op == nir_op_b32fcsel_mdg;
887 op = is_float ? (mixed ? midgard_alu_op_fcsel_v : midgard_alu_op_fcsel)
888 : (mixed ? midgard_alu_op_icsel_v : midgard_alu_op_icsel);
889
890 int index = nir_src_index(ctx, &instr->src[0].src);
891 emit_explicit_constant(ctx, index);
892
893 break;
894 }
895
896 case nir_op_unpack_32_2x16:
897 case nir_op_unpack_32_4x8:
898 case nir_op_pack_32_2x16:
899 case nir_op_pack_32_4x8: {
900 op = midgard_alu_op_imov;
901 break;
902 }
903
904 case nir_op_unpack_64_2x32:
905 case nir_op_unpack_64_4x16:
906 case nir_op_pack_64_2x32:
907 case nir_op_pack_64_4x16: {
908 op = midgard_alu_op_imov;
909 break;
910 }
911
912 default:
913 mesa_loge("Unhandled ALU op %s\n", nir_op_infos[instr->op].name);
914 assert(0);
915 return;
916 }
917
918 /* Promote imov to fmov if it might help inline a constant */
919 if (op == midgard_alu_op_imov && nir_src_is_const(instr->src[0].src) &&
920 nir_src_bit_size(instr->src[0].src) == 32 &&
921 nir_is_same_comp_swizzle(instr->src[0].swizzle,
922 nir_src_num_components(instr->src[0].src))) {
923 op = midgard_alu_op_fmov;
924 }
925
926 /* Midgard can perform certain modifiers on output of an ALU op */
927
928 unsigned outmod = 0;
929 bool is_int = midgard_is_integer_op(op);
930
931 if (instr->op == nir_op_umul_high || instr->op == nir_op_imul_high) {
932 outmod = midgard_outmod_keephi;
933 } else if (midgard_is_integer_out_op(op)) {
934 outmod = midgard_outmod_keeplo;
935 } else if (instr->op == nir_op_fsat) {
936 outmod = midgard_outmod_clamp_0_1;
937 } else if (instr->op == nir_op_fsat_signed_mali) {
938 outmod = midgard_outmod_clamp_m1_1;
939 } else if (instr->op == nir_op_fclamp_pos_mali) {
940 outmod = midgard_outmod_clamp_0_inf;
941 }
942
943 /* Fetch unit, quirks, etc information */
944 unsigned opcode_props = alu_opcode_props[op].props;
945 bool quirk_flipped_r24 = opcode_props & QUIRK_FLIPPED_R24;
946
947 midgard_instruction ins = {
948 .type = TAG_ALU_4,
949 .dest_type = nir_op_infos[instr->op].output_type | dst_bitsize,
950 .roundmode = roundmode,
951 };
952
953 ins.dest = nir_def_index_with_mask(&instr->def, &ins.mask);
954
955 for (unsigned i = nr_inputs; i < ARRAY_SIZE(ins.src); ++i)
956 ins.src[i] = ~0;
957
958 if (quirk_flipped_r24) {
959 ins.src[0] = ~0;
960 mir_copy_src(&ins, instr, 0, 1, is_int, broadcast_swizzle);
961 } else {
962 for (unsigned i = 0; i < nr_inputs; ++i) {
963 unsigned to = i;
964
965 if (instr->op == nir_op_b32csel || instr->op == nir_op_b32fcsel_mdg) {
966 /* The condition is the first argument; move
967 * the other arguments up one to be a binary
968 * instruction for Midgard with the condition
969 * last */
970
971 if (i == 0)
972 to = 2;
973 else if (flip_src12)
974 to = 2 - i;
975 else
976 to = i - 1;
977 } else if (flip_src12) {
978 to = 1 - to;
979 }
980
981 mir_copy_src(&ins, instr, i, to, is_int, broadcast_swizzle);
982 }
983 }
984
985 if (instr->op == nir_op_fneg || instr->op == nir_op_fabs) {
986 /* Lowered to move */
987 if (instr->op == nir_op_fneg)
988 ins.src_neg[1] ^= true;
989
990 if (instr->op == nir_op_fabs)
991 ins.src_abs[1] = true;
992 }
993
994 ins.op = op;
995 ins.outmod = outmod;
996
997 /* Late fixup for emulated instructions */
998
999 if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) {
1000 /* Presently, our second argument is an inline #0 constant.
1001 * Switch over to an embedded 1.0 constant (that can't fit
1002 * inline, since we're 32-bit, not 16-bit like the inline
1003 * constants) */
1004
1005 ins.has_inline_constant = false;
1006 ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1007 ins.src_types[1] = nir_type_float32;
1008 ins.has_constants = true;
1009
1010 if (instr->op == nir_op_b2f32)
1011 ins.constants.f32[0] = 1.0f;
1012 else
1013 ins.constants.i32[0] = 1;
1014
1015 for (unsigned c = 0; c < 16; ++c)
1016 ins.swizzle[1][c] = 0;
1017 } else if (instr->op == nir_op_b2f16) {
1018 ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1019 ins.src_types[1] = nir_type_float16;
1020 ins.has_constants = true;
1021 ins.constants.i16[0] = _mesa_float_to_half(1.0);
1022
1023 for (unsigned c = 0; c < 16; ++c)
1024 ins.swizzle[1][c] = 0;
1025 } else if (nr_inputs == 1 && !quirk_flipped_r24) {
1026 /* Lots of instructions need a 0 plonked in */
1027 ins.has_inline_constant = false;
1028 ins.src[1] = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
1029 ins.src_types[1] = ins.src_types[0];
1030 ins.has_constants = true;
1031 ins.constants.u32[0] = 0;
1032
1033 for (unsigned c = 0; c < 16; ++c)
1034 ins.swizzle[1][c] = 0;
1035 } else if (instr->op == nir_op_pack_32_2x16) {
1036 ins.dest_type = nir_type_uint16;
1037 ins.mask = mask_of(nr_components * 2);
1038 ins.is_pack = true;
1039 } else if (instr->op == nir_op_pack_32_4x8) {
1040 ins.dest_type = nir_type_uint8;
1041 ins.mask = mask_of(nr_components * 4);
1042 ins.is_pack = true;
1043 } else if (instr->op == nir_op_unpack_32_2x16) {
1044 ins.dest_type = nir_type_uint32;
1045 ins.mask = mask_of(nr_components >> 1);
1046 ins.is_pack = true;
1047 } else if (instr->op == nir_op_unpack_32_4x8) {
1048 ins.dest_type = nir_type_uint32;
1049 ins.mask = mask_of(nr_components >> 2);
1050 ins.is_pack = true;
1051 }
1052
1053 emit_mir_instruction(ctx, ins);
1054 }
1055
1056 #undef ALU_CASE
1057
1058 static void
mir_set_intr_mask(nir_instr * instr,midgard_instruction * ins,bool is_read)1059 mir_set_intr_mask(nir_instr *instr, midgard_instruction *ins, bool is_read)
1060 {
1061 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
1062 unsigned nir_mask = 0;
1063 unsigned dsize = 0;
1064
1065 if (is_read) {
1066 nir_mask = mask_of(nir_intrinsic_dest_components(intr));
1067
1068 /* Extension is mandatory for 8/16-bit loads */
1069 dsize = intr->def.bit_size == 64 ? 64 : 32;
1070 } else {
1071 nir_mask = nir_intrinsic_write_mask(intr);
1072 dsize = OP_IS_COMMON_STORE(ins->op) ? nir_src_bit_size(intr->src[0]) : 32;
1073 }
1074
1075 /* Once we have the NIR mask, we need to normalize to work in 32-bit space */
1076 unsigned bytemask = pan_to_bytemask(dsize, nir_mask);
1077 ins->dest_type = nir_type_uint | dsize;
1078 mir_set_bytemask(ins, bytemask);
1079 }
1080
1081 /* Uniforms and UBOs use a shared code path, as uniforms are just (slightly
1082 * optimized) versions of UBO #0 */
1083
1084 static midgard_instruction *
emit_ubo_read(compiler_context * ctx,nir_instr * instr,unsigned dest,unsigned offset,nir_src * indirect_offset,unsigned indirect_shift,unsigned index,unsigned nr_comps)1085 emit_ubo_read(compiler_context *ctx, nir_instr *instr, unsigned dest,
1086 unsigned offset, nir_src *indirect_offset,
1087 unsigned indirect_shift, unsigned index, unsigned nr_comps)
1088 {
1089 midgard_instruction ins;
1090
1091 unsigned dest_size = (instr->type == nir_instr_type_intrinsic)
1092 ? nir_instr_as_intrinsic(instr)->def.bit_size
1093 : 32;
1094
1095 unsigned bitsize = dest_size * nr_comps;
1096
1097 /* Pick the smallest intrinsic to avoid out-of-bounds reads */
1098 if (bitsize <= 8)
1099 ins = m_ld_ubo_u8(dest, 0);
1100 else if (bitsize <= 16)
1101 ins = m_ld_ubo_u16(dest, 0);
1102 else if (bitsize <= 32)
1103 ins = m_ld_ubo_32(dest, 0);
1104 else if (bitsize <= 64)
1105 ins = m_ld_ubo_64(dest, 0);
1106 else if (bitsize <= 128)
1107 ins = m_ld_ubo_128(dest, 0);
1108 else
1109 unreachable("Invalid UBO read size");
1110
1111 ins.constants.u32[0] = offset;
1112
1113 if (instr->type == nir_instr_type_intrinsic)
1114 mir_set_intr_mask(instr, &ins, true);
1115
1116 if (indirect_offset) {
1117 ins.src[2] = nir_src_index(ctx, indirect_offset);
1118 ins.src_types[2] = nir_type_uint32;
1119 ins.load_store.index_shift = indirect_shift;
1120
1121 /* X component for the whole swizzle to prevent register
1122 * pressure from ballooning from the extra components */
1123 for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[2]); ++i)
1124 ins.swizzle[2][i] = 0;
1125 } else {
1126 ins.load_store.index_reg = REGISTER_LDST_ZERO;
1127 }
1128
1129 if (indirect_offset && !indirect_shift)
1130 mir_set_ubo_offset(&ins, indirect_offset, offset);
1131
1132 midgard_pack_ubo_index_imm(&ins.load_store, index);
1133
1134 return emit_mir_instruction(ctx, ins);
1135 }
1136
1137 /* Globals are like UBOs if you squint. And shared memory is like globals if
1138 * you squint even harder */
1139
1140 static void
emit_global(compiler_context * ctx,nir_instr * instr,bool is_read,unsigned srcdest,nir_src * offset,unsigned seg)1141 emit_global(compiler_context *ctx, nir_instr *instr, bool is_read,
1142 unsigned srcdest, nir_src *offset, unsigned seg)
1143 {
1144 midgard_instruction ins;
1145
1146 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
1147 if (is_read) {
1148 unsigned bitsize = intr->def.bit_size * intr->def.num_components;
1149
1150 switch (bitsize) {
1151 case 8:
1152 ins = m_ld_u8(srcdest, 0);
1153 break;
1154 case 16:
1155 ins = m_ld_u16(srcdest, 0);
1156 break;
1157 case 32:
1158 ins = m_ld_32(srcdest, 0);
1159 break;
1160 case 64:
1161 ins = m_ld_64(srcdest, 0);
1162 break;
1163 case 128:
1164 ins = m_ld_128(srcdest, 0);
1165 break;
1166 default:
1167 unreachable("Invalid global read size");
1168 }
1169
1170 mir_set_intr_mask(instr, &ins, is_read);
1171
1172 /* For anything not aligned on 32bit, make sure we write full
1173 * 32 bits registers. */
1174 if (bitsize & 31) {
1175 unsigned comps_per_32b = 32 / intr->def.bit_size;
1176
1177 for (unsigned c = 0; c < 4 * comps_per_32b; c += comps_per_32b) {
1178 if (!(ins.mask & BITFIELD_RANGE(c, comps_per_32b)))
1179 continue;
1180
1181 unsigned base = ~0;
1182 for (unsigned i = 0; i < comps_per_32b; i++) {
1183 if (ins.mask & BITFIELD_BIT(c + i)) {
1184 base = ins.swizzle[0][c + i];
1185 break;
1186 }
1187 }
1188
1189 assert(base != ~0);
1190
1191 for (unsigned i = 0; i < comps_per_32b; i++) {
1192 if (!(ins.mask & BITFIELD_BIT(c + i))) {
1193 ins.swizzle[0][c + i] = base + i;
1194 ins.mask |= BITFIELD_BIT(c + i);
1195 }
1196 assert(ins.swizzle[0][c + i] == base + i);
1197 }
1198 }
1199 }
1200 } else {
1201 unsigned bitsize =
1202 nir_src_bit_size(intr->src[0]) * nir_src_num_components(intr->src[0]);
1203
1204 if (bitsize == 8)
1205 ins = m_st_u8(srcdest, 0);
1206 else if (bitsize == 16)
1207 ins = m_st_u16(srcdest, 0);
1208 else if (bitsize <= 32)
1209 ins = m_st_32(srcdest, 0);
1210 else if (bitsize <= 64)
1211 ins = m_st_64(srcdest, 0);
1212 else if (bitsize <= 128)
1213 ins = m_st_128(srcdest, 0);
1214 else
1215 unreachable("Invalid global store size");
1216
1217 mir_set_intr_mask(instr, &ins, is_read);
1218 }
1219
1220 mir_set_offset(ctx, &ins, offset, seg);
1221
1222 /* Set a valid swizzle for masked out components */
1223 assert(ins.mask);
1224 unsigned first_component = __builtin_ffs(ins.mask) - 1;
1225
1226 for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[0]); ++i) {
1227 if (!(ins.mask & (1 << i)))
1228 ins.swizzle[0][i] = first_component;
1229 }
1230
1231 emit_mir_instruction(ctx, ins);
1232 }
1233
1234 static midgard_load_store_op
translate_atomic_op(nir_atomic_op op)1235 translate_atomic_op(nir_atomic_op op)
1236 {
1237 /* clang-format off */
1238 switch (op) {
1239 case nir_atomic_op_xchg: return midgard_op_atomic_xchg;
1240 case nir_atomic_op_cmpxchg: return midgard_op_atomic_cmpxchg;
1241 case nir_atomic_op_iadd: return midgard_op_atomic_add;
1242 case nir_atomic_op_iand: return midgard_op_atomic_and;
1243 case nir_atomic_op_imax: return midgard_op_atomic_imax;
1244 case nir_atomic_op_imin: return midgard_op_atomic_imin;
1245 case nir_atomic_op_ior: return midgard_op_atomic_or;
1246 case nir_atomic_op_umax: return midgard_op_atomic_umax;
1247 case nir_atomic_op_umin: return midgard_op_atomic_umin;
1248 case nir_atomic_op_ixor: return midgard_op_atomic_xor;
1249 default: unreachable("Unexpected atomic");
1250 }
1251 /* clang-format on */
1252 }
1253
1254 /* Emit an atomic to shared memory or global memory. */
1255 static void
emit_atomic(compiler_context * ctx,nir_intrinsic_instr * instr)1256 emit_atomic(compiler_context *ctx, nir_intrinsic_instr *instr)
1257 {
1258 midgard_load_store_op op =
1259 translate_atomic_op(nir_intrinsic_atomic_op(instr));
1260
1261 nir_alu_type type =
1262 (op == midgard_op_atomic_imin || op == midgard_op_atomic_imax)
1263 ? nir_type_int
1264 : nir_type_uint;
1265
1266 bool is_shared = (instr->intrinsic == nir_intrinsic_shared_atomic) ||
1267 (instr->intrinsic == nir_intrinsic_shared_atomic_swap);
1268
1269 unsigned dest = nir_def_index(&instr->def);
1270 unsigned val = nir_src_index(ctx, &instr->src[1]);
1271 unsigned bitsize = nir_src_bit_size(instr->src[1]);
1272 emit_explicit_constant(ctx, val);
1273
1274 midgard_instruction ins = {
1275 .type = TAG_LOAD_STORE_4,
1276 .mask = 0xF,
1277 .dest = dest,
1278 .src = {~0, ~0, ~0, val},
1279 .src_types = {0, 0, 0, type | bitsize},
1280 .op = op,
1281 };
1282
1283 nir_src *src_offset = nir_get_io_offset_src(instr);
1284
1285 if (op == midgard_op_atomic_cmpxchg) {
1286 unsigned xchg_val = nir_src_index(ctx, &instr->src[2]);
1287 emit_explicit_constant(ctx, xchg_val);
1288
1289 ins.src[2] = val;
1290 ins.src_types[2] = type | bitsize;
1291 ins.src[3] = xchg_val;
1292
1293 if (is_shared) {
1294 ins.load_store.arg_reg = REGISTER_LDST_LOCAL_STORAGE_PTR;
1295 ins.load_store.arg_comp = COMPONENT_Z;
1296 ins.load_store.bitsize_toggle = true;
1297 } else {
1298 for (unsigned i = 0; i < 2; ++i)
1299 ins.swizzle[1][i] = i;
1300
1301 ins.src[1] = nir_src_index(ctx, src_offset);
1302 ins.src_types[1] = nir_type_uint64;
1303 }
1304 } else
1305 mir_set_offset(ctx, &ins, src_offset,
1306 is_shared ? LDST_SHARED : LDST_GLOBAL);
1307
1308 mir_set_intr_mask(&instr->instr, &ins, true);
1309
1310 emit_mir_instruction(ctx, ins);
1311 }
1312
1313 static void
emit_varying_read(compiler_context * ctx,unsigned dest,unsigned offset,unsigned nr_comp,unsigned component,nir_src * indirect_offset,nir_alu_type type,bool flat)1314 emit_varying_read(compiler_context *ctx, unsigned dest, unsigned offset,
1315 unsigned nr_comp, unsigned component,
1316 nir_src *indirect_offset, nir_alu_type type, bool flat)
1317 {
1318 midgard_instruction ins = m_ld_vary_32(dest, PACK_LDST_ATTRIB_OFS(offset));
1319 ins.mask = mask_of(nr_comp);
1320 ins.dest_type = type;
1321
1322 if (type == nir_type_float16) {
1323 /* Ensure we are aligned so we can pack it later */
1324 ins.mask = mask_of(ALIGN_POT(nr_comp, 2));
1325 }
1326
1327 for (unsigned i = 0; i < ARRAY_SIZE(ins.swizzle[0]); ++i)
1328 ins.swizzle[0][i] = MIN2(i + component, COMPONENT_W);
1329
1330 midgard_varying_params p = {
1331 .flat_shading = flat,
1332 .perspective_correction = 1,
1333 .interpolate_sample = true,
1334 };
1335 midgard_pack_varying_params(&ins.load_store, p);
1336
1337 if (indirect_offset) {
1338 ins.src[2] = nir_src_index(ctx, indirect_offset);
1339 ins.src_types[2] = nir_type_uint32;
1340 } else
1341 ins.load_store.index_reg = REGISTER_LDST_ZERO;
1342
1343 ins.load_store.arg_reg = REGISTER_LDST_ZERO;
1344 ins.load_store.index_format = midgard_index_address_u32;
1345
1346 /* For flat shading, for GPUs supporting auto32, we always use .u32 and
1347 * require 32-bit mode. For smooth shading, we use the appropriate
1348 * floating-point type.
1349 *
1350 * This could be optimized, but it makes it easy to check correctness.
1351 */
1352 if (ctx->quirks & MIDGARD_NO_AUTO32) {
1353 switch (type) {
1354 case nir_type_uint32:
1355 case nir_type_bool32:
1356 ins.op = midgard_op_ld_vary_32u;
1357 break;
1358 case nir_type_int32:
1359 ins.op = midgard_op_ld_vary_32i;
1360 break;
1361 case nir_type_float32:
1362 ins.op = midgard_op_ld_vary_32;
1363 break;
1364 case nir_type_float16:
1365 ins.op = midgard_op_ld_vary_16;
1366 break;
1367 default:
1368 unreachable("Attempted to load unknown type");
1369 break;
1370 }
1371 } else if (flat) {
1372 assert(nir_alu_type_get_type_size(type) == 32);
1373 ins.op = midgard_op_ld_vary_32u;
1374 } else {
1375 assert(nir_alu_type_get_base_type(type) == nir_type_float);
1376
1377 ins.op = (nir_alu_type_get_type_size(type) == 32) ? midgard_op_ld_vary_32
1378 : midgard_op_ld_vary_16;
1379 }
1380
1381 emit_mir_instruction(ctx, ins);
1382 }
1383
1384 static midgard_instruction
emit_image_op(compiler_context * ctx,nir_intrinsic_instr * instr)1385 emit_image_op(compiler_context *ctx, nir_intrinsic_instr *instr)
1386 {
1387 enum glsl_sampler_dim dim = nir_intrinsic_image_dim(instr);
1388 unsigned nr_dim = glsl_get_sampler_dim_coordinate_components(dim);
1389 bool is_array = nir_intrinsic_image_array(instr);
1390 bool is_store = instr->intrinsic == nir_intrinsic_image_store;
1391
1392 assert(dim != GLSL_SAMPLER_DIM_MS && "MSAA'd image not lowered");
1393
1394 unsigned coord_reg = nir_src_index(ctx, &instr->src[1]);
1395 emit_explicit_constant(ctx, coord_reg);
1396
1397 nir_src *index = &instr->src[0];
1398 bool is_direct = nir_src_is_const(*index);
1399
1400 /* For image opcodes, address is used as an index into the attribute
1401 * descriptor */
1402 unsigned address = is_direct ? nir_src_as_uint(*index) : 0;
1403
1404 midgard_instruction ins;
1405 if (is_store) { /* emit st_image_* */
1406 unsigned val = nir_src_index(ctx, &instr->src[3]);
1407 emit_explicit_constant(ctx, val);
1408
1409 nir_alu_type type = nir_intrinsic_src_type(instr);
1410 ins = st_image(type, val, PACK_LDST_ATTRIB_OFS(address));
1411 nir_alu_type base_type = nir_alu_type_get_base_type(type);
1412 ins.src_types[0] = base_type | nir_src_bit_size(instr->src[3]);
1413 } else if (instr->intrinsic == nir_intrinsic_image_texel_address) {
1414 ins =
1415 m_lea_image(nir_def_index(&instr->def), PACK_LDST_ATTRIB_OFS(address));
1416 ins.mask = mask_of(2); /* 64-bit memory address */
1417 } else { /* emit ld_image_* */
1418 nir_alu_type type = nir_intrinsic_dest_type(instr);
1419 ins = ld_image(type, nir_def_index(&instr->def),
1420 PACK_LDST_ATTRIB_OFS(address));
1421 ins.mask = mask_of(nir_intrinsic_dest_components(instr));
1422 ins.dest_type = type;
1423 }
1424
1425 /* Coord reg */
1426 ins.src[1] = coord_reg;
1427 ins.src_types[1] = nir_type_uint16;
1428 if (nr_dim == 3 || is_array) {
1429 ins.load_store.bitsize_toggle = true;
1430 }
1431
1432 /* Image index reg */
1433 if (!is_direct) {
1434 ins.src[2] = nir_src_index(ctx, index);
1435 ins.src_types[2] = nir_type_uint32;
1436 } else
1437 ins.load_store.index_reg = REGISTER_LDST_ZERO;
1438
1439 emit_mir_instruction(ctx, ins);
1440
1441 return ins;
1442 }
1443
1444 static void
emit_attr_read(compiler_context * ctx,unsigned dest,unsigned offset,unsigned nr_comp,nir_alu_type t)1445 emit_attr_read(compiler_context *ctx, unsigned dest, unsigned offset,
1446 unsigned nr_comp, nir_alu_type t)
1447 {
1448 midgard_instruction ins = m_ld_attr_32(dest, PACK_LDST_ATTRIB_OFS(offset));
1449 ins.load_store.arg_reg = REGISTER_LDST_ZERO;
1450 ins.load_store.index_reg = REGISTER_LDST_ZERO;
1451 ins.mask = mask_of(nr_comp);
1452
1453 /* Use the type appropriate load */
1454 switch (t) {
1455 case nir_type_uint:
1456 case nir_type_bool:
1457 ins.op = midgard_op_ld_attr_32u;
1458 break;
1459 case nir_type_int:
1460 ins.op = midgard_op_ld_attr_32i;
1461 break;
1462 case nir_type_float:
1463 ins.op = midgard_op_ld_attr_32;
1464 break;
1465 default:
1466 unreachable("Attempted to load unknown type");
1467 break;
1468 }
1469
1470 emit_mir_instruction(ctx, ins);
1471 }
1472
1473 static unsigned
compute_builtin_arg(nir_intrinsic_op op)1474 compute_builtin_arg(nir_intrinsic_op op)
1475 {
1476 switch (op) {
1477 case nir_intrinsic_load_workgroup_id:
1478 return REGISTER_LDST_GROUP_ID;
1479 case nir_intrinsic_load_local_invocation_id:
1480 return REGISTER_LDST_LOCAL_THREAD_ID;
1481 case nir_intrinsic_load_global_invocation_id:
1482 return REGISTER_LDST_GLOBAL_THREAD_ID;
1483 default:
1484 unreachable("Invalid compute paramater loaded");
1485 }
1486 }
1487
1488 static void
emit_fragment_store(compiler_context * ctx,unsigned src,unsigned src_z,unsigned src_s,enum midgard_rt_id rt,unsigned sample_iter)1489 emit_fragment_store(compiler_context *ctx, unsigned src, unsigned src_z,
1490 unsigned src_s, enum midgard_rt_id rt, unsigned sample_iter)
1491 {
1492 assert(rt < ARRAY_SIZE(ctx->writeout_branch));
1493 assert(sample_iter < ARRAY_SIZE(ctx->writeout_branch[0]));
1494
1495 midgard_instruction *br = ctx->writeout_branch[rt][sample_iter];
1496
1497 assert(!br);
1498
1499 emit_explicit_constant(ctx, src);
1500
1501 struct midgard_instruction ins = v_branch(false, false);
1502
1503 bool depth_only = (rt == MIDGARD_ZS_RT);
1504
1505 ins.writeout = depth_only ? 0 : PAN_WRITEOUT_C;
1506
1507 /* Add dependencies */
1508 ins.src[0] = src;
1509 ins.src_types[0] = nir_type_uint32;
1510
1511 if (depth_only)
1512 ins.constants.u32[0] = 0xFF;
1513 else
1514 ins.constants.u32[0] = ((rt - MIDGARD_COLOR_RT0) << 8) | sample_iter;
1515
1516 for (int i = 0; i < 4; ++i)
1517 ins.swizzle[0][i] = i;
1518
1519 if (~src_z) {
1520 emit_explicit_constant(ctx, src_z);
1521 ins.src[2] = src_z;
1522 ins.src_types[2] = nir_type_uint32;
1523 ins.writeout |= PAN_WRITEOUT_Z;
1524 }
1525 if (~src_s) {
1526 emit_explicit_constant(ctx, src_s);
1527 ins.src[3] = src_s;
1528 ins.src_types[3] = nir_type_uint32;
1529 ins.writeout |= PAN_WRITEOUT_S;
1530 }
1531
1532 /* Emit the branch */
1533 br = emit_mir_instruction(ctx, ins);
1534 schedule_barrier(ctx);
1535 ctx->writeout_branch[rt][sample_iter] = br;
1536
1537 /* Push our current location = current block count - 1 = where we'll
1538 * jump to. Maybe a bit too clever for my own good */
1539
1540 br->branch.target_block = ctx->block_count - 1;
1541 }
1542
1543 static void
emit_compute_builtin(compiler_context * ctx,nir_intrinsic_instr * instr)1544 emit_compute_builtin(compiler_context *ctx, nir_intrinsic_instr *instr)
1545 {
1546 unsigned reg = nir_def_index(&instr->def);
1547 midgard_instruction ins = m_ldst_mov(reg, 0);
1548 ins.mask = mask_of(3);
1549 ins.swizzle[0][3] = COMPONENT_X; /* xyzx */
1550 ins.load_store.arg_reg = compute_builtin_arg(instr->intrinsic);
1551 emit_mir_instruction(ctx, ins);
1552 }
1553
1554 static unsigned
vertex_builtin_arg(nir_intrinsic_op op)1555 vertex_builtin_arg(nir_intrinsic_op op)
1556 {
1557 switch (op) {
1558 case nir_intrinsic_load_vertex_id_zero_base:
1559 return PAN_VERTEX_ID;
1560 case nir_intrinsic_load_instance_id:
1561 return PAN_INSTANCE_ID;
1562 default:
1563 unreachable("Invalid vertex builtin");
1564 }
1565 }
1566
1567 static void
emit_vertex_builtin(compiler_context * ctx,nir_intrinsic_instr * instr)1568 emit_vertex_builtin(compiler_context *ctx, nir_intrinsic_instr *instr)
1569 {
1570 unsigned reg = nir_def_index(&instr->def);
1571 emit_attr_read(ctx, reg, vertex_builtin_arg(instr->intrinsic), 1,
1572 nir_type_int);
1573 }
1574
1575 static void
emit_special(compiler_context * ctx,nir_intrinsic_instr * instr,unsigned idx)1576 emit_special(compiler_context *ctx, nir_intrinsic_instr *instr, unsigned idx)
1577 {
1578 unsigned reg = nir_def_index(&instr->def);
1579
1580 midgard_instruction ld = m_ld_tilebuffer_raw(reg, 0);
1581 ld.op = midgard_op_ld_special_32u;
1582 ld.load_store.signed_offset = PACK_LDST_SELECTOR_OFS(idx);
1583 ld.load_store.index_reg = REGISTER_LDST_ZERO;
1584
1585 for (int i = 0; i < 4; ++i)
1586 ld.swizzle[0][i] = COMPONENT_X;
1587
1588 emit_mir_instruction(ctx, ld);
1589 }
1590
1591 static void
emit_control_barrier(compiler_context * ctx)1592 emit_control_barrier(compiler_context *ctx)
1593 {
1594 midgard_instruction ins = {
1595 .type = TAG_TEXTURE_4,
1596 .dest = ~0,
1597 .src = {~0, ~0, ~0, ~0},
1598 .op = midgard_tex_op_barrier,
1599 };
1600
1601 emit_mir_instruction(ctx, ins);
1602 }
1603
1604 static uint8_t
output_load_rt_addr(compiler_context * ctx,nir_intrinsic_instr * instr)1605 output_load_rt_addr(compiler_context *ctx, nir_intrinsic_instr *instr)
1606 {
1607 unsigned loc = nir_intrinsic_io_semantics(instr).location;
1608
1609 if (loc >= FRAG_RESULT_DATA0)
1610 return loc - FRAG_RESULT_DATA0;
1611
1612 if (loc == FRAG_RESULT_DEPTH)
1613 return 0x1F;
1614 if (loc == FRAG_RESULT_STENCIL)
1615 return 0x1E;
1616
1617 unreachable("Invalid RT to load from");
1618 }
1619
1620 static void
emit_intrinsic(compiler_context * ctx,nir_intrinsic_instr * instr)1621 emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr)
1622 {
1623 unsigned offset = 0, reg;
1624
1625 switch (instr->intrinsic) {
1626 case nir_intrinsic_decl_reg:
1627 case nir_intrinsic_store_reg:
1628 /* Always fully consumed */
1629 break;
1630
1631 case nir_intrinsic_load_reg: {
1632 /* NIR guarantees that, for typical isel, this will always be fully
1633 * consumed. However, we also do our own nir_scalar chasing for
1634 * address arithmetic, bypassing the source chasing helpers. So we can end
1635 * up with unconsumed load_register instructions. Translate them here. 99%
1636 * of the time, these moves will be DCE'd away.
1637 */
1638 nir_def *handle = instr->src[0].ssa;
1639
1640 midgard_instruction ins =
1641 v_mov(nir_reg_index(handle), nir_def_index(&instr->def));
1642
1643 ins.dest_type = ins.src_types[1] = nir_type_uint | instr->def.bit_size;
1644
1645 ins.mask = BITFIELD_MASK(instr->def.num_components);
1646 emit_mir_instruction(ctx, ins);
1647 break;
1648 }
1649
1650 case nir_intrinsic_terminate_if:
1651 case nir_intrinsic_terminate: {
1652 bool conditional = instr->intrinsic == nir_intrinsic_terminate_if;
1653 struct midgard_instruction discard = v_branch(conditional, false);
1654 discard.branch.target_type = TARGET_DISCARD;
1655
1656 if (conditional) {
1657 discard.src[0] = nir_src_index(ctx, &instr->src[0]);
1658 discard.src_types[0] = nir_type_uint32;
1659 }
1660
1661 emit_mir_instruction(ctx, discard);
1662 schedule_barrier(ctx);
1663
1664 break;
1665 }
1666
1667 case nir_intrinsic_image_load:
1668 case nir_intrinsic_image_store:
1669 case nir_intrinsic_image_texel_address:
1670 emit_image_op(ctx, instr);
1671 break;
1672
1673 case nir_intrinsic_load_ubo:
1674 case nir_intrinsic_load_global:
1675 case nir_intrinsic_load_global_constant:
1676 case nir_intrinsic_load_shared:
1677 case nir_intrinsic_load_scratch:
1678 case nir_intrinsic_load_input:
1679 case nir_intrinsic_load_interpolated_input: {
1680 bool is_ubo = instr->intrinsic == nir_intrinsic_load_ubo;
1681 bool is_global = instr->intrinsic == nir_intrinsic_load_global ||
1682 instr->intrinsic == nir_intrinsic_load_global_constant;
1683 bool is_shared = instr->intrinsic == nir_intrinsic_load_shared;
1684 bool is_scratch = instr->intrinsic == nir_intrinsic_load_scratch;
1685 bool is_flat = instr->intrinsic == nir_intrinsic_load_input;
1686 bool is_interp =
1687 instr->intrinsic == nir_intrinsic_load_interpolated_input;
1688
1689 /* Get the base type of the intrinsic */
1690 /* TODO: Infer type? Does it matter? */
1691 nir_alu_type t = (is_interp) ? nir_type_float
1692 : (is_flat) ? nir_intrinsic_dest_type(instr)
1693 : nir_type_uint;
1694
1695 t = nir_alu_type_get_base_type(t);
1696
1697 if (!(is_ubo || is_global || is_scratch)) {
1698 offset = nir_intrinsic_base(instr);
1699 }
1700
1701 unsigned nr_comp = nir_intrinsic_dest_components(instr);
1702
1703 nir_src *src_offset = nir_get_io_offset_src(instr);
1704
1705 bool direct = nir_src_is_const(*src_offset);
1706 nir_src *indirect_offset = direct ? NULL : src_offset;
1707
1708 if (direct)
1709 offset += nir_src_as_uint(*src_offset);
1710
1711 /* We may need to apply a fractional offset */
1712 int component =
1713 (is_flat || is_interp) ? nir_intrinsic_component(instr) : 0;
1714 reg = nir_def_index(&instr->def);
1715
1716 if (is_ubo) {
1717 nir_src index = instr->src[0];
1718
1719 /* TODO: Is indirect block number possible? */
1720 assert(nir_src_is_const(index));
1721
1722 uint32_t uindex = nir_src_as_uint(index);
1723 emit_ubo_read(ctx, &instr->instr, reg, offset, indirect_offset, 0,
1724 uindex, nr_comp);
1725 } else if (is_global || is_shared || is_scratch) {
1726 unsigned seg =
1727 is_global ? LDST_GLOBAL : (is_shared ? LDST_SHARED : LDST_SCRATCH);
1728 emit_global(ctx, &instr->instr, true, reg, src_offset, seg);
1729 } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->inputs->is_blend) {
1730 emit_varying_read(ctx, reg, offset, nr_comp, component,
1731 indirect_offset, t | instr->def.bit_size, is_flat);
1732 } else if (ctx->inputs->is_blend) {
1733 /* ctx->blend_input will be precoloured to r0/r2, where
1734 * the input is preloaded */
1735
1736 unsigned *input = offset ? &ctx->blend_src1 : &ctx->blend_input;
1737
1738 if (*input == ~0)
1739 *input = reg;
1740 else
1741 emit_mir_instruction(ctx, v_mov(*input, reg));
1742 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1743 emit_attr_read(ctx, reg, offset, nr_comp, t);
1744 } else {
1745 unreachable("Unknown load");
1746 }
1747
1748 break;
1749 }
1750
1751 /* Handled together with load_interpolated_input */
1752 case nir_intrinsic_load_barycentric_pixel:
1753 case nir_intrinsic_load_barycentric_centroid:
1754 case nir_intrinsic_load_barycentric_sample:
1755 break;
1756
1757 /* Reads 128-bit value raw off the tilebuffer during blending, tasty */
1758
1759 case nir_intrinsic_load_raw_output_pan: {
1760 reg = nir_def_index(&instr->def);
1761
1762 /* T720 and below use different blend opcodes with slightly
1763 * different semantics than T760 and up */
1764
1765 midgard_instruction ld = m_ld_tilebuffer_raw(reg, 0);
1766
1767 unsigned target = output_load_rt_addr(ctx, instr);
1768 ld.load_store.index_comp = target & 0x3;
1769 ld.load_store.index_reg = target >> 2;
1770
1771 if (nir_src_is_const(instr->src[0])) {
1772 unsigned sample = nir_src_as_uint(instr->src[0]);
1773 ld.load_store.arg_comp = sample & 0x3;
1774 ld.load_store.arg_reg = sample >> 2;
1775 } else {
1776 /* Enable sample index via register. */
1777 ld.load_store.signed_offset |= 1;
1778 ld.src[1] = nir_src_index(ctx, &instr->src[0]);
1779 ld.src_types[1] = nir_type_int32;
1780 }
1781
1782 if (ctx->quirks & MIDGARD_OLD_BLEND) {
1783 ld.op = midgard_op_ld_special_32u;
1784 ld.load_store.signed_offset = PACK_LDST_SELECTOR_OFS(16);
1785 ld.load_store.index_reg = REGISTER_LDST_ZERO;
1786 }
1787
1788 emit_mir_instruction(ctx, ld);
1789 break;
1790 }
1791
1792 case nir_intrinsic_load_output: {
1793 reg = nir_def_index(&instr->def);
1794
1795 unsigned bits = instr->def.bit_size;
1796
1797 midgard_instruction ld;
1798 if (bits == 16)
1799 ld = m_ld_tilebuffer_16f(reg, 0);
1800 else
1801 ld = m_ld_tilebuffer_32f(reg, 0);
1802
1803 unsigned index = output_load_rt_addr(ctx, instr);
1804 ld.load_store.index_comp = index & 0x3;
1805 ld.load_store.index_reg = index >> 2;
1806
1807 for (unsigned c = 4; c < 16; ++c)
1808 ld.swizzle[0][c] = 0;
1809
1810 if (ctx->quirks & MIDGARD_OLD_BLEND) {
1811 if (bits == 16)
1812 ld.op = midgard_op_ld_special_16f;
1813 else
1814 ld.op = midgard_op_ld_special_32f;
1815 ld.load_store.signed_offset = PACK_LDST_SELECTOR_OFS(1);
1816 ld.load_store.index_reg = REGISTER_LDST_ZERO;
1817 }
1818
1819 emit_mir_instruction(ctx, ld);
1820 break;
1821 }
1822
1823 case nir_intrinsic_store_output:
1824 case nir_intrinsic_store_combined_output_pan:
1825 assert(nir_src_is_const(instr->src[1]) && "no indirect outputs");
1826
1827 reg = nir_src_index(ctx, &instr->src[0]);
1828
1829 if (ctx->stage == MESA_SHADER_FRAGMENT) {
1830 bool combined =
1831 instr->intrinsic == nir_intrinsic_store_combined_output_pan;
1832
1833 enum midgard_rt_id rt;
1834
1835 unsigned reg_z = ~0, reg_s = ~0, reg_2 = ~0;
1836 unsigned writeout = PAN_WRITEOUT_C;
1837 if (combined) {
1838 writeout = nir_intrinsic_component(instr);
1839 if (writeout & PAN_WRITEOUT_Z)
1840 reg_z = nir_src_index(ctx, &instr->src[2]);
1841 if (writeout & PAN_WRITEOUT_S)
1842 reg_s = nir_src_index(ctx, &instr->src[3]);
1843 if (writeout & PAN_WRITEOUT_2)
1844 reg_2 = nir_src_index(ctx, &instr->src[4]);
1845 }
1846
1847 if (writeout & PAN_WRITEOUT_C) {
1848 nir_io_semantics sem = nir_intrinsic_io_semantics(instr);
1849
1850 rt = MIDGARD_COLOR_RT0 + (sem.location - FRAG_RESULT_DATA0);
1851 } else {
1852 rt = MIDGARD_ZS_RT;
1853 }
1854
1855 /* Dual-source blend writeout is done by leaving the
1856 * value in r2 for the blend shader to use. */
1857 if (~reg_2) {
1858 emit_explicit_constant(ctx, reg_2);
1859
1860 unsigned out = make_compiler_temp(ctx);
1861
1862 midgard_instruction ins = v_mov(reg_2, out);
1863 emit_mir_instruction(ctx, ins);
1864
1865 ctx->blend_src1 = out;
1866 }
1867
1868 emit_fragment_store(ctx, reg, reg_z, reg_s, rt, 0);
1869 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1870 assert(instr->intrinsic == nir_intrinsic_store_output);
1871
1872 /* We should have been vectorized, though we don't
1873 * currently check that st_vary is emitted only once
1874 * per slot (this is relevant, since there's not a mask
1875 * parameter available on the store [set to 0 by the
1876 * blob]). We do respect the component by adjusting the
1877 * swizzle. If this is a constant source, we'll need to
1878 * emit that explicitly. */
1879
1880 emit_explicit_constant(ctx, reg);
1881
1882 offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]);
1883
1884 unsigned dst_component = nir_intrinsic_component(instr);
1885 unsigned nr_comp = nir_src_num_components(instr->src[0]);
1886
1887 /* ABI: Format controlled by the attribute descriptor.
1888 * This simplifies flat shading, although it prevents
1889 * certain (unimplemented) 16-bit optimizations.
1890 *
1891 * In particular, it lets the driver handle internal
1892 * TGSI shaders that set flat in the VS but smooth in
1893 * the FS. This matches our handling on Bifrost.
1894 */
1895 bool auto32 = true;
1896 assert(nir_alu_type_get_type_size(nir_intrinsic_src_type(instr)) ==
1897 32);
1898
1899 /* ABI: varyings in the secondary attribute table */
1900 bool secondary_table = true;
1901
1902 midgard_instruction st =
1903 m_st_vary_32(reg, PACK_LDST_ATTRIB_OFS(offset));
1904 st.load_store.arg_reg = REGISTER_LDST_ZERO;
1905 st.load_store.index_reg = REGISTER_LDST_ZERO;
1906
1907 /* Attribute instruction uses these 2-bits for the
1908 * a32 and table bits, pack this specially.
1909 */
1910 st.load_store.index_format =
1911 (auto32 ? (1 << 0) : 0) | (secondary_table ? (1 << 1) : 0);
1912
1913 /* nir_intrinsic_component(store_intr) encodes the
1914 * destination component start. Source component offset
1915 * adjustment is taken care of in
1916 * install_registers_instr(), when offset_swizzle() is
1917 * called.
1918 */
1919 unsigned src_component = COMPONENT_X;
1920
1921 assert(nr_comp > 0);
1922 for (unsigned i = 0; i < ARRAY_SIZE(st.swizzle); ++i) {
1923 st.swizzle[0][i] = src_component;
1924 if (i >= dst_component && i < dst_component + nr_comp - 1)
1925 src_component++;
1926 }
1927
1928 emit_mir_instruction(ctx, st);
1929 } else {
1930 unreachable("Unknown store");
1931 }
1932
1933 break;
1934
1935 /* Special case of store_output for lowered blend shaders */
1936 case nir_intrinsic_store_raw_output_pan: {
1937 assert(ctx->stage == MESA_SHADER_FRAGMENT);
1938 reg = nir_src_index(ctx, &instr->src[0]);
1939
1940 nir_io_semantics sem = nir_intrinsic_io_semantics(instr);
1941 assert(sem.location >= FRAG_RESULT_DATA0);
1942 unsigned rt = sem.location - FRAG_RESULT_DATA0;
1943
1944 emit_fragment_store(ctx, reg, ~0, ~0, rt + MIDGARD_COLOR_RT0,
1945 nir_intrinsic_base(instr));
1946 break;
1947 }
1948
1949 case nir_intrinsic_store_global:
1950 case nir_intrinsic_store_shared:
1951 case nir_intrinsic_store_scratch:
1952 reg = nir_src_index(ctx, &instr->src[0]);
1953 emit_explicit_constant(ctx, reg);
1954
1955 unsigned seg;
1956 if (instr->intrinsic == nir_intrinsic_store_global)
1957 seg = LDST_GLOBAL;
1958 else if (instr->intrinsic == nir_intrinsic_store_shared)
1959 seg = LDST_SHARED;
1960 else
1961 seg = LDST_SCRATCH;
1962
1963 emit_global(ctx, &instr->instr, false, reg, &instr->src[1], seg);
1964 break;
1965
1966 case nir_intrinsic_load_workgroup_id:
1967 case nir_intrinsic_load_local_invocation_id:
1968 case nir_intrinsic_load_global_invocation_id:
1969 emit_compute_builtin(ctx, instr);
1970 break;
1971
1972 case nir_intrinsic_load_vertex_id_zero_base:
1973 case nir_intrinsic_load_instance_id:
1974 emit_vertex_builtin(ctx, instr);
1975 break;
1976
1977 case nir_intrinsic_load_sample_mask_in:
1978 emit_special(ctx, instr, 96);
1979 break;
1980
1981 case nir_intrinsic_load_sample_id:
1982 emit_special(ctx, instr, 97);
1983 break;
1984
1985 case nir_intrinsic_barrier:
1986 if (nir_intrinsic_execution_scope(instr) != SCOPE_NONE) {
1987 schedule_barrier(ctx);
1988 emit_control_barrier(ctx);
1989 schedule_barrier(ctx);
1990 } else if (nir_intrinsic_memory_scope(instr) != SCOPE_NONE) {
1991 /* Midgard doesn't seem to want special handling, though we do need to
1992 * take care when scheduling to avoid incorrect reordering.
1993 *
1994 * Note this is an "else if" since the handling for the execution scope
1995 * case already covers the case when both scopes are present.
1996 */
1997 schedule_barrier(ctx);
1998 }
1999 break;
2000
2001 case nir_intrinsic_shared_atomic:
2002 case nir_intrinsic_shared_atomic_swap:
2003 case nir_intrinsic_global_atomic:
2004 case nir_intrinsic_global_atomic_swap:
2005 emit_atomic(ctx, instr);
2006 break;
2007
2008 case nir_intrinsic_ddx:
2009 case nir_intrinsic_ddy:
2010 midgard_emit_derivatives(ctx, instr);
2011 break;
2012
2013 default:
2014 fprintf(stderr, "Unhandled intrinsic %s\n",
2015 nir_intrinsic_infos[instr->intrinsic].name);
2016 assert(0);
2017 break;
2018 }
2019 }
2020
2021 /* Returns dimension with 0 special casing cubemaps */
2022 static unsigned
midgard_tex_format(enum glsl_sampler_dim dim)2023 midgard_tex_format(enum glsl_sampler_dim dim)
2024 {
2025 switch (dim) {
2026 case GLSL_SAMPLER_DIM_1D:
2027 case GLSL_SAMPLER_DIM_BUF:
2028 return 1;
2029
2030 case GLSL_SAMPLER_DIM_2D:
2031 case GLSL_SAMPLER_DIM_MS:
2032 case GLSL_SAMPLER_DIM_EXTERNAL:
2033 case GLSL_SAMPLER_DIM_RECT:
2034 return 2;
2035
2036 case GLSL_SAMPLER_DIM_3D:
2037 return 3;
2038
2039 case GLSL_SAMPLER_DIM_CUBE:
2040 return 0;
2041
2042 default:
2043 unreachable("Unknown sampler dim type");
2044 }
2045 }
2046
2047 /* Tries to attach an explicit LOD or bias as a constant. Returns whether this
2048 * was successful */
2049
2050 static bool
pan_attach_constant_bias(compiler_context * ctx,nir_src lod,midgard_texture_word * word)2051 pan_attach_constant_bias(compiler_context *ctx, nir_src lod,
2052 midgard_texture_word *word)
2053 {
2054 /* To attach as constant, it has to *be* constant */
2055
2056 if (!nir_src_is_const(lod))
2057 return false;
2058
2059 float f = nir_src_as_float(lod);
2060
2061 /* Break into fixed-point */
2062 signed lod_int = f;
2063 float lod_frac = f - lod_int;
2064
2065 /* Carry over negative fractions */
2066 if (lod_frac < 0.0) {
2067 lod_int--;
2068 lod_frac += 1.0;
2069 }
2070
2071 /* Encode */
2072 word->bias = float_to_ubyte(lod_frac);
2073 word->bias_int = lod_int;
2074
2075 return true;
2076 }
2077
2078 static enum mali_texture_mode
mdg_texture_mode(nir_tex_instr * instr)2079 mdg_texture_mode(nir_tex_instr *instr)
2080 {
2081 if (instr->op == nir_texop_tg4 && instr->is_shadow)
2082 return TEXTURE_GATHER_SHADOW;
2083 else if (instr->op == nir_texop_tg4)
2084 return TEXTURE_GATHER_X + instr->component;
2085 else if (instr->is_shadow)
2086 return TEXTURE_SHADOW;
2087 else
2088 return TEXTURE_NORMAL;
2089 }
2090
2091 static void
set_tex_coord(compiler_context * ctx,nir_tex_instr * instr,midgard_instruction * ins)2092 set_tex_coord(compiler_context *ctx, nir_tex_instr *instr,
2093 midgard_instruction *ins)
2094 {
2095 int coord_idx = nir_tex_instr_src_index(instr, nir_tex_src_coord);
2096
2097 assert(coord_idx >= 0);
2098
2099 int comparator_idx = nir_tex_instr_src_index(instr, nir_tex_src_comparator);
2100 int ms_idx = nir_tex_instr_src_index(instr, nir_tex_src_ms_index);
2101 assert(comparator_idx < 0 || ms_idx < 0);
2102 int ms_or_comparator_idx = ms_idx >= 0 ? ms_idx : comparator_idx;
2103
2104 unsigned coords = nir_src_index(ctx, &instr->src[coord_idx].src);
2105
2106 emit_explicit_constant(ctx, coords);
2107
2108 ins->src_types[1] = nir_tex_instr_src_type(instr, coord_idx) |
2109 nir_src_bit_size(instr->src[coord_idx].src);
2110
2111 unsigned nr_comps = instr->coord_components;
2112 unsigned written_mask = 0, write_mask = 0;
2113
2114 /* Initialize all components to coord.x which is expected to always be
2115 * present. Swizzle is updated below based on the texture dimension
2116 * and extra attributes that are packed in the coordinate argument.
2117 */
2118 for (unsigned c = 0; c < MIR_VEC_COMPONENTS; c++)
2119 ins->swizzle[1][c] = COMPONENT_X;
2120
2121 /* Shadow ref value is part of the coordinates if there's no comparator
2122 * source, in that case it's always placed in the last component.
2123 * Midgard wants the ref value in coord.z.
2124 */
2125 if (instr->is_shadow && comparator_idx < 0) {
2126 ins->swizzle[1][COMPONENT_Z] = --nr_comps;
2127 write_mask |= 1 << COMPONENT_Z;
2128 }
2129
2130 /* The array index is the last component if there's no shadow ref value
2131 * or second last if there's one. We already decremented the number of
2132 * components to account for the shadow ref value above.
2133 * Midgard wants the array index in coord.w.
2134 */
2135 if (instr->is_array) {
2136 ins->swizzle[1][COMPONENT_W] = --nr_comps;
2137 write_mask |= 1 << COMPONENT_W;
2138 }
2139
2140 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) {
2141 /* texelFetch is undefined on samplerCube */
2142 assert(ins->op != midgard_tex_op_fetch);
2143
2144 ins->src[1] = make_compiler_temp_reg(ctx);
2145
2146 /* For cubemaps, we use a special ld/st op to select the face
2147 * and copy the xy into the texture register
2148 */
2149 midgard_instruction ld = m_ld_cubemap_coords(ins->src[1], 0);
2150 ld.src[1] = coords;
2151 ld.src_types[1] = ins->src_types[1];
2152 ld.mask = 0x3; /* xy */
2153 ld.load_store.bitsize_toggle = true;
2154 ld.swizzle[1][3] = COMPONENT_X;
2155 emit_mir_instruction(ctx, ld);
2156
2157 /* We packed cube coordiates (X,Y,Z) into (X,Y), update the
2158 * written mask accordingly and decrement the number of
2159 * components
2160 */
2161 nr_comps--;
2162 written_mask |= 3;
2163 }
2164
2165 /* Now flag tex coord components that have not been written yet */
2166 write_mask |= mask_of(nr_comps) & ~written_mask;
2167 for (unsigned c = 0; c < nr_comps; c++)
2168 ins->swizzle[1][c] = c;
2169
2170 /* Sample index and shadow ref are expected in coord.z */
2171 if (ms_or_comparator_idx >= 0) {
2172 assert(!((write_mask | written_mask) & (1 << COMPONENT_Z)));
2173
2174 unsigned sample_or_ref =
2175 nir_src_index(ctx, &instr->src[ms_or_comparator_idx].src);
2176
2177 emit_explicit_constant(ctx, sample_or_ref);
2178
2179 if (ins->src[1] == ~0)
2180 ins->src[1] = make_compiler_temp_reg(ctx);
2181
2182 midgard_instruction mov = v_mov(sample_or_ref, ins->src[1]);
2183
2184 for (unsigned c = 0; c < MIR_VEC_COMPONENTS; c++)
2185 mov.swizzle[1][c] = COMPONENT_X;
2186
2187 mov.mask = 1 << COMPONENT_Z;
2188 written_mask |= 1 << COMPONENT_Z;
2189 ins->swizzle[1][COMPONENT_Z] = COMPONENT_Z;
2190 emit_mir_instruction(ctx, mov);
2191 }
2192
2193 /* Texelfetch coordinates uses all four elements (xyz/index) regardless
2194 * of texture dimensionality, which means it's necessary to zero the
2195 * unused components to keep everything happy.
2196 */
2197 if (ins->op == midgard_tex_op_fetch && (written_mask | write_mask) != 0xF) {
2198 if (ins->src[1] == ~0)
2199 ins->src[1] = make_compiler_temp_reg(ctx);
2200
2201 /* mov index.zw, #0, or generalized */
2202 midgard_instruction mov =
2203 v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), ins->src[1]);
2204 mov.has_constants = true;
2205 mov.mask = (written_mask | write_mask) ^ 0xF;
2206 emit_mir_instruction(ctx, mov);
2207 for (unsigned c = 0; c < MIR_VEC_COMPONENTS; c++) {
2208 if (mov.mask & (1 << c))
2209 ins->swizzle[1][c] = c;
2210 }
2211 }
2212
2213 if (ins->src[1] == ~0) {
2214 /* No temporary reg created, use the src coords directly */
2215 ins->src[1] = coords;
2216 } else if (write_mask) {
2217 /* Move the remaining coordinates to the temporary reg */
2218 midgard_instruction mov = v_mov(coords, ins->src[1]);
2219
2220 for (unsigned c = 0; c < MIR_VEC_COMPONENTS; c++) {
2221 if ((1 << c) & write_mask) {
2222 mov.swizzle[1][c] = ins->swizzle[1][c];
2223 ins->swizzle[1][c] = c;
2224 } else {
2225 mov.swizzle[1][c] = COMPONENT_X;
2226 }
2227 }
2228
2229 mov.mask = write_mask;
2230 emit_mir_instruction(ctx, mov);
2231 }
2232 }
2233
2234 static void
emit_texop_native(compiler_context * ctx,nir_tex_instr * instr,unsigned midgard_texop)2235 emit_texop_native(compiler_context *ctx, nir_tex_instr *instr,
2236 unsigned midgard_texop)
2237 {
2238 int texture_index = instr->texture_index;
2239 int sampler_index = instr->sampler_index;
2240
2241 /* If txf is used, we assume there is a valid sampler bound at index 0. Use
2242 * it for txf operations, since there may be no other valid samplers. This is
2243 * a workaround: txf does not require a sampler in NIR (so sampler_index is
2244 * undefined) but we need one in the hardware. This is ABI with the driver.
2245 */
2246 if (!nir_tex_instr_need_sampler(instr))
2247 sampler_index = 0;
2248
2249 midgard_instruction ins = {
2250 .type = TAG_TEXTURE_4,
2251 .mask = 0xF,
2252 .dest = nir_def_index(&instr->def),
2253 .src = {~0, ~0, ~0, ~0},
2254 .dest_type = instr->dest_type,
2255 .swizzle = SWIZZLE_IDENTITY_4,
2256 .outmod = midgard_outmod_none,
2257 .op = midgard_texop,
2258 .texture = {
2259 .format = midgard_tex_format(instr->sampler_dim),
2260 .texture_handle = texture_index,
2261 .sampler_handle = sampler_index,
2262 .mode = mdg_texture_mode(instr),
2263 }};
2264
2265 if (instr->is_shadow && !instr->is_new_style_shadow &&
2266 instr->op != nir_texop_tg4)
2267 for (int i = 0; i < 4; ++i)
2268 ins.swizzle[0][i] = COMPONENT_X;
2269
2270 for (unsigned i = 0; i < instr->num_srcs; ++i) {
2271 int index = nir_src_index(ctx, &instr->src[i].src);
2272 unsigned sz = nir_src_bit_size(instr->src[i].src);
2273 nir_alu_type T = nir_tex_instr_src_type(instr, i) | sz;
2274
2275 switch (instr->src[i].src_type) {
2276 case nir_tex_src_coord:
2277 set_tex_coord(ctx, instr, &ins);
2278 break;
2279
2280 case nir_tex_src_bias:
2281 case nir_tex_src_lod: {
2282 /* Try as a constant if we can */
2283
2284 bool is_txf = midgard_texop == midgard_tex_op_fetch;
2285 if (!is_txf &&
2286 pan_attach_constant_bias(ctx, instr->src[i].src, &ins.texture))
2287 break;
2288
2289 ins.texture.lod_register = true;
2290 ins.src[2] = index;
2291 ins.src_types[2] = T;
2292
2293 for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c)
2294 ins.swizzle[2][c] = COMPONENT_X;
2295
2296 emit_explicit_constant(ctx, index);
2297
2298 break;
2299 };
2300
2301 case nir_tex_src_offset: {
2302 ins.texture.offset_register = true;
2303 ins.src[3] = index;
2304 ins.src_types[3] = T;
2305
2306 for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c)
2307 ins.swizzle[3][c] = (c > COMPONENT_Z) ? 0 : c;
2308
2309 emit_explicit_constant(ctx, index);
2310 break;
2311 };
2312
2313 case nir_tex_src_comparator:
2314 case nir_tex_src_ms_index:
2315 /* Nothing to do, handled in set_tex_coord() */
2316 break;
2317
2318 default: {
2319 fprintf(stderr, "Unknown texture source type: %d\n",
2320 instr->src[i].src_type);
2321 assert(0);
2322 }
2323 }
2324 }
2325
2326 emit_mir_instruction(ctx, ins);
2327 }
2328
2329 static void
emit_tex(compiler_context * ctx,nir_tex_instr * instr)2330 emit_tex(compiler_context *ctx, nir_tex_instr *instr)
2331 {
2332 switch (instr->op) {
2333 case nir_texop_tex:
2334 case nir_texop_txb:
2335 emit_texop_native(ctx, instr, midgard_tex_op_normal);
2336 break;
2337 case nir_texop_txl:
2338 case nir_texop_tg4:
2339 emit_texop_native(ctx, instr, midgard_tex_op_gradient);
2340 break;
2341 case nir_texop_txf:
2342 case nir_texop_txf_ms:
2343 emit_texop_native(ctx, instr, midgard_tex_op_fetch);
2344 break;
2345 default: {
2346 fprintf(stderr, "Unhandled texture op: %d\n", instr->op);
2347 assert(0);
2348 }
2349 }
2350 }
2351
2352 static void
emit_jump(compiler_context * ctx,nir_jump_instr * instr)2353 emit_jump(compiler_context *ctx, nir_jump_instr *instr)
2354 {
2355 switch (instr->type) {
2356 case nir_jump_break: {
2357 /* Emit a branch out of the loop */
2358 struct midgard_instruction br = v_branch(false, false);
2359 br.branch.target_type = TARGET_BREAK;
2360 br.branch.target_break = ctx->current_loop_depth;
2361 emit_mir_instruction(ctx, br);
2362 break;
2363 }
2364
2365 default:
2366 unreachable("Unhandled jump");
2367 }
2368 }
2369
2370 static void
emit_instr(compiler_context * ctx,struct nir_instr * instr)2371 emit_instr(compiler_context *ctx, struct nir_instr *instr)
2372 {
2373 switch (instr->type) {
2374 case nir_instr_type_load_const:
2375 emit_load_const(ctx, nir_instr_as_load_const(instr));
2376 break;
2377
2378 case nir_instr_type_intrinsic:
2379 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
2380 break;
2381
2382 case nir_instr_type_alu:
2383 emit_alu(ctx, nir_instr_as_alu(instr));
2384 break;
2385
2386 case nir_instr_type_tex:
2387 emit_tex(ctx, nir_instr_as_tex(instr));
2388 break;
2389
2390 case nir_instr_type_jump:
2391 emit_jump(ctx, nir_instr_as_jump(instr));
2392 break;
2393
2394 case nir_instr_type_undef:
2395 /* Spurious */
2396 break;
2397
2398 default:
2399 unreachable("Unhandled instruction type");
2400 }
2401 }
2402
2403 /* ALU instructions can inline or embed constants, which decreases register
2404 * pressure and saves space. */
2405
2406 #define CONDITIONAL_ATTACH(idx) \
2407 { \
2408 void *entry = \
2409 _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[idx] + 1); \
2410 \
2411 if (entry) { \
2412 attach_constants(ctx, alu, entry, alu->src[idx] + 1); \
2413 alu->src[idx] = SSA_FIXED_REGISTER(REGISTER_CONSTANT); \
2414 } \
2415 }
2416
2417 static void
inline_alu_constants(compiler_context * ctx,midgard_block * block)2418 inline_alu_constants(compiler_context *ctx, midgard_block *block)
2419 {
2420 mir_foreach_instr_in_block(block, alu) {
2421 /* Other instructions cannot inline constants */
2422 if (alu->type != TAG_ALU_4)
2423 continue;
2424 if (alu->compact_branch)
2425 continue;
2426
2427 /* If there is already a constant here, we can do nothing */
2428 if (alu->has_constants)
2429 continue;
2430
2431 CONDITIONAL_ATTACH(0);
2432
2433 if (!alu->has_constants) {
2434 CONDITIONAL_ATTACH(1)
2435 } else if (!alu->inline_constant) {
2436 /* Corner case: _two_ vec4 constants, for instance with a
2437 * csel. For this case, we can only use a constant
2438 * register for one, we'll have to emit a move for the
2439 * other. */
2440
2441 void *entry =
2442 _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[1] + 1);
2443 unsigned scratch = make_compiler_temp(ctx);
2444
2445 if (entry) {
2446 midgard_instruction ins =
2447 v_mov(SSA_FIXED_REGISTER(REGISTER_CONSTANT), scratch);
2448 attach_constants(ctx, &ins, entry, alu->src[1] + 1);
2449
2450 /* Set the source */
2451 alu->src[1] = scratch;
2452
2453 /* Inject us -before- the last instruction which set r31, if
2454 * possible.
2455 */
2456 midgard_instruction *first = list_first_entry(
2457 &block->base.instructions, midgard_instruction, link);
2458
2459 if (alu == first) {
2460 mir_insert_instruction_before(ctx, alu, ins);
2461 } else {
2462 mir_insert_instruction_before(ctx, mir_prev_op(alu), ins);
2463 }
2464 }
2465 }
2466 }
2467 }
2468
2469 unsigned
max_bitsize_for_alu(midgard_instruction * ins)2470 max_bitsize_for_alu(midgard_instruction *ins)
2471 {
2472 unsigned max_bitsize = 0;
2473 for (int i = 0; i < MIR_SRC_COUNT; i++) {
2474 if (ins->src[i] == ~0)
2475 continue;
2476 unsigned src_bitsize = nir_alu_type_get_type_size(ins->src_types[i]);
2477 max_bitsize = MAX2(src_bitsize, max_bitsize);
2478 }
2479 unsigned dst_bitsize = nir_alu_type_get_type_size(ins->dest_type);
2480 max_bitsize = MAX2(dst_bitsize, max_bitsize);
2481
2482 /* We emulate 8-bit as 16-bit for simplicity of packing */
2483 max_bitsize = MAX2(max_bitsize, 16);
2484
2485 /* We don't have fp16 LUTs, so we'll want to emit code like:
2486 *
2487 * vlut.fsinr hr0, hr0
2488 *
2489 * where both input and output are 16-bit but the operation is carried
2490 * out in 32-bit
2491 */
2492
2493 switch (ins->op) {
2494 case midgard_alu_op_fsqrt:
2495 case midgard_alu_op_frcp:
2496 case midgard_alu_op_frsqrt:
2497 case midgard_alu_op_fsinpi:
2498 case midgard_alu_op_fcospi:
2499 case midgard_alu_op_fexp2:
2500 case midgard_alu_op_flog2:
2501 max_bitsize = MAX2(max_bitsize, 32);
2502 break;
2503
2504 default:
2505 break;
2506 }
2507
2508 /* High implies computing at a higher bitsize, e.g umul_high of 32-bit
2509 * requires computing at 64-bit */
2510 if (midgard_is_integer_out_op(ins->op) &&
2511 ins->outmod == midgard_outmod_keephi) {
2512 max_bitsize *= 2;
2513 assert(max_bitsize <= 64);
2514 }
2515
2516 return max_bitsize;
2517 }
2518
2519 midgard_reg_mode
reg_mode_for_bitsize(unsigned bitsize)2520 reg_mode_for_bitsize(unsigned bitsize)
2521 {
2522 switch (bitsize) {
2523 /* use 16 pipe for 8 since we don't support vec16 yet */
2524 case 8:
2525 case 16:
2526 return midgard_reg_mode_16;
2527 case 32:
2528 return midgard_reg_mode_32;
2529 case 64:
2530 return midgard_reg_mode_64;
2531 default:
2532 unreachable("invalid bit size");
2533 }
2534 }
2535
2536 /* Midgard supports two types of constants, embedded constants (128-bit) and
2537 * inline constants (16-bit). Sometimes, especially with scalar ops, embedded
2538 * constants can be demoted to inline constants, for space savings and
2539 * sometimes a performance boost */
2540
2541 static void
embedded_to_inline_constant(compiler_context * ctx,midgard_block * block)2542 embedded_to_inline_constant(compiler_context *ctx, midgard_block *block)
2543 {
2544 mir_foreach_instr_in_block(block, ins) {
2545 if (!ins->has_constants)
2546 continue;
2547 if (ins->has_inline_constant)
2548 continue;
2549
2550 unsigned max_bitsize = max_bitsize_for_alu(ins);
2551
2552 /* We can inline 32-bit (sometimes) or 16-bit (usually) */
2553 bool is_16 = max_bitsize == 16;
2554 bool is_32 = max_bitsize == 32;
2555
2556 if (!(is_16 || is_32))
2557 continue;
2558
2559 /* src1 cannot be an inline constant due to encoding
2560 * restrictions. So, if possible we try to flip the arguments
2561 * in that case */
2562
2563 int op = ins->op;
2564
2565 if (ins->src[0] == SSA_FIXED_REGISTER(REGISTER_CONSTANT) &&
2566 alu_opcode_props[op].props & OP_COMMUTES) {
2567 mir_flip(ins);
2568 }
2569
2570 if (ins->src[1] == SSA_FIXED_REGISTER(REGISTER_CONSTANT)) {
2571 /* Component is from the swizzle. Take a nonzero component */
2572 assert(ins->mask);
2573 unsigned first_comp = ffs(ins->mask) - 1;
2574 unsigned component = ins->swizzle[1][first_comp];
2575
2576 /* Scale constant appropriately, if we can legally */
2577 int16_t scaled_constant = 0;
2578
2579 if (is_16) {
2580 scaled_constant = ins->constants.u16[component];
2581 } else if (midgard_is_integer_op(op)) {
2582 scaled_constant = ins->constants.u32[component];
2583
2584 /* Constant overflow after resize */
2585 if (scaled_constant != ins->constants.u32[component])
2586 continue;
2587 } else {
2588 float original = ins->constants.f32[component];
2589 scaled_constant = _mesa_float_to_half(original);
2590
2591 /* Check for loss of precision. If this is
2592 * mediump, we don't care, but for a highp
2593 * shader, we need to pay attention. NIR
2594 * doesn't yet tell us which mode we're in!
2595 * Practically this prevents most constants
2596 * from being inlined, sadly. */
2597
2598 float fp32 = _mesa_half_to_float(scaled_constant);
2599
2600 if (fp32 != original)
2601 continue;
2602 }
2603
2604 /* Should've been const folded */
2605 if (ins->src_abs[1] || ins->src_neg[1])
2606 continue;
2607
2608 /* Make sure that the constant is not itself a vector
2609 * by checking if all accessed values are the same. */
2610
2611 const midgard_constants *cons = &ins->constants;
2612 uint32_t value = is_16 ? cons->u16[component] : cons->u32[component];
2613
2614 bool is_vector = false;
2615 unsigned mask = effective_writemask(ins->op, ins->mask);
2616
2617 for (unsigned c = 0; c < MIR_VEC_COMPONENTS; ++c) {
2618 /* We only care if this component is actually used */
2619 if (!(mask & (1 << c)))
2620 continue;
2621
2622 uint32_t test = is_16 ? cons->u16[ins->swizzle[1][c]]
2623 : cons->u32[ins->swizzle[1][c]];
2624
2625 if (test != value) {
2626 is_vector = true;
2627 break;
2628 }
2629 }
2630
2631 if (is_vector)
2632 continue;
2633
2634 /* Get rid of the embedded constant */
2635 ins->has_constants = false;
2636 ins->src[1] = ~0;
2637 ins->has_inline_constant = true;
2638 ins->inline_constant = scaled_constant;
2639 }
2640 }
2641 }
2642
2643 /* Dead code elimination for branches at the end of a block - only one branch
2644 * per block is legal semantically */
2645
2646 static void
midgard_cull_dead_branch(compiler_context * ctx,midgard_block * block)2647 midgard_cull_dead_branch(compiler_context *ctx, midgard_block *block)
2648 {
2649 bool branched = false;
2650
2651 mir_foreach_instr_in_block_safe(block, ins) {
2652 if (!midgard_is_branch_unit(ins->unit))
2653 continue;
2654
2655 if (branched)
2656 mir_remove_instruction(ins);
2657
2658 branched = true;
2659 }
2660 }
2661
2662 /* We want to force the invert on AND/OR to the second slot to legalize into
2663 * iandnot/iornot. The relevant patterns are for AND (and OR respectively)
2664 *
2665 * ~a & #b = ~a & ~(#~b)
2666 * ~a & b = b & ~a
2667 */
2668
2669 static void
midgard_legalize_invert(compiler_context * ctx,midgard_block * block)2670 midgard_legalize_invert(compiler_context *ctx, midgard_block *block)
2671 {
2672 mir_foreach_instr_in_block(block, ins) {
2673 if (ins->type != TAG_ALU_4)
2674 continue;
2675
2676 if (ins->op != midgard_alu_op_iand && ins->op != midgard_alu_op_ior)
2677 continue;
2678
2679 if (ins->src_invert[1] || !ins->src_invert[0])
2680 continue;
2681
2682 if (ins->has_inline_constant) {
2683 /* ~(#~a) = ~(~#a) = a, so valid, and forces both
2684 * inverts on */
2685 ins->inline_constant = ~ins->inline_constant;
2686 ins->src_invert[1] = true;
2687 } else {
2688 /* Flip to the right invert order. Note
2689 * has_inline_constant false by assumption on the
2690 * branch, so flipping makes sense. */
2691 mir_flip(ins);
2692 }
2693 }
2694 }
2695
2696 static unsigned
emit_fragment_epilogue(compiler_context * ctx,unsigned rt,unsigned sample_iter)2697 emit_fragment_epilogue(compiler_context *ctx, unsigned rt, unsigned sample_iter)
2698 {
2699 /* Loop to ourselves */
2700 midgard_instruction *br = ctx->writeout_branch[rt][sample_iter];
2701 struct midgard_instruction ins = v_branch(false, false);
2702 ins.writeout = br->writeout;
2703 ins.branch.target_block = ctx->block_count - 1;
2704 ins.constants.u32[0] = br->constants.u32[0];
2705 memcpy(&ins.src_types, &br->src_types, sizeof(ins.src_types));
2706 emit_mir_instruction(ctx, ins);
2707
2708 ctx->current_block->epilogue = true;
2709 schedule_barrier(ctx);
2710 return ins.branch.target_block;
2711 }
2712
2713 static midgard_block *
emit_block_init(compiler_context * ctx)2714 emit_block_init(compiler_context *ctx)
2715 {
2716 midgard_block *this_block = ctx->after_block;
2717 ctx->after_block = NULL;
2718
2719 if (!this_block)
2720 this_block = create_empty_block(ctx);
2721
2722 list_addtail(&this_block->base.link, &ctx->blocks);
2723
2724 this_block->scheduled = false;
2725 ++ctx->block_count;
2726
2727 /* Set up current block */
2728 list_inithead(&this_block->base.instructions);
2729 ctx->current_block = this_block;
2730
2731 return this_block;
2732 }
2733
2734 static midgard_block *
emit_block(compiler_context * ctx,nir_block * block)2735 emit_block(compiler_context *ctx, nir_block *block)
2736 {
2737 midgard_block *this_block = emit_block_init(ctx);
2738
2739 nir_foreach_instr(instr, block) {
2740 emit_instr(ctx, instr);
2741 ++ctx->instruction_count;
2742 }
2743
2744 return this_block;
2745 }
2746
2747 static midgard_block *emit_cf_list(struct compiler_context *ctx,
2748 struct exec_list *list);
2749
2750 static void
emit_if(struct compiler_context * ctx,nir_if * nif)2751 emit_if(struct compiler_context *ctx, nir_if *nif)
2752 {
2753 midgard_block *before_block = ctx->current_block;
2754
2755 /* Speculatively emit the branch, but we can't fill it in until later */
2756 EMIT(branch, true, true);
2757 midgard_instruction *then_branch = mir_last_in_block(ctx->current_block);
2758 then_branch->src[0] = nir_src_index(ctx, &nif->condition);
2759 then_branch->src_types[0] = nir_type_uint32;
2760
2761 /* Emit the two subblocks. */
2762 midgard_block *then_block = emit_cf_list(ctx, &nif->then_list);
2763 midgard_block *end_then_block = ctx->current_block;
2764
2765 /* Emit a jump from the end of the then block to the end of the else */
2766 EMIT(branch, false, false);
2767 midgard_instruction *then_exit = mir_last_in_block(ctx->current_block);
2768
2769 /* Emit second block, and check if it's empty */
2770
2771 int else_idx = ctx->block_count;
2772 int count_in = ctx->instruction_count;
2773 midgard_block *else_block = emit_cf_list(ctx, &nif->else_list);
2774 midgard_block *end_else_block = ctx->current_block;
2775 int after_else_idx = ctx->block_count;
2776
2777 /* Now that we have the subblocks emitted, fix up the branches */
2778
2779 assert(then_block);
2780 assert(else_block);
2781
2782 if (ctx->instruction_count == count_in) {
2783 /* The else block is empty, so don't emit an exit jump */
2784 mir_remove_instruction(then_exit);
2785 then_branch->branch.target_block = after_else_idx;
2786 } else {
2787 then_branch->branch.target_block = else_idx;
2788 then_exit->branch.target_block = after_else_idx;
2789 }
2790
2791 /* Wire up the successors */
2792
2793 ctx->after_block = create_empty_block(ctx);
2794
2795 pan_block_add_successor(&before_block->base, &then_block->base);
2796 pan_block_add_successor(&before_block->base, &else_block->base);
2797
2798 pan_block_add_successor(&end_then_block->base, &ctx->after_block->base);
2799 pan_block_add_successor(&end_else_block->base, &ctx->after_block->base);
2800 }
2801
2802 static void
emit_loop(struct compiler_context * ctx,nir_loop * nloop)2803 emit_loop(struct compiler_context *ctx, nir_loop *nloop)
2804 {
2805 assert(!nir_loop_has_continue_construct(nloop));
2806
2807 /* Remember where we are */
2808 midgard_block *start_block = ctx->current_block;
2809
2810 /* Allocate a loop number, growing the current inner loop depth */
2811 int loop_idx = ++ctx->current_loop_depth;
2812
2813 /* Get index from before the body so we can loop back later */
2814 int start_idx = ctx->block_count;
2815
2816 /* Emit the body itself */
2817 midgard_block *loop_block = emit_cf_list(ctx, &nloop->body);
2818
2819 /* Branch back to loop back */
2820 struct midgard_instruction br_back = v_branch(false, false);
2821 br_back.branch.target_block = start_idx;
2822 emit_mir_instruction(ctx, br_back);
2823
2824 /* Mark down that branch in the graph. */
2825 pan_block_add_successor(&start_block->base, &loop_block->base);
2826 pan_block_add_successor(&ctx->current_block->base, &loop_block->base);
2827
2828 /* Find the index of the block about to follow us (note: we don't add
2829 * one; blocks are 0-indexed so we get a fencepost problem) */
2830 int break_block_idx = ctx->block_count;
2831
2832 /* Fix up the break statements we emitted to point to the right place,
2833 * now that we can allocate a block number for them */
2834 ctx->after_block = create_empty_block(ctx);
2835
2836 mir_foreach_block_from(ctx, start_block, _block) {
2837 mir_foreach_instr_in_block(((midgard_block *)_block), ins) {
2838 if (ins->type != TAG_ALU_4)
2839 continue;
2840 if (!ins->compact_branch)
2841 continue;
2842
2843 /* We found a branch -- check the type to see if we need to do anything
2844 */
2845 if (ins->branch.target_type != TARGET_BREAK)
2846 continue;
2847
2848 /* It's a break! Check if it's our break */
2849 if (ins->branch.target_break != loop_idx)
2850 continue;
2851
2852 /* Okay, cool, we're breaking out of this loop.
2853 * Rewrite from a break to a goto */
2854
2855 ins->branch.target_type = TARGET_GOTO;
2856 ins->branch.target_block = break_block_idx;
2857
2858 pan_block_add_successor(_block, &ctx->after_block->base);
2859 }
2860 }
2861
2862 /* Now that we've finished emitting the loop, free up the depth again
2863 * so we play nice with recursion amid nested loops */
2864 --ctx->current_loop_depth;
2865
2866 /* Dump loop stats */
2867 ++ctx->loop_count;
2868 }
2869
2870 static midgard_block *
emit_cf_list(struct compiler_context * ctx,struct exec_list * list)2871 emit_cf_list(struct compiler_context *ctx, struct exec_list *list)
2872 {
2873 midgard_block *start_block = NULL;
2874
2875 foreach_list_typed(nir_cf_node, node, node, list) {
2876 switch (node->type) {
2877 case nir_cf_node_block: {
2878 midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node));
2879
2880 if (!start_block)
2881 start_block = block;
2882
2883 break;
2884 }
2885
2886 case nir_cf_node_if:
2887 emit_if(ctx, nir_cf_node_as_if(node));
2888 break;
2889
2890 case nir_cf_node_loop:
2891 emit_loop(ctx, nir_cf_node_as_loop(node));
2892 break;
2893
2894 case nir_cf_node_function:
2895 assert(0);
2896 break;
2897 }
2898 }
2899
2900 return start_block;
2901 }
2902
2903 /* Due to lookahead, we need to report the first tag executed in the command
2904 * stream and in branch targets. An initial block might be empty, so iterate
2905 * until we find one that 'works' */
2906
2907 unsigned
midgard_get_first_tag_from_block(compiler_context * ctx,unsigned block_idx)2908 midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx)
2909 {
2910 midgard_block *initial_block = mir_get_block(ctx, block_idx);
2911
2912 mir_foreach_block_from(ctx, initial_block, _v) {
2913 midgard_block *v = (midgard_block *)_v;
2914 if (v->quadword_count) {
2915 midgard_bundle *initial_bundle =
2916 util_dynarray_element(&v->bundles, midgard_bundle, 0);
2917
2918 return initial_bundle->tag;
2919 }
2920 }
2921
2922 /* Default to a tag 1 which will break from the shader, in case we jump
2923 * to the exit block (i.e. `return` in a compute shader) */
2924
2925 return 1;
2926 }
2927
2928 /* For each fragment writeout instruction, generate a writeout loop to
2929 * associate with it */
2930
2931 static void
mir_add_writeout_loops(compiler_context * ctx)2932 mir_add_writeout_loops(compiler_context *ctx)
2933 {
2934 for (unsigned rt = 0; rt < ARRAY_SIZE(ctx->writeout_branch); ++rt) {
2935 for (unsigned s = 0; s < MIDGARD_MAX_SAMPLE_ITER; ++s) {
2936 midgard_instruction *br = ctx->writeout_branch[rt][s];
2937 if (!br)
2938 continue;
2939
2940 unsigned popped = br->branch.target_block;
2941 pan_block_add_successor(&(mir_get_block(ctx, popped - 1)->base),
2942 &ctx->current_block->base);
2943 br->branch.target_block = emit_fragment_epilogue(ctx, rt, s);
2944 br->branch.target_type = TARGET_GOTO;
2945
2946 /* If we have more RTs, we'll need to restore back after our
2947 * loop terminates */
2948 midgard_instruction *next_br = NULL;
2949
2950 if ((s + 1) < MIDGARD_MAX_SAMPLE_ITER)
2951 next_br = ctx->writeout_branch[rt][s + 1];
2952
2953 if (!next_br && (rt + 1) < ARRAY_SIZE(ctx->writeout_branch))
2954 next_br = ctx->writeout_branch[rt + 1][0];
2955
2956 if (next_br) {
2957 midgard_instruction uncond = v_branch(false, false);
2958 uncond.branch.target_block = popped;
2959 uncond.branch.target_type = TARGET_GOTO;
2960 emit_mir_instruction(ctx, uncond);
2961 pan_block_add_successor(&ctx->current_block->base,
2962 &(mir_get_block(ctx, popped)->base));
2963 schedule_barrier(ctx);
2964 } else {
2965 /* We're last, so we can terminate here */
2966 br->last_writeout = true;
2967 }
2968 }
2969 }
2970 }
2971
2972 void
midgard_compile_shader_nir(nir_shader * nir,const struct panfrost_compile_inputs * inputs,struct util_dynarray * binary,struct pan_shader_info * info)2973 midgard_compile_shader_nir(nir_shader *nir,
2974 const struct panfrost_compile_inputs *inputs,
2975 struct util_dynarray *binary,
2976 struct pan_shader_info *info)
2977 {
2978 midgard_debug = debug_get_option_midgard_debug();
2979
2980 /* TODO: Bound against what? */
2981 compiler_context *ctx = rzalloc(NULL, compiler_context);
2982
2983 ctx->inputs = inputs;
2984 ctx->nir = nir;
2985 ctx->info = info;
2986 ctx->stage = nir->info.stage;
2987 ctx->blend_input = ~0;
2988 ctx->blend_src1 = ~0;
2989 ctx->quirks = midgard_get_quirks(inputs->gpu_id);
2990
2991 /* Initialize at a global (not block) level hash tables */
2992
2993 ctx->ssa_constants = _mesa_hash_table_u64_create(ctx);
2994
2995 /* Collect varyings after lowering I/O */
2996 info->quirk_no_auto32 = (ctx->quirks & MIDGARD_NO_AUTO32);
2997 pan_nir_collect_varyings(nir, info);
2998
2999 /* Optimisation passes */
3000 optimise_nir(nir, ctx->quirks, inputs->is_blend);
3001
3002 bool skip_internal = nir->info.internal;
3003 skip_internal &= !(midgard_debug & MIDGARD_DBG_INTERNAL);
3004
3005 if (midgard_debug & MIDGARD_DBG_SHADERS && !skip_internal)
3006 nir_print_shader(nir, stdout);
3007
3008 info->tls_size = nir->scratch_size;
3009
3010 nir_foreach_function_with_impl(func, impl, nir) {
3011 list_inithead(&ctx->blocks);
3012 ctx->block_count = 0;
3013 ctx->func = func;
3014
3015 if (nir->info.outputs_read && !inputs->is_blend) {
3016 emit_block_init(ctx);
3017
3018 struct midgard_instruction wait = v_branch(false, false);
3019 wait.branch.target_type = TARGET_TILEBUF_WAIT;
3020
3021 emit_mir_instruction(ctx, wait);
3022
3023 ++ctx->instruction_count;
3024 }
3025
3026 emit_cf_list(ctx, &impl->body);
3027 break; /* TODO: Multi-function shaders */
3028 }
3029
3030 /* Per-block lowering before opts */
3031
3032 mir_foreach_block(ctx, _block) {
3033 midgard_block *block = (midgard_block *)_block;
3034 inline_alu_constants(ctx, block);
3035 embedded_to_inline_constant(ctx, block);
3036 }
3037 /* MIR-level optimizations */
3038
3039 bool progress = false;
3040
3041 do {
3042 progress = false;
3043 progress |= midgard_opt_dead_code_eliminate(ctx);
3044 progress |= midgard_opt_prop(ctx);
3045
3046 mir_foreach_block(ctx, _block) {
3047 midgard_block *block = (midgard_block *)_block;
3048 progress |= midgard_opt_copy_prop(ctx, block);
3049 progress |= midgard_opt_combine_projection(ctx, block);
3050 progress |= midgard_opt_varying_projection(ctx, block);
3051 }
3052 } while (progress);
3053
3054 mir_foreach_block(ctx, _block) {
3055 midgard_block *block = (midgard_block *)_block;
3056 midgard_lower_derivatives(ctx, block);
3057 midgard_legalize_invert(ctx, block);
3058 midgard_cull_dead_branch(ctx, block);
3059 }
3060
3061 if (ctx->stage == MESA_SHADER_FRAGMENT)
3062 mir_add_writeout_loops(ctx);
3063
3064 /* Analyze now that the code is known but before scheduling creates
3065 * pipeline registers which are harder to track */
3066 mir_analyze_helper_requirements(ctx);
3067
3068 if (midgard_debug & MIDGARD_DBG_SHADERS && !skip_internal)
3069 mir_print_shader(ctx);
3070
3071 /* Schedule! */
3072 midgard_schedule_program(ctx);
3073 mir_ra(ctx);
3074
3075 if (midgard_debug & MIDGARD_DBG_SHADERS && !skip_internal)
3076 mir_print_shader(ctx);
3077
3078 /* Analyze after scheduling since this is order-dependent */
3079 mir_analyze_helper_terminate(ctx);
3080
3081 /* Emit flat binary from the instruction arrays. Iterate each block in
3082 * sequence. Save instruction boundaries such that lookahead tags can
3083 * be assigned easily */
3084
3085 /* Cache _all_ bundles in source order for lookahead across failed branches */
3086
3087 int bundle_count = 0;
3088 mir_foreach_block(ctx, _block) {
3089 midgard_block *block = (midgard_block *)_block;
3090 bundle_count += block->bundles.size / sizeof(midgard_bundle);
3091 }
3092 midgard_bundle **source_order_bundles =
3093 malloc(sizeof(midgard_bundle *) * bundle_count);
3094 int bundle_idx = 0;
3095 mir_foreach_block(ctx, _block) {
3096 midgard_block *block = (midgard_block *)_block;
3097 util_dynarray_foreach(&block->bundles, midgard_bundle, bundle) {
3098 source_order_bundles[bundle_idx++] = bundle;
3099 }
3100 }
3101
3102 int current_bundle = 0;
3103
3104 /* Midgard prefetches instruction types, so during emission we
3105 * need to lookahead. Unless this is the last instruction, in
3106 * which we return 1. */
3107
3108 mir_foreach_block(ctx, _block) {
3109 midgard_block *block = (midgard_block *)_block;
3110 mir_foreach_bundle_in_block(block, bundle) {
3111 int lookahead = 1;
3112
3113 if (!bundle->last_writeout && (current_bundle + 1 < bundle_count))
3114 lookahead = source_order_bundles[current_bundle + 1]->tag;
3115
3116 emit_binary_bundle(ctx, block, bundle, binary, lookahead);
3117 ++current_bundle;
3118 }
3119
3120 /* TODO: Free deeper */
3121 // util_dynarray_fini(&block->instructions);
3122 }
3123
3124 free(source_order_bundles);
3125
3126 /* Report the very first tag executed */
3127 info->midgard.first_tag = midgard_get_first_tag_from_block(ctx, 0);
3128
3129 info->ubo_mask = ctx->ubo_mask & ((1 << ctx->nir->info.num_ubos) - 1);
3130
3131 if (midgard_debug & MIDGARD_DBG_SHADERS && !skip_internal) {
3132 disassemble_midgard(stdout, binary->data, binary->size, inputs->gpu_id,
3133 midgard_debug & MIDGARD_DBG_VERBOSE);
3134 fflush(stdout);
3135 }
3136
3137 /* A shader ending on a 16MB boundary causes INSTR_INVALID_PC faults,
3138 * workaround by adding some padding to the end of the shader. (The
3139 * kernel makes sure shader BOs can't cross 16MB boundaries.) */
3140 if (binary->size)
3141 memset(util_dynarray_grow(binary, uint8_t, 16), 0, 16);
3142
3143 if ((midgard_debug & MIDGARD_DBG_SHADERDB || inputs->debug) &&
3144 !nir->info.internal) {
3145 unsigned nr_bundles = 0, nr_ins = 0;
3146
3147 /* Count instructions and bundles */
3148
3149 mir_foreach_block(ctx, _block) {
3150 midgard_block *block = (midgard_block *)_block;
3151 nr_bundles +=
3152 util_dynarray_num_elements(&block->bundles, midgard_bundle);
3153
3154 mir_foreach_bundle_in_block(block, bun)
3155 nr_ins += bun->instruction_count;
3156 }
3157
3158 /* Calculate thread count. There are certain cutoffs by
3159 * register count for thread count */
3160
3161 unsigned nr_registers = info->work_reg_count;
3162
3163 unsigned nr_threads = (nr_registers <= 4) ? 4
3164 : (nr_registers <= 8) ? 2
3165 : 1;
3166
3167 char *shaderdb = NULL;
3168
3169 /* Dump stats */
3170
3171 asprintf(&shaderdb,
3172 "%s shader: "
3173 "%u inst, %u bundles, %u quadwords, "
3174 "%u registers, %u threads, %u loops, "
3175 "%u:%u spills:fills",
3176 ctx->inputs->is_blend ? "PAN_SHADER_BLEND"
3177 : gl_shader_stage_name(ctx->stage),
3178 nr_ins, nr_bundles, ctx->quadword_count, nr_registers,
3179 nr_threads, ctx->loop_count, ctx->spills, ctx->fills);
3180
3181 if (midgard_debug & MIDGARD_DBG_SHADERDB)
3182 fprintf(stderr, "SHADER-DB: %s\n", shaderdb);
3183
3184 if (inputs->debug)
3185 util_debug_message(inputs->debug, SHADER_INFO, "%s", shaderdb);
3186
3187 free(shaderdb);
3188 }
3189
3190 _mesa_hash_table_u64_destroy(ctx->ssa_constants);
3191 ralloc_free(ctx);
3192 }
3193