1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 #include "draw_gs.h"
29
30 #include "draw_private.h"
31 #include "draw_context.h"
32 #if DRAW_LLVM_AVAILABLE
33 #include "draw_llvm.h"
34 #endif
35
36 #include "tgsi/tgsi_parse.h"
37 #include "tgsi/tgsi_exec.h"
38 #include "nir/nir_to_tgsi_info.h"
39 #include "compiler/nir/nir.h"
40 #include "pipe/p_shader_tokens.h"
41
42 #include "util/u_math.h"
43 #include "util/u_memory.h"
44 #include "util/u_prim.h"
45 #include "util/ralloc.h"
46 /* fixme: move it from here */
47 #define MAX_PRIMITIVES 64
48
49
50 static inline int
draw_gs_get_input_index(int semantic,int index,const struct tgsi_shader_info * input_info)51 draw_gs_get_input_index(int semantic, int index,
52 const struct tgsi_shader_info *input_info)
53 {
54 const uint8_t *input_semantic_names = input_info->output_semantic_name;
55 const uint8_t *input_semantic_indices = input_info->output_semantic_index;
56 for (int i = 0; i < PIPE_MAX_SHADER_OUTPUTS; i++) {
57 if (input_semantic_names[i] == semantic &&
58 input_semantic_indices[i] == index)
59 return i;
60 }
61 return -1;
62 }
63
64
65 /**
66 * We execute geometry shaders in the SOA mode, so ideally we want to
67 * flush when the number of currently fetched primitives is equal to
68 * the number of elements in the SOA vector. This ensures that the
69 * throughput is optimized for the given vector instruction set.
70 */
71 static inline bool
draw_gs_should_flush(struct draw_geometry_shader * shader)72 draw_gs_should_flush(struct draw_geometry_shader *shader)
73 {
74 return (shader->fetched_prim_count == shader->vector_length || shader->num_invocations > 1);
75 }
76
77
78 /*#define DEBUG_OUTPUTS 1*/
79 static void
tgsi_fetch_gs_outputs(struct draw_geometry_shader * shader,unsigned stream,unsigned num_primitives,float (** p_output)[4])80 tgsi_fetch_gs_outputs(struct draw_geometry_shader *shader,
81 unsigned stream,
82 unsigned num_primitives,
83 float (**p_output)[4])
84 {
85 struct tgsi_exec_machine *machine = shader->machine;
86 float (*output)[4];
87
88 output = *p_output;
89
90 /* Unswizzle all output results.
91 */
92
93 for (unsigned prim_idx = 0; prim_idx < num_primitives; ++prim_idx) {
94 unsigned num_verts_per_prim = machine->Primitives[stream][prim_idx];
95 unsigned prim_offset = machine->PrimitiveOffsets[stream][prim_idx];
96 shader->stream[stream].primitive_lengths[prim_idx + shader->stream[stream].emitted_primitives] =
97 machine->Primitives[stream][prim_idx];
98 shader->stream[stream].emitted_vertices += num_verts_per_prim;
99
100 for (unsigned j = 0; j < num_verts_per_prim; j++) {
101 int idx = prim_offset + j * shader->info.num_outputs;
102 #ifdef DEBUG_OUTPUTS
103 debug_printf("%d/%d) Output vert:\n", stream, idx / shader->info.num_outputs);
104 #endif
105 for (unsigned slot = 0; slot < shader->info.num_outputs; slot++) {
106 output[slot][0] = machine->Outputs[idx + slot].xyzw[0].f[0];
107 output[slot][1] = machine->Outputs[idx + slot].xyzw[1].f[0];
108 output[slot][2] = machine->Outputs[idx + slot].xyzw[2].f[0];
109 output[slot][3] = machine->Outputs[idx + slot].xyzw[3].f[0];
110 #ifdef DEBUG_OUTPUTS
111 debug_printf("\t%d: %f %f %f %f\n", slot,
112 output[slot][0],
113 output[slot][1],
114 output[slot][2],
115 output[slot][3]);
116 #endif
117 }
118 output = (float (*)[4])((char *)output + shader->vertex_size);
119 }
120 }
121 *p_output = output;
122 shader->stream[stream].emitted_primitives += num_primitives;
123 }
124
125
126 #define DEBUG_INPUTS 0
127 static void
tgsi_fetch_gs_input(struct draw_geometry_shader * shader,unsigned * indices,unsigned num_vertices,unsigned prim_idx)128 tgsi_fetch_gs_input(struct draw_geometry_shader *shader,
129 unsigned *indices,
130 unsigned num_vertices,
131 unsigned prim_idx)
132 {
133 struct tgsi_exec_machine *machine = shader->machine;
134 const unsigned input_vertex_stride = shader->input_vertex_stride;
135
136 int primid_sv = machine->SysSemanticToIndex[TGSI_SEMANTIC_PRIMID];
137 if (primid_sv != -1) {
138 for (unsigned j = 0; j < TGSI_QUAD_SIZE; j++)
139 machine->SystemValue[primid_sv].xyzw[0].i[j] = shader->in_prim_idx;
140 }
141
142 const float (*input_ptr)[4] = shader->input;
143
144 for (unsigned i = 0; i < num_vertices; ++i) {
145 #if DEBUG_INPUTS
146 debug_printf("%d) vertex index = %d (prim idx = %d)\n",
147 i, indices[i], prim_idx);
148 #endif
149 const float (*input)[4] = (const float (*)[4])
150 ((const char *)input_ptr + (indices[i] * input_vertex_stride));
151 for (unsigned slot = 0; slot < shader->info.num_inputs; ++slot) {
152 unsigned idx = i * TGSI_EXEC_MAX_INPUT_ATTRIBS + slot;
153 if (shader->info.input_semantic_name[slot] == TGSI_SEMANTIC_PRIMID) {
154 machine->Inputs[idx].xyzw[0].u[prim_idx] = shader->in_prim_idx;
155 machine->Inputs[idx].xyzw[1].u[prim_idx] = shader->in_prim_idx;
156 machine->Inputs[idx].xyzw[2].u[prim_idx] = shader->in_prim_idx;
157 machine->Inputs[idx].xyzw[3].u[prim_idx] = shader->in_prim_idx;
158 } else {
159 /* TODO: Move this call out of the for(i) loop */
160 int vs_slot = draw_gs_get_input_index(
161 shader->info.input_semantic_name[slot],
162 shader->info.input_semantic_index[slot],
163 shader->input_info);
164 if (vs_slot < 0) {
165 debug_printf("VS/GS signature mismatch!\n");
166 machine->Inputs[idx].xyzw[0].f[prim_idx] = 0;
167 machine->Inputs[idx].xyzw[1].f[prim_idx] = 0;
168 machine->Inputs[idx].xyzw[2].f[prim_idx] = 0;
169 machine->Inputs[idx].xyzw[3].f[prim_idx] = 0;
170 } else {
171 #if DEBUG_INPUTS
172 debug_printf("\tSlot = %d, vs_slot = %d, idx = %d:\n",
173 slot, vs_slot, idx);
174 assert(!util_is_inf_or_nan(input[vs_slot][0]));
175 assert(!util_is_inf_or_nan(input[vs_slot][1]));
176 assert(!util_is_inf_or_nan(input[vs_slot][2]));
177 assert(!util_is_inf_or_nan(input[vs_slot][3]));
178 #endif
179 machine->Inputs[idx].xyzw[0].f[prim_idx] = input[vs_slot][0];
180 machine->Inputs[idx].xyzw[1].f[prim_idx] = input[vs_slot][1];
181 machine->Inputs[idx].xyzw[2].f[prim_idx] = input[vs_slot][2];
182 machine->Inputs[idx].xyzw[3].f[prim_idx] = input[vs_slot][3];
183 #if DEBUG_INPUTS
184 debug_printf("\t\t%f %f %f %f\n",
185 machine->Inputs[idx].xyzw[0].f[prim_idx],
186 machine->Inputs[idx].xyzw[1].f[prim_idx],
187 machine->Inputs[idx].xyzw[2].f[prim_idx],
188 machine->Inputs[idx].xyzw[3].f[prim_idx]);
189 #endif
190 }
191 }
192 }
193 }
194 }
195
196
197 static void
tgsi_gs_prepare(struct draw_geometry_shader * shader,const struct draw_buffer_info * constants)198 tgsi_gs_prepare(struct draw_geometry_shader *shader,
199 const struct draw_buffer_info *constants)
200 {
201 struct tgsi_exec_machine *machine = shader->machine;
202 tgsi_exec_set_constant_buffers(machine, PIPE_MAX_CONSTANT_BUFFERS,
203 (const struct tgsi_exec_consts_info *)constants);
204 }
205
206
207 static void
tgsi_gs_run(struct draw_geometry_shader * shader,unsigned input_primitives,unsigned * out_prims)208 tgsi_gs_run(struct draw_geometry_shader *shader,
209 unsigned input_primitives,
210 unsigned *out_prims)
211 {
212 struct tgsi_exec_machine *machine = shader->machine;
213
214 if (shader->info.uses_invocationid) {
215 unsigned i = machine->SysSemanticToIndex[TGSI_SEMANTIC_INVOCATIONID];
216 for (int j = 0; j < TGSI_QUAD_SIZE; j++)
217 machine->SystemValue[i].xyzw[0].i[j] = shader->invocation_id;
218 }
219
220 /* run interpreter */
221 tgsi_exec_machine_run(machine, 0);
222
223 for (int i = 0; i < 4; i++)
224 out_prims[i] = machine->OutputPrimCount[i];
225 }
226
227
228 #if DRAW_LLVM_AVAILABLE
229
230 /*
231 * Fetch the vertex attribute values for one primitive.
232 * num_vertices is vertices/prim (1 for points, 2 for lines, 3 for tris)
233 */
234 static void
llvm_fetch_gs_input(struct draw_geometry_shader * shader,unsigned * indices,unsigned num_vertices,unsigned prim_idx)235 llvm_fetch_gs_input(struct draw_geometry_shader *shader,
236 unsigned *indices,
237 unsigned num_vertices,
238 unsigned prim_idx)
239 {
240 const unsigned input_vertex_stride = shader->input_vertex_stride;
241 float (*input_data)[6][PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS][TGSI_NUM_CHANNELS] = &shader->gs_input->data;
242
243 shader->llvm_prim_ids[shader->fetched_prim_count] = shader->in_prim_idx;
244
245 const float (*input_ptr)[4] = shader->input;
246
247 for (unsigned i = 0; i < num_vertices; ++i) {
248 #if DEBUG_INPUTS
249 debug_printf("%d) vertex index = %d (prim idx = %d)\n",
250 i, indices[i], prim_idx);
251 #endif
252 const float (*input)[4] = (const float (*)[4])
253 ((const char *)input_ptr + (indices[i] * input_vertex_stride));
254 for (unsigned slot = 0; slot < shader->info.num_inputs; ++slot) {
255 if (shader->info.input_semantic_name[slot] == TGSI_SEMANTIC_PRIMID) {
256 /* skip. we handle system values through gallivm */
257 /* NOTE: If we hit this case here it's an ordinary input not a sv,
258 * even though it probably should be a sv.
259 * Not sure how to set it up as regular input however if that even,
260 * would make sense so hack around this later in gallivm.
261 */
262 } else {
263 int vs_slot = draw_gs_get_input_index(
264 shader->info.input_semantic_name[slot],
265 shader->info.input_semantic_index[slot],
266 shader->input_info);
267 if (vs_slot < 0) {
268 debug_printf("VS/GS signature mismatch!\n");
269 (*input_data)[i][slot][0][prim_idx] = 0;
270 (*input_data)[i][slot][1][prim_idx] = 0;
271 (*input_data)[i][slot][2][prim_idx] = 0;
272 (*input_data)[i][slot][3][prim_idx] = 0;
273 } else {
274 #if DEBUG_INPUTS
275 debug_printf("\tSlot = %d, vs_slot = %d, i = %d:\n",
276 slot, vs_slot, i);
277 assert(!util_is_inf_or_nan(input[vs_slot][0]));
278 assert(!util_is_inf_or_nan(input[vs_slot][1]));
279 assert(!util_is_inf_or_nan(input[vs_slot][2]));
280 assert(!util_is_inf_or_nan(input[vs_slot][3]));
281 #endif
282 (*input_data)[i][slot][0][prim_idx] = input[vs_slot][0];
283 (*input_data)[i][slot][1][prim_idx] = input[vs_slot][1];
284 (*input_data)[i][slot][2][prim_idx] = input[vs_slot][2];
285 (*input_data)[i][slot][3][prim_idx] = input[vs_slot][3];
286 #if DEBUG_INPUTS
287 debug_printf("\t\t%f %f %f %f\n",
288 (*input_data)[i][slot][0][prim_idx],
289 (*input_data)[i][slot][1][prim_idx],
290 (*input_data)[i][slot][2][prim_idx],
291 (*input_data)[i][slot][3][prim_idx]);
292 #endif
293 }
294 }
295 }
296 }
297 }
298
299
300 static void
llvm_fetch_gs_outputs(struct draw_geometry_shader * shader,unsigned stream,unsigned num_primitives,float (** p_output)[4])301 llvm_fetch_gs_outputs(struct draw_geometry_shader *shader,
302 unsigned stream,
303 unsigned num_primitives,
304 float (**p_output)[4])
305 {
306 int total_verts = 0;
307 int vertex_count = 0;
308 int total_prims = 0;
309 int max_prims_per_invocation = 0;
310 char *output_ptr = (char*)shader->gs_output[stream];
311 int prim_idx;
312 unsigned next_prim_boundary = shader->primitive_boundary;
313
314 for (int i = 0; i < shader->vector_length; ++i) {
315 int prims = shader->llvm_emitted_primitives[i + (stream * shader->vector_length)];
316 total_prims += prims;
317 max_prims_per_invocation = MAX2(max_prims_per_invocation, prims);
318 }
319 for (int i = 0; i < shader->vector_length; ++i) {
320 total_verts += shader->llvm_emitted_vertices[i + (stream * shader->vector_length)];
321 }
322
323 output_ptr += shader->stream[stream].emitted_vertices * shader->vertex_size;
324 for (int i = 0; i < shader->vector_length - 1; ++i) {
325 int current_verts = shader->llvm_emitted_vertices[i + (stream * shader->vector_length)];
326 int next_verts = shader->llvm_emitted_vertices[i + 1 + (stream * shader->vector_length)];
327 #if 0
328 int j;
329 for (j = 0; j < current_verts; ++j) {
330 struct vertex_header *vh = (struct vertex_header *)
331 (output_ptr + shader->vertex_size * (i * next_prim_boundary + j));
332 debug_printf("--- %d) [%f, %f, %f, %f]\n", j + vertex_count,
333 vh->data[0][0], vh->data[0][1], vh->data[0][2], vh->data[0][3]);
334
335 }
336 #endif
337 assert(current_verts <= shader->max_output_vertices);
338 assert(next_verts <= shader->max_output_vertices);
339 if (next_verts) {
340 memmove(output_ptr + (vertex_count + current_verts) * shader->vertex_size,
341 output_ptr + ((i + 1) * next_prim_boundary) * shader->vertex_size,
342 shader->vertex_size * next_verts);
343 }
344 vertex_count += current_verts;
345 }
346
347 #if 0
348 {
349 for (int i = 0; i < total_verts; ++i) {
350 struct vertex_header *vh = (struct vertex_header *)(output_ptr + shader->vertex_size * i);
351 debug_printf("%d) Vertex:\n", i);
352 for (j = 0; j < shader->info.num_outputs; ++j) {
353 unsigned *udata = (unsigned*)vh->data[j];
354 debug_printf(" %d) [%f, %f, %f, %f] [%d, %d, %d, %d]\n", j,
355 vh->data[j][0], vh->data[j][1], vh->data[j][2], vh->data[j][3],
356 udata[0], udata[1], udata[2], udata[3]);
357 }
358
359 }
360 }
361 #endif
362
363 prim_idx = 0;
364 for (int i = 0; i < shader->vector_length; ++i) {
365 int num_prims = shader->llvm_emitted_primitives[i + (stream * shader->vector_length)];
366 for (int j = 0; j < num_prims; ++j) {
367 int prim_length =
368 shader->llvm_prim_lengths[j * shader->num_vertex_streams + stream][i];
369 shader->stream[stream].primitive_lengths[shader->stream[stream].emitted_primitives + prim_idx] =
370 prim_length;
371 ++prim_idx;
372 }
373 }
374
375 shader->stream[stream].emitted_primitives += total_prims;
376 shader->stream[stream].emitted_vertices += total_verts;
377 }
378
379
380 static void
llvm_gs_prepare(struct draw_geometry_shader * shader,const struct draw_buffer_info * constants)381 llvm_gs_prepare(struct draw_geometry_shader *shader,
382 const struct draw_buffer_info *constants)
383 {
384 }
385
386
387 static void
llvm_gs_run(struct draw_geometry_shader * shader,unsigned input_primitives,unsigned * out_prims)388 llvm_gs_run(struct draw_geometry_shader *shader,
389 unsigned input_primitives, unsigned *out_prims)
390 {
391 struct vertex_header *input[PIPE_MAX_VERTEX_STREAMS];
392 for (unsigned i = 0; i < shader->num_vertex_streams; i++) {
393 char *tmp = (char *)shader->gs_output[i];
394 tmp += shader->stream[i].emitted_vertices * shader->vertex_size;
395 input[i] = (struct vertex_header *)tmp;
396 }
397
398 shader->current_variant->jit_func(shader->jit_context,
399 shader->jit_resources,
400 shader->gs_input->data,
401 input,
402 input_primitives,
403 shader->draw->instance_id,
404 shader->llvm_prim_ids,
405 shader->invocation_id,
406 shader->draw->pt.user.viewid);
407
408 for (unsigned i = 0; i < shader->num_vertex_streams; i++) {
409 out_prims[i] = shader->jit_context->emitted_prims[i];
410 }
411 }
412
413 #endif
414
415
416 static void
gs_flush(struct draw_geometry_shader * shader)417 gs_flush(struct draw_geometry_shader *shader)
418 {
419 unsigned out_prim_count[TGSI_MAX_VERTEX_STREAMS];
420 unsigned i;
421 unsigned input_primitives = shader->fetched_prim_count;
422
423 if (shader->draw->collect_statistics) {
424 shader->draw->statistics.gs_invocations += input_primitives;
425 }
426
427 assert(input_primitives > 0 &&
428 input_primitives <= 4);
429
430 for (unsigned invocation = 0; invocation < shader->num_invocations; invocation++) {
431 shader->invocation_id = invocation;
432 shader->run(shader, input_primitives, out_prim_count);
433 for (i = 0; i < shader->num_vertex_streams; i++) {
434 shader->fetch_outputs(shader, i, out_prim_count[i],
435 &shader->stream[i].tmp_output);
436 }
437 }
438
439 #if 0
440 for (i = 0; i < shader->num_vertex_streams; i++) {
441 debug_printf("stream %d: PRIM emitted prims = %d (verts=%d), cur prim count = %d\n",
442 i,
443 shader->stream[i].emitted_primitives, shader->stream[i].emitted_vertices,
444 out_prim_count[i]);
445 }
446 #endif
447
448 shader->fetched_prim_count = 0;
449 }
450
451
452 static void
gs_point(struct draw_geometry_shader * shader,int idx)453 gs_point(struct draw_geometry_shader *shader, int idx)
454 {
455 unsigned indices[1];
456
457 indices[0] = idx;
458
459 shader->fetch_inputs(shader, indices, 1,
460 shader->fetched_prim_count);
461 ++shader->in_prim_idx;
462 ++shader->fetched_prim_count;
463
464 if (draw_gs_should_flush(shader))
465 gs_flush(shader);
466 }
467
468
469 static void
gs_line(struct draw_geometry_shader * shader,int i0,int i1)470 gs_line(struct draw_geometry_shader *shader, int i0, int i1)
471 {
472 unsigned indices[2];
473
474 indices[0] = i0;
475 indices[1] = i1;
476
477 shader->fetch_inputs(shader, indices, 2,
478 shader->fetched_prim_count);
479 ++shader->in_prim_idx;
480 ++shader->fetched_prim_count;
481
482 if (draw_gs_should_flush(shader))
483 gs_flush(shader);
484 }
485
486
487 static void
gs_line_adj(struct draw_geometry_shader * shader,int i0,int i1,int i2,int i3)488 gs_line_adj(struct draw_geometry_shader *shader,
489 int i0, int i1, int i2, int i3)
490 {
491 unsigned indices[4];
492
493 indices[0] = i0;
494 indices[1] = i1;
495 indices[2] = i2;
496 indices[3] = i3;
497
498 shader->fetch_inputs(shader, indices, 4,
499 shader->fetched_prim_count);
500 ++shader->in_prim_idx;
501 ++shader->fetched_prim_count;
502
503 if (draw_gs_should_flush(shader))
504 gs_flush(shader);
505 }
506
507
508 static void
gs_tri(struct draw_geometry_shader * shader,int i0,int i1,int i2)509 gs_tri(struct draw_geometry_shader *shader,
510 int i0, int i1, int i2)
511 {
512 unsigned indices[3];
513
514 indices[0] = i0;
515 indices[1] = i1;
516 indices[2] = i2;
517
518 shader->fetch_inputs(shader, indices, 3,
519 shader->fetched_prim_count);
520 ++shader->in_prim_idx;
521 ++shader->fetched_prim_count;
522
523 if (draw_gs_should_flush(shader))
524 gs_flush(shader);
525 }
526
527
528 static void
gs_tri_adj(struct draw_geometry_shader * shader,int i0,int i1,int i2,int i3,int i4,int i5)529 gs_tri_adj(struct draw_geometry_shader *shader,
530 int i0, int i1, int i2,
531 int i3, int i4, int i5)
532 {
533 unsigned indices[6];
534
535 indices[0] = i0;
536 indices[1] = i1;
537 indices[2] = i2;
538 indices[3] = i3;
539 indices[4] = i4;
540 indices[5] = i5;
541
542 shader->fetch_inputs(shader, indices, 6,
543 shader->fetched_prim_count);
544 ++shader->in_prim_idx;
545 ++shader->fetched_prim_count;
546
547 if (draw_gs_should_flush(shader))
548 gs_flush(shader);
549 }
550
551 #define FUNC gs_run
552 #define GET_ELT(idx) (idx)
553 #include "draw_gs_tmp.h"
554
555
556 #define FUNC gs_run_elts
557 #define LOCAL_VARS const uint16_t *elts = input_prims->elts;
558 #define GET_ELT(idx) (elts[idx])
559 #include "draw_gs_tmp.h"
560
561
562 /**
563 * Execute geometry shader.
564 */
565 void
draw_geometry_shader_run(struct draw_geometry_shader * shader,const struct draw_buffer_info * constants,const struct draw_vertex_info * input_verts,const struct draw_prim_info * input_prim,const struct tgsi_shader_info * input_info,struct draw_vertex_info * output_verts,struct draw_prim_info * output_prims)566 draw_geometry_shader_run(struct draw_geometry_shader *shader,
567 const struct draw_buffer_info *constants,
568 const struct draw_vertex_info *input_verts,
569 const struct draw_prim_info *input_prim,
570 const struct tgsi_shader_info *input_info,
571 struct draw_vertex_info *output_verts,
572 struct draw_prim_info *output_prims)
573 {
574 const float (*input)[4] = (const float (*)[4])input_verts->verts->data;
575 const unsigned input_stride = input_verts->vertex_size;
576 const unsigned num_outputs = draw_total_gs_outputs(shader->draw);
577 const unsigned vertex_size =
578 sizeof(struct vertex_header) + num_outputs * 4 * sizeof(float);
579 const unsigned num_input_verts =
580 input_prim->linear ? input_verts->count : input_prim->count;
581 const unsigned num_in_primitives =
582 align(MAX2(u_decomposed_prims_for_vertices(input_prim->prim,
583 num_input_verts),
584 u_decomposed_prims_for_vertices(shader->input_primitive,
585 num_input_verts)),
586 shader->vector_length);
587 //Assume at least one primitive
588 const unsigned max_out_prims =
589 MAX2(1, u_decomposed_prims_for_vertices(shader->output_primitive,
590 shader->max_output_vertices)
591 * num_in_primitives);
592
593 /* we allocate exactly one extra vertex per primitive to allow the GS to
594 * emit overflown vertices into some area where they won't harm anyone */
595 const unsigned total_verts_per_buffer =
596 shader->primitive_boundary * num_in_primitives;
597
598
599 for (int i = 0; i < shader->num_vertex_streams; i++) {
600 /* write all the vertex data into all the streams */
601 output_verts[i].vertex_size = vertex_size;
602 output_verts[i].stride = output_verts[i].vertex_size;
603 output_verts[i].verts =
604 (struct vertex_header *) MALLOC(output_verts[i].vertex_size *
605 total_verts_per_buffer *
606 shader->num_invocations +
607 DRAW_EXTRA_VERTICES_PADDING);
608 assert(output_verts[i].verts);
609 }
610
611 #if 0
612 debug_printf("%s count = %d (in prims # = %d, invocs = %d, streams = %d)\n",
613 __func__, num_input_verts, num_in_primitives,
614 shader->num_invocations, shader->num_vertex_streams);
615 debug_printf("\tlinear = %d, prim_info->count = %d\n",
616 input_prim->linear, input_prim->count);
617 debug_printf("\tprim pipe = %s, shader in = %s, shader out = %s\n",
618 u_prim_name(input_prim->prim),
619 u_prim_name(shader->input_primitive),
620 u_prim_name(shader->output_primitive));
621 debug_printf("\tmaxv = %d, maxp = %d, primitive_boundary = %d, "
622 "vertex_size = %d, tverts = %d\n",
623 shader->max_output_vertices, max_out_prims,
624 shader->primitive_boundary, output_verts->vertex_size,
625 total_verts_per_buffer);
626 #endif
627
628 for (int i = 0; i < shader->num_vertex_streams; i++) {
629 shader->stream[i].emitted_vertices = 0;
630 shader->stream[i].emitted_primitives = 0;
631 FREE(shader->stream[i].primitive_lengths);
632 shader->stream[i].primitive_lengths =
633 MALLOC(max_out_prims * sizeof(unsigned) * shader->num_invocations);
634 shader->stream[i].tmp_output = (float (*)[4])output_verts[i].verts->data;
635 }
636 shader->vertex_size = vertex_size;
637 shader->fetched_prim_count = 0;
638 shader->input_vertex_stride = input_stride;
639 shader->input = input;
640 shader->input_info = input_info;
641
642 #if DRAW_LLVM_AVAILABLE
643 if (shader->draw->llvm) {
644 for (int i = 0; i < shader->num_vertex_streams; i++) {
645 shader->gs_output[i] = output_verts[i].verts;
646 }
647 if (max_out_prims > shader->max_out_prims) {
648 if (shader->llvm_prim_lengths) {
649 for (unsigned i = 0; i < shader->num_vertex_streams * shader->max_out_prims; ++i) {
650 align_free(shader->llvm_prim_lengths[i]);
651 }
652 FREE(shader->llvm_prim_lengths);
653 }
654
655 shader->llvm_prim_lengths = MALLOC(shader->num_vertex_streams * max_out_prims * sizeof(unsigned*));
656 for (unsigned i = 0; i < shader->num_vertex_streams * max_out_prims; ++i) {
657 int vector_size = shader->vector_length * sizeof(unsigned);
658 shader->llvm_prim_lengths[i] =
659 align_malloc(vector_size, vector_size);
660 }
661
662 shader->max_out_prims = max_out_prims;
663 }
664 shader->jit_context->prim_lengths = shader->llvm_prim_lengths;
665 shader->jit_context->emitted_vertices = shader->llvm_emitted_vertices;
666 shader->jit_context->emitted_prims = shader->llvm_emitted_primitives;
667 }
668 #endif
669
670 shader->prepare(shader, constants);
671
672 if (input_prim->linear)
673 gs_run(shader, input_prim, input_verts,
674 output_prims, output_verts);
675 else
676 gs_run_elts(shader, input_prim, input_verts,
677 output_prims, output_verts);
678
679 /* Flush the remaining primitives. Will happen if
680 * num_input_primitives % 4 != 0
681 */
682 if (shader->fetched_prim_count > 0) {
683 gs_flush(shader);
684 }
685 assert(shader->fetched_prim_count == 0);
686
687 /* Update prim_info:
688 */
689 for (int i = 0; i < shader->num_vertex_streams; i++) {
690 output_prims[i].linear = true;
691 output_prims[i].elts = NULL;
692 output_prims[i].start = 0;
693 output_prims[i].count = shader->stream[i].emitted_vertices;
694 output_prims[i].prim = shader->output_primitive;
695 output_prims[i].flags = 0x0;
696 output_prims[i].primitive_lengths = shader->stream[i].primitive_lengths;
697 output_prims[i].primitive_count = shader->stream[i].emitted_primitives;
698 output_verts[i].count = shader->stream[i].emitted_vertices;
699
700 if (shader->draw->collect_statistics) {
701 for (unsigned j = 0; j < shader->stream[i].emitted_primitives; ++j) {
702 shader->draw->statistics.gs_primitives +=
703 u_decomposed_prims_for_vertices(shader->output_primitive,
704 shader->stream[i].primitive_lengths[j]);
705 }
706 }
707 }
708
709 #if 0
710 debug_printf("GS finished\n");
711 for (int i = 0; i < 4; i++)
712 debug_printf("stream %d: prims = %d verts = %d\n", i,
713 output_prims[i].primitive_count, output_verts[i].count);
714 #endif
715 }
716
717
718 void
draw_geometry_shader_prepare(struct draw_geometry_shader * shader,struct draw_context * draw)719 draw_geometry_shader_prepare(struct draw_geometry_shader *shader,
720 struct draw_context *draw)
721 {
722 bool use_llvm = draw->llvm != NULL;
723 if (!use_llvm &&
724 shader && shader->machine->Tokens != shader->state.tokens) {
725 tgsi_exec_machine_bind_shader(shader->machine,
726 shader->state.tokens,
727 draw->gs.tgsi.sampler,
728 draw->gs.tgsi.image,
729 draw->gs.tgsi.buffer);
730 }
731 }
732
733
734 bool
draw_gs_init(struct draw_context * draw)735 draw_gs_init(struct draw_context *draw)
736 {
737 if (!draw->llvm) {
738 draw->gs.tgsi.machine = tgsi_exec_machine_create(PIPE_SHADER_GEOMETRY);
739
740 for (unsigned i = 0; i < TGSI_MAX_VERTEX_STREAMS; i++) {
741 draw->gs.tgsi.machine->Primitives[i] = align_malloc(
742 MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector), 16);
743 draw->gs.tgsi.machine->PrimitiveOffsets[i] = align_malloc(
744 MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector), 16);
745 if (!draw->gs.tgsi.machine->Primitives[i] ||
746 !draw->gs.tgsi.machine->PrimitiveOffsets[i])
747 return false;
748 memset(draw->gs.tgsi.machine->Primitives[i], 0,
749 MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector));
750 memset(draw->gs.tgsi.machine->PrimitiveOffsets[i], 0,
751 MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector));
752 }
753 }
754
755 return true;
756 }
757
758
759 void
draw_gs_destroy(struct draw_context * draw)760 draw_gs_destroy(struct draw_context *draw)
761 {
762 if (draw->gs.tgsi.machine) {
763 for (int i = 0; i < TGSI_MAX_VERTEX_STREAMS; i++) {
764 align_free(draw->gs.tgsi.machine->Primitives[i]);
765 align_free(draw->gs.tgsi.machine->PrimitiveOffsets[i]);
766 }
767 tgsi_exec_machine_destroy(draw->gs.tgsi.machine);
768 }
769 }
770
771
772 /*
773 * num_vertices is vertices/prim (1 for points, 2 for lines, 3 for tris)
774 */
775 struct draw_geometry_shader *
draw_create_geometry_shader(struct draw_context * draw,const struct pipe_shader_state * state)776 draw_create_geometry_shader(struct draw_context *draw,
777 const struct pipe_shader_state *state)
778 {
779 #if DRAW_LLVM_AVAILABLE
780 bool use_llvm = draw->llvm != NULL;
781 struct llvm_geometry_shader *llvm_gs = NULL;
782 #endif
783 struct draw_geometry_shader *gs;
784
785 #if DRAW_LLVM_AVAILABLE
786 if (use_llvm) {
787 llvm_gs = CALLOC_STRUCT(llvm_geometry_shader);
788
789 if (!llvm_gs)
790 return NULL;
791
792 gs = &llvm_gs->base;
793
794 list_inithead(&llvm_gs->variants.list);
795 } else
796 #endif
797 {
798 gs = CALLOC_STRUCT(draw_geometry_shader);
799 }
800
801 if (!gs)
802 return NULL;
803
804 gs->draw = draw;
805 gs->state = *state;
806
807 if (state->type == PIPE_SHADER_IR_TGSI) {
808 gs->state.tokens = tgsi_dup_tokens(state->tokens);
809 if (!gs->state.tokens) {
810 FREE(gs);
811 return NULL;
812 }
813
814 tgsi_scan_shader(state->tokens, &gs->info);
815 gs->num_vertex_streams = 1;
816 for (unsigned i = 0; i < gs->state.stream_output.num_outputs; i++) {
817 if (gs->state.stream_output.output[i].stream >= gs->num_vertex_streams)
818 gs->num_vertex_streams = gs->state.stream_output.output[i].stream + 1;
819 }
820 } else {
821 nir_tgsi_scan_shader(state->ir.nir, &gs->info, true);
822 nir_shader *nir = state->ir.nir;
823 gs->num_vertex_streams = util_last_bit(nir->info.gs.active_stream_mask);
824 }
825
826 /* setup the defaults */
827 gs->max_out_prims = 0;
828
829 #if DRAW_LLVM_AVAILABLE
830 if (use_llvm) {
831 /* TODO: change the input array to handle the following
832 vector length, instead of the currently hardcoded
833 TGSI_NUM_CHANNELS
834 gs->vector_length = lp_native_vector_width / 32;*/
835 gs->vector_length = TGSI_NUM_CHANNELS;
836 } else
837 #endif
838 {
839 gs->vector_length = 1;
840 }
841
842 gs->input_primitive =
843 gs->info.properties[TGSI_PROPERTY_GS_INPUT_PRIM];
844 gs->output_primitive =
845 gs->info.properties[TGSI_PROPERTY_GS_OUTPUT_PRIM];
846 gs->max_output_vertices =
847 gs->info.properties[TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES];
848 gs->num_invocations =
849 gs->info.properties[TGSI_PROPERTY_GS_INVOCATIONS];
850 if (!gs->max_output_vertices)
851 gs->max_output_vertices = 32;
852
853 /* Primitive boundary is bigger than max_output_vertices by one, because
854 * the specification says that the geometry shader should exit if the
855 * number of emitted vertices is bigger or equal to max_output_vertices and
856 * we can't do that because we're running in the SoA mode, which means that
857 * our storing routines will keep getting called on channels that have
858 * overflown.
859 * So we need some scratch area where we can keep writing the overflown
860 * vertices without overwriting anything important or crashing.
861 */
862 gs->primitive_boundary = gs->max_output_vertices + 1;
863
864 gs->position_output = -1;
865 bool found_clipvertex = false;
866 for (unsigned i = 0; i < gs->info.num_outputs; i++) {
867 if (gs->info.output_semantic_name[i] == TGSI_SEMANTIC_POSITION &&
868 gs->info.output_semantic_index[i] == 0)
869 gs->position_output = i;
870 if (gs->info.output_semantic_name[i] == TGSI_SEMANTIC_VIEWPORT_INDEX)
871 gs->viewport_index_output = i;
872 if (gs->info.output_semantic_name[i] == TGSI_SEMANTIC_CLIPVERTEX &&
873 gs->info.output_semantic_index[i] == 0) {
874 found_clipvertex = true;
875 gs->clipvertex_output = i;
876 }
877 if (gs->info.output_semantic_name[i] == TGSI_SEMANTIC_CLIPDIST) {
878 assert(gs->info.output_semantic_index[i] <
879 PIPE_MAX_CLIP_OR_CULL_DISTANCE_ELEMENT_COUNT);
880 gs->ccdistance_output[gs->info.output_semantic_index[i]] = i;
881 }
882 }
883
884 if (!found_clipvertex)
885 gs->clipvertex_output = gs->position_output;
886
887 gs->machine = draw->gs.tgsi.machine;
888
889 #if DRAW_LLVM_AVAILABLE
890 if (use_llvm) {
891 int vector_size = gs->vector_length * sizeof(float);
892 gs->gs_input = align_malloc(sizeof(struct draw_gs_inputs), 16);
893 memset(gs->gs_input, 0, sizeof(struct draw_gs_inputs));
894 gs->llvm_prim_lengths = 0;
895
896 gs->llvm_emitted_primitives = align_malloc(vector_size * gs->num_vertex_streams, vector_size);
897 gs->llvm_emitted_vertices = align_malloc(vector_size * gs->num_vertex_streams, vector_size);
898 gs->llvm_prim_ids = align_calloc(vector_size, vector_size);
899
900 gs->fetch_outputs = llvm_fetch_gs_outputs;
901 gs->fetch_inputs = llvm_fetch_gs_input;
902 gs->prepare = llvm_gs_prepare;
903 gs->run = llvm_gs_run;
904
905 gs->jit_context = &draw->llvm->gs_jit_context;
906 gs->jit_resources = &draw->llvm->jit_resources[PIPE_SHADER_GEOMETRY];
907
908 llvm_gs->variant_key_size =
909 draw_gs_llvm_variant_key_size(
910 gs->info.file_max[TGSI_FILE_SAMPLER]+1,
911 gs->info.file_max[TGSI_FILE_SAMPLER_VIEW]+1,
912 gs->info.file_max[TGSI_FILE_IMAGE]+1);
913 } else
914 #endif
915 {
916 gs->fetch_outputs = tgsi_fetch_gs_outputs;
917 gs->fetch_inputs = tgsi_fetch_gs_input;
918 gs->prepare = tgsi_gs_prepare;
919 gs->run = tgsi_gs_run;
920 }
921
922 return gs;
923 }
924
925
926 void
draw_bind_geometry_shader(struct draw_context * draw,struct draw_geometry_shader * dgs)927 draw_bind_geometry_shader(struct draw_context *draw,
928 struct draw_geometry_shader *dgs)
929 {
930 draw_do_flush(draw, DRAW_FLUSH_STATE_CHANGE);
931
932 if (dgs) {
933 draw->gs.geometry_shader = dgs;
934 draw->gs.num_gs_outputs = dgs->info.num_outputs;
935 draw->gs.position_output = dgs->position_output;
936 draw->gs.clipvertex_output = dgs->clipvertex_output;
937 draw_geometry_shader_prepare(dgs, draw);
938 } else {
939 draw->gs.geometry_shader = NULL;
940 draw->gs.num_gs_outputs = 0;
941 }
942 }
943
944
945 void
draw_delete_geometry_shader(struct draw_context * draw,struct draw_geometry_shader * dgs)946 draw_delete_geometry_shader(struct draw_context *draw,
947 struct draw_geometry_shader *dgs)
948 {
949 if (!dgs) {
950 return;
951 }
952 #if DRAW_LLVM_AVAILABLE
953 if (draw->llvm) {
954 struct llvm_geometry_shader *shader = llvm_geometry_shader(dgs);
955 struct draw_gs_llvm_variant_list_item *li, *next;
956
957 LIST_FOR_EACH_ENTRY_SAFE(li, next, &shader->variants.list, list) {
958 draw_gs_llvm_destroy_variant(li->base);
959 }
960
961 assert(shader->variants_cached == 0);
962
963 if (dgs->llvm_prim_lengths) {
964 for (unsigned i = 0; i < dgs->num_vertex_streams * dgs->max_out_prims; ++i) {
965 align_free(dgs->llvm_prim_lengths[i]);
966 }
967 FREE(dgs->llvm_prim_lengths);
968 }
969 align_free(dgs->llvm_emitted_primitives);
970 align_free(dgs->llvm_emitted_vertices);
971 align_free(dgs->llvm_prim_ids);
972
973 align_free(dgs->gs_input);
974 }
975 #endif
976
977 if (draw->gs.tgsi.machine && draw->gs.tgsi.machine->Tokens == dgs->state.tokens)
978 draw->gs.tgsi.machine->Tokens = NULL;
979
980 for (unsigned i = 0; i < TGSI_MAX_VERTEX_STREAMS; i++)
981 FREE(dgs->stream[i].primitive_lengths);
982
983 if (dgs->state.type == PIPE_SHADER_IR_NIR && dgs->state.ir.nir)
984 ralloc_free(dgs->state.ir.nir);
985 FREE((void*) dgs->state.tokens);
986 FREE(dgs);
987 }
988
989
990 #if DRAW_LLVM_AVAILABLE
991 void
draw_gs_set_current_variant(struct draw_geometry_shader * shader,struct draw_gs_llvm_variant * variant)992 draw_gs_set_current_variant(struct draw_geometry_shader *shader,
993 struct draw_gs_llvm_variant *variant)
994 {
995 shader->current_variant = variant;
996 }
997 #endif
998
999 /*
1000 * Called at the very begin of the draw call with a new instance
1001 * Used to reset state that should persist between primitive restart.
1002 */
1003 void
draw_geometry_shader_new_instance(struct draw_geometry_shader * gs)1004 draw_geometry_shader_new_instance(struct draw_geometry_shader *gs)
1005 {
1006 if (!gs)
1007 return;
1008
1009 gs->in_prim_idx = 0;
1010 }
1011