1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 /**
24 * @file iris_draw.c
25 *
26 * The main driver hooks for drawing and launching compute shaders.
27 */
28
29 #include <stdio.h>
30 #include <errno.h>
31 #include "pipe/p_defines.h"
32 #include "pipe/p_state.h"
33 #include "pipe/p_context.h"
34 #include "pipe/p_screen.h"
35 #include "util/u_draw.h"
36 #include "util/u_inlines.h"
37 #include "util/u_transfer.h"
38 #include "util/u_upload_mgr.h"
39 #include "compiler/shader_info.h"
40 #include "iris_context.h"
41 #include "iris_defines.h"
42
43 static bool
prim_is_points_or_lines(const struct pipe_draw_info * draw)44 prim_is_points_or_lines(const struct pipe_draw_info *draw)
45 {
46 /* We don't need to worry about adjacency - it can only be used with
47 * geometry shaders, and we don't care about this info when GS is on.
48 */
49 return draw->mode == MESA_PRIM_POINTS ||
50 draw->mode == MESA_PRIM_LINES ||
51 draw->mode == MESA_PRIM_LINE_LOOP ||
52 draw->mode == MESA_PRIM_LINE_STRIP;
53 }
54
55 /**
56 * Record the current primitive mode and restart information, flagging
57 * related packets as dirty if necessary.
58 *
59 * This must be called before updating compiled shaders, because the patch
60 * information informs the TCS key.
61 */
62 static void
iris_update_draw_info(struct iris_context * ice,const struct pipe_draw_info * info)63 iris_update_draw_info(struct iris_context *ice,
64 const struct pipe_draw_info *info)
65 {
66 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
67 const struct intel_device_info *devinfo = screen->devinfo;
68
69 if (ice->state.prim_mode != info->mode) {
70 ice->state.prim_mode = info->mode;
71 ice->state.dirty |= IRIS_DIRTY_VF_TOPOLOGY;
72
73
74 /* For XY Clip enables */
75 bool points_or_lines = prim_is_points_or_lines(info);
76 if (points_or_lines != ice->state.prim_is_points_or_lines) {
77 ice->state.prim_is_points_or_lines = points_or_lines;
78 ice->state.dirty |= IRIS_DIRTY_CLIP;
79 }
80 }
81
82 if (info->mode == MESA_PRIM_PATCHES &&
83 ice->state.vertices_per_patch != ice->state.patch_vertices) {
84 ice->state.vertices_per_patch = ice->state.patch_vertices;
85 ice->state.dirty |= IRIS_DIRTY_VF_TOPOLOGY;
86
87 /* MULTI_PATCH TCS needs this for key->input_vertices */
88 if (iris_use_tcs_multi_patch(screen))
89 ice->state.stage_dirty |= IRIS_STAGE_DIRTY_UNCOMPILED_TCS;
90
91 /* Flag constants dirty for gl_PatchVerticesIn if needed. */
92 const struct shader_info *tcs_info =
93 iris_get_shader_info(ice, MESA_SHADER_TESS_CTRL);
94 if (tcs_info &&
95 BITSET_TEST(tcs_info->system_values_read, SYSTEM_VALUE_VERTICES_IN)) {
96 ice->state.stage_dirty |= IRIS_STAGE_DIRTY_CONSTANTS_TCS;
97 ice->state.shaders[MESA_SHADER_TESS_CTRL].sysvals_need_upload = true;
98 }
99 }
100
101 /* Track restart_index changes only if primitive_restart is true */
102 const unsigned cut_index = info->primitive_restart ? info->restart_index :
103 ice->state.cut_index;
104 if (ice->state.primitive_restart != info->primitive_restart ||
105 ice->state.cut_index != cut_index) {
106 ice->state.dirty |= IRIS_DIRTY_VF;
107 ice->state.cut_index = cut_index;
108 ice->state.dirty |=
109 ((ice->state.primitive_restart != info->primitive_restart) &&
110 devinfo->verx10 >= 125) ? IRIS_DIRTY_VFG : 0;
111 ice->state.primitive_restart = info->primitive_restart;
112 }
113 }
114
115 /**
116 * Update shader draw parameters, flagging VF packets as dirty if necessary.
117 */
118 static void
iris_update_draw_parameters(struct iris_context * ice,const struct pipe_draw_info * info,unsigned drawid_offset,const struct pipe_draw_indirect_info * indirect,const struct pipe_draw_start_count_bias * draw)119 iris_update_draw_parameters(struct iris_context *ice,
120 const struct pipe_draw_info *info,
121 unsigned drawid_offset,
122 const struct pipe_draw_indirect_info *indirect,
123 const struct pipe_draw_start_count_bias *draw)
124 {
125 bool changed = false;
126
127 if (ice->state.vs_uses_draw_params) {
128 struct iris_state_ref *draw_params = &ice->draw.draw_params;
129
130 if (indirect && indirect->buffer) {
131 pipe_resource_reference(&draw_params->res, indirect->buffer);
132 draw_params->offset =
133 indirect->offset + (info->index_size ? 12 : 8);
134
135 changed = true;
136 ice->draw.params_valid = false;
137 } else {
138 int firstvertex = info->index_size ? draw->index_bias : draw->start;
139
140 if (!ice->draw.params_valid ||
141 ice->draw.params.firstvertex != firstvertex ||
142 ice->draw.params.baseinstance != info->start_instance) {
143
144 changed = true;
145 ice->draw.params.firstvertex = firstvertex;
146 ice->draw.params.baseinstance = info->start_instance;
147 ice->draw.params_valid = true;
148
149 u_upload_data(ice->ctx.const_uploader, 0,
150 sizeof(ice->draw.params), 4, &ice->draw.params,
151 &draw_params->offset, &draw_params->res);
152 }
153 }
154 }
155
156 if (ice->state.vs_uses_derived_draw_params) {
157 struct iris_state_ref *derived_params = &ice->draw.derived_draw_params;
158 int is_indexed_draw = info->index_size ? -1 : 0;
159
160 if (ice->draw.derived_params.drawid != drawid_offset ||
161 ice->draw.derived_params.is_indexed_draw != is_indexed_draw) {
162
163 changed = true;
164 ice->draw.derived_params.drawid = drawid_offset;
165 ice->draw.derived_params.is_indexed_draw = is_indexed_draw;
166
167 u_upload_data(ice->ctx.const_uploader, 0,
168 sizeof(ice->draw.derived_params), 4,
169 &ice->draw.derived_params,
170 &derived_params->offset, &derived_params->res);
171 }
172 }
173
174 if (changed) {
175 ice->state.dirty |= IRIS_DIRTY_VERTEX_BUFFERS |
176 IRIS_DIRTY_VERTEX_ELEMENTS |
177 IRIS_DIRTY_VF_SGVS;
178 }
179 }
180
181 static void
iris_simple_draw_vbo(struct iris_context * ice,const struct pipe_draw_info * draw,unsigned drawid_offset,const struct pipe_draw_indirect_info * indirect,const struct pipe_draw_start_count_bias * sc)182 iris_simple_draw_vbo(struct iris_context *ice,
183 const struct pipe_draw_info *draw,
184 unsigned drawid_offset,
185 const struct pipe_draw_indirect_info *indirect,
186 const struct pipe_draw_start_count_bias *sc)
187 {
188 struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
189
190 iris_batch_maybe_flush(batch, 1500);
191
192 iris_update_draw_parameters(ice, draw, drawid_offset, indirect, sc);
193
194 batch->screen->vtbl.upload_render_state(ice, batch, draw, drawid_offset, indirect, sc);
195 }
196
197 static inline bool
iris_use_draw_indirect_generation(const struct iris_screen * screen,const struct pipe_draw_indirect_info * dindirect)198 iris_use_draw_indirect_generation(const struct iris_screen *screen,
199 const struct pipe_draw_indirect_info *dindirect)
200 {
201 return dindirect != NULL &&
202 dindirect->draw_count >= screen->driconf.generated_indirect_threshold;
203 }
204
205 static void
iris_indirect_draw_vbo(struct iris_context * ice,const struct pipe_draw_info * dinfo,unsigned drawid_offset,const struct pipe_draw_indirect_info * dindirect,const struct pipe_draw_start_count_bias * draw)206 iris_indirect_draw_vbo(struct iris_context *ice,
207 const struct pipe_draw_info *dinfo,
208 unsigned drawid_offset,
209 const struct pipe_draw_indirect_info *dindirect,
210 const struct pipe_draw_start_count_bias *draw)
211 {
212 struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
213 struct iris_screen *screen = batch->screen;
214 struct pipe_draw_info info = *dinfo;
215 struct pipe_draw_indirect_info indirect = *dindirect;
216 const bool use_predicate =
217 ice->state.predicate == IRIS_PREDICATE_STATE_USE_BIT;
218
219 const uint64_t orig_dirty = ice->state.dirty;
220 const uint64_t orig_stage_dirty = ice->state.stage_dirty;
221
222 if (iris_execute_indirect_draw_supported(ice, &indirect, &info)) {
223 iris_batch_maybe_flush(batch, 1500);
224
225 iris_update_draw_parameters(ice, &info, drawid_offset, &indirect, draw);
226
227 screen->vtbl.upload_indirect_render_state(ice, &info, &indirect, draw);
228 } else if (iris_use_draw_indirect_generation(screen, &indirect)) {
229 iris_batch_maybe_flush(batch, 1500);
230
231 iris_update_draw_parameters(ice, &info, drawid_offset, &indirect, draw);
232
233 screen->vtbl.upload_indirect_shader_render_state(
234 ice, &info, &indirect, draw);
235 } else {
236 iris_emit_buffer_barrier_for(batch, iris_resource_bo(indirect.buffer),
237 IRIS_DOMAIN_VF_READ);
238
239 if (indirect.indirect_draw_count) {
240 struct iris_bo *draw_count_bo =
241 iris_resource_bo(indirect.indirect_draw_count);
242 iris_emit_buffer_barrier_for(batch, draw_count_bo,
243 IRIS_DOMAIN_OTHER_READ);
244 }
245
246 if (use_predicate) {
247 /* Upload MI_PREDICATE_RESULT to GPR15.*/
248 screen->vtbl.load_register_reg64(batch, CS_GPR(15), MI_PREDICATE_RESULT);
249 }
250
251 for (int i = 0; i < indirect.draw_count; i++) {
252 iris_simple_draw_vbo(ice, &info, drawid_offset + i, &indirect, draw);
253
254 ice->state.dirty &= ~IRIS_ALL_DIRTY_FOR_RENDER;
255 ice->state.stage_dirty &= ~IRIS_ALL_STAGE_DIRTY_FOR_RENDER;
256
257 indirect.offset += indirect.stride;
258 }
259
260 if (use_predicate) {
261 /* Restore MI_PREDICATE_RESULT. */
262 screen->vtbl.load_register_reg64(batch, MI_PREDICATE_RESULT, CS_GPR(15));
263 }
264 }
265
266 /* Put this back for post-draw resolves, we'll clear it again after. */
267 ice->state.dirty = orig_dirty;
268 ice->state.stage_dirty = orig_stage_dirty;
269 }
270
271 /**
272 * The pipe->draw_vbo() driver hook. Performs a draw on the GPU.
273 */
274 void
iris_draw_vbo(struct pipe_context * ctx,const struct pipe_draw_info * info,unsigned drawid_offset,const struct pipe_draw_indirect_info * indirect,const struct pipe_draw_start_count_bias * draws,unsigned num_draws)275 iris_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info,
276 unsigned drawid_offset,
277 const struct pipe_draw_indirect_info *indirect,
278 const struct pipe_draw_start_count_bias *draws,
279 unsigned num_draws)
280 {
281 if (num_draws > 1) {
282 util_draw_multi(ctx, info, drawid_offset, indirect, draws, num_draws);
283 return;
284 }
285
286 if (!indirect && (!draws[0].count || !info->instance_count))
287 return;
288
289 struct iris_context *ice = (struct iris_context *) ctx;
290 struct iris_screen *screen = (struct iris_screen*)ice->ctx.screen;
291 const struct intel_device_info *devinfo = screen->devinfo;
292 struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
293
294 if (ice->state.predicate == IRIS_PREDICATE_STATE_DONT_RENDER)
295 return;
296
297 if (INTEL_DEBUG(DEBUG_REEMIT)) {
298 ice->state.dirty |= IRIS_ALL_DIRTY_FOR_RENDER;
299 ice->state.stage_dirty |= IRIS_ALL_STAGE_DIRTY_FOR_RENDER;
300 }
301
302 iris_update_draw_info(ice, info);
303
304 if (devinfo->ver == 9)
305 gfx9_toggle_preemption(ice, batch, info);
306
307 iris_update_compiled_shaders(ice);
308
309 if (ice->state.dirty & IRIS_DIRTY_RENDER_RESOLVES_AND_FLUSHES) {
310 bool draw_aux_buffer_disabled[IRIS_MAX_DRAW_BUFFERS] = { };
311 for (gl_shader_stage stage = 0; stage < MESA_SHADER_COMPUTE; stage++) {
312 if (ice->shaders.prog[stage])
313 iris_predraw_resolve_inputs(ice, batch, draw_aux_buffer_disabled,
314 stage, true);
315 }
316 iris_predraw_resolve_framebuffer(ice, batch, draw_aux_buffer_disabled);
317 }
318
319 if (ice->state.dirty & IRIS_DIRTY_RENDER_MISC_BUFFER_FLUSHES) {
320 for (gl_shader_stage stage = 0; stage < MESA_SHADER_COMPUTE; stage++)
321 iris_predraw_flush_buffers(ice, batch, stage);
322 }
323
324 /* If we're going to use the generation shader, we need to allocate a
325 * binding table entry for it on <= Gfx9 because that platform does not
326 * have a null-rendertarget bit in the send message to the render cache,
327 * the EOT message might pollute later writes to the actual RT of the
328 * draws.
329 *
330 * The generation will call iris_binder_reserve_3d() after the generation
331 * draw call.
332 */
333 if (iris_use_draw_indirect_generation(screen, indirect) && devinfo->ver <= 9)
334 iris_binder_reserve_gen(ice);
335 else
336 iris_binder_reserve_3d(ice);
337
338 batch->screen->vtbl.update_binder_address(batch, &ice->state.binder);
339
340 iris_handle_always_flush_cache(batch);
341
342 if (indirect && indirect->buffer)
343 iris_indirect_draw_vbo(ice, info, drawid_offset, indirect, &draws[0]);
344 else
345 iris_simple_draw_vbo(ice, info, drawid_offset, indirect, &draws[0]);
346
347 iris_handle_always_flush_cache(batch);
348
349 iris_postdraw_update_resolve_tracking(ice);
350
351 ice->state.dirty &= ~IRIS_ALL_DIRTY_FOR_RENDER;
352 ice->state.stage_dirty &= ~IRIS_ALL_STAGE_DIRTY_FOR_RENDER;
353 }
354
355 static void
iris_update_grid_size_resource(struct iris_context * ice,const struct pipe_grid_info * grid)356 iris_update_grid_size_resource(struct iris_context *ice,
357 const struct pipe_grid_info *grid)
358 {
359 const struct iris_screen *screen = (void *) ice->ctx.screen;
360 const struct isl_device *isl_dev = &screen->isl_dev;
361 struct iris_state_ref *grid_ref = &ice->state.grid_size;
362 struct iris_state_ref *state_ref = &ice->state.grid_surf_state;
363
364 const struct iris_compiled_shader *shader = ice->shaders.prog[MESA_SHADER_COMPUTE];
365 bool grid_needs_surface = shader->bt.used_mask[IRIS_SURFACE_GROUP_CS_WORK_GROUPS];
366 bool grid_updated = false;
367
368 if (grid->indirect) {
369 pipe_resource_reference(&grid_ref->res, grid->indirect);
370 grid_ref->offset = grid->indirect_offset;
371
372 /* Zero out the grid size so that the next non-indirect grid launch will
373 * re-upload it properly.
374 */
375 memset(ice->state.last_grid, 0, sizeof(ice->state.last_grid));
376 grid_updated = true;
377 } else if (memcmp(ice->state.last_grid, grid->grid, sizeof(grid->grid)) != 0) {
378 memcpy(ice->state.last_grid, grid->grid, sizeof(grid->grid));
379 u_upload_data(ice->state.dynamic_uploader, 0, sizeof(grid->grid), 4,
380 grid->grid, &grid_ref->offset, &grid_ref->res);
381 grid_updated = true;
382 }
383
384 /* If we changed the grid, the old surface state is invalid. */
385 if (grid_updated)
386 pipe_resource_reference(&state_ref->res, NULL);
387
388 /* Skip surface upload if we don't need it or we already have one */
389 if (!grid_needs_surface || state_ref->res)
390 return;
391
392 struct iris_bo *grid_bo = iris_resource_bo(grid_ref->res);
393
394 void *surf_map = NULL;
395 u_upload_alloc(ice->state.surface_uploader, 0, isl_dev->ss.size,
396 isl_dev->ss.align, &state_ref->offset, &state_ref->res,
397 &surf_map);
398 state_ref->offset +=
399 iris_bo_offset_from_base_address(iris_resource_bo(state_ref->res));
400 isl_buffer_fill_state(&screen->isl_dev, surf_map,
401 .address = grid_ref->offset + grid_bo->address,
402 .size_B = sizeof(grid->grid),
403 .format = ISL_FORMAT_RAW,
404 .stride_B = 1,
405 .mocs = iris_mocs(grid_bo, isl_dev,
406 ISL_SURF_USAGE_CONSTANT_BUFFER_BIT));
407
408 ice->state.stage_dirty |= IRIS_STAGE_DIRTY_BINDINGS_CS;
409 }
410
411 void
iris_launch_grid(struct pipe_context * ctx,const struct pipe_grid_info * grid)412 iris_launch_grid(struct pipe_context *ctx, const struct pipe_grid_info *grid)
413 {
414 struct iris_context *ice = (struct iris_context *) ctx;
415 struct iris_screen *screen = (struct iris_screen *) ctx->screen;
416 const struct intel_device_info *devinfo = screen->devinfo;
417 struct iris_batch *batch = &ice->batches[IRIS_BATCH_COMPUTE];
418
419 if (ice->state.predicate == IRIS_PREDICATE_STATE_DONT_RENDER)
420 return;
421
422 if (INTEL_DEBUG(DEBUG_REEMIT)) {
423 ice->state.dirty |= IRIS_ALL_DIRTY_FOR_COMPUTE;
424 ice->state.stage_dirty |= IRIS_ALL_STAGE_DIRTY_FOR_COMPUTE;
425 }
426
427 if (ice->state.dirty & IRIS_DIRTY_COMPUTE_RESOLVES_AND_FLUSHES)
428 iris_predraw_resolve_inputs(ice, batch, NULL, MESA_SHADER_COMPUTE, false);
429
430 if (ice->state.dirty & IRIS_DIRTY_COMPUTE_MISC_BUFFER_FLUSHES)
431 iris_predraw_flush_buffers(ice, batch, MESA_SHADER_COMPUTE);
432
433 iris_batch_maybe_flush(batch, 1500);
434
435 iris_update_compiled_compute_shader(ice);
436
437 if (memcmp(ice->state.last_block, grid->block, sizeof(grid->block)) != 0) {
438 memcpy(ice->state.last_block, grid->block, sizeof(grid->block));
439 ice->state.stage_dirty |= IRIS_STAGE_DIRTY_CONSTANTS_CS;
440 ice->state.shaders[MESA_SHADER_COMPUTE].sysvals_need_upload = true;
441 }
442
443 if (ice->state.last_grid_dim != grid->work_dim) {
444 ice->state.last_grid_dim = grid->work_dim;
445 ice->state.stage_dirty |= IRIS_STAGE_DIRTY_CONSTANTS_CS;
446 ice->state.shaders[MESA_SHADER_COMPUTE].sysvals_need_upload = true;
447 }
448
449 iris_update_grid_size_resource(ice, grid);
450
451 iris_binder_reserve_compute(ice);
452 batch->screen->vtbl.update_binder_address(batch, &ice->state.binder);
453
454 if (ice->state.compute_predicate) {
455 batch->screen->vtbl.load_register_mem32(batch, MI_PREDICATE_RESULT,
456 ice->state.compute_predicate, 0);
457 ice->state.compute_predicate = NULL;
458 }
459
460 iris_handle_always_flush_cache(batch);
461
462 batch->screen->vtbl.upload_compute_state(ice, batch, grid);
463
464 iris_handle_always_flush_cache(batch);
465
466 ice->state.dirty &= ~IRIS_ALL_DIRTY_FOR_COMPUTE;
467 ice->state.stage_dirty &= ~IRIS_ALL_STAGE_DIRTY_FOR_COMPUTE;
468
469 if (devinfo->ver >= 12)
470 iris_postdraw_update_image_resolve_tracking(ice, MESA_SHADER_COMPUTE);
471 }
472