1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 #include <stdio.h>
24 #include "pipe/p_defines.h"
25 #include "pipe/p_state.h"
26 #include "pipe/p_context.h"
27 #include "pipe/p_screen.h"
28 #include "util/format/u_format.h"
29 #include "util/u_inlines.h"
30 #include "util/u_surface.h"
31 #include "util/ralloc.h"
32 #include "intel/blorp/blorp.h"
33 #include "crocus_context.h"
34 #include "crocus_resource.h"
35 #include "crocus_screen.h"
36
crocus_blitter_begin(struct crocus_context * ice,enum crocus_blitter_op op,bool render_cond)37 void crocus_blitter_begin(struct crocus_context *ice, enum crocus_blitter_op op, bool render_cond)
38 {
39 util_blitter_save_vertex_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_VERTEX]);
40 util_blitter_save_tessctrl_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_TESS_CTRL]);
41 util_blitter_save_tesseval_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL]);
42 util_blitter_save_geometry_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_GEOMETRY]);
43 util_blitter_save_so_targets(ice->blitter, ice->state.so_targets,
44 (struct pipe_stream_output_target**)ice->state.so_target);
45 util_blitter_save_vertex_buffers(ice->blitter, ice->state.vertex_buffers,
46 util_last_bit(ice->state.bound_vertex_buffers));
47 util_blitter_save_vertex_elements(ice->blitter, (void *)ice->state.cso_vertex_elements);
48 if (op & CROCUS_SAVE_FRAGMENT_STATE) {
49 util_blitter_save_blend(ice->blitter, ice->state.cso_blend);
50 util_blitter_save_depth_stencil_alpha(ice->blitter, ice->state.cso_zsa);
51 util_blitter_save_stencil_ref(ice->blitter, &ice->state.stencil_ref);
52 util_blitter_save_fragment_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_FRAGMENT]);
53 util_blitter_save_sample_mask(ice->blitter, ice->state.sample_mask, 0);
54 util_blitter_save_rasterizer(ice->blitter, ice->state.cso_rast);
55 util_blitter_save_scissor(ice->blitter, &ice->state.scissors[0]);
56 util_blitter_save_viewport(ice->blitter, &ice->state.viewports[0]);
57 util_blitter_save_fragment_constant_buffer_slot(ice->blitter, &ice->state.shaders[MESA_SHADER_FRAGMENT].constbufs[0]);
58 }
59
60 if (!render_cond)
61 util_blitter_save_render_condition(ice->blitter,
62 (struct pipe_query *)ice->condition.query,
63 ice->condition.condition,
64 ice->condition.mode);
65
66 // util_blitter_save_scissor(ice->blitter, &ice->scissors[0]);
67 if (op & CROCUS_SAVE_FRAMEBUFFER)
68 util_blitter_save_framebuffer(ice->blitter, &ice->state.framebuffer);
69
70 if (op & CROCUS_SAVE_TEXTURES) {
71 util_blitter_save_fragment_sampler_states(ice->blitter, 1, (void **)ice->state.shaders[MESA_SHADER_FRAGMENT].samplers);
72 util_blitter_save_fragment_sampler_views(ice->blitter, 1, (struct pipe_sampler_view **)ice->state.shaders[MESA_SHADER_FRAGMENT].textures);
73 }
74 }
75
76 /**
77 * Helper function for handling mirror image blits.
78 *
79 * If coord0 > coord1, swap them and return "true" (mirrored).
80 */
81 static bool
apply_mirror(float * coord0,float * coord1)82 apply_mirror(float *coord0, float *coord1)
83 {
84 if (*coord0 > *coord1) {
85 float tmp = *coord0;
86 *coord0 = *coord1;
87 *coord1 = tmp;
88 return true;
89 }
90 return false;
91 }
92
93 /**
94 * Compute the number of pixels to clip for each side of a rect
95 *
96 * \param x0 The rect's left coordinate
97 * \param y0 The rect's bottom coordinate
98 * \param x1 The rect's right coordinate
99 * \param y1 The rect's top coordinate
100 * \param min_x The clipping region's left coordinate
101 * \param min_y The clipping region's bottom coordinate
102 * \param max_x The clipping region's right coordinate
103 * \param max_y The clipping region's top coordinate
104 * \param clipped_x0 The number of pixels to clip from the left side
105 * \param clipped_y0 The number of pixels to clip from the bottom side
106 * \param clipped_x1 The number of pixels to clip from the right side
107 * \param clipped_y1 The number of pixels to clip from the top side
108 *
109 * \return false if we clip everything away, true otherwise
110 */
111 static inline bool
compute_pixels_clipped(float x0,float y0,float x1,float y1,float min_x,float min_y,float max_x,float max_y,float * clipped_x0,float * clipped_y0,float * clipped_x1,float * clipped_y1)112 compute_pixels_clipped(float x0, float y0, float x1, float y1,
113 float min_x, float min_y, float max_x, float max_y,
114 float *clipped_x0, float *clipped_y0,
115 float *clipped_x1, float *clipped_y1)
116 {
117 /* If we are going to clip everything away, stop. */
118 if (!(min_x <= max_x &&
119 min_y <= max_y &&
120 x0 <= max_x &&
121 y0 <= max_y &&
122 min_x <= x1 &&
123 min_y <= y1 &&
124 x0 <= x1 &&
125 y0 <= y1)) {
126 return false;
127 }
128
129 if (x0 < min_x)
130 *clipped_x0 = min_x - x0;
131 else
132 *clipped_x0 = 0;
133 if (max_x < x1)
134 *clipped_x1 = x1 - max_x;
135 else
136 *clipped_x1 = 0;
137
138 if (y0 < min_y)
139 *clipped_y0 = min_y - y0;
140 else
141 *clipped_y0 = 0;
142 if (max_y < y1)
143 *clipped_y1 = y1 - max_y;
144 else
145 *clipped_y1 = 0;
146
147 return true;
148 }
149
150 /**
151 * Clips a coordinate (left, right, top or bottom) for the src or dst rect
152 * (whichever requires the largest clip) and adjusts the coordinate
153 * for the other rect accordingly.
154 *
155 * \param mirror true if mirroring is required
156 * \param src the source rect coordinate (for example src_x0)
157 * \param dst0 the dst rect coordinate (for example dst_x0)
158 * \param dst1 the opposite dst rect coordinate (for example dst_x1)
159 * \param clipped_dst0 number of pixels to clip from the dst coordinate
160 * \param clipped_dst1 number of pixels to clip from the opposite dst coordinate
161 * \param scale the src vs dst scale involved for that coordinate
162 * \param is_left_or_bottom true if we are clipping the left or bottom sides
163 * of the rect.
164 */
165 static void
clip_coordinates(bool mirror,float * src,float * dst0,float * dst1,float clipped_dst0,float clipped_dst1,float scale,bool is_left_or_bottom)166 clip_coordinates(bool mirror,
167 float *src, float *dst0, float *dst1,
168 float clipped_dst0,
169 float clipped_dst1,
170 float scale,
171 bool is_left_or_bottom)
172 {
173 /* When clipping we need to add or subtract pixels from the original
174 * coordinates depending on whether we are acting on the left/bottom
175 * or right/top sides of the rect respectively. We assume we have to
176 * add them in the code below, and multiply by -1 when we should
177 * subtract.
178 */
179 int mult = is_left_or_bottom ? 1 : -1;
180
181 if (!mirror) {
182 *dst0 += clipped_dst0 * mult;
183 *src += clipped_dst0 * scale * mult;
184 } else {
185 *dst1 -= clipped_dst1 * mult;
186 *src += clipped_dst1 * scale * mult;
187 }
188 }
189
190 /**
191 * Apply a scissor rectangle to blit coordinates.
192 *
193 * Returns true if the blit was entirely scissored away.
194 */
195 static bool
apply_blit_scissor(const struct pipe_scissor_state * scissor,float * src_x0,float * src_y0,float * src_x1,float * src_y1,float * dst_x0,float * dst_y0,float * dst_x1,float * dst_y1,bool mirror_x,bool mirror_y)196 apply_blit_scissor(const struct pipe_scissor_state *scissor,
197 float *src_x0, float *src_y0,
198 float *src_x1, float *src_y1,
199 float *dst_x0, float *dst_y0,
200 float *dst_x1, float *dst_y1,
201 bool mirror_x, bool mirror_y)
202 {
203 float clip_dst_x0, clip_dst_x1, clip_dst_y0, clip_dst_y1;
204
205 /* Compute number of pixels to scissor away. */
206 if (!compute_pixels_clipped(*dst_x0, *dst_y0, *dst_x1, *dst_y1,
207 scissor->minx, scissor->miny,
208 scissor->maxx, scissor->maxy,
209 &clip_dst_x0, &clip_dst_y0,
210 &clip_dst_x1, &clip_dst_y1))
211 return true;
212
213 // XXX: comments assume source clipping, which we don't do
214
215 /* When clipping any of the two rects we need to adjust the coordinates
216 * in the other rect considering the scaling factor involved. To obtain
217 * the best precision we want to make sure that we only clip once per
218 * side to avoid accumulating errors due to the scaling adjustment.
219 *
220 * For example, if src_x0 and dst_x0 need both to be clipped we want to
221 * avoid the situation where we clip src_x0 first, then adjust dst_x0
222 * accordingly but then we realize that the resulting dst_x0 still needs
223 * to be clipped, so we clip dst_x0 and adjust src_x0 again. Because we are
224 * applying scaling factors to adjust the coordinates in each clipping
225 * pass we lose some precision and that can affect the results of the
226 * blorp blit operation slightly. What we want to do here is detect the
227 * rect that we should clip first for each side so that when we adjust
228 * the other rect we ensure the resulting coordinate does not need to be
229 * clipped again.
230 *
231 * The code below implements this by comparing the number of pixels that
232 * we need to clip for each side of both rects considering the scales
233 * involved. For example, clip_src_x0 represents the number of pixels
234 * to be clipped for the src rect's left side, so if clip_src_x0 = 5,
235 * clip_dst_x0 = 4 and scale_x = 2 it means that we are clipping more
236 * from the dst rect so we should clip dst_x0 only and adjust src_x0.
237 * This is because clipping 4 pixels in the dst is equivalent to
238 * clipping 4 * 2 = 8 > 5 in the src.
239 */
240
241 if (*src_x0 == *src_x1 || *src_y0 == *src_y1
242 || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1)
243 return true;
244
245 float scale_x = (float) (*src_x1 - *src_x0) / (*dst_x1 - *dst_x0);
246 float scale_y = (float) (*src_y1 - *src_y0) / (*dst_y1 - *dst_y0);
247
248 /* Clip left side */
249 clip_coordinates(mirror_x, src_x0, dst_x0, dst_x1,
250 clip_dst_x0, clip_dst_x1, scale_x, true);
251
252 /* Clip right side */
253 clip_coordinates(mirror_x, src_x1, dst_x1, dst_x0,
254 clip_dst_x1, clip_dst_x0, scale_x, false);
255
256 /* Clip bottom side */
257 clip_coordinates(mirror_y, src_y0, dst_y0, dst_y1,
258 clip_dst_y0, clip_dst_y1, scale_y, true);
259
260 /* Clip top side */
261 clip_coordinates(mirror_y, src_y1, dst_y1, dst_y0,
262 clip_dst_y1, clip_dst_y0, scale_y, false);
263
264 /* Check for invalid bounds
265 * Can't blit for 0-dimensions
266 */
267 return *src_x0 == *src_x1 || *src_y0 == *src_y1
268 || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1;
269 }
270
271 void
crocus_blorp_surf_for_resource(struct crocus_vtable * vtbl,struct isl_device * isl_dev,struct blorp_surf * surf,struct pipe_resource * p_res,enum isl_aux_usage aux_usage,unsigned level,bool is_render_target)272 crocus_blorp_surf_for_resource(struct crocus_vtable *vtbl,
273 struct isl_device *isl_dev,
274 struct blorp_surf *surf,
275 struct pipe_resource *p_res,
276 enum isl_aux_usage aux_usage,
277 unsigned level,
278 bool is_render_target)
279 {
280 struct crocus_resource *res = (void *) p_res;
281
282 if (isl_aux_usage_has_hiz(aux_usage) &&
283 !crocus_resource_level_has_hiz(res, level))
284 aux_usage = ISL_AUX_USAGE_NONE;
285
286 *surf = (struct blorp_surf) {
287 .surf = &res->surf,
288 .addr = (struct blorp_address) {
289 .buffer = res->bo,
290 .offset = res->offset,
291 .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
292 .mocs = crocus_mocs(res->bo, isl_dev),
293 },
294 .aux_usage = aux_usage,
295 };
296
297 if (aux_usage != ISL_AUX_USAGE_NONE) {
298 surf->aux_surf = &res->aux.surf;
299 surf->aux_addr = (struct blorp_address) {
300 .buffer = res->aux.bo,
301 .offset = res->aux.offset,
302 .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
303 .mocs = crocus_mocs(res->bo, isl_dev),
304 };
305 surf->clear_color =
306 crocus_resource_get_clear_color(res);
307 }
308 }
309
310 static void
tex_cache_flush_hack(struct crocus_batch * batch,enum isl_format view_format,enum isl_format surf_format)311 tex_cache_flush_hack(struct crocus_batch *batch,
312 enum isl_format view_format,
313 enum isl_format surf_format)
314 {
315 /* The WaSamplerCacheFlushBetweenRedescribedSurfaceReads workaround says:
316 *
317 * "Currently Sampler assumes that a surface would not have two
318 * different format associate with it. It will not properly cache
319 * the different views in the MT cache, causing a data corruption."
320 *
321 * We may need to handle this for texture views in general someday, but
322 * for now we handle it here, as it hurts copies and blits particularly
323 * badly because they ofter reinterpret formats.
324 *
325 * If the BO hasn't been referenced yet this batch, we assume that the
326 * texture cache doesn't contain any relevant data nor need flushing.
327 *
328 * Icelake (Gen11+) claims to fix this issue, but seems to still have
329 * issues with ASTC formats.
330 */
331 bool need_flush = view_format != surf_format;
332 if (!need_flush)
333 return;
334
335 const char *reason =
336 "workaround: WaSamplerCacheFlushBetweenRedescribedSurfaceReads";
337
338 crocus_emit_pipe_control_flush(batch, reason, PIPE_CONTROL_CS_STALL);
339 crocus_emit_pipe_control_flush(batch, reason,
340 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
341 }
342
343 static struct crocus_resource *
crocus_resource_for_aspect(const struct intel_device_info * devinfo,struct pipe_resource * p_res,unsigned pipe_mask)344 crocus_resource_for_aspect(const struct intel_device_info *devinfo,
345 struct pipe_resource *p_res, unsigned pipe_mask)
346 {
347 if (pipe_mask == PIPE_MASK_S) {
348 struct crocus_resource *junk, *s_res;
349 crocus_get_depth_stencil_resources(devinfo, p_res, &junk, &s_res);
350 return s_res;
351 } else {
352 return (struct crocus_resource *)p_res;
353 }
354 }
355
356 static enum pipe_format
pipe_format_for_aspect(enum pipe_format format,unsigned pipe_mask)357 pipe_format_for_aspect(enum pipe_format format, unsigned pipe_mask)
358 {
359 if (pipe_mask == PIPE_MASK_S) {
360 return util_format_stencil_only(format);
361 } else if (pipe_mask == PIPE_MASK_Z) {
362 return util_format_get_depth_only(format);
363 } else {
364 return format;
365 }
366 }
367
368 static void
crocus_u_blitter(struct crocus_context * ice,const struct pipe_blit_info * info)369 crocus_u_blitter(struct crocus_context *ice,
370 const struct pipe_blit_info *info)
371 {
372 struct pipe_blit_info dinfo = *info;
373 if (!util_format_has_alpha(dinfo.dst.resource->format))
374 dinfo.mask &= ~PIPE_MASK_A;
375 crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable);
376 util_blitter_blit(ice->blitter, &dinfo, NULL);
377 }
378
379 /**
380 * The pipe->blit() driver hook.
381 *
382 * This performs a blit between two surfaces, which copies data but may
383 * also perform format conversion, scaling, flipping, and so on.
384 */
385 static void
crocus_blit(struct pipe_context * ctx,const struct pipe_blit_info * info)386 crocus_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
387 {
388 struct crocus_context *ice = (void *) ctx;
389 struct crocus_screen *screen = (struct crocus_screen *)ctx->screen;
390 const struct intel_device_info *devinfo = &screen->devinfo;
391 struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
392 enum blorp_batch_flags blorp_flags = 0;
393
394 /* We don't support color masking. */
395 assert((info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA ||
396 (info->mask & PIPE_MASK_RGBA) == 0);
397
398 if (info->render_condition_enable)
399 if (!crocus_check_conditional_render(ice))
400 return;
401
402 if (devinfo->ver <= 5) {
403 if (!screen->vtbl.blit_blt(batch, info)) {
404
405 if (!util_format_is_depth_or_stencil(info->src.resource->format) &&
406 info->dst.resource->target != PIPE_TEXTURE_3D)
407 goto use_blorp;
408
409 if (!util_blitter_is_blit_supported(ice->blitter, info)) {
410 if (util_format_is_depth_or_stencil(info->src.resource->format)) {
411
412 struct pipe_blit_info depth_blit = *info;
413 depth_blit.mask = PIPE_MASK_Z;
414 crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable);
415 util_blitter_blit(ice->blitter, &depth_blit, NULL);
416
417 struct pipe_surface *dst_view, dst_templ;
418 util_blitter_default_dst_texture(&dst_templ, info->dst.resource, info->dst.level, info->dst.box.z);
419 dst_view = ctx->create_surface(ctx, info->dst.resource, &dst_templ);
420
421 crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable);
422
423 util_blitter_clear_depth_stencil(ice->blitter, dst_view, PIPE_CLEAR_STENCIL,
424 0, 0, info->dst.box.x, info->dst.box.y,
425 info->dst.box.width, info->dst.box.height);
426 crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable);
427 util_blitter_stencil_fallback(ice->blitter,
428 info->dst.resource,
429 info->dst.level,
430 &info->dst.box,
431 info->src.resource,
432 info->src.level,
433 &info->src.box, NULL);
434
435 pipe_surface_release(ctx, &dst_view);
436 }
437 return;
438 }
439
440 crocus_u_blitter(ice, info);
441 }
442 return;
443 }
444
445 if (devinfo->ver == 6) {
446 if (info->src.resource->target == PIPE_TEXTURE_3D &&
447 info->dst.resource->target == PIPE_TEXTURE_3D) {
448 crocus_u_blitter(ice, info);
449 return;
450 }
451 }
452
453 use_blorp:
454 if (info->render_condition_enable) {
455 if (ice->state.predicate == CROCUS_PREDICATE_STATE_USE_BIT)
456 blorp_flags |= BLORP_BATCH_PREDICATE_ENABLE;
457 }
458
459 float src_x0 = info->src.box.x;
460 float src_x1 = info->src.box.x + info->src.box.width;
461 float src_y0 = info->src.box.y;
462 float src_y1 = info->src.box.y + info->src.box.height;
463 float dst_x0 = info->dst.box.x;
464 float dst_x1 = info->dst.box.x + info->dst.box.width;
465 float dst_y0 = info->dst.box.y;
466 float dst_y1 = info->dst.box.y + info->dst.box.height;
467 bool mirror_x = apply_mirror(&src_x0, &src_x1);
468 bool mirror_y = apply_mirror(&src_y0, &src_y1);
469 enum blorp_filter filter;
470
471 if (info->scissor_enable) {
472 bool noop = apply_blit_scissor(&info->scissor,
473 &src_x0, &src_y0, &src_x1, &src_y1,
474 &dst_x0, &dst_y0, &dst_x1, &dst_y1,
475 mirror_x, mirror_y);
476 if (noop)
477 return;
478 }
479
480 if (abs(info->dst.box.width) == abs(info->src.box.width) &&
481 abs(info->dst.box.height) == abs(info->src.box.height)) {
482 if (info->src.resource->nr_samples > 1 &&
483 info->dst.resource->nr_samples <= 1) {
484 /* The OpenGL ES 3.2 specification, section 16.2.1, says:
485 *
486 * "If the read framebuffer is multisampled (its effective
487 * value of SAMPLE_BUFFERS is one) and the draw framebuffer
488 * is not (its value of SAMPLE_BUFFERS is zero), the samples
489 * corresponding to each pixel location in the source are
490 * converted to a single sample before being written to the
491 * destination. The filter parameter is ignored. If the
492 * source formats are integer types or stencil values, a
493 * single sample’s value is selected for each pixel. If the
494 * source formats are floating-point or normalized types,
495 * the sample values for each pixel are resolved in an
496 * implementation-dependent manner. If the source formats
497 * are depth values, sample values are resolved in an
498 * implementation-dependent manner where the result will be
499 * between the minimum and maximum depth values in the pixel."
500 *
501 * When selecting a single sample, we always choose sample 0.
502 */
503 if (util_format_is_depth_or_stencil(info->src.format) ||
504 util_format_is_pure_integer(info->src.format)) {
505 filter = BLORP_FILTER_SAMPLE_0;
506 } else {
507 filter = BLORP_FILTER_AVERAGE;
508 }
509 } else {
510 /* The OpenGL 4.6 specification, section 18.3.1, says:
511 *
512 * "If the source and destination dimensions are identical,
513 * no filtering is applied."
514 *
515 * Using BLORP_FILTER_NONE will also handle the upsample case by
516 * replicating the one value in the source to all values in the
517 * destination.
518 */
519 filter = BLORP_FILTER_NONE;
520 }
521 } else if (info->filter == PIPE_TEX_FILTER_LINEAR) {
522 filter = BLORP_FILTER_BILINEAR;
523 } else {
524 filter = BLORP_FILTER_NEAREST;
525 }
526
527 struct blorp_batch blorp_batch;
528 blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags);
529
530 float src_z_step = (float)info->src.box.depth / (float)info->dst.box.depth;
531
532 /* There is no interpolation to the pixel center during rendering, so
533 * add the 0.5 offset ourselves here.
534 */
535 float depth_center_offset = 0;
536 if (info->src.resource->target == PIPE_TEXTURE_3D)
537 depth_center_offset = 0.5 / info->dst.box.depth * info->src.box.depth;
538
539 /* Perform a blit for each aspect requested by the caller. PIPE_MASK_R is
540 * used to represent the color aspect. */
541 unsigned aspect_mask = info->mask & (PIPE_MASK_R | PIPE_MASK_ZS);
542 while (aspect_mask) {
543 unsigned aspect = 1 << u_bit_scan(&aspect_mask);
544
545 struct crocus_resource *src_res =
546 crocus_resource_for_aspect(devinfo, info->src.resource, aspect);
547 struct crocus_resource *dst_res =
548 crocus_resource_for_aspect(devinfo, info->dst.resource, aspect);
549
550 enum pipe_format src_pfmt =
551 pipe_format_for_aspect(info->src.format, aspect);
552 enum pipe_format dst_pfmt =
553 pipe_format_for_aspect(info->dst.format, aspect);
554
555 struct crocus_format_info src_fmt =
556 crocus_format_for_usage(devinfo, src_pfmt, ISL_SURF_USAGE_TEXTURE_BIT);
557 enum isl_aux_usage src_aux_usage =
558 crocus_resource_texture_aux_usage(src_res);
559
560 crocus_resource_prepare_texture(ice, src_res, src_fmt.fmt,
561 info->src.level, 1, info->src.box.z,
562 info->src.box.depth);
563 // crocus_emit_buffer_barrier_for(batch, src_res->bo,
564 // CROCUS_DOMAIN_OTHER_READ);
565
566 bool dst_aux_disable = false;
567 /* on SNB blorp will use render target instead of depth
568 * so disable HiZ.
569 */
570 if (devinfo->ver <= 6 && util_format_is_depth_or_stencil(dst_pfmt))
571 dst_aux_disable = true;
572 struct crocus_format_info dst_fmt =
573 crocus_format_for_usage(devinfo, dst_pfmt,
574 ISL_SURF_USAGE_RENDER_TARGET_BIT);
575 enum isl_aux_usage dst_aux_usage =
576 crocus_resource_render_aux_usage(ice, dst_res, info->dst.level,
577 dst_fmt.fmt, dst_aux_disable);
578
579 struct blorp_surf src_surf, dst_surf;
580 crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &src_surf,
581 &src_res->base.b, src_aux_usage,
582 info->src.level, false);
583 crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &dst_surf,
584 &dst_res->base.b, dst_aux_usage,
585 info->dst.level, true);
586
587 crocus_resource_prepare_render(ice, dst_res, info->dst.level,
588 info->dst.box.z, info->dst.box.depth,
589 dst_aux_usage);
590 // crocus_emit_buffer_barrier_for(batch, dst_res->bo,
591 // CROCUS_DOMAIN_RENDER_WRITE);
592
593 if (crocus_batch_references(batch, src_res->bo))
594 tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format);
595
596 if (dst_res->base.b.target == PIPE_BUFFER) {
597 util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range,
598 dst_x0, dst_x1);
599 }
600
601 struct isl_swizzle src_swiz = pipe_to_isl_swizzles(src_fmt.swizzles);
602 struct isl_swizzle dst_swiz = pipe_to_isl_swizzles(dst_fmt.swizzles);
603
604 for (int slice = 0; slice < info->dst.box.depth; slice++) {
605 unsigned dst_z = info->dst.box.z + slice;
606 float src_z = info->src.box.z + slice * src_z_step +
607 depth_center_offset;
608
609 crocus_batch_maybe_flush(batch, 1500);
610
611 blorp_blit(&blorp_batch,
612 &src_surf, info->src.level, src_z,
613 src_fmt.fmt, src_swiz,
614 &dst_surf, info->dst.level, dst_z,
615 dst_fmt.fmt, dst_swiz,
616 src_x0, src_y0, src_x1, src_y1,
617 dst_x0, dst_y0, dst_x1, dst_y1,
618 filter, mirror_x, mirror_y);
619
620 }
621
622 tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format);
623
624 crocus_resource_finish_render(ice, dst_res, info->dst.level,
625 info->dst.box.z, info->dst.box.depth,
626 dst_aux_usage);
627 }
628
629 blorp_batch_finish(&blorp_batch);
630
631 crocus_flush_and_dirty_for_history(ice, batch, (struct crocus_resource *)
632 info->dst.resource,
633 PIPE_CONTROL_RENDER_TARGET_FLUSH,
634 "cache history: post-blit");
635 }
636
637 static void
get_copy_region_aux_settings(struct crocus_resource * res,enum isl_aux_usage * out_aux_usage,bool is_render_target)638 get_copy_region_aux_settings(struct crocus_resource *res,
639 enum isl_aux_usage *out_aux_usage,
640 bool is_render_target)
641 {
642 switch (res->aux.usage) {
643 case ISL_AUX_USAGE_MCS:
644 /* A stencil resolve operation must be performed prior to doing resource
645 * copies or used by CPU.
646 * (see HSD 1209978162)
647 */
648 if (is_render_target && isl_surf_usage_is_stencil(res->surf.usage)) {
649 *out_aux_usage = ISL_AUX_USAGE_NONE;
650 } else {
651 *out_aux_usage = res->aux.usage;
652 }
653 break;
654 default:
655 *out_aux_usage = ISL_AUX_USAGE_NONE;
656 break;
657 }
658 }
659
660 /**
661 * Perform a GPU-based raw memory copy between compatible view classes.
662 *
663 * Does not perform any flushing - the new data may still be left in the
664 * render cache, and old data may remain in other caches.
665 *
666 * Wraps blorp_copy() and blorp_buffer_copy().
667 */
668 void
crocus_copy_region(struct blorp_context * blorp,struct crocus_batch * batch,struct pipe_resource * dst,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,struct pipe_resource * src,unsigned src_level,const struct pipe_box * src_box)669 crocus_copy_region(struct blorp_context *blorp,
670 struct crocus_batch *batch,
671 struct pipe_resource *dst,
672 unsigned dst_level,
673 unsigned dstx, unsigned dsty, unsigned dstz,
674 struct pipe_resource *src,
675 unsigned src_level,
676 const struct pipe_box *src_box)
677 {
678 struct blorp_batch blorp_batch;
679 struct crocus_context *ice = blorp->driver_ctx;
680 struct crocus_screen *screen = (void *) ice->ctx.screen;
681 const struct intel_device_info *devinfo = &screen->devinfo;
682 struct crocus_resource *src_res = (void *) src;
683 struct crocus_resource *dst_res = (void *) dst;
684
685 if (devinfo->ver <= 5) {
686 if (screen->vtbl.copy_region_blt(batch, dst_res,
687 dst_level, dstx, dsty, dstz,
688 src_res, src_level, src_box))
689 return;
690 }
691 enum isl_aux_usage src_aux_usage, dst_aux_usage;
692 get_copy_region_aux_settings(src_res, &src_aux_usage,
693 false);
694 get_copy_region_aux_settings(dst_res, &dst_aux_usage,
695 true);
696
697 if (crocus_batch_references(batch, src_res->bo))
698 tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
699
700 if (dst->target == PIPE_BUFFER)
701 util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range, dstx, dstx + src_box->width);
702
703 if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
704 struct blorp_address src_addr = {
705 .buffer = crocus_resource_bo(src), .offset = src_box->x,
706 .mocs = crocus_mocs(src_res->bo, &screen->isl_dev),
707 };
708 struct blorp_address dst_addr = {
709 .buffer = crocus_resource_bo(dst), .offset = dstx,
710 .mocs = crocus_mocs(dst_res->bo, &screen->isl_dev),
711 .reloc_flags = EXEC_OBJECT_WRITE,
712 };
713
714 crocus_batch_maybe_flush(batch, 1500);
715
716 blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
717 blorp_buffer_copy(&blorp_batch, src_addr, dst_addr, src_box->width);
718 blorp_batch_finish(&blorp_batch);
719 } else {
720 // XXX: what about one surface being a buffer and not the other?
721
722 struct blorp_surf src_surf, dst_surf;
723 crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &src_surf,
724 src, src_aux_usage, src_level, false);
725 crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &dst_surf,
726 dst, dst_aux_usage, dst_level, true);
727
728 crocus_resource_prepare_access(ice, src_res, src_level, 1,
729 src_box->z, src_box->depth,
730 src_aux_usage, false);
731 crocus_resource_prepare_access(ice, dst_res, dst_level, 1,
732 dstz, src_box->depth,
733 dst_aux_usage, false);
734
735 blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
736
737 for (int slice = 0; slice < src_box->depth; slice++) {
738 crocus_batch_maybe_flush(batch, 1500);
739
740 blorp_copy(&blorp_batch, &src_surf, src_level, src_box->z + slice,
741 &dst_surf, dst_level, dstz + slice,
742 src_box->x, src_box->y, dstx, dsty,
743 src_box->width, src_box->height);
744 }
745 blorp_batch_finish(&blorp_batch);
746
747 crocus_resource_finish_write(ice, dst_res, dst_level, dstz,
748 src_box->depth, dst_aux_usage);
749 }
750
751 tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
752 }
753
754 /**
755 * The pipe->resource_copy_region() driver hook.
756 *
757 * This implements ARB_copy_image semantics - a raw memory copy between
758 * compatible view classes.
759 */
760 static void
crocus_resource_copy_region(struct pipe_context * ctx,struct pipe_resource * p_dst,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,struct pipe_resource * p_src,unsigned src_level,const struct pipe_box * src_box)761 crocus_resource_copy_region(struct pipe_context *ctx,
762 struct pipe_resource *p_dst,
763 unsigned dst_level,
764 unsigned dstx, unsigned dsty, unsigned dstz,
765 struct pipe_resource *p_src,
766 unsigned src_level,
767 const struct pipe_box *src_box)
768 {
769 struct crocus_context *ice = (void *) ctx;
770 struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER];
771 struct crocus_screen *screen = (struct crocus_screen *)ctx->screen;
772 const struct intel_device_info *devinfo = &screen->devinfo;
773 struct crocus_resource *dst = (void *) p_dst;
774
775 if (devinfo->ver < 6 && util_format_is_depth_or_stencil(p_dst->format)) {
776 util_resource_copy_region(ctx, p_dst, dst_level, dstx, dsty, dstz,
777 p_src, src_level, src_box);
778 return;
779 }
780 crocus_copy_region(&ice->blorp, batch, p_dst, dst_level, dstx, dsty, dstz,
781 p_src, src_level, src_box);
782
783 if (util_format_is_depth_and_stencil(p_dst->format) &&
784 util_format_has_stencil(util_format_description(p_src->format)) &&
785 devinfo->ver >= 6) {
786 struct crocus_resource *junk, *s_src_res, *s_dst_res;
787 crocus_get_depth_stencil_resources(devinfo, p_src, &junk, &s_src_res);
788 crocus_get_depth_stencil_resources(devinfo, p_dst, &junk, &s_dst_res);
789
790 crocus_copy_region(&ice->blorp, batch, &s_dst_res->base.b, dst_level, dstx,
791 dsty, dstz, &s_src_res->base.b, src_level, src_box);
792 }
793
794 crocus_flush_and_dirty_for_history(ice, batch, dst,
795 PIPE_CONTROL_RENDER_TARGET_FLUSH,
796 "cache history: post copy_region");
797 }
798
799 void
crocus_init_blit_functions(struct pipe_context * ctx)800 crocus_init_blit_functions(struct pipe_context *ctx)
801 {
802 ctx->blit = crocus_blit;
803 ctx->resource_copy_region = crocus_resource_copy_region;
804 }
805